datago 2026.1.2__tar.gz → 2026.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. {datago-2026.1.2 → datago-2026.2.1}/.github/workflows/ci-cd.yml +35 -11
  2. datago-2026.2.1/.pre-commit-config.yaml +25 -0
  3. {datago-2026.1.2 → datago-2026.2.1}/Cargo.lock +797 -785
  4. {datago-2026.1.2 → datago-2026.2.1}/Cargo.toml +3 -3
  5. {datago-2026.1.2 → datago-2026.2.1}/PKG-INFO +3 -2
  6. datago-2026.2.1/assets/zen3_ssd.png +0 -0
  7. {datago-2026.1.2 → datago-2026.2.1}/pyproject.toml +22 -2
  8. {datago-2026.1.2 → datago-2026.2.1}/python/benchmark_filesystem.py +4 -4
  9. {datago-2026.1.2 → datago-2026.2.1}/python/benchmark_webdataset.py +17 -13
  10. {datago-2026.1.2 → datago-2026.2.1}/python/dataset.py +9 -31
  11. {datago-2026.1.2 → datago-2026.2.1}/python/test_datago_db.py +0 -3
  12. {datago-2026.1.2 → datago-2026.2.1}/python/test_datago_edge_cases.py +0 -1
  13. {datago-2026.1.2 → datago-2026.2.1}/python/test_datago_wds.py +17 -6
  14. {datago-2026.1.2 → datago-2026.2.1}/src/client.rs +14 -5
  15. {datago-2026.1.2 → datago-2026.2.1}/src/image_processing.rs +106 -1
  16. {datago-2026.1.2 → datago-2026.2.1}/src/structs.rs +100 -21
  17. {datago-2026.1.2 → datago-2026.2.1}/src/worker_files.rs +170 -0
  18. {datago-2026.1.2 → datago-2026.2.1}/src/worker_http.rs +2 -5
  19. datago-2026.2.1/uv.lock +2468 -0
  20. datago-2026.1.2/.pre-commit-config.yaml +0 -14
  21. datago-2026.1.2/assets/zen3_ssd.png +0 -0
  22. datago-2026.1.2/requirements-dev.txt +0 -3
  23. datago-2026.1.2/requirements-tests.txt +0 -8
  24. datago-2026.1.2/requirements.txt +0 -1
  25. {datago-2026.1.2 → datago-2026.2.1}/.github/workflows/rust.yml +0 -0
  26. {datago-2026.1.2 → datago-2026.2.1}/.gitignore +0 -0
  27. {datago-2026.1.2 → datago-2026.2.1}/LICENSE +0 -0
  28. {datago-2026.1.2 → datago-2026.2.1}/README.md +0 -0
  29. {datago-2026.1.2 → datago-2026.2.1}/assets/447175851-2277afcb-8abf-4d17-b2db-dae27c6056d0.png +0 -0
  30. {datago-2026.1.2 → datago-2026.2.1}/assets/epyc_vast.png +0 -0
  31. {datago-2026.1.2 → datago-2026.2.1}/assets/epyc_wds_pd12m.png +0 -0
  32. {datago-2026.1.2 → datago-2026.2.1}/assets/zen3_wds_fakein.png.png +0 -0
  33. {datago-2026.1.2 → datago-2026.2.1}/assets/zen3_wds_pd12m.png +0 -0
  34. {datago-2026.1.2 → datago-2026.2.1}/assets/zen3_wds_pd12m_processing.png +0 -0
  35. {datago-2026.1.2 → datago-2026.2.1}/python/benchmark_db.py +0 -0
  36. {datago-2026.1.2 → datago-2026.2.1}/python/benchmark_defaults.py +0 -0
  37. {datago-2026.1.2 → datago-2026.2.1}/python/raw_types.py +0 -0
  38. {datago-2026.1.2 → datago-2026.2.1}/python/test_datago_client.py +0 -0
  39. {datago-2026.1.2 → datago-2026.2.1}/python/test_datago_filesystem.py +0 -0
  40. {datago-2026.1.2 → datago-2026.2.1}/python/test_pil_implicit_conversion.py +0 -0
  41. {datago-2026.1.2 → datago-2026.2.1}/src/generator_files.rs +0 -0
  42. {datago-2026.1.2 → datago-2026.2.1}/src/generator_http.rs +0 -0
  43. {datago-2026.1.2 → datago-2026.2.1}/src/generator_wds.rs +0 -0
  44. {datago-2026.1.2 → datago-2026.2.1}/src/lib.rs +0 -0
  45. {datago-2026.1.2 → datago-2026.2.1}/src/main.rs +0 -0
  46. {datago-2026.1.2 → datago-2026.2.1}/src/worker_wds.rs +0 -0
@@ -13,8 +13,30 @@ on:
13
13
 
14
14
  permissions:
15
15
  contents: read
16
+ env:
17
+ UV_SYSTEM_PYTHON: 1
16
18
 
17
19
  jobs:
20
+ prek:
21
+ runs-on: ubuntu-latest
22
+ steps:
23
+ - uses: actions/checkout@v6
24
+
25
+ - uses: actions/setup-python@v6
26
+ with:
27
+ python-version: "3.13"
28
+
29
+ - name: Install uv
30
+ uses: astral-sh/setup-uv@v7
31
+
32
+ - name: Install ty
33
+ run: uv tool install ty@latest
34
+
35
+ - name: Install ruff
36
+ run: uv tool install ruff@latest
37
+
38
+ - uses: j178/prek-action@v1
39
+
18
40
  linux:
19
41
  runs-on: ${{ matrix.platform.runner }}
20
42
  strategy:
@@ -22,23 +44,26 @@ jobs:
22
44
  platform:
23
45
  - runner: ubuntu-latest
24
46
  target: x86_64
25
- python-version: ['3.10', '3.11', '3.12', '3.13']
47
+ python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
26
48
 
27
49
  environment:
28
50
  name: release
29
51
  url: https://pypi.org/p/datago
30
52
 
31
53
  steps:
32
- - uses: actions/checkout@v4
54
+ - uses: actions/checkout@v6
33
55
  - run: git fetch --prune --unshallow
34
56
 
35
57
  - uses: actions/setup-python@v5
36
58
  with:
37
59
  python-version: ${{ matrix.python-version }}
38
60
 
61
+ - name: Install uv
62
+ uses: astral-sh/setup-uv@v7
63
+
39
64
  - name: Install deps
40
65
  run: |
41
- python3 -m pip install maturin twine
66
+ uv tool install maturin
42
67
  sudo snap install zig --classic --beta
43
68
 
44
69
  # Gather the name of the latest tag on the current main branch
@@ -56,14 +81,13 @@ jobs:
56
81
  DATAROOM_TEST_SOURCE: ${{ secrets.DATAROOM_TEST_SOURCE }}
57
82
  DATAROOM_API_URL: ${{ secrets.DATAROOM_API_URL }}
58
83
  run: |
59
- if [ "${{ matrix.python-version }}" == "3.11" ]; then
60
- python3 -m pip install dist/*.whl
61
- python3 -m pip install -r requirements-tests.txt
62
- cd python
63
- python3 -m pytest -v .
64
- else
65
- echo "Skipping tests for Python ${{ matrix.python-version }}, only testing Python 3.11"
66
- fi
84
+ if [ "${{ matrix.python-version }}" == "3.11" ]; then
85
+ uv pip install dist/*.whl
86
+ cd python
87
+ uv run --group test pytest -v .
88
+ else
89
+ echo "Skipping tests for Python ${{ matrix.python-version }}, only testing Python 3.11"
90
+ fi
67
91
 
68
92
  - name: Upload wheels
69
93
  uses: actions/upload-artifact@v4
@@ -0,0 +1,25 @@
1
+ repos:
2
+ - repo: builtin
3
+ hooks:
4
+ - id: trailing-whitespace
5
+ - id: end-of-file-fixer
6
+ - id: check-yaml
7
+ - repo: https://github.com/doublify/pre-commit-rust
8
+ rev: v1.0
9
+ hooks:
10
+ - id: fmt
11
+ - id: cargo-check
12
+ - id: clippy
13
+ args: ["--", "-D", "warnings"]
14
+ - repo: https://github.com/charliermarsh/ruff-pre-commit
15
+ rev: "v0.14.14"
16
+ hooks:
17
+ - id: ruff
18
+ name: ruff
19
+ args: [--fix, --exit-non-zero-on-fix]
20
+ - id: ruff-format
21
+
22
+ - repo: https://github.com/NSPBot911/ty-pre-commit
23
+ rev: v0.0.14
24
+ hooks:
25
+ - id: ty-check