zoopipe 2026.1.20__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of zoopipe might be problematic. Click here for more details.

Files changed (146) hide show
  1. zoopipe-2026.1.20/.cargo/config.toml +5 -0
  2. zoopipe-2026.1.20/.github/workflows/ci.yml +66 -0
  3. zoopipe-2026.1.20/.github/workflows/release.yml +141 -0
  4. zoopipe-2026.1.20/.gitignore +29 -0
  5. zoopipe-2026.1.20/.pre-commit-config.yaml +31 -0
  6. zoopipe-2026.1.20/.python-version +1 -0
  7. zoopipe-2026.1.20/.rayignore +19 -0
  8. zoopipe-2026.1.20/.readthedocs.yaml +17 -0
  9. zoopipe-2026.1.20/.vscode/settings.json +7 -0
  10. zoopipe-2026.1.20/Cargo.lock +4640 -0
  11. zoopipe-2026.1.20/Cargo.toml +43 -0
  12. zoopipe-2026.1.20/LICENSE +21 -0
  13. zoopipe-2026.1.20/PKG-INFO +231 -0
  14. zoopipe-2026.1.20/README.md +211 -0
  15. zoopipe-2026.1.20/benches/zoopipe_benchmarks.rs +510 -0
  16. zoopipe-2026.1.20/docs/ADR/ADR-000-architectural-decision-records.md +27 -0
  17. zoopipe-2026.1.20/docs/ADR/ADR-001-hybrid-python-rust-architecture.md +24 -0
  18. zoopipe-2026.1.20/docs/ADR/ADR-002-batch-based-pipeline-processing.md +23 -0
  19. zoopipe-2026.1.20/docs/ADR/ADR-003-pluggable-adapter-system.md +25 -0
  20. zoopipe-2026.1.20/docs/ADR/ADR-004-lifecycle-hooks.md +25 -0
  21. zoopipe-2026.1.20/docs/ADR/ADR-005-pydantic-validation.md +23 -0
  22. zoopipe-2026.1.20/docs/ADR/ADR-006-multi-process-parallelism.md +23 -0
  23. zoopipe-2026.1.20/docs/ADR/ADR-007-error-routing.md +24 -0
  24. zoopipe-2026.1.20/docs/arrow.md +440 -0
  25. zoopipe-2026.1.20/docs/assets/benchmark.svg +131 -0
  26. zoopipe-2026.1.20/docs/assets/favicon.svg +23 -0
  27. zoopipe-2026.1.20/docs/assets/logo-dark.svg +65 -0
  28. zoopipe-2026.1.20/docs/assets/logo-light.svg +54 -0
  29. zoopipe-2026.1.20/docs/cloud-storage.md +565 -0
  30. zoopipe-2026.1.20/docs/csv.md +280 -0
  31. zoopipe-2026.1.20/docs/dask.md +62 -0
  32. zoopipe-2026.1.20/docs/duckdb.md +433 -0
  33. zoopipe-2026.1.20/docs/excel.md +73 -0
  34. zoopipe-2026.1.20/docs/executors.md +150 -0
  35. zoopipe-2026.1.20/docs/hooks.md +156 -0
  36. zoopipe-2026.1.20/docs/index.md +208 -0
  37. zoopipe-2026.1.20/docs/json.md +326 -0
  38. zoopipe-2026.1.20/docs/kafka.md +92 -0
  39. zoopipe-2026.1.20/docs/parquet.md +472 -0
  40. zoopipe-2026.1.20/docs/pipemanager.md +310 -0
  41. zoopipe-2026.1.20/docs/pygen.md +567 -0
  42. zoopipe-2026.1.20/docs/ray.md +67 -0
  43. zoopipe-2026.1.20/docs/sql.md +289 -0
  44. zoopipe-2026.1.20/docs-requirements.txt +3 -0
  45. zoopipe-2026.1.20/examples/01_basic_csv.py +49 -0
  46. zoopipe-2026.1.20/examples/02_jsonl_to_csv.py +37 -0
  47. zoopipe-2026.1.20/examples/03_executor_comparison.py +70 -0
  48. zoopipe-2026.1.20/examples/04_csv_to_duckdb.py +40 -0
  49. zoopipe-2026.1.20/examples/05_duckdb_to_jsonl.py +41 -0
  50. zoopipe-2026.1.20/examples/06_csv_to_arrow.py +38 -0
  51. zoopipe-2026.1.20/examples/07_arrow_to_jsonl.py +40 -0
  52. zoopipe-2026.1.20/examples/08_csv_to_sql.py +44 -0
  53. zoopipe-2026.1.20/examples/09_sql_to_jsonl.py +45 -0
  54. zoopipe-2026.1.20/examples/10_csv_to_generator.py +39 -0
  55. zoopipe-2026.1.20/examples/11_csv_to_parquet.py +37 -0
  56. zoopipe-2026.1.20/examples/12_parquet_to_jsonl.py +37 -0
  57. zoopipe-2026.1.20/examples/13_pipemanager.py +95 -0
  58. zoopipe-2026.1.20/examples/14_basic_excel.py +53 -0
  59. zoopipe-2026.1.20/examples/15_csv_to_excel.py +62 -0
  60. zoopipe-2026.1.20/examples/16_kafka_adapter.py +65 -0
  61. zoopipe-2026.1.20/examples/17_cursor_pagination.py +86 -0
  62. zoopipe-2026.1.20/examples/README.md +189 -0
  63. zoopipe-2026.1.20/examples/__init__.py +0 -0
  64. zoopipe-2026.1.20/lint.sh +3 -0
  65. zoopipe-2026.1.20/mkdocs.yml +72 -0
  66. zoopipe-2026.1.20/pyproject.toml +56 -0
  67. zoopipe-2026.1.20/src/zoopipe/__init__.py +72 -0
  68. zoopipe-2026.1.20/src/zoopipe/engines/__init__.py +4 -0
  69. zoopipe-2026.1.20/src/zoopipe/engines/base.py +45 -0
  70. zoopipe-2026.1.20/src/zoopipe/engines/dask.py +225 -0
  71. zoopipe-2026.1.20/src/zoopipe/engines/local.py +215 -0
  72. zoopipe-2026.1.20/src/zoopipe/engines/ray.py +252 -0
  73. zoopipe-2026.1.20/src/zoopipe/hooks/__init__.py +4 -0
  74. zoopipe-2026.1.20/src/zoopipe/hooks/base.py +70 -0
  75. zoopipe-2026.1.20/src/zoopipe/hooks/sql.py +94 -0
  76. zoopipe-2026.1.20/src/zoopipe/input_adapter/__init__.py +24 -0
  77. zoopipe-2026.1.20/src/zoopipe/input_adapter/arrow.py +38 -0
  78. zoopipe-2026.1.20/src/zoopipe/input_adapter/base.py +48 -0
  79. zoopipe-2026.1.20/src/zoopipe/input_adapter/csv.py +144 -0
  80. zoopipe-2026.1.20/src/zoopipe/input_adapter/duckdb.py +54 -0
  81. zoopipe-2026.1.20/src/zoopipe/input_adapter/excel.py +51 -0
  82. zoopipe-2026.1.20/src/zoopipe/input_adapter/json.py +73 -0
  83. zoopipe-2026.1.20/src/zoopipe/input_adapter/kafka.py +39 -0
  84. zoopipe-2026.1.20/src/zoopipe/input_adapter/parquet.py +85 -0
  85. zoopipe-2026.1.20/src/zoopipe/input_adapter/pygen.py +37 -0
  86. zoopipe-2026.1.20/src/zoopipe/input_adapter/sql.py +103 -0
  87. zoopipe-2026.1.20/src/zoopipe/manager.py +211 -0
  88. zoopipe-2026.1.20/src/zoopipe/output_adapter/__init__.py +23 -0
  89. zoopipe-2026.1.20/src/zoopipe/output_adapter/arrow.py +50 -0
  90. zoopipe-2026.1.20/src/zoopipe/output_adapter/base.py +41 -0
  91. zoopipe-2026.1.20/src/zoopipe/output_adapter/csv.py +71 -0
  92. zoopipe-2026.1.20/src/zoopipe/output_adapter/duckdb.py +46 -0
  93. zoopipe-2026.1.20/src/zoopipe/output_adapter/excel.py +42 -0
  94. zoopipe-2026.1.20/src/zoopipe/output_adapter/json.py +66 -0
  95. zoopipe-2026.1.20/src/zoopipe/output_adapter/kafka.py +39 -0
  96. zoopipe-2026.1.20/src/zoopipe/output_adapter/parquet.py +49 -0
  97. zoopipe-2026.1.20/src/zoopipe/output_adapter/pygen.py +29 -0
  98. zoopipe-2026.1.20/src/zoopipe/output_adapter/sql.py +43 -0
  99. zoopipe-2026.1.20/src/zoopipe/pipe.py +263 -0
  100. zoopipe-2026.1.20/src/zoopipe/protocols.py +37 -0
  101. zoopipe-2026.1.20/src/zoopipe/py.typed +0 -0
  102. zoopipe-2026.1.20/src/zoopipe/report.py +173 -0
  103. zoopipe-2026.1.20/src/zoopipe/utils/__init__.py +0 -0
  104. zoopipe-2026.1.20/src/zoopipe/utils/dependency.py +78 -0
  105. zoopipe-2026.1.20/src/zoopipe_core/src/error.rs +132 -0
  106. zoopipe-2026.1.20/src/zoopipe_core/src/executor/mod.rs +5 -0
  107. zoopipe-2026.1.20/src/zoopipe_core/src/executor/multithread.rs +141 -0
  108. zoopipe-2026.1.20/src/zoopipe_core/src/executor/strategy.rs +145 -0
  109. zoopipe-2026.1.20/src/zoopipe_core/src/io/mod.rs +641 -0
  110. zoopipe-2026.1.20/src/zoopipe_core/src/io/smart_reader.rs +58 -0
  111. zoopipe-2026.1.20/src/zoopipe_core/src/io/storage.rs +140 -0
  112. zoopipe-2026.1.20/src/zoopipe_core/src/lib.rs +73 -0
  113. zoopipe-2026.1.20/src/zoopipe_core/src/parsers/arrow.rs +234 -0
  114. zoopipe-2026.1.20/src/zoopipe_core/src/parsers/arrow_utils.rs +135 -0
  115. zoopipe-2026.1.20/src/zoopipe_core/src/parsers/csv.rs +558 -0
  116. zoopipe-2026.1.20/src/zoopipe_core/src/parsers/duckdb.rs +411 -0
  117. zoopipe-2026.1.20/src/zoopipe_core/src/parsers/excel.rs +398 -0
  118. zoopipe-2026.1.20/src/zoopipe_core/src/parsers/json.rs +430 -0
  119. zoopipe-2026.1.20/src/zoopipe_core/src/parsers/kafka.rs +283 -0
  120. zoopipe-2026.1.20/src/zoopipe_core/src/parsers/mod.rs +22 -0
  121. zoopipe-2026.1.20/src/zoopipe_core/src/parsers/parquet.rs +422 -0
  122. zoopipe-2026.1.20/src/zoopipe_core/src/parsers/pygen.rs +186 -0
  123. zoopipe-2026.1.20/src/zoopipe_core/src/parsers/sql.rs +433 -0
  124. zoopipe-2026.1.20/src/zoopipe_core/src/pipeline.rs +467 -0
  125. zoopipe-2026.1.20/src/zoopipe_core/src/utils/interning.rs +38 -0
  126. zoopipe-2026.1.20/src/zoopipe_core/src/utils/mod.rs +111 -0
  127. zoopipe-2026.1.20/tests/__init__.py +0 -0
  128. zoopipe-2026.1.20/tests/test_arrow.py +59 -0
  129. zoopipe-2026.1.20/tests/test_core.py +329 -0
  130. zoopipe-2026.1.20/tests/test_csv.py +152 -0
  131. zoopipe-2026.1.20/tests/test_dask_engine.py +132 -0
  132. zoopipe-2026.1.20/tests/test_duckdb.py +95 -0
  133. zoopipe-2026.1.20/tests/test_engines.py +122 -0
  134. zoopipe-2026.1.20/tests/test_excel.py +253 -0
  135. zoopipe-2026.1.20/tests/test_executors.py +26 -0
  136. zoopipe-2026.1.20/tests/test_integration.py +92 -0
  137. zoopipe-2026.1.20/tests/test_json.py +178 -0
  138. zoopipe-2026.1.20/tests/test_kafka_adapter.py +48 -0
  139. zoopipe-2026.1.20/tests/test_parquet.py +104 -0
  140. zoopipe-2026.1.20/tests/test_pipemanager.py +570 -0
  141. zoopipe-2026.1.20/tests/test_pygen.py +67 -0
  142. zoopipe-2026.1.20/tests/test_ray_engine.py +104 -0
  143. zoopipe-2026.1.20/tests/test_report.py +33 -0
  144. zoopipe-2026.1.20/tests/test_shutdown.py +98 -0
  145. zoopipe-2026.1.20/tests/test_sql.py +88 -0
  146. zoopipe-2026.1.20/uv.lock +1417 -0
@@ -0,0 +1,5 @@
1
+ [target.aarch64-apple-darwin]
2
+ rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
3
+
4
+ [target.x86_64-apple-darwin]
5
+ rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
@@ -0,0 +1,66 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [ main, master, dev ]
6
+ pull_request:
7
+ branches: [ main, master, dev ]
8
+ workflow_dispatch:
9
+
10
+ permissions:
11
+ contents: read
12
+
13
+ jobs:
14
+ rust:
15
+ name: Rust
16
+ runs-on: ubuntu-latest
17
+ steps:
18
+ - uses: actions/checkout@v4
19
+
20
+ - name: Install Rust
21
+ uses: dtolnay/rust-toolchain@stable
22
+ with:
23
+ components: clippy
24
+
25
+ - name: Rust Cache
26
+ uses: Swatinem/rust-cache@v2
27
+
28
+ - name: Run Clippy
29
+ run: cargo clippy --all-targets --all-features -- -D warnings
30
+
31
+ - name: Run Rust Tests
32
+ run: cargo test --all-features
33
+
34
+ python:
35
+ name: Python
36
+ runs-on: ubuntu-latest
37
+ needs: rust
38
+ steps:
39
+ - uses: actions/checkout@v4
40
+
41
+ - name: Install Rust
42
+ uses: dtolnay/rust-toolchain@stable
43
+
44
+ - name: Rust Cache
45
+ uses: Swatinem/rust-cache@v2
46
+
47
+ - name: Install uv
48
+ uses: astral-sh/setup-uv@v5
49
+ with:
50
+ enable-cache: true
51
+ cache-dependency-glob: "uv.lock"
52
+
53
+ - name: Set up Python
54
+ run: uv python install 3.10
55
+
56
+ - name: Install dependencies
57
+ run: uv sync --all-extras --dev
58
+
59
+ - name: Build extension (maturin develop)
60
+ run: uv run maturin develop
61
+
62
+ - name: Lint with Ruff
63
+ run: uv run ruff check .
64
+
65
+ - name: Run tests
66
+ run: uv run pytest
@@ -0,0 +1,141 @@
1
+ name: Release
2
+
3
+ on:
4
+ release:
5
+ types: [published]
6
+ workflow_dispatch:
7
+
8
+ permissions:
9
+ contents: read
10
+
11
+ jobs:
12
+ linux:
13
+ runs-on: ubuntu-latest
14
+ strategy:
15
+ matrix:
16
+ include:
17
+ - target: x86_64
18
+ sccache: 'true'
19
+ - target: aarch64
20
+ sccache: 'true'
21
+ steps:
22
+ - uses: actions/checkout@v4
23
+ - name: Rust Cache
24
+ uses: Swatinem/rust-cache@v2
25
+ - name: Build wheels
26
+ uses: PyO3/maturin-action@v1.49.4
27
+ with:
28
+ target: ${{ matrix.target }}
29
+ args: --release --out dist --find-interpreter
30
+ sccache: ${{ matrix.sccache }}
31
+ manylinux: 2_28
32
+ zig: true
33
+ uv: true
34
+ before-script-linux: |
35
+ if command -v microdnf &> /dev/null; then
36
+ microdnf install -y perl-IPC-Cmd perl-Time-Piece
37
+ elif command -v dnf &> /dev/null; then
38
+ dnf install -y perl-IPC-Cmd perl-Time-Piece
39
+ elif command -v yum &> /dev/null; then
40
+ yum install -y perl-IPC-Cmd perl-Time-Piece
41
+ elif command -v apt-get &> /dev/null; then
42
+ apt-get update && apt-get install -y perl-modules libtime-piece-perl
43
+ else
44
+ echo "No supported package manager found"
45
+ exit 1
46
+ fi
47
+ - name: Upload wheels
48
+ uses: actions/upload-artifact@v4
49
+ with:
50
+ name: wheels-linux-${{ matrix.target }}
51
+ path: dist
52
+
53
+ windows:
54
+ runs-on: windows-latest
55
+ strategy:
56
+ matrix:
57
+ target: [x64]
58
+ steps:
59
+ - uses: actions/checkout@v4
60
+ - uses: actions/setup-python@v5
61
+ with:
62
+ python-version: '3.10'
63
+ - name: Rust Cache
64
+ uses: Swatinem/rust-cache@v2
65
+ - name: Build wheels
66
+ uses: PyO3/maturin-action@v1.49.4
67
+ with:
68
+ target: ${{ matrix.target }}
69
+ args: --release --out dist --find-interpreter
70
+ sccache: 'true'
71
+ uv: true
72
+ - name: Upload wheels
73
+ uses: actions/upload-artifact@v4
74
+ with:
75
+ name: wheels-windows-${{ matrix.target }}
76
+ path: dist
77
+
78
+ macos:
79
+ runs-on: macos-latest
80
+ strategy:
81
+ matrix:
82
+ target: [x86_64, aarch64]
83
+ steps:
84
+ - uses: actions/checkout@v4
85
+ - uses: actions/setup-python@v5
86
+ with:
87
+ python-version: '3.10'
88
+ - name: Rust Cache
89
+ uses: Swatinem/rust-cache@v2
90
+ - name: Build wheels
91
+ uses: PyO3/maturin-action@v1.49.4
92
+ with:
93
+ target: ${{ matrix.target }}
94
+ args: --release --out dist --find-interpreter
95
+ sccache: 'true'
96
+ uv: true
97
+ - name: Upload wheels
98
+ uses: actions/upload-artifact@v4
99
+ with:
100
+ name: wheels-macos-${{ matrix.target }}
101
+ path: dist
102
+
103
+ sdist:
104
+ runs-on: ubuntu-latest
105
+ steps:
106
+ - uses: actions/checkout@v4
107
+ - name: Build sdist
108
+ uses: PyO3/maturin-action@v1.49.4
109
+ with:
110
+ command: sdist
111
+ args: --out dist
112
+ uv: true
113
+ - name: Upload sdist
114
+ uses: actions/upload-artifact@v4
115
+ with:
116
+ name: sdist
117
+ path: dist
118
+
119
+ release:
120
+ name: Release
121
+ runs-on: ubuntu-latest
122
+ if: "startsWith(github.ref, 'refs/tags/')"
123
+ needs: [linux, windows, macos, sdist]
124
+ permissions:
125
+ id-token: write # Mandatory for trusted publishing
126
+ steps:
127
+ - name: Download all wheels
128
+ uses: actions/download-artifact@v4
129
+ with:
130
+ path: dist
131
+ pattern: wheels-*
132
+ merge-multiple: true
133
+ - name: Download sdist
134
+ uses: actions/download-artifact@v4
135
+ with:
136
+ name: sdist
137
+ path: dist
138
+ - name: Publish to PyPI
139
+ uses: pypa/gh-action-pypi-publish@release/v1
140
+ with:
141
+ packages-dir: dist/
@@ -0,0 +1,29 @@
1
+ # Python-generated files
2
+ __pycache__/
3
+ *.py[oc]
4
+ build/
5
+ dist/
6
+ wheels/
7
+ *.egg-info
8
+ .pytest_cache/
9
+ .ruff_cache/
10
+ target/
11
+
12
+ # Virtual environments
13
+ .venv
14
+ .env
15
+
16
+ # IDEs
17
+ .vscode/
18
+ .idea/
19
+ .DS_Store
20
+
21
+ # Test files
22
+ examples/output_data/*
23
+ examples/sample_data/*
24
+
25
+ # Rust Core
26
+ src/zoopipe/zoopipe_rust_core.abi3.so
27
+
28
+ # Documentation
29
+ site/
@@ -0,0 +1,31 @@
1
+ repos:
2
+ - repo: local
3
+ hooks:
4
+ - id: python-lint
5
+ name: Python Lint
6
+ entry: sh lint.sh
7
+ language: system
8
+ types: [python]
9
+ pass_filenames: false
10
+
11
+ - id: python-tests
12
+ name: Python Tests
13
+ entry: uv run pytest
14
+ language: system
15
+ types: [python]
16
+ pass_filenames: false
17
+ # Ejecutar tests puede ser lento, pero el usuario lo pidió explícitamente en el pre-commit
18
+
19
+ - id: rust-lint
20
+ name: Rust Lint
21
+ entry: cargo clippy --all-targets --all-features -- -D warnings
22
+ language: system
23
+ types_or: [rust, toml] # Correr si cambian archivos rust o Cargo.toml
24
+ pass_filenames: false
25
+
26
+ - id: rust-tests
27
+ name: Rust Tests
28
+ entry: cargo test
29
+ language: system
30
+ types_or: [rust, toml]
31
+ pass_filenames: false
@@ -0,0 +1 @@
1
+ 3.10
@@ -0,0 +1,19 @@
1
+ .git
2
+ .venv
3
+ target
4
+ dist
5
+ __pycache__
6
+ .pytest_cache
7
+ .ruff_cache
8
+ build
9
+
10
+ # Exclude build manifests to stop Ray from attempting to install/build the package
11
+ pyproject.toml
12
+ uv.lock
13
+ Cargo.toml
14
+ Cargo.lock
15
+ maturin.toml
16
+
17
+ # Do NOT exclude .so files, we want to ship the pre-compiled binary
18
+ !**/*.so
19
+ !**/*.abi3.so
@@ -0,0 +1,17 @@
1
+ version: 2
2
+
3
+ mkdocs:
4
+ configuration: mkdocs.yml
5
+
6
+ build:
7
+ os: ubuntu-22.04
8
+ tools:
9
+ python: "3.10"
10
+ rust: "latest"
11
+ jobs:
12
+ post_install:
13
+ - pip install uv
14
+ - uv pip install -r docs-requirements.txt
15
+ build:
16
+ html:
17
+ - mkdocs build -d $READTHEDOCS_OUTPUT/html
@@ -0,0 +1,7 @@
1
+ {
2
+ "python.testing.pytestArgs": [
3
+ "."
4
+ ],
5
+ "python.testing.unittestEnabled": false,
6
+ "python.testing.pytestEnabled": true
7
+ }