de-onager 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,25 @@
1
+ Metadata-Version: 2.5
2
+ Name: de-onager
3
+ Version: 0.2.2
4
+ Summary: Onager — safe local Parquet small-file compaction
5
+ Project-URL: Homepage, https://github.com/KooEric/de-arsenal
6
+ Project-URL: Repository, https://github.com/KooEric/de-arsenal
7
+ License-Expression: Apache-2.0
8
+ Classifier: License :: OSI Approved :: Apache Software License
9
+ Classifier: Operating System :: OS Independent
10
+ Classifier: Programming Language :: Python :: 3
11
+ Classifier: Programming Language :: Python :: 3.11
12
+ Classifier: Programming Language :: Python :: 3.12
13
+ Requires-Python: >=3.11
14
+ Requires-Dist: arsenal-core<0.3,>=0.2
15
+ Requires-Dist: de-scutum<0.3,>=0.2
16
+ Requires-Dist: duckdb>=1.0
17
+ Requires-Dist: typer>=0.12
18
+ Description-Content-Type: text/markdown
19
+
20
+ # Onager
21
+
22
+ 로컬 Parquet small-file compaction과 명시적 promote 방식의 격리 백필을 제공한다.
23
+ 실패한 백필 workspace는 상태 증거와 함께 남는다.
24
+
25
+ PyPI 배포명은 `de-onager`이며 Python import와 CLI 명령은 `onager`를 유지한다.
@@ -0,0 +1,9 @@
1
+ onager/__init__.py,sha256=c3f-BnPyj18Bx4-FaqEcIl7P0UApiW6pATyV_YXFzMY,626
2
+ onager/backfill.py,sha256=n2smc1sGclgP4stpGIDKHdhSyNgjdFUAFQKfwcwPxsM,3569
3
+ onager/cli.py,sha256=rpGtJV360dvOK1IJQxI4sD1bNvqjwM2B20udljSKm9o,1209
4
+ onager/compact.py,sha256=-wWOKUCCfjAoDecTjrvbWeBZhZtU-WYbZKbhSCMw-FI,7114
5
+ onager/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
6
+ de_onager-0.2.2.dist-info/METADATA,sha256=gMaPzTAXCgXhCRQrHmaYUbNGtVKyRrTpPDbDfnwJlWU,989
7
+ de_onager-0.2.2.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
8
+ de_onager-0.2.2.dist-info/entry_points.txt,sha256=ouipTbKcK_UNx5PxzwdrhdSNB24QC3yBTK5lL3RTJkk,42
9
+ de_onager-0.2.2.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.32.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ onager = onager.cli:app
onager/__init__.py ADDED
@@ -0,0 +1,26 @@
1
+ """Onager — safe small-file compaction for local Parquet datasets."""
2
+
3
+ from importlib.metadata import PackageNotFoundError, version
4
+
5
+ try:
6
+ __version__ = version("de-onager")
7
+ except PackageNotFoundError:
8
+ __version__ = "0.2.2"
9
+
10
+ from onager.backfill import (
11
+ BackfillWorkspace,
12
+ create_backfill_workspace,
13
+ promote_backfill,
14
+ run_backfill,
15
+ )
16
+ from onager.compact import CompactionReport, compact_dataset, plan_compaction
17
+
18
+ __all__ = [
19
+ "BackfillWorkspace",
20
+ "CompactionReport",
21
+ "compact_dataset",
22
+ "create_backfill_workspace",
23
+ "plan_compaction",
24
+ "promote_backfill",
25
+ "run_backfill",
26
+ ]
onager/backfill.py ADDED
@@ -0,0 +1,109 @@
1
+ """Isolated Parquet backfill workspaces with explicit promotion."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import os
7
+ import shutil
8
+ from collections.abc import Callable
9
+ from dataclasses import dataclass
10
+ from pathlib import Path
11
+
12
+ from scutum import FileLock
13
+
14
+ from arsenal_core.errors import FatalError
15
+
16
+
17
+ @dataclass(frozen=True)
18
+ class BackfillWorkspace:
19
+ run_id: str
20
+ root: Path
21
+ input: Path
22
+ output: Path
23
+ status: str
24
+
25
+
26
+ def create_backfill_workspace(
27
+ dataset: Path,
28
+ run_id: str,
29
+ *,
30
+ root: Path | None = None,
31
+ ) -> BackfillWorkspace:
32
+ """Snapshot ``dataset`` into an isolated, not-yet-active workspace."""
33
+ _validate_dataset(dataset)
34
+ _validate_run_id(run_id)
35
+ workspace_root = root or dataset.parent / ".onager" / "backfills"
36
+ workspace = workspace_root / run_id
37
+ if workspace.exists():
38
+ raise FatalError(f"backfill workspace already exists: {workspace}")
39
+ input_dir = workspace / "input"
40
+ output_dir = workspace / "output"
41
+ workspace.mkdir(parents=True)
42
+ shutil.copytree(dataset, input_dir)
43
+ output_dir.mkdir()
44
+ _write_status(workspace, "created")
45
+ return BackfillWorkspace(run_id, workspace, input_dir, output_dir, "created")
46
+
47
+
48
+ def run_backfill(
49
+ dataset: Path,
50
+ run_id: str,
51
+ transform: Callable[[Path, Path], None],
52
+ *,
53
+ root: Path | None = None,
54
+ ) -> BackfillWorkspace:
55
+ """Run a transform against the snapshot and retain success/failure evidence."""
56
+ workspace = create_backfill_workspace(dataset, run_id, root=root)
57
+ _write_status(workspace.root, "running")
58
+ try:
59
+ transform(workspace.input, workspace.output)
60
+ except Exception:
61
+ _write_status(workspace.root, "failed")
62
+ raise
63
+ _write_status(workspace.root, "ready")
64
+ return BackfillWorkspace(
65
+ workspace.run_id, workspace.root, workspace.input, workspace.output, "ready"
66
+ )
67
+
68
+
69
+ def promote_backfill(workspace: BackfillWorkspace, dataset: Path) -> None:
70
+ """Atomically promote a ready backfill output after an explicit operator action."""
71
+ if workspace.status != "ready" or not workspace.output.is_dir():
72
+ raise FatalError("only a ready backfill workspace can be promoted")
73
+ _validate_dataset(dataset)
74
+ lock_path = dataset.parent / f".{dataset.name}.onager.lock"
75
+ backup = dataset.parent / f".{dataset.name}.backfill-backup"
76
+ if backup.exists():
77
+ raise FatalError(f"stale backfill backup exists; remove it manually: {backup}")
78
+ with FileLock(lock_path):
79
+ os.replace(dataset, backup)
80
+ try:
81
+ os.replace(workspace.output, dataset)
82
+ except OSError:
83
+ os.replace(backup, dataset)
84
+ raise
85
+ shutil.rmtree(backup)
86
+ _write_status(workspace.root, "promoted")
87
+
88
+
89
+ def _validate_dataset(dataset: Path) -> None:
90
+ first = dataset.parts[0].lower() if dataset.parts else ""
91
+ if str(dataset).lower().startswith(("s3://", "gs://", "gcs://")) or first.rstrip(":") in {
92
+ "s3",
93
+ "gs",
94
+ "gcs",
95
+ }:
96
+ raise FatalError("Onager backfill currently supports local datasets only")
97
+ if not dataset.is_dir():
98
+ raise FatalError(f"dataset directory not found: {dataset}")
99
+
100
+
101
+ def _validate_run_id(run_id: str) -> None:
102
+ if not run_id or run_id in {".", ".."} or "/" in run_id or "\\" in run_id:
103
+ raise FatalError("backfill run_id must be a non-empty path-safe name")
104
+
105
+
106
+ def _write_status(root: Path, status: str) -> None:
107
+ (root / "status.json").write_text(
108
+ json.dumps({"status": status}, sort_keys=True) + "\n", encoding="utf-8"
109
+ )
onager/cli.py ADDED
@@ -0,0 +1,41 @@
1
+ """Onager CLI."""
2
+
3
+ from pathlib import Path
4
+
5
+ import typer
6
+
7
+ from arsenal_core.errors import ArsenalError
8
+ from onager.compact import compact_dataset
9
+
10
+ app = typer.Typer(help="Onager — safe Parquet small-file compaction.")
11
+
12
+
13
+ @app.command()
14
+ def compact(
15
+ dataset: Path,
16
+ target_bytes: int = typer.Option(128 * 1024 * 1024, "--target-bytes"),
17
+ min_files: int = typer.Option(2, "--min-files"),
18
+ max_input_bytes: int | None = typer.Option(None, "--max-input-bytes"),
19
+ dry_run: bool = typer.Option(False, "--dry-run"),
20
+ ) -> None:
21
+ """Compact selected local Parquet files; inspect the plan with --dry-run."""
22
+ try:
23
+ report = compact_dataset(
24
+ dataset,
25
+ target_bytes=target_bytes,
26
+ min_files=min_files,
27
+ max_input_bytes=max_input_bytes,
28
+ dry_run=dry_run,
29
+ )
30
+ except ArsenalError as e:
31
+ typer.echo(f"error: {e}", err=True)
32
+ raise typer.Exit(1) from e
33
+ typer.echo(
34
+ f"selected={report.selected_files} preserved={report.preserved_files} "
35
+ f"input_bytes={report.input_bytes} output_bytes={report.output_bytes} "
36
+ f"compacted={report.compacted}"
37
+ )
38
+
39
+
40
+ if __name__ == "__main__":
41
+ app()
onager/compact.py ADDED
@@ -0,0 +1,215 @@
1
+ """Small-file compaction with dry-run and recoverable directory replacement."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import os
8
+ import shutil
9
+ import tempfile
10
+ from dataclasses import dataclass
11
+ from pathlib import Path
12
+
13
+ import duckdb
14
+ from scutum import FileLock
15
+
16
+ from arsenal_core.errors import FatalError
17
+
18
+
19
+ @dataclass(frozen=True)
20
+ class CompactionPlan:
21
+ dataset: Path
22
+ candidates: tuple[Path, ...]
23
+ preserved: tuple[Path, ...]
24
+ target_bytes: int
25
+ min_files: int
26
+ max_input_bytes: int | None
27
+
28
+ @property
29
+ def selected(self) -> bool:
30
+ return len(self.candidates) >= self.min_files and (
31
+ self.max_input_bytes is None or self.input_bytes <= self.max_input_bytes
32
+ )
33
+
34
+ @property
35
+ def input_bytes(self) -> int:
36
+ return sum(path.stat().st_size for path in self.candidates)
37
+
38
+
39
+ @dataclass(frozen=True)
40
+ class CompactionReport:
41
+ dataset: Path
42
+ dry_run: bool
43
+ selected_files: int
44
+ preserved_files: int
45
+ input_bytes: int
46
+ output_bytes: int
47
+ compacted: bool
48
+
49
+
50
+ def plan_compaction(
51
+ dataset: Path,
52
+ *,
53
+ target_bytes: int = 128 * 1024 * 1024,
54
+ min_files: int = 2,
55
+ max_input_bytes: int | None = None,
56
+ ) -> CompactionPlan:
57
+ """Select local Parquet files smaller than ``target_bytes``."""
58
+ _validate_options(dataset, target_bytes, min_files, max_input_bytes)
59
+ files = tuple(sorted(path for path in dataset.rglob("*.parquet") if path.is_file()))
60
+ candidates = tuple(path for path in files if path.stat().st_size < target_bytes)
61
+ preserved = tuple(path for path in files if path not in candidates)
62
+ return CompactionPlan(dataset, candidates, preserved, target_bytes, min_files, max_input_bytes)
63
+
64
+
65
+ def compact_dataset(
66
+ dataset: Path,
67
+ *,
68
+ target_bytes: int = 128 * 1024 * 1024,
69
+ min_files: int = 2,
70
+ max_input_bytes: int | None = None,
71
+ dry_run: bool = False,
72
+ ) -> CompactionReport:
73
+ _validate_options(dataset, target_bytes, min_files, max_input_bytes)
74
+ lock_path = dataset.parent / f".{dataset.name}.onager.lock"
75
+ with FileLock(lock_path):
76
+ return _compact_dataset_locked(
77
+ dataset,
78
+ target_bytes=target_bytes,
79
+ min_files=min_files,
80
+ max_input_bytes=max_input_bytes,
81
+ dry_run=dry_run,
82
+ )
83
+
84
+
85
+ def _compact_dataset_locked(
86
+ dataset: Path,
87
+ *,
88
+ target_bytes: int,
89
+ min_files: int,
90
+ max_input_bytes: int | None,
91
+ dry_run: bool,
92
+ ) -> CompactionReport:
93
+ """Compact selected files into one Parquet part and replace the dataset safely.
94
+
95
+ Only local directories are accepted. The existing dataset is moved to a
96
+ sibling backup before the new directory is installed; a failed install
97
+ restores the original directory. Backups are removed only after success.
98
+ """
99
+ plan = plan_compaction(
100
+ dataset,
101
+ target_bytes=target_bytes,
102
+ min_files=min_files,
103
+ max_input_bytes=max_input_bytes,
104
+ )
105
+ input_bytes = plan.input_bytes
106
+ if not plan.selected or dry_run:
107
+ report = CompactionReport(
108
+ dataset=dataset,
109
+ dry_run=dry_run,
110
+ selected_files=len(plan.candidates),
111
+ preserved_files=len(plan.preserved),
112
+ input_bytes=input_bytes,
113
+ output_bytes=0,
114
+ compacted=False,
115
+ )
116
+ _record_compaction(report, plan)
117
+ return report
118
+
119
+ parent = dataset.parent
120
+ temp_dir = Path(tempfile.mkdtemp(prefix=f".{dataset.name}.onager-", dir=parent))
121
+ backup = parent / f".{dataset.name}.onager-backup"
122
+ compact_name = f"part-compact-{_plan_hash(plan):s}.parquet"
123
+ try:
124
+ for path in plan.preserved:
125
+ relative = path.relative_to(dataset)
126
+ target = temp_dir / relative
127
+ target.parent.mkdir(parents=True, exist_ok=True)
128
+ shutil.copy2(path, target)
129
+ input_sql = ", ".join(_quote_literal(str(path)) for path in plan.candidates)
130
+ output = temp_dir / compact_name
131
+ con = duckdb.connect()
132
+ try:
133
+ con.execute(
134
+ f"COPY (SELECT * FROM read_parquet([{input_sql}], union_by_name=true)) "
135
+ f"TO {_quote_literal(str(output))} (FORMAT PARQUET, COMPRESSION ZSTD)"
136
+ )
137
+ except duckdb.Error as e:
138
+ raise FatalError(f"compaction failed for {dataset}: {e}") from e
139
+ finally:
140
+ con.close()
141
+
142
+ if backup.exists():
143
+ raise FatalError(f"stale Onager backup exists; remove it manually: {backup}")
144
+ os.replace(dataset, backup)
145
+ try:
146
+ os.replace(temp_dir, dataset)
147
+ except OSError:
148
+ os.replace(backup, dataset)
149
+ raise
150
+ shutil.rmtree(backup)
151
+ output_bytes = sum(path.stat().st_size for path in dataset.rglob("*.parquet"))
152
+ report = CompactionReport(
153
+ dataset=dataset,
154
+ dry_run=False,
155
+ selected_files=len(plan.candidates),
156
+ preserved_files=len(plan.preserved),
157
+ input_bytes=input_bytes,
158
+ output_bytes=output_bytes,
159
+ compacted=True,
160
+ )
161
+ _record_compaction(report, plan)
162
+ return report
163
+ finally:
164
+ if temp_dir.exists():
165
+ shutil.rmtree(temp_dir)
166
+
167
+
168
+ def _validate_options(
169
+ dataset: Path,
170
+ target_bytes: int,
171
+ min_files: int,
172
+ max_input_bytes: int | None,
173
+ ) -> None:
174
+ first_part = dataset.parts[0].lower() if dataset.parts else ""
175
+ if str(dataset).lower().startswith(("s3://", "gs://", "gcs://")) or first_part.rstrip(":") in {
176
+ "s3",
177
+ "gs",
178
+ "gcs",
179
+ }:
180
+ raise FatalError("Onager compaction currently supports local Parquet directories only")
181
+ if not dataset.exists() or not dataset.is_dir():
182
+ raise FatalError(f"dataset directory not found: {dataset}")
183
+ if target_bytes <= 0:
184
+ raise FatalError("target_bytes must be positive")
185
+ if min_files < 2:
186
+ raise FatalError("min_files must be at least 2")
187
+ if max_input_bytes is not None and max_input_bytes <= 0:
188
+ raise FatalError("max_input_bytes must be positive")
189
+
190
+
191
+ def _plan_hash(plan: CompactionPlan) -> str:
192
+ payload = "|".join(f"{path}:{path.stat().st_size}" for path in plan.candidates)
193
+ return hashlib.sha256(payload.encode()).hexdigest()[:12]
194
+
195
+
196
+ def _quote_literal(value: str) -> str:
197
+ return "'" + value.replace("'", "''") + "'"
198
+
199
+
200
+ def _record_compaction(report: CompactionReport, plan: CompactionPlan) -> None:
201
+ log_dir = report.dataset.parent / ".onager"
202
+ log_dir.mkdir(parents=True, exist_ok=True)
203
+ entry = {
204
+ "dataset": str(report.dataset),
205
+ "selected_files": report.selected_files,
206
+ "preserved_files": report.preserved_files,
207
+ "input_bytes": report.input_bytes,
208
+ "output_bytes": report.output_bytes,
209
+ "compacted": report.compacted,
210
+ "dry_run": report.dry_run,
211
+ "target_bytes": plan.target_bytes,
212
+ "max_input_bytes": plan.max_input_bytes,
213
+ }
214
+ with (log_dir / "compaction-log.jsonl").open("a", encoding="utf-8") as handle:
215
+ handle.write(json.dumps(entry, sort_keys=True) + "\n")
onager/py.typed ADDED
File without changes