rollingpebble 0.6.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rollingpebble/__init__.py +5 -0
- rollingpebble/adapters/__init__.py +0 -0
- rollingpebble/adapters/pylrclib_adapter.py +255 -0
- rollingpebble/adapters/pyroller_adapter.py +368 -0
- rollingpebble/cli.py +298 -0
- rollingpebble/config.py +54 -0
- rollingpebble/frontend_dist/assets/index-ByboMpdf.css +1 -0
- rollingpebble/frontend_dist/assets/index-yyBbsxuf.js +157 -0
- rollingpebble/frontend_dist/assets/index-yyBbsxuf.js.map +1 -0
- rollingpebble/frontend_dist/assets/ja-D7_rhcbo.js +2 -0
- rollingpebble/frontend_dist/assets/ja-D7_rhcbo.js.map +1 -0
- rollingpebble/frontend_dist/assets/ko-KR-DD0vrR3m.js +2 -0
- rollingpebble/frontend_dist/assets/ko-KR-DD0vrR3m.js.map +1 -0
- rollingpebble/frontend_dist/assets/ncmc-worker-BpoEUDiM.ts +210 -0
- rollingpebble/frontend_dist/assets/pl-PL-CSfhlz8C.js +2 -0
- rollingpebble/frontend_dist/assets/pl-PL-CSfhlz8C.js.map +1 -0
- rollingpebble/frontend_dist/assets/pt-BR-BYOQPXu3.js +2 -0
- rollingpebble/frontend_dist/assets/pt-BR-BYOQPXu3.js.map +1 -0
- rollingpebble/frontend_dist/assets/sk-SK-ATjCPXMQ.js +2 -0
- rollingpebble/frontend_dist/assets/sk-SK-ATjCPXMQ.js.map +1 -0
- rollingpebble/frontend_dist/assets/smooth-scroll-Cyy-OSkS.js +2 -0
- rollingpebble/frontend_dist/assets/smooth-scroll-Cyy-OSkS.js.map +1 -0
- rollingpebble/frontend_dist/assets/zh-CN-y2rje7l9.js +2 -0
- rollingpebble/frontend_dist/assets/zh-CN-y2rje7l9.js.map +1 -0
- rollingpebble/frontend_dist/assets/zh-HK-BvMz0uGi.js +2 -0
- rollingpebble/frontend_dist/assets/zh-HK-BvMz0uGi.js.map +1 -0
- rollingpebble/frontend_dist/assets/zh-TW-o2O3CXEw.js +2 -0
- rollingpebble/frontend_dist/assets/zh-TW-o2O3CXEw.js.map +1 -0
- rollingpebble/frontend_dist/favicons/android-chrome-192x192.png +0 -0
- rollingpebble/frontend_dist/favicons/android-chrome-512x512.png +0 -0
- rollingpebble/frontend_dist/favicons/apple-touch-icon.png +0 -0
- rollingpebble/frontend_dist/favicons/browserconfig.xml +10 -0
- rollingpebble/frontend_dist/favicons/favicon-16x16.png +0 -0
- rollingpebble/frontend_dist/favicons/favicon-32x32.png +0 -0
- rollingpebble/frontend_dist/favicons/favicon.ico +0 -0
- rollingpebble/frontend_dist/favicons/mstile-150x150.png +0 -0
- rollingpebble/frontend_dist/favicons/mstile-310x310.png +0 -0
- rollingpebble/frontend_dist/favicons/safari-pinned-tab.svg +1 -0
- rollingpebble/frontend_dist/img/rollingpebble-avatar.png +0 -0
- rollingpebble/frontend_dist/img/rollingpebble-avatar.webp +0 -0
- rollingpebble/frontend_dist/img/rollingpebble-workspace-bg.webp +0 -0
- rollingpebble/frontend_dist/index.html +37 -0
- rollingpebble/frontend_dist/site.webmanifest +31 -0
- rollingpebble/jobs.py +573 -0
- rollingpebble/lyrics_utils.py +101 -0
- rollingpebble/main.py +555 -0
- rollingpebble/messages.py +147 -0
- rollingpebble/models.py +673 -0
- rollingpebble/paths.py +81 -0
- rollingpebble/runtime_constants.py +7 -0
- rollingpebble/runtime_dependencies.py +49 -0
- rollingpebble/runtime_environment.py +43 -0
- rollingpebble/runtime_installer.py +265 -0
- rollingpebble/runtime_python.py +80 -0
- rollingpebble/runtime_recipe.py +61 -0
- rollingpebble/services/__init__.py +0 -0
- rollingpebble/services/local_dialog.py +85 -0
- rollingpebble/services/lrclib_service.py +29 -0
- rollingpebble/services/netease_service.py +369 -0
- rollingpebble/services/project_service.py +193 -0
- rollingpebble/services/roller_service.py +421 -0
- rollingpebble/services/runtime_manager.py +250 -0
- rollingpebble/services/runtime_service.py +335 -0
- rollingpebble/services/storage_service.py +1309 -0
- rollingpebble/services/upload_service.py +44 -0
- rollingpebble/storage/__init__.py +0 -0
- rollingpebble/storage/app_settings.py +41 -0
- rollingpebble/storage/files.py +149 -0
- rollingpebble/version.py +27 -0
- rollingpebble-0.6.2.dist-info/METADATA +158 -0
- rollingpebble-0.6.2.dist-info/RECORD +74 -0
- rollingpebble-0.6.2.dist-info/WHEEL +5 -0
- rollingpebble-0.6.2.dist-info/entry_points.txt +2 -0
- rollingpebble-0.6.2.dist-info/top_level.txt +1 -0
|
File without changes
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import importlib.metadata
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from rollingpebble.version import app_version
|
|
8
|
+
from rollingpebble.models import (
|
|
9
|
+
LrclibGetResponse,
|
|
10
|
+
LrclibSearchResponse,
|
|
11
|
+
LyricsRecordModel,
|
|
12
|
+
MetaModel,
|
|
13
|
+
UploadPlanResponse,
|
|
14
|
+
)
|
|
15
|
+
from rollingpebble.messages import message_from_text
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _common_options():
|
|
19
|
+
from pylrclib.config import CommonOptions, LRCLIB_BASE, MAX_HTTP_RETRIES_DEFAULT, PREVIEW_LINES_DEFAULT
|
|
20
|
+
|
|
21
|
+
return CommonOptions(
|
|
22
|
+
lang="en",
|
|
23
|
+
preview_lines=PREVIEW_LINES_DEFAULT,
|
|
24
|
+
max_http_retries=MAX_HTTP_RETRIES_DEFAULT,
|
|
25
|
+
user_agent=f"rollingpebble/{app_version()} pylrclib/embedded",
|
|
26
|
+
lrclib_base=LRCLIB_BASE,
|
|
27
|
+
interactive=False,
|
|
28
|
+
assume_yes=True,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _track_meta(meta: MetaModel):
|
|
33
|
+
from pylrclib.models import TrackMeta
|
|
34
|
+
|
|
35
|
+
return TrackMeta(
|
|
36
|
+
path=None,
|
|
37
|
+
track=meta.track.strip(),
|
|
38
|
+
artist=meta.artist.strip(),
|
|
39
|
+
album=meta.album.strip(),
|
|
40
|
+
duration=int(meta.duration or 0),
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def dependency_status() -> tuple[bool, str | None, str | None]:
|
|
45
|
+
try:
|
|
46
|
+
import pylrclib # noqa: F401
|
|
47
|
+
except Exception as exc: # pragma: no cover - env dependent
|
|
48
|
+
return False, None, str(exc)
|
|
49
|
+
version = getattr(pylrclib, "__version__", None)
|
|
50
|
+
try:
|
|
51
|
+
package_version = importlib.metadata.version("pylrclib-cli")
|
|
52
|
+
if version and version != package_version:
|
|
53
|
+
version = f"module {version}, package {package_version}"
|
|
54
|
+
else:
|
|
55
|
+
version = package_version or version
|
|
56
|
+
except importlib.metadata.PackageNotFoundError:
|
|
57
|
+
pass
|
|
58
|
+
return True, version, None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _record_model(record: Any) -> LyricsRecordModel:
|
|
62
|
+
return LyricsRecordModel(
|
|
63
|
+
id=record.lrclib_id,
|
|
64
|
+
track_name=record.track_name,
|
|
65
|
+
artist_name=record.artist_name,
|
|
66
|
+
album_name=record.album_name,
|
|
67
|
+
duration=record.duration,
|
|
68
|
+
plain_lyrics=record.plain,
|
|
69
|
+
synced_lyrics=record.synced,
|
|
70
|
+
instrumental=record.instrumental,
|
|
71
|
+
has_plain=bool(record.plain.strip()),
|
|
72
|
+
has_synced=bool(record.synced.strip()),
|
|
73
|
+
label=record.label,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass(slots=True)
|
|
78
|
+
class PylrclibAdapter:
|
|
79
|
+
def _client(self):
|
|
80
|
+
from pylrclib.api import ApiClient
|
|
81
|
+
|
|
82
|
+
return ApiClient(_common_options())
|
|
83
|
+
|
|
84
|
+
def search(
|
|
85
|
+
self,
|
|
86
|
+
*,
|
|
87
|
+
query: str | None,
|
|
88
|
+
track_name: str | None,
|
|
89
|
+
artist_name: str | None,
|
|
90
|
+
album_name: str | None,
|
|
91
|
+
limit: int,
|
|
92
|
+
) -> LrclibSearchResponse:
|
|
93
|
+
client = self._client()
|
|
94
|
+
records = client.search(
|
|
95
|
+
query=query or None,
|
|
96
|
+
track_name=track_name or None,
|
|
97
|
+
artist_name=artist_name or None,
|
|
98
|
+
album_name=album_name or None,
|
|
99
|
+
)
|
|
100
|
+
return LrclibSearchResponse(results=[_record_model(record) for record in records[: max(1, limit)]])
|
|
101
|
+
|
|
102
|
+
def get_cached_then_external(self, meta: MetaModel) -> LrclibGetResponse:
|
|
103
|
+
client = self._client()
|
|
104
|
+
track = _track_meta(meta)
|
|
105
|
+
result = client.get_cached(track)
|
|
106
|
+
if not result.record:
|
|
107
|
+
result = client.get_external(track)
|
|
108
|
+
return LrclibGetResponse(
|
|
109
|
+
record=_record_model(result.record) if result.record else None,
|
|
110
|
+
duration_diff=result.duration_diff,
|
|
111
|
+
duration_ok=result.duration_ok,
|
|
112
|
+
source=result.source,
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
def get_external(self, meta: MetaModel) -> LrclibGetResponse:
|
|
116
|
+
client = self._client()
|
|
117
|
+
result = client.get_external(_track_meta(meta))
|
|
118
|
+
return LrclibGetResponse(
|
|
119
|
+
record=_record_model(result.record) if result.record else None,
|
|
120
|
+
duration_diff=result.duration_diff,
|
|
121
|
+
duration_ok=result.duration_ok,
|
|
122
|
+
source=result.source,
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
def cleanse_lrc_text(self, text: str, *, remove_translations: bool = True) -> dict[str, Any]:
|
|
126
|
+
from pylrclib.lrc import parse_lrc_text
|
|
127
|
+
|
|
128
|
+
parsed = parse_lrc_text(text, remove_translations=remove_translations)
|
|
129
|
+
if not parsed.has_valid_timestamps:
|
|
130
|
+
return {
|
|
131
|
+
"status": "invalid",
|
|
132
|
+
"cleaned_text": None,
|
|
133
|
+
"plain_lyrics": parsed.plain or "",
|
|
134
|
+
"is_instrumental": parsed.is_instrumental,
|
|
135
|
+
"has_valid_timestamps": False,
|
|
136
|
+
"warnings": parsed.warnings,
|
|
137
|
+
"reason": "no_valid_timestamps",
|
|
138
|
+
}
|
|
139
|
+
cleaned = parsed.synced
|
|
140
|
+
status = "unchanged" if cleaned == text else "updated"
|
|
141
|
+
if text.strip() and not cleaned.strip() and not parsed.is_instrumental:
|
|
142
|
+
return {
|
|
143
|
+
"status": "invalid",
|
|
144
|
+
"cleaned_text": None,
|
|
145
|
+
"plain_lyrics": parsed.plain or "",
|
|
146
|
+
"is_instrumental": parsed.is_instrumental,
|
|
147
|
+
"has_valid_timestamps": True,
|
|
148
|
+
"warnings": parsed.warnings,
|
|
149
|
+
"reason": "empty_after_cleanse",
|
|
150
|
+
}
|
|
151
|
+
return {
|
|
152
|
+
"status": status,
|
|
153
|
+
"cleaned_text": cleaned,
|
|
154
|
+
"plain_lyrics": parsed.plain or "",
|
|
155
|
+
"is_instrumental": parsed.is_instrumental,
|
|
156
|
+
"has_valid_timestamps": parsed.has_valid_timestamps,
|
|
157
|
+
"warnings": parsed.warnings,
|
|
158
|
+
"reason": None,
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
def get_by_id(self, lrclib_id: int) -> LyricsRecordModel | None:
|
|
162
|
+
client = self._client()
|
|
163
|
+
record = client.get_by_id(lrclib_id)
|
|
164
|
+
return _record_model(record) if record else None
|
|
165
|
+
|
|
166
|
+
def build_upload_plan(
|
|
167
|
+
self,
|
|
168
|
+
*,
|
|
169
|
+
meta: MetaModel,
|
|
170
|
+
plain: str,
|
|
171
|
+
synced: str,
|
|
172
|
+
mode: str,
|
|
173
|
+
allow_derived_plain: bool,
|
|
174
|
+
) -> UploadPlanResponse:
|
|
175
|
+
from pylrclib.models import LyricsBundle
|
|
176
|
+
from pylrclib.workflows.up import build_upload_plan
|
|
177
|
+
|
|
178
|
+
instrumental = mode == "instrumental"
|
|
179
|
+
bundle_kind = "instrumental" if instrumental else "mixed" if plain and synced else "synced" if synced else "plain" if plain else "empty"
|
|
180
|
+
bundle = LyricsBundle(
|
|
181
|
+
kind=bundle_kind,
|
|
182
|
+
plain=plain or "",
|
|
183
|
+
synced=synced or "",
|
|
184
|
+
instrumental=instrumental,
|
|
185
|
+
warnings=[],
|
|
186
|
+
)
|
|
187
|
+
plan = build_upload_plan(bundle, mode=mode, allow_derived_plain=allow_derived_plain)
|
|
188
|
+
warnings: list[str] = []
|
|
189
|
+
if not meta.track.strip():
|
|
190
|
+
warnings.append("missing_track")
|
|
191
|
+
if not meta.artist.strip():
|
|
192
|
+
warnings.append("missing_artist")
|
|
193
|
+
if not meta.duration:
|
|
194
|
+
warnings.append("missing_duration")
|
|
195
|
+
if plan.mode == "lyrics" and not ((plan.plain or "").strip() or (plan.synced or "").strip()):
|
|
196
|
+
warnings.append("empty_lyrics")
|
|
197
|
+
can_upload = plan.mode in {"lyrics", "instrumental"} and not any(w.startswith("missing_") for w in warnings)
|
|
198
|
+
payload_preview: dict[str, Any] = {
|
|
199
|
+
"trackName": meta.track,
|
|
200
|
+
"artistName": meta.artist,
|
|
201
|
+
"albumName": meta.album,
|
|
202
|
+
"duration": meta.duration,
|
|
203
|
+
}
|
|
204
|
+
if plan.mode == "lyrics":
|
|
205
|
+
payload_preview.update(
|
|
206
|
+
{
|
|
207
|
+
"plainLyrics": bool((plan.plain or "").strip()),
|
|
208
|
+
"syncedLyrics": bool((plan.synced or "").strip()),
|
|
209
|
+
}
|
|
210
|
+
)
|
|
211
|
+
return UploadPlanResponse(
|
|
212
|
+
can_upload=can_upload,
|
|
213
|
+
mode=plan.mode,
|
|
214
|
+
reason=plan.reason,
|
|
215
|
+
reason_message=message_from_text(plan.reason, default_code="upload.plan.reason"),
|
|
216
|
+
plain_lines=len((plan.plain or "").splitlines()) if plan.plain else 0,
|
|
217
|
+
synced_lines=len((plan.synced or "").splitlines()) if plan.synced else 0,
|
|
218
|
+
warnings=warnings,
|
|
219
|
+
warning_messages=[message_from_text(warning, default_code="upload.plan.warning") for warning in warnings],
|
|
220
|
+
payload_preview=payload_preview,
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
def upload(
|
|
224
|
+
self,
|
|
225
|
+
*,
|
|
226
|
+
meta: MetaModel,
|
|
227
|
+
plain: str,
|
|
228
|
+
synced: str,
|
|
229
|
+
mode: str,
|
|
230
|
+
allow_derived_plain: bool,
|
|
231
|
+
) -> tuple[bool, str]:
|
|
232
|
+
plan = self.build_upload_plan(
|
|
233
|
+
meta=meta,
|
|
234
|
+
plain=plain,
|
|
235
|
+
synced=synced,
|
|
236
|
+
mode=mode,
|
|
237
|
+
allow_derived_plain=allow_derived_plain,
|
|
238
|
+
)
|
|
239
|
+
if not plan.can_upload:
|
|
240
|
+
return False, f"Upload skipped: {plan.reason}; warnings={','.join(plan.warnings)}"
|
|
241
|
+
|
|
242
|
+
client = self._client()
|
|
243
|
+
track = _track_meta(meta)
|
|
244
|
+
if plan.mode == "instrumental":
|
|
245
|
+
ok = client.upload_instrumental(track)
|
|
246
|
+
return ok, "instrumental uploaded" if ok else "instrumental upload failed"
|
|
247
|
+
|
|
248
|
+
# Recompute through pylrclib so synced-only/derived-plain semantics stay aligned.
|
|
249
|
+
from pylrclib.models import LyricsBundle
|
|
250
|
+
from pylrclib.workflows.up import build_upload_plan
|
|
251
|
+
|
|
252
|
+
bundle = LyricsBundle(kind="mixed" if plain and synced else "synced" if synced else "plain", plain=plain, synced=synced)
|
|
253
|
+
pylrc_plan = build_upload_plan(bundle, mode=mode, allow_derived_plain=allow_derived_plain)
|
|
254
|
+
ok = client.upload_lyrics(track, pylrc_plan.plain or "", pylrc_plan.synced or "")
|
|
255
|
+
return ok, "lyrics uploaded" if ok else "lyrics upload failed"
|
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import importlib.metadata
|
|
4
|
+
import os
|
|
5
|
+
import shlex
|
|
6
|
+
import shutil
|
|
7
|
+
import sys
|
|
8
|
+
import tempfile
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from urllib.parse import urlparse
|
|
11
|
+
|
|
12
|
+
import yaml
|
|
13
|
+
|
|
14
|
+
from rollingpebble.models import BatchRollRequest, RollRequest
|
|
15
|
+
|
|
16
|
+
PIPELINE_ORDER = ("s", "f", "t", "p", "a", "w")
|
|
17
|
+
PIPELINE_INDEX = {stage: index for index, stage in enumerate(PIPELINE_ORDER)}
|
|
18
|
+
|
|
19
|
+
ARTIFACTS_DIR_NAME = "artifacts"
|
|
20
|
+
VOCAL_AUDIO_NAME = "vocal.wav"
|
|
21
|
+
FILTERED_AUDIO_NAME = "filtered.wav"
|
|
22
|
+
TIMED_UNITS_NAME = "timed_units.json"
|
|
23
|
+
PARSED_LYRICS_NAME = "parsed_lyrics.json"
|
|
24
|
+
ALIGNMENT_RESULT_NAME = "alignment_result.json"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
_PROXY_ENV_KEYS = (
|
|
28
|
+
"HTTP_PROXY",
|
|
29
|
+
"HTTPS_PROXY",
|
|
30
|
+
"ALL_PROXY",
|
|
31
|
+
"http_proxy",
|
|
32
|
+
"https_proxy",
|
|
33
|
+
"all_proxy",
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _proxy_can_be_exported_to_stdlib_env(proxy: str) -> bool:
|
|
38
|
+
"""Return whether the proxy is safe for stdlib urllib/torch.hub.
|
|
39
|
+
|
|
40
|
+
Hugging Face downloads receive --transcriber-hf-proxy directly from
|
|
41
|
+
py-roller, but Demucs uses torch.hub -> urllib.request for model
|
|
42
|
+
downloads. urllib understands HTTP(S) proxy CONNECT, but it does not
|
|
43
|
+
implement SOCKS. Exporting socks5/socks5h as HTTPS_PROXY makes urllib
|
|
44
|
+
talk HTTP CONNECT to a SOCKS server, which fails with connection reset.
|
|
45
|
+
"""
|
|
46
|
+
scheme = urlparse(proxy).scheme.lower()
|
|
47
|
+
return scheme in {"http", "https"}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
# Map rollingpebble UI language codes to py-roller PYROLLER_LANG locale codes.
|
|
51
|
+
# py-roller v0.6.0+ uses this env var to select its display language.
|
|
52
|
+
_UI_LANG_TO_PYROLLER_LANG: dict[str, str] = {
|
|
53
|
+
"zh-CN": "zh",
|
|
54
|
+
"zh-HK": "zh_Hant_HK",
|
|
55
|
+
"zh-TW": "zh_Hant",
|
|
56
|
+
"ja": "ja",
|
|
57
|
+
"ko-KR": "ko",
|
|
58
|
+
"pl-PL": "pl",
|
|
59
|
+
"pt-BR": "pt",
|
|
60
|
+
"sk-SK": "sk",
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def build_pyroller_env(request: RollRequest, base_env: dict[str, str] | None = None) -> dict[str, str] | None:
|
|
65
|
+
"""Return subprocess environment overrides for py-roller downloads.
|
|
66
|
+
|
|
67
|
+
py-roller accepts Hugging Face download options as CLI flags, but some
|
|
68
|
+
backends and their dependencies also consult process-level environment
|
|
69
|
+
variables. In particular Transformers / huggingface_hub based phoneme
|
|
70
|
+
backends may open their own HTTP clients while resolving model assets.
|
|
71
|
+
Exporting the same settings here keeps rollingpebble's UI proxy field
|
|
72
|
+
effective for all py-roller transcriber backends.
|
|
73
|
+
"""
|
|
74
|
+
stages = set(normalize_stages(request.stages))
|
|
75
|
+
env = dict(base_env if base_env is not None else os.environ)
|
|
76
|
+
changed = False
|
|
77
|
+
|
|
78
|
+
if request.ui_lang:
|
|
79
|
+
pyroller_lang = _UI_LANG_TO_PYROLLER_LANG.get(request.ui_lang)
|
|
80
|
+
if pyroller_lang:
|
|
81
|
+
env["PYROLLER_LANG"] = pyroller_lang
|
|
82
|
+
changed = True
|
|
83
|
+
|
|
84
|
+
if "t" not in stages:
|
|
85
|
+
return env if changed else None
|
|
86
|
+
|
|
87
|
+
proxy = str(request.transcriber_hf_proxy or "").strip()
|
|
88
|
+
if proxy and _proxy_can_be_exported_to_stdlib_env(proxy):
|
|
89
|
+
for key in _PROXY_ENV_KEYS:
|
|
90
|
+
env[key] = proxy
|
|
91
|
+
changed = True
|
|
92
|
+
|
|
93
|
+
if request.transcriber_local_files_only:
|
|
94
|
+
env["HF_HUB_OFFLINE"] = "1"
|
|
95
|
+
env["TRANSFORMERS_OFFLINE"] = "1"
|
|
96
|
+
changed = True
|
|
97
|
+
|
|
98
|
+
if request.transcriber_hf_xet == "off":
|
|
99
|
+
env["HF_HUB_DISABLE_XET"] = "1"
|
|
100
|
+
changed = True
|
|
101
|
+
elif request.transcriber_hf_xet == "on":
|
|
102
|
+
env["HF_HUB_DISABLE_XET"] = "0"
|
|
103
|
+
changed = True
|
|
104
|
+
|
|
105
|
+
if request.transcriber_hf_etag_timeout is not None:
|
|
106
|
+
env["HF_HUB_ETAG_TIMEOUT"] = str(request.transcriber_hf_etag_timeout)
|
|
107
|
+
changed = True
|
|
108
|
+
if request.transcriber_hf_download_timeout is not None:
|
|
109
|
+
env["HF_HUB_DOWNLOAD_TIMEOUT"] = str(request.transcriber_hf_download_timeout)
|
|
110
|
+
changed = True
|
|
111
|
+
|
|
112
|
+
return env if changed else None
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def dependency_status() -> tuple[bool, str | None, str | None]:
|
|
116
|
+
cli = shutil.which("py-roller")
|
|
117
|
+
version = None
|
|
118
|
+
detail = None
|
|
119
|
+
try:
|
|
120
|
+
version = importlib.metadata.version("py-roller")
|
|
121
|
+
except importlib.metadata.PackageNotFoundError:
|
|
122
|
+
pass
|
|
123
|
+
if cli is None:
|
|
124
|
+
return False, version, "py-roller CLI not found on PATH"
|
|
125
|
+
return True, version, cli
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def cli_path() -> str | None:
|
|
129
|
+
return shutil.which("py-roller")
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def command_text(command: list[str]) -> str:
|
|
133
|
+
return " ".join(shlex.quote(part) for part in command)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def default_model_store() -> Path:
|
|
137
|
+
return Path.home() / ".cache" / "py-roller" / "models" / "transcriber"
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def python_executable() -> str:
|
|
141
|
+
return sys.executable
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def default_artifacts_dir(project_root: Path) -> Path:
|
|
145
|
+
return project_root / ARTIFACTS_DIR_NAME
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def artifacts_for(project_root: Path) -> dict[str, Path]:
|
|
149
|
+
root = default_artifacts_dir(project_root)
|
|
150
|
+
return {
|
|
151
|
+
"vocal_audio": root / VOCAL_AUDIO_NAME,
|
|
152
|
+
"filtered_audio": root / FILTERED_AUDIO_NAME,
|
|
153
|
+
"timed_units": root / TIMED_UNITS_NAME,
|
|
154
|
+
"parsed_lyrics": root / PARSED_LYRICS_NAME,
|
|
155
|
+
"alignment_result": root / ALIGNMENT_RESULT_NAME,
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def normalize_stages(stages: str | None) -> list[str]:
|
|
160
|
+
items = [item.strip() for item in (stages or "s,f,t,p,a,w").split(",") if item.strip()]
|
|
161
|
+
if not items:
|
|
162
|
+
items = ["s", "f", "t", "p", "a", "w"]
|
|
163
|
+
unknown = [stage for stage in items if stage not in PIPELINE_INDEX]
|
|
164
|
+
if unknown:
|
|
165
|
+
raise ValueError(f"Unknown py-roller stage(s): {', '.join(unknown)}")
|
|
166
|
+
indexes = [PIPELINE_INDEX[stage] for stage in items]
|
|
167
|
+
if indexes != list(range(indexes[0], indexes[0] + len(indexes))):
|
|
168
|
+
raise ValueError(
|
|
169
|
+
"py-roller stages must be a continuous subsequence of "
|
|
170
|
+
f"{','.join(PIPELINE_ORDER)}; got {','.join(items)}"
|
|
171
|
+
)
|
|
172
|
+
return items
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def normalized_stage_text(stages: str | None) -> str:
|
|
176
|
+
return ",".join(normalize_stages(stages))
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _add_option(command: list[str], name: str, value: object | None) -> None:
|
|
180
|
+
if value is not None and not (isinstance(value, str) and not value.strip()):
|
|
181
|
+
command.extend([name, str(value)])
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def build_pyroller_command(
|
|
185
|
+
*,
|
|
186
|
+
audio_path: Path,
|
|
187
|
+
lyrics_path: Path,
|
|
188
|
+
output_path: Path,
|
|
189
|
+
intermediate_dir: Path,
|
|
190
|
+
request: RollRequest,
|
|
191
|
+
artifacts_dir: Path | None = None,
|
|
192
|
+
command_prefix: list[str] | None = None,
|
|
193
|
+
default_model_store: Path | None = None,
|
|
194
|
+
) -> list[str]:
|
|
195
|
+
stages = normalize_stages(request.stages)
|
|
196
|
+
stage_set = set(stages)
|
|
197
|
+
first_stage = stages[0]
|
|
198
|
+
artifacts = artifacts_for(artifacts_dir.parent) if artifacts_dir else artifacts_for(output_path.parent)
|
|
199
|
+
if artifacts_dir:
|
|
200
|
+
artifacts = {
|
|
201
|
+
"vocal_audio": artifacts_dir / VOCAL_AUDIO_NAME,
|
|
202
|
+
"filtered_audio": artifacts_dir / FILTERED_AUDIO_NAME,
|
|
203
|
+
"timed_units": artifacts_dir / TIMED_UNITS_NAME,
|
|
204
|
+
"parsed_lyrics": artifacts_dir / PARSED_LYRICS_NAME,
|
|
205
|
+
"alignment_result": artifacts_dir / ALIGNMENT_RESULT_NAME,
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
command = [
|
|
209
|
+
*(command_prefix or ["py-roller"]),
|
|
210
|
+
"run",
|
|
211
|
+
"--stages",
|
|
212
|
+
",".join(stages),
|
|
213
|
+
"--language",
|
|
214
|
+
request.language,
|
|
215
|
+
"--cleanup",
|
|
216
|
+
request.cleanup,
|
|
217
|
+
"--intermediate",
|
|
218
|
+
str(intermediate_dir),
|
|
219
|
+
"--log-level",
|
|
220
|
+
request.log_level,
|
|
221
|
+
"--progress-format",
|
|
222
|
+
"jsonl",
|
|
223
|
+
]
|
|
224
|
+
|
|
225
|
+
if stage_set.intersection({"s", "f", "t"}):
|
|
226
|
+
command.extend(["--audio", str(audio_path)])
|
|
227
|
+
if "p" in stage_set:
|
|
228
|
+
command.extend(["--lyrics", str(lyrics_path)])
|
|
229
|
+
if first_stage == "a":
|
|
230
|
+
command.extend(["--timed-units", str(artifacts["timed_units"])])
|
|
231
|
+
command.extend(["--parsed-lyrics", str(artifacts["parsed_lyrics"])])
|
|
232
|
+
if first_stage == "w":
|
|
233
|
+
command.extend(["--alignment-result", str(artifacts["alignment_result"])])
|
|
234
|
+
if "w" in stage_set:
|
|
235
|
+
command.extend(["--output-roller", str(output_path)])
|
|
236
|
+
|
|
237
|
+
if "s" in stage_set:
|
|
238
|
+
_add_option(command, "--splitter-backend", request.splitter_backend)
|
|
239
|
+
_add_option(command, "--splitter-demucs-model", request.splitter_demucs_model)
|
|
240
|
+
_add_option(command, "--splitter-demucs-device", request.splitter_demucs_device)
|
|
241
|
+
_add_option(command, "--splitter-demucs-jobs", request.splitter_demucs_jobs)
|
|
242
|
+
_add_option(command, "--splitter-demucs-overlap", request.splitter_demucs_overlap)
|
|
243
|
+
_add_option(command, "--splitter-demucs-segment", request.splitter_demucs_segment)
|
|
244
|
+
command.extend(["--output-vocal-audio", str(artifacts["vocal_audio"])])
|
|
245
|
+
|
|
246
|
+
if "f" in stage_set:
|
|
247
|
+
_add_option(command, "--filter-chain", request.filter_chain)
|
|
248
|
+
command.extend(["--output-filtered-audio", str(artifacts["filtered_audio"])])
|
|
249
|
+
|
|
250
|
+
if "t" in stage_set:
|
|
251
|
+
_add_option(command, "--transcriber-backend", request.transcriber_backend)
|
|
252
|
+
_add_option(command, "--transcriber-device", request.transcriber_device)
|
|
253
|
+
_add_option(command, "--transcriber-model-name", request.transcriber_model_name)
|
|
254
|
+
model_path_value = request.transcriber_model_path or (str(default_model_store) if default_model_store else "")
|
|
255
|
+
if model_path_value:
|
|
256
|
+
model_path = Path(model_path_value).expanduser()
|
|
257
|
+
command.extend(["--transcriber-model-path", str(model_path)])
|
|
258
|
+
if request.transcriber_local_files_only:
|
|
259
|
+
command.append("--transcriber-local-files-only")
|
|
260
|
+
_add_option(command, "--transcriber-compute-type", request.transcriber_compute_type)
|
|
261
|
+
_add_option(command, "--transcriber-batch-size", request.transcriber_batch_size)
|
|
262
|
+
_add_option(command, "--transcriber-hf-xet", request.transcriber_hf_xet)
|
|
263
|
+
_add_option(command, "--transcriber-hf-proxy", request.transcriber_hf_proxy)
|
|
264
|
+
_add_option(command, "--transcriber-hf-etag-timeout", request.transcriber_hf_etag_timeout)
|
|
265
|
+
_add_option(command, "--transcriber-hf-download-timeout", request.transcriber_hf_download_timeout)
|
|
266
|
+
_add_option(command, "--transcriber-hf-max-workers", request.transcriber_hf_max_workers)
|
|
267
|
+
if request.transcriber_vad_filter:
|
|
268
|
+
command.append("--transcriber-vad-filter")
|
|
269
|
+
command.extend(["--output-timed-units", str(artifacts["timed_units"])])
|
|
270
|
+
|
|
271
|
+
if "p" in stage_set:
|
|
272
|
+
_add_option(command, "--parser-lyrics-encoding", request.parser_lyrics_encoding)
|
|
273
|
+
command.extend(["--output-parsed-lyrics", str(artifacts["parsed_lyrics"])])
|
|
274
|
+
|
|
275
|
+
if "a" in stage_set:
|
|
276
|
+
_add_option(command, "--aligner-backend", request.aligner_backend)
|
|
277
|
+
_add_option(command, "--aligner-min-gap", request.aligner_min_gap)
|
|
278
|
+
_add_option(command, "--aligner-repetition", request.aligner_repetition)
|
|
279
|
+
command.extend(["--output-alignment-result", str(artifacts["alignment_result"])])
|
|
280
|
+
|
|
281
|
+
if "w" in stage_set:
|
|
282
|
+
_add_option(command, "--writer-backend", request.writer_backend)
|
|
283
|
+
_add_option(command, "--writer-spacing", request.writer_spacing)
|
|
284
|
+
_add_option(command, "--writer-by-tag", request.writer_by_tag)
|
|
285
|
+
_add_option(command, "--writer-ass-karaoke-tag-type", request.writer_ass_karaoke_tag_type)
|
|
286
|
+
return command
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def build_pyroller_batch_command(
|
|
290
|
+
request: BatchRollRequest,
|
|
291
|
+
tasks: list[dict[str, str]],
|
|
292
|
+
*,
|
|
293
|
+
default_model_store: str | None = None,
|
|
294
|
+
) -> tuple[list[str], str]:
|
|
295
|
+
"""Build a py-roller batch command with a YAML manifest.
|
|
296
|
+
|
|
297
|
+
Returns (command, manifest_yaml_text) so callers can log the manifest.
|
|
298
|
+
"""
|
|
299
|
+
manifest = {"tasks": tasks}
|
|
300
|
+
manifest_text = yaml.safe_dump(manifest, default_flow_style=False, allow_unicode=True,
|
|
301
|
+
sort_keys=False)
|
|
302
|
+
|
|
303
|
+
# Write manifest to a temp file alongside the first project's intermediate dir
|
|
304
|
+
# so it lives in the rollingpebble data directory.
|
|
305
|
+
tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", prefix="batch_manifest_",
|
|
306
|
+
delete=False, encoding="utf-8")
|
|
307
|
+
tmp.write(manifest_text)
|
|
308
|
+
tmp.close()
|
|
309
|
+
manifest_path = tmp.name
|
|
310
|
+
|
|
311
|
+
stage_set = set(normalize_stages(request.stages or "s,f,t,p,a,w"))
|
|
312
|
+
command = ["py-roller", "batch", "--manifest", manifest_path,
|
|
313
|
+
"--progress-format", "jsonl",
|
|
314
|
+
"--stages", ",".join(sorted(stage_set, key=lambda s: PIPELINE_INDEX.get(s, 99))),
|
|
315
|
+
"--language", request.language or "zh",
|
|
316
|
+
"--cleanup", request.cleanup or "on-success",
|
|
317
|
+
"--log-level", request.log_level or "INFO"]
|
|
318
|
+
|
|
319
|
+
if request.continue_on_error:
|
|
320
|
+
command.append("--continue-on-error")
|
|
321
|
+
if request.skip_existing:
|
|
322
|
+
command.append("--skip-existing")
|
|
323
|
+
|
|
324
|
+
if "s" in stage_set:
|
|
325
|
+
_add_option(command, "--splitter-backend", request.splitter_backend)
|
|
326
|
+
_add_option(command, "--splitter-demucs-model", request.splitter_demucs_model)
|
|
327
|
+
_add_option(command, "--splitter-demucs-device", request.splitter_demucs_device)
|
|
328
|
+
_add_option(command, "--splitter-demucs-jobs", request.splitter_demucs_jobs)
|
|
329
|
+
_add_option(command, "--splitter-demucs-overlap", request.splitter_demucs_overlap)
|
|
330
|
+
_add_option(command, "--splitter-demucs-segment", request.splitter_demucs_segment)
|
|
331
|
+
|
|
332
|
+
if "f" in stage_set:
|
|
333
|
+
_add_option(command, "--filter-chain", request.filter_chain)
|
|
334
|
+
|
|
335
|
+
if "t" in stage_set:
|
|
336
|
+
_add_option(command, "--transcriber-backend", request.transcriber_backend)
|
|
337
|
+
_add_option(command, "--transcriber-device", request.transcriber_device)
|
|
338
|
+
_add_option(command, "--transcriber-model-name", request.transcriber_model_name)
|
|
339
|
+
model_path_value = request.transcriber_model_path or (str(default_model_store) if default_model_store else "")
|
|
340
|
+
if model_path_value:
|
|
341
|
+
command.extend(["--transcriber-model-path", str(Path(model_path_value).expanduser())])
|
|
342
|
+
if request.transcriber_local_files_only:
|
|
343
|
+
command.append("--transcriber-local-files-only")
|
|
344
|
+
if request.transcriber_vad_filter:
|
|
345
|
+
command.append("--transcriber-vad-filter")
|
|
346
|
+
_add_option(command, "--transcriber-compute-type", request.transcriber_compute_type)
|
|
347
|
+
_add_option(command, "--transcriber-batch-size", request.transcriber_batch_size)
|
|
348
|
+
_add_option(command, "--transcriber-hf-xet", request.transcriber_hf_xet)
|
|
349
|
+
_add_option(command, "--transcriber-hf-proxy", request.transcriber_hf_proxy)
|
|
350
|
+
_add_option(command, "--transcriber-hf-etag-timeout", request.transcriber_hf_etag_timeout)
|
|
351
|
+
_add_option(command, "--transcriber-hf-download-timeout", request.transcriber_hf_download_timeout)
|
|
352
|
+
_add_option(command, "--transcriber-hf-max-workers", request.transcriber_hf_max_workers)
|
|
353
|
+
|
|
354
|
+
if "p" in stage_set:
|
|
355
|
+
_add_option(command, "--parser-lyrics-encoding", request.parser_lyrics_encoding)
|
|
356
|
+
|
|
357
|
+
if "a" in stage_set:
|
|
358
|
+
_add_option(command, "--aligner-backend", request.aligner_backend)
|
|
359
|
+
_add_option(command, "--aligner-min-gap", request.aligner_min_gap)
|
|
360
|
+
_add_option(command, "--aligner-repetition", request.aligner_repetition)
|
|
361
|
+
|
|
362
|
+
if "w" in stage_set:
|
|
363
|
+
_add_option(command, "--writer-backend", request.writer_backend)
|
|
364
|
+
_add_option(command, "--writer-spacing", request.writer_spacing)
|
|
365
|
+
_add_option(command, "--writer-by-tag", request.writer_by_tag)
|
|
366
|
+
_add_option(command, "--writer-ass-karaoke-tag-type", request.writer_ass_karaoke_tag_type)
|
|
367
|
+
|
|
368
|
+
return command, manifest_text
|