libavalon 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- avalon/__init__.py +1 -0
- avalon/analysis/__init__.py +0 -0
- avalon/analysis/essentia_analyzer.py +140 -0
- avalon/analysis/model_cache.py +140 -0
- avalon/cli.py +413 -0
- avalon/constants.py +137 -0
- avalon/conversion/__init__.py +0 -0
- avalon/conversion/converter.py +159 -0
- avalon/models.py +53 -0
- avalon/pathing.py +83 -0
- avalon/pipeline.py +251 -0
- avalon/state.py +65 -0
- avalon/tagging/__init__.py +0 -0
- avalon/tagging/analysis_blob.py +155 -0
- avalon/tagging/cover_art.py +90 -0
- avalon/tagging/tag_writer.py +249 -0
- avalon/watcher.py +117 -0
- libavalon-0.0.1.dist-info/METADATA +117 -0
- libavalon-0.0.1.dist-info/RECORD +22 -0
- libavalon-0.0.1.dist-info/WHEEL +4 -0
- libavalon-0.0.1.dist-info/entry_points.txt +2 -0
- libavalon-0.0.1.dist-info/licenses/LICENSE +7 -0
avalon/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
|
File without changes
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""Runs the full Essentia extraction roster against audio files.
|
|
2
|
+
|
|
3
|
+
One embedding pass (discogs-effnet) feeds every classifier head, so
|
|
4
|
+
breadth is cheap -- this is the "extract as much information as we can
|
|
5
|
+
from essentia" implementation, not a curated subset.
|
|
6
|
+
|
|
7
|
+
Loading a TensorFlow graph has real fixed overhead (~1-2s each); with 14
|
|
8
|
+
classifier heads plus the embedding extractor, reloading all of them for
|
|
9
|
+
every file measured at ~25s/track. Loading them once and reusing the
|
|
10
|
+
loaded graphs across files drops that to ~1.1s/track (measured), which is
|
|
11
|
+
the difference between a usable batch tool and one that takes days on a
|
|
12
|
+
real library. `EssentiaAnalyzer` is therefore a reusable object, not a
|
|
13
|
+
stateless function: construct one per CLI run and call `.analyze()` per
|
|
14
|
+
file.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import logging
|
|
20
|
+
import os
|
|
21
|
+
|
|
22
|
+
import numpy as np
|
|
23
|
+
|
|
24
|
+
from avalon.analysis import model_cache
|
|
25
|
+
from avalon.constants import to_camelot
|
|
26
|
+
from avalon.models import Label, TrackAnalysis
|
|
27
|
+
|
|
28
|
+
logger = logging.getLogger(__name__)
|
|
29
|
+
|
|
30
|
+
_ANALYSIS_SAMPLE_RATE = 44100
|
|
31
|
+
_EMBEDDING_SAMPLE_RATE = 16000
|
|
32
|
+
_TOP_N_BY_MODEL = {"genre_discogs400": 3, "mtg_jamendo_moodtheme": 5}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _clean_label(raw: str) -> str:
|
|
36
|
+
""" "Electronic---Techno" -> "Electronic / Techno" for readability."""
|
|
37
|
+
return raw.replace("---", " / ")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class EssentiaAnalyzer:
|
|
41
|
+
"""Loads every model once; call `analyze(path)` per file."""
|
|
42
|
+
|
|
43
|
+
def __init__(self) -> None:
|
|
44
|
+
os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "3")
|
|
45
|
+
|
|
46
|
+
import essentia
|
|
47
|
+
import essentia.standard as es
|
|
48
|
+
|
|
49
|
+
essentia.log.warningActive = False
|
|
50
|
+
essentia.log.infoActive = False
|
|
51
|
+
|
|
52
|
+
self._es = es
|
|
53
|
+
embedding_pb, embedding_output = model_cache.get_embedding_model()
|
|
54
|
+
self._embedding_algo = es.TensorflowPredictEffnetDiscogs(
|
|
55
|
+
graphFilename=embedding_pb, output=embedding_output
|
|
56
|
+
)
|
|
57
|
+
self._classifiers: dict[str, tuple[object, model_cache.ModelMeta]] = {}
|
|
58
|
+
for spec in model_cache.CLASSIFIER_HEADS:
|
|
59
|
+
meta = model_cache.get_classifier(spec)
|
|
60
|
+
algo = es.TensorflowPredict2D(
|
|
61
|
+
graphFilename=meta.pb_path,
|
|
62
|
+
input=meta.input_name,
|
|
63
|
+
output=meta.output_name,
|
|
64
|
+
)
|
|
65
|
+
self._classifiers[spec.name] = (algo, meta)
|
|
66
|
+
|
|
67
|
+
def analyze(self, path: str) -> TrackAnalysis:
|
|
68
|
+
es = self._es
|
|
69
|
+
|
|
70
|
+
rhythm_audio = es.MonoLoader(filename=path, sampleRate=_ANALYSIS_SAMPLE_RATE)()
|
|
71
|
+
bpm, _beats, bpm_confidence, _estimates, _intervals = es.RhythmExtractor2013(
|
|
72
|
+
method="multifeature"
|
|
73
|
+
)(rhythm_audio)
|
|
74
|
+
key, scale, key_strength = es.KeyExtractor()(rhythm_audio)
|
|
75
|
+
dynamic_complexity, loudness = es.DynamicComplexity()(rhythm_audio)
|
|
76
|
+
|
|
77
|
+
embedding_audio = es.Resample(
|
|
78
|
+
inputSampleRate=_ANALYSIS_SAMPLE_RATE,
|
|
79
|
+
outputSampleRate=_EMBEDDING_SAMPLE_RATE,
|
|
80
|
+
quality=4,
|
|
81
|
+
)(rhythm_audio)
|
|
82
|
+
embeddings = self._embedding_algo(embedding_audio)
|
|
83
|
+
|
|
84
|
+
binary_probs: dict[str, float] = {}
|
|
85
|
+
categorical: dict[str, tuple[str, float]] = {}
|
|
86
|
+
multilabel: dict[str, list[Label]] = {}
|
|
87
|
+
|
|
88
|
+
for spec in model_cache.CLASSIFIER_HEADS:
|
|
89
|
+
algo, meta = self._classifiers[spec.name]
|
|
90
|
+
track_scores = np.mean(algo(embeddings), axis=0)
|
|
91
|
+
|
|
92
|
+
if spec.kind == "binary":
|
|
93
|
+
binary_probs[spec.name] = float(
|
|
94
|
+
track_scores[meta.classes.index(spec.positive_label)]
|
|
95
|
+
)
|
|
96
|
+
elif spec.kind == "categorical":
|
|
97
|
+
idx = int(np.argmax(track_scores))
|
|
98
|
+
categorical[spec.name] = (meta.classes[idx], float(track_scores[idx]))
|
|
99
|
+
elif spec.kind == "multilabel":
|
|
100
|
+
top_n = _TOP_N_BY_MODEL[spec.name]
|
|
101
|
+
order = np.argsort(track_scores)[::-1][:top_n]
|
|
102
|
+
multilabel[spec.name] = [
|
|
103
|
+
Label(
|
|
104
|
+
name=_clean_label(meta.classes[i]),
|
|
105
|
+
confidence=float(track_scores[i]),
|
|
106
|
+
)
|
|
107
|
+
for i in order
|
|
108
|
+
]
|
|
109
|
+
else:
|
|
110
|
+
raise ValueError(f"Unknown model kind: {spec.kind!r}")
|
|
111
|
+
|
|
112
|
+
gender_label, gender_confidence = categorical["gender"]
|
|
113
|
+
timbre_label, timbre_confidence = categorical["timbre"]
|
|
114
|
+
|
|
115
|
+
return TrackAnalysis(
|
|
116
|
+
bpm=float(bpm),
|
|
117
|
+
bpm_confidence=float(bpm_confidence),
|
|
118
|
+
key=key,
|
|
119
|
+
scale=scale,
|
|
120
|
+
camelot=to_camelot(key, scale),
|
|
121
|
+
key_strength=float(key_strength),
|
|
122
|
+
loudness=float(loudness),
|
|
123
|
+
dynamic_complexity=float(dynamic_complexity),
|
|
124
|
+
mood_aggressive=binary_probs["mood_aggressive"],
|
|
125
|
+
mood_happy=binary_probs["mood_happy"],
|
|
126
|
+
mood_sad=binary_probs["mood_sad"],
|
|
127
|
+
mood_relaxed=binary_probs["mood_relaxed"],
|
|
128
|
+
mood_party=binary_probs["mood_party"],
|
|
129
|
+
danceability=binary_probs["danceability"],
|
|
130
|
+
mood_acoustic=binary_probs["mood_acoustic"],
|
|
131
|
+
mood_electronic=binary_probs["mood_electronic"],
|
|
132
|
+
voice_probability=binary_probs["voice_instrumental"],
|
|
133
|
+
gender=gender_label,
|
|
134
|
+
gender_confidence=gender_confidence,
|
|
135
|
+
tonal_probability=binary_probs["tonal_atonal"],
|
|
136
|
+
timbre=timbre_label,
|
|
137
|
+
timbre_confidence=timbre_confidence,
|
|
138
|
+
genres=multilabel["genre_discogs400"],
|
|
139
|
+
mood_themes=multilabel["mtg_jamendo_moodtheme"],
|
|
140
|
+
)
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""Downloads and caches the Essentia pretrained models avalon uses.
|
|
2
|
+
|
|
3
|
+
Each model's input/output tensor names and class label order come from its
|
|
4
|
+
`.json` sidecar at run time rather than being hardcoded: node names vary
|
|
5
|
+
per model (e.g. genre_discogs400 uses a different input node than the
|
|
6
|
+
mood/character heads), and -- more importantly -- class order is *not*
|
|
7
|
+
consistent (`mood_sad` is `["non_sad", "sad"]` while `mood_happy` is
|
|
8
|
+
`["happy", "non_happy"]`). Trusting the declared schema avoids silently
|
|
9
|
+
inverted probabilities.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import logging
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
import requests
|
|
20
|
+
|
|
21
|
+
from avalon.constants import MODEL_CACHE_DIRNAME
|
|
22
|
+
|
|
23
|
+
logger = logging.getLogger(__name__)
|
|
24
|
+
|
|
25
|
+
_BASE_URL = "https://essentia.upf.edu/models"
|
|
26
|
+
_EMBEDDING_SUBDIR = "feature-extractors/discogs-effnet"
|
|
27
|
+
_EMBEDDING_STEM = "discogs-effnet-bs64-1"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True, slots=True)
|
|
31
|
+
class ModelSpec:
|
|
32
|
+
"""One classifier head. `kind` drives how essentia_analyzer reads its
|
|
33
|
+
output vector:
|
|
34
|
+
binary -> single probability of `positive_label`
|
|
35
|
+
categorical -> whichever of `labels` scores highest, plus its confidence
|
|
36
|
+
multilabel -> independent (sigmoid) scores, no single "positive" class
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
name: str
|
|
40
|
+
subdir: str
|
|
41
|
+
kind: str
|
|
42
|
+
positive_label: str | None = None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
CLASSIFIER_HEADS: tuple[ModelSpec, ...] = (
|
|
46
|
+
ModelSpec("danceability", "danceability", "binary", positive_label="danceable"),
|
|
47
|
+
ModelSpec("mood_acoustic", "mood_acoustic", "binary", positive_label="acoustic"),
|
|
48
|
+
ModelSpec(
|
|
49
|
+
"mood_aggressive", "mood_aggressive", "binary", positive_label="aggressive"
|
|
50
|
+
),
|
|
51
|
+
ModelSpec(
|
|
52
|
+
"mood_electronic", "mood_electronic", "binary", positive_label="electronic"
|
|
53
|
+
),
|
|
54
|
+
ModelSpec("mood_happy", "mood_happy", "binary", positive_label="happy"),
|
|
55
|
+
ModelSpec("mood_sad", "mood_sad", "binary", positive_label="sad"),
|
|
56
|
+
ModelSpec("mood_relaxed", "mood_relaxed", "binary", positive_label="relaxed"),
|
|
57
|
+
ModelSpec("mood_party", "mood_party", "binary", positive_label="party"),
|
|
58
|
+
ModelSpec(
|
|
59
|
+
"voice_instrumental", "voice_instrumental", "binary", positive_label="voice"
|
|
60
|
+
),
|
|
61
|
+
ModelSpec("tonal_atonal", "tonal_atonal", "binary", positive_label="tonal"),
|
|
62
|
+
ModelSpec("gender", "gender", "categorical"),
|
|
63
|
+
ModelSpec("timbre", "timbre", "categorical"),
|
|
64
|
+
ModelSpec("genre_discogs400", "genre_discogs400", "multilabel"),
|
|
65
|
+
ModelSpec("mtg_jamendo_moodtheme", "mtg_jamendo_moodtheme", "multilabel"),
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass(frozen=True, slots=True)
|
|
70
|
+
class ModelMeta:
|
|
71
|
+
"""Parsed `.json` sidecar: local file path plus the schema bits needed
|
|
72
|
+
to run inference generically."""
|
|
73
|
+
|
|
74
|
+
pb_path: str
|
|
75
|
+
input_name: str
|
|
76
|
+
output_name: str
|
|
77
|
+
classes: tuple[str, ...]
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _cache_dir() -> Path:
|
|
81
|
+
path = Path.home() / ".cache" / MODEL_CACHE_DIRNAME / "models"
|
|
82
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
83
|
+
return path
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _download(url: str, dest: Path) -> None:
|
|
87
|
+
if dest.exists():
|
|
88
|
+
return
|
|
89
|
+
logger.info("Downloading model file %s", url)
|
|
90
|
+
response = requests.get(url, timeout=60)
|
|
91
|
+
response.raise_for_status()
|
|
92
|
+
tmp = dest.with_suffix(dest.suffix + ".part")
|
|
93
|
+
tmp.write_bytes(response.content)
|
|
94
|
+
tmp.rename(dest)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _fetch(subdir: str, filename_stem: str) -> tuple[Path, dict]:
|
|
98
|
+
cache = _cache_dir()
|
|
99
|
+
pb_dest = cache / f"{filename_stem}.pb"
|
|
100
|
+
json_dest = cache / f"{filename_stem}.json"
|
|
101
|
+
base = f"{_BASE_URL}/{subdir}/{filename_stem}"
|
|
102
|
+
_download(f"{base}.pb", pb_dest)
|
|
103
|
+
_download(f"{base}.json", json_dest)
|
|
104
|
+
return pb_dest, json.loads(json_dest.read_text())
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _output_by_purpose(outputs: list[dict], purpose: str) -> str:
|
|
108
|
+
for out in outputs:
|
|
109
|
+
if out.get("output_purpose") == purpose:
|
|
110
|
+
return out["name"]
|
|
111
|
+
return outputs[0]["name"]
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def get_embedding_model() -> tuple[str, str]:
|
|
115
|
+
"""Downloads (if needed) the shared discogs-effnet embedding extractor.
|
|
116
|
+
|
|
117
|
+
Returns (pb_path, embedding_output_node_name).
|
|
118
|
+
"""
|
|
119
|
+
pb_path, meta = _fetch(_EMBEDDING_SUBDIR, _EMBEDDING_STEM)
|
|
120
|
+
output_name = _output_by_purpose(meta["schema"]["outputs"], "embeddings")
|
|
121
|
+
return str(pb_path), output_name
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def get_classifier(spec: ModelSpec) -> ModelMeta:
|
|
125
|
+
filename_stem = f"{spec.name}-discogs-effnet-1"
|
|
126
|
+
pb_path, meta = _fetch(f"classification-heads/{spec.subdir}", filename_stem)
|
|
127
|
+
schema = meta["schema"]
|
|
128
|
+
return ModelMeta(
|
|
129
|
+
pb_path=str(pb_path),
|
|
130
|
+
input_name=schema["inputs"][0]["name"],
|
|
131
|
+
output_name=_output_by_purpose(schema["outputs"], "predictions"),
|
|
132
|
+
classes=tuple(meta["classes"]),
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def prefetch_all() -> None:
|
|
137
|
+
"""Downloads every model avalon needs, up front (~26.5MB total)."""
|
|
138
|
+
get_embedding_model()
|
|
139
|
+
for spec in CLASSIFIER_HEADS:
|
|
140
|
+
get_classifier(spec)
|
avalon/cli.py
ADDED
|
@@ -0,0 +1,413 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import logging
|
|
5
|
+
import os
|
|
6
|
+
import random
|
|
7
|
+
import sys
|
|
8
|
+
from collections.abc import Callable, Iterator
|
|
9
|
+
from concurrent.futures import FIRST_COMPLETED, Future, ProcessPoolExecutor, wait
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from avalon.constants import AUDIO_EXTENSIONS
|
|
13
|
+
from avalon.models import ProcessResult
|
|
14
|
+
from avalon.pathing import DEFAULT_TEMPLATE
|
|
15
|
+
from avalon.pipeline import Pipeline, PipelineOptions, init_worker, process_planned_in_worker
|
|
16
|
+
from avalon.tagging import analysis_blob, tag_writer
|
|
17
|
+
from avalon import state as state_module
|
|
18
|
+
from avalon import watcher
|
|
19
|
+
|
|
20
|
+
logger = logging.getLogger(__name__)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
24
|
+
parser = argparse.ArgumentParser(
|
|
25
|
+
prog="avalon", description="Audio analysis, tagging, and organization"
|
|
26
|
+
)
|
|
27
|
+
parser.add_argument("--debug", action="store_true", help="Enable debug logging")
|
|
28
|
+
parser.add_argument(
|
|
29
|
+
"-v", "--verbose", action="store_true", help="Enable verbose (info) logging"
|
|
30
|
+
)
|
|
31
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
32
|
+
|
|
33
|
+
_add_pipeline_flags(_add_analyze_parser(subparsers))
|
|
34
|
+
_add_pipeline_flags(_add_watch_parser(subparsers))
|
|
35
|
+
_add_inspect_parser(subparsers)
|
|
36
|
+
return parser
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _add_analyze_parser(subparsers) -> argparse.ArgumentParser:
|
|
40
|
+
parser = subparsers.add_parser(
|
|
41
|
+
"analyze", help="Analyze/tag/convert a file or folder"
|
|
42
|
+
)
|
|
43
|
+
parser.add_argument(
|
|
44
|
+
"sources", nargs="+", help="Audio file(s) or folder(s) to process"
|
|
45
|
+
)
|
|
46
|
+
parser.add_argument(
|
|
47
|
+
"--recursive", action="store_true", help="Recurse into subfolders"
|
|
48
|
+
)
|
|
49
|
+
parser.add_argument(
|
|
50
|
+
"--dry-run",
|
|
51
|
+
action="store_true",
|
|
52
|
+
help="Show what would happen without writing anything",
|
|
53
|
+
)
|
|
54
|
+
parser.add_argument(
|
|
55
|
+
"--random",
|
|
56
|
+
type=int,
|
|
57
|
+
default=None,
|
|
58
|
+
metavar="N",
|
|
59
|
+
help="Process a random sample of N files instead of everything found "
|
|
60
|
+
"(fast, repeatable smoke-testing on a large library)",
|
|
61
|
+
)
|
|
62
|
+
parser.add_argument(
|
|
63
|
+
"--workers",
|
|
64
|
+
type=int,
|
|
65
|
+
default=1,
|
|
66
|
+
metavar="N",
|
|
67
|
+
help="Process N files concurrently in separate worker processes (default: 1, sequential)",
|
|
68
|
+
)
|
|
69
|
+
parser.set_defaults(func=run_analyze)
|
|
70
|
+
return parser
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _add_watch_parser(subparsers) -> argparse.ArgumentParser:
|
|
74
|
+
parser = subparsers.add_parser(
|
|
75
|
+
"watch", help="Watch folder(s) and process new/changed files"
|
|
76
|
+
)
|
|
77
|
+
parser.add_argument("sources", nargs="+", help="Folder(s) to watch")
|
|
78
|
+
parser.add_argument(
|
|
79
|
+
"--debounce-seconds",
|
|
80
|
+
type=int,
|
|
81
|
+
default=5,
|
|
82
|
+
help="Quiet period before processing a file (default: 5)",
|
|
83
|
+
)
|
|
84
|
+
parser.add_argument(
|
|
85
|
+
"--no-backfill",
|
|
86
|
+
action="store_true",
|
|
87
|
+
help="Skip processing pre-existing files on startup",
|
|
88
|
+
)
|
|
89
|
+
parser.set_defaults(func=run_watch)
|
|
90
|
+
return parser
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _headline_fields_type(raw: str) -> tuple[str, ...]:
|
|
94
|
+
try:
|
|
95
|
+
return analysis_blob.parse_headline_fields(raw)
|
|
96
|
+
except ValueError as exc:
|
|
97
|
+
raise argparse.ArgumentTypeError(str(exc)) from exc
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _add_pipeline_flags(parser: argparse.ArgumentParser) -> None:
|
|
101
|
+
parser.add_argument(
|
|
102
|
+
"--dest", type=str, default=None, help="Destination root; omit to tag in place"
|
|
103
|
+
)
|
|
104
|
+
parser.add_argument("--path-template", type=str, default=DEFAULT_TEMPLATE)
|
|
105
|
+
parser.add_argument(
|
|
106
|
+
"--convert-lossless-to",
|
|
107
|
+
type=str,
|
|
108
|
+
default=None,
|
|
109
|
+
help="Re-encode lossless sources (FLAC/ALAC/WAV/AIFF/...) to this format (e.g. aiff); "
|
|
110
|
+
"lossy sources (mp3, aac, ...) are left untouched regardless",
|
|
111
|
+
)
|
|
112
|
+
parser.add_argument("--max-sample-rate", type=int, default=None)
|
|
113
|
+
parser.add_argument("--max-bit-depth", type=int, default=None)
|
|
114
|
+
parser.add_argument(
|
|
115
|
+
"--no-analyze", action="store_true", help="Skip essentia analysis"
|
|
116
|
+
)
|
|
117
|
+
parser.add_argument(
|
|
118
|
+
"--no-convert", action="store_true", help="Skip format/rate/depth conversion"
|
|
119
|
+
)
|
|
120
|
+
parser.add_argument(
|
|
121
|
+
"--force-reanalyze",
|
|
122
|
+
action="store_true",
|
|
123
|
+
help="Re-run analysis even if already current",
|
|
124
|
+
)
|
|
125
|
+
parser.add_argument(
|
|
126
|
+
"--overwrite", action="store_true", help="Overwrite existing destination files"
|
|
127
|
+
)
|
|
128
|
+
parser.add_argument(
|
|
129
|
+
"--overwrite-description",
|
|
130
|
+
action="store_true",
|
|
131
|
+
help="Replace the headline tag instead of merging into it",
|
|
132
|
+
)
|
|
133
|
+
parser.add_argument(
|
|
134
|
+
"--headline-tag",
|
|
135
|
+
type=str,
|
|
136
|
+
default=None,
|
|
137
|
+
help="Tag/field name to write the headline to (default: COMM for MP3/AIFF/WAV, "
|
|
138
|
+
"DESCRIPTION for FLAC, desc for MP4). A name other than the default becomes a "
|
|
139
|
+
"TXXX frame (ID3-family) or freeform atom (MP4) rather than the native comment field",
|
|
140
|
+
)
|
|
141
|
+
parser.add_argument(
|
|
142
|
+
"--headline-format",
|
|
143
|
+
type=_headline_fields_type,
|
|
144
|
+
default=analysis_blob.DEFAULT_HEADLINE_FIELDS,
|
|
145
|
+
help="Comma-separated fields (and order) for the headline tag. Available: "
|
|
146
|
+
f"{', '.join(analysis_blob.HEADLINE_FIELD_VALUES)} "
|
|
147
|
+
f"(default: {','.join(analysis_blob.DEFAULT_HEADLINE_FIELDS)})",
|
|
148
|
+
)
|
|
149
|
+
parser.add_argument(
|
|
150
|
+
"--delete-original",
|
|
151
|
+
action="store_true",
|
|
152
|
+
help="Delete the source file after successful processing",
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _add_inspect_parser(subparsers) -> None:
|
|
157
|
+
parser = subparsers.add_parser(
|
|
158
|
+
"inspect", help="Show a file's parsed canonical + analysis tags"
|
|
159
|
+
)
|
|
160
|
+
parser.add_argument("path", help="Audio file to inspect")
|
|
161
|
+
parser.set_defaults(func=run_inspect)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def setup_logging(debug: bool, verbose: bool) -> None:
|
|
165
|
+
level = logging.DEBUG if debug else (logging.INFO if verbose else logging.WARNING)
|
|
166
|
+
logging.basicConfig(
|
|
167
|
+
level=level, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
|
168
|
+
)
|
|
169
|
+
logging.getLogger("urllib3").setLevel(logging.INFO)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _is_audio_file(path: Path) -> bool:
|
|
173
|
+
return path.suffix.lower() in AUDIO_EXTENSIONS and not path.name.startswith("._")
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _iter_files_shuffled(root: Path, recursive: bool) -> Iterator[Path]:
|
|
177
|
+
with os.scandir(root) as it:
|
|
178
|
+
entries = list(it)
|
|
179
|
+
random.shuffle(entries)
|
|
180
|
+
subdirs = []
|
|
181
|
+
for entry in entries:
|
|
182
|
+
if entry.is_file():
|
|
183
|
+
yield Path(entry.path)
|
|
184
|
+
elif recursive and entry.is_dir():
|
|
185
|
+
subdirs.append(entry.path)
|
|
186
|
+
for subdir in subdirs:
|
|
187
|
+
yield from _iter_files_shuffled(Path(subdir), recursive)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def gather_files(
|
|
191
|
+
sources: list[str], recursive: bool, sample_size: int | None = None
|
|
192
|
+
) -> Iterator[Path]:
|
|
193
|
+
found = 0
|
|
194
|
+
for source in sources:
|
|
195
|
+
path = Path(source)
|
|
196
|
+
if path.is_file():
|
|
197
|
+
if _is_audio_file(path):
|
|
198
|
+
yield path
|
|
199
|
+
found += 1
|
|
200
|
+
if sample_size is not None and found >= sample_size:
|
|
201
|
+
return
|
|
202
|
+
continue
|
|
203
|
+
if not path.is_dir():
|
|
204
|
+
logger.warning("Source not found: %s", path)
|
|
205
|
+
continue
|
|
206
|
+
walker = (
|
|
207
|
+
_iter_files_shuffled(path, recursive)
|
|
208
|
+
if sample_size is not None
|
|
209
|
+
else (path.rglob("*") if recursive else path.glob("*"))
|
|
210
|
+
)
|
|
211
|
+
for p in walker:
|
|
212
|
+
if p.is_file() and _is_audio_file(p):
|
|
213
|
+
yield p
|
|
214
|
+
found += 1
|
|
215
|
+
if sample_size is not None and found >= sample_size:
|
|
216
|
+
return
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _default_state_dir(first_source: str) -> Path:
|
|
220
|
+
path = Path(first_source).resolve()
|
|
221
|
+
return path if path.is_dir() else path.parent
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _pipeline_options_from_args(args: argparse.Namespace) -> PipelineOptions:
|
|
225
|
+
return PipelineOptions(
|
|
226
|
+
dest_root=Path(args.dest) if args.dest else None,
|
|
227
|
+
path_template=args.path_template,
|
|
228
|
+
convert_lossless_to=args.convert_lossless_to,
|
|
229
|
+
max_sample_rate=args.max_sample_rate,
|
|
230
|
+
max_bit_depth=args.max_bit_depth,
|
|
231
|
+
do_analyze=not args.no_analyze,
|
|
232
|
+
do_convert=not args.no_convert,
|
|
233
|
+
force_reanalyze=args.force_reanalyze,
|
|
234
|
+
overwrite=args.overwrite,
|
|
235
|
+
overwrite_description=args.overwrite_description,
|
|
236
|
+
headline_tag=args.headline_tag,
|
|
237
|
+
headline_fields=args.headline_format,
|
|
238
|
+
delete_original=args.delete_original,
|
|
239
|
+
dry_run=getattr(args, "dry_run", False),
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _process_parallel(
|
|
244
|
+
pipeline: Pipeline,
|
|
245
|
+
paths: Iterator[Path],
|
|
246
|
+
workers: int,
|
|
247
|
+
handle: Callable[[Path, ProcessResult], None],
|
|
248
|
+
) -> None:
|
|
249
|
+
max_in_flight = workers * 2
|
|
250
|
+
with ProcessPoolExecutor(
|
|
251
|
+
max_workers=workers, initializer=init_worker, initargs=(pipeline.options,)
|
|
252
|
+
) as executor:
|
|
253
|
+
in_flight: dict[Future, Path] = {}
|
|
254
|
+
|
|
255
|
+
def submit_next() -> bool:
|
|
256
|
+
path = next(paths, None)
|
|
257
|
+
if path is None:
|
|
258
|
+
return False
|
|
259
|
+
try:
|
|
260
|
+
planned = pipeline.plan(path)
|
|
261
|
+
except Exception as exc:
|
|
262
|
+
handle(
|
|
263
|
+
path,
|
|
264
|
+
ProcessResult(
|
|
265
|
+
source_path=str(path),
|
|
266
|
+
output_path=str(path),
|
|
267
|
+
analyzed=False,
|
|
268
|
+
converted=False,
|
|
269
|
+
error=str(exc),
|
|
270
|
+
),
|
|
271
|
+
)
|
|
272
|
+
return True
|
|
273
|
+
in_flight[executor.submit(process_planned_in_worker, planned)] = path
|
|
274
|
+
return True
|
|
275
|
+
|
|
276
|
+
while len(in_flight) < max_in_flight and submit_next():
|
|
277
|
+
pass
|
|
278
|
+
while in_flight:
|
|
279
|
+
done, _ = wait(in_flight, return_when=FIRST_COMPLETED)
|
|
280
|
+
for future in done:
|
|
281
|
+
path = in_flight.pop(future)
|
|
282
|
+
handle(path, future.result())
|
|
283
|
+
submit_next()
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def run_analyze(args: argparse.Namespace) -> int:
|
|
287
|
+
options = _pipeline_options_from_args(args)
|
|
288
|
+
pipeline = Pipeline(options)
|
|
289
|
+
dest_root = options.dest_root or _default_state_dir(args.sources[0])
|
|
290
|
+
state = state_module.load(dest_root)
|
|
291
|
+
skip_fast_path = args.dry_run or args.force_reanalyze
|
|
292
|
+
|
|
293
|
+
def unprocessed() -> Iterator[Path]:
|
|
294
|
+
for path in gather_files(args.sources, args.recursive, sample_size=args.random):
|
|
295
|
+
if not skip_fast_path and state_module.is_unchanged(state, path):
|
|
296
|
+
logger.debug("Unchanged, skipping: %s", path)
|
|
297
|
+
continue
|
|
298
|
+
yield path
|
|
299
|
+
|
|
300
|
+
failures: list[tuple[Path, str]] = []
|
|
301
|
+
processed = 0
|
|
302
|
+
|
|
303
|
+
def handle(path: Path, result: ProcessResult) -> None:
|
|
304
|
+
nonlocal processed
|
|
305
|
+
if result.error:
|
|
306
|
+
failures.append((path, result.error))
|
|
307
|
+
logger.error("Failed: %s: %s", path, result.error)
|
|
308
|
+
return
|
|
309
|
+
processed += 1
|
|
310
|
+
logger.info(
|
|
311
|
+
"%s%s -> %s",
|
|
312
|
+
"[dry-run] " if args.dry_run else "",
|
|
313
|
+
path,
|
|
314
|
+
result.output_path,
|
|
315
|
+
)
|
|
316
|
+
if not args.dry_run:
|
|
317
|
+
state_module.record(state, path)
|
|
318
|
+
|
|
319
|
+
if args.workers > 1:
|
|
320
|
+
_process_parallel(pipeline, unprocessed(), args.workers, handle)
|
|
321
|
+
else:
|
|
322
|
+
for path in unprocessed():
|
|
323
|
+
handle(path, pipeline.process_file(path))
|
|
324
|
+
|
|
325
|
+
if not args.dry_run:
|
|
326
|
+
state_module.save(dest_root, state)
|
|
327
|
+
|
|
328
|
+
logger.info("Processed %d file(s), %d failure(s)", processed, len(failures))
|
|
329
|
+
return 0 if not failures else 2
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def run_watch(args: argparse.Namespace) -> int:
|
|
333
|
+
source_roots = [Path(s).resolve() for s in args.sources]
|
|
334
|
+
missing = [s for s in source_roots if not s.is_dir()]
|
|
335
|
+
if missing:
|
|
336
|
+
for path in missing:
|
|
337
|
+
logger.error("Source folder not found: %s", path)
|
|
338
|
+
return 1
|
|
339
|
+
|
|
340
|
+
options = _pipeline_options_from_args(args)
|
|
341
|
+
pipeline = Pipeline(options)
|
|
342
|
+
dest_root = options.dest_root or source_roots[0]
|
|
343
|
+
state = state_module.load(dest_root)
|
|
344
|
+
|
|
345
|
+
skip_fast_path = args.force_reanalyze
|
|
346
|
+
|
|
347
|
+
def handle(path: Path) -> None:
|
|
348
|
+
if not skip_fast_path and state_module.is_unchanged(state, path):
|
|
349
|
+
return
|
|
350
|
+
result = pipeline.process_file(path)
|
|
351
|
+
if result.error:
|
|
352
|
+
logger.error("Failed: %s: %s", path, result.error)
|
|
353
|
+
else:
|
|
354
|
+
logger.info("%s -> %s", path, result.output_path)
|
|
355
|
+
state_module.record(state, path)
|
|
356
|
+
state_module.save(dest_root, state)
|
|
357
|
+
|
|
358
|
+
if not args.no_backfill:
|
|
359
|
+
logger.info("Backfilling existing files")
|
|
360
|
+
for path in gather_files([str(root) for root in source_roots], recursive=True):
|
|
361
|
+
handle(path)
|
|
362
|
+
|
|
363
|
+
watcher.watch(source_roots, handle, debounce_seconds=args.debounce_seconds)
|
|
364
|
+
return 0
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def run_inspect(args: argparse.Namespace) -> int:
|
|
368
|
+
path = args.path
|
|
369
|
+
file_format = tag_writer.detect_format(path)
|
|
370
|
+
audio = tag_writer.load(path, file_format)
|
|
371
|
+
|
|
372
|
+
print(f"path: {path}")
|
|
373
|
+
print(f"format: {file_format.value}")
|
|
374
|
+
print("\ncanonical fields:")
|
|
375
|
+
for key, value in tag_writer.read_canonical(audio, file_format).items():
|
|
376
|
+
print(f" {key}: {value}")
|
|
377
|
+
|
|
378
|
+
headline = tag_writer.read_headline(audio, file_format)
|
|
379
|
+
print(f"\nheadline tag (raw): {headline}")
|
|
380
|
+
parsed_headline = analysis_blob.parse_headline(headline)
|
|
381
|
+
if parsed_headline:
|
|
382
|
+
print("headline tag (parsed):")
|
|
383
|
+
for key, value in parsed_headline.items():
|
|
384
|
+
print(f" {key}: {value}")
|
|
385
|
+
|
|
386
|
+
extended = tag_writer.read_extended(audio, file_format)
|
|
387
|
+
print(f"\nextended tag (raw): {extended}")
|
|
388
|
+
if extended:
|
|
389
|
+
print("extended tag (parsed):")
|
|
390
|
+
for key, value in analysis_blob.decode_extended(extended).items():
|
|
391
|
+
print(f" {key}: {value}")
|
|
392
|
+
for field in ("genre", "moodtheme"):
|
|
393
|
+
labels = analysis_blob.decode_extended_labels(extended, field)
|
|
394
|
+
if labels:
|
|
395
|
+
print(
|
|
396
|
+
f" {field} labels: "
|
|
397
|
+
+ ", ".join(
|
|
398
|
+
f"{label.name} ({label.confidence:.2f})" for label in labels
|
|
399
|
+
)
|
|
400
|
+
)
|
|
401
|
+
return 0
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def main() -> None:
|
|
405
|
+
parser = build_parser()
|
|
406
|
+
args = parser.parse_args()
|
|
407
|
+
setup_logging(args.debug, args.verbose)
|
|
408
|
+
exit_code = args.func(args)
|
|
409
|
+
sys.exit(exit_code)
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
if __name__ == "__main__":
|
|
413
|
+
main()
|