collapsarr 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- collapsarr/__init__.py +15 -0
- collapsarr/__main__.py +25 -0
- collapsarr/arr/__init__.py +76 -0
- collapsarr/arr/client.py +71 -0
- collapsarr/arr/files.py +211 -0
- collapsarr/arr/models.py +147 -0
- collapsarr/arr/routes.py +290 -0
- collapsarr/arr/service.py +222 -0
- collapsarr/arr/webhooks.py +200 -0
- collapsarr/auth.py +83 -0
- collapsarr/config.py +95 -0
- collapsarr/database.py +82 -0
- collapsarr/downmix/__init__.py +54 -0
- collapsarr/downmix/apply.py +241 -0
- collapsarr/downmix/pipeline.py +235 -0
- collapsarr/downmix/probe.py +310 -0
- collapsarr/downmix/remux.py +284 -0
- collapsarr/downmix/targets.py +145 -0
- collapsarr/frontend.py +61 -0
- collapsarr/health.py +111 -0
- collapsarr/jobs/__init__.py +49 -0
- collapsarr/jobs/failure_notify.py +145 -0
- collapsarr/jobs/history.py +153 -0
- collapsarr/jobs/models.py +90 -0
- collapsarr/jobs/queue.py +382 -0
- collapsarr/jobs/routes.py +193 -0
- collapsarr/jobs/scheduler.py +402 -0
- collapsarr/main.py +212 -0
- collapsarr/media/__init__.py +38 -0
- collapsarr/media/models.py +132 -0
- collapsarr/media/routes.py +88 -0
- collapsarr/media/service.py +241 -0
- collapsarr/notify/__init__.py +37 -0
- collapsarr/notify/dispatch.py +162 -0
- collapsarr/notify/models.py +63 -0
- collapsarr/notify/routes.py +105 -0
- collapsarr/notify/service.py +80 -0
- collapsarr/settings/__init__.py +29 -0
- collapsarr/settings/models.py +107 -0
- collapsarr/settings/routes.py +157 -0
- collapsarr/settings/service.py +158 -0
- collapsarr/static/apple-touch-icon.png +0 -0
- collapsarr/static/assets/index-Dfozi1oM.js +68 -0
- collapsarr/static/assets/index-J2qCuzXj.css +1 -0
- collapsarr/static/favicon-32.png +0 -0
- collapsarr/static/favicon.svg +10 -0
- collapsarr/static/index.html +18 -0
- collapsarr-0.1.3.dist-info/METADATA +218 -0
- collapsarr-0.1.3.dist-info/RECORD +52 -0
- collapsarr-0.1.3.dist-info/WHEEL +4 -0
- collapsarr-0.1.3.dist-info/entry_points.txt +2 -0
- collapsarr-0.1.3.dist-info/licenses/LICENSE +674 -0
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""ORM model for persisted job run history (COL-21).
|
|
2
|
+
|
|
3
|
+
:mod:`collapsarr.jobs.queue` (COL-20) runs each enqueued file through the
|
|
4
|
+
downmix pipeline and captures the outcome onto its own in-memory
|
|
5
|
+
:class:`~collapsarr.jobs.queue.Job` -- ``status``, ``result``/``error``, and
|
|
6
|
+
(as of COL-21) ``started_at``/``ended_at``. That in-memory state doesn't
|
|
7
|
+
survive a process restart and isn't queryable, which is what this module
|
|
8
|
+
fixes: :class:`JobHistory` is the durable row a completed (or in-flight)
|
|
9
|
+
:class:`~collapsarr.jobs.queue.Job` gets persisted into, ready for a future
|
|
10
|
+
Activity/History view.
|
|
11
|
+
|
|
12
|
+
Deliberately reuses :class:`~collapsarr.jobs.queue.JobStatus` for the
|
|
13
|
+
``status`` column rather than inventing a parallel status vocabulary --
|
|
14
|
+
``JobStatus.PENDING`` is this ticket's "queued" per the acceptance criteria
|
|
15
|
+
(the job is enqueued and has not started running yet).
|
|
16
|
+
|
|
17
|
+
:mod:`collapsarr.jobs.history` is the service layer (record/list/get) built
|
|
18
|
+
on top of this model; nothing in this module touches a session.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
from datetime import UTC, datetime
|
|
24
|
+
|
|
25
|
+
from sqlalchemy import Enum as SAEnum
|
|
26
|
+
from sqlalchemy import Integer, String, Text
|
|
27
|
+
from sqlalchemy.orm import Mapped, mapped_column
|
|
28
|
+
|
|
29
|
+
from collapsarr.database import Base
|
|
30
|
+
from collapsarr.jobs.queue import JobStatus
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _utcnow() -> datetime:
|
|
34
|
+
return datetime.now(UTC)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class JobHistory(Base):
|
|
38
|
+
"""One persisted record of a single job run.
|
|
39
|
+
|
|
40
|
+
``job_id`` is the string form of the originating
|
|
41
|
+
:attr:`~collapsarr.jobs.queue.Job.id` and is unique -- :func:`collapsarr.
|
|
42
|
+
jobs.history.record_job_history` upserts by it, so the same row is
|
|
43
|
+
updated in place as a job progresses from queued -> running ->
|
|
44
|
+
succeeded/failed rather than accumulating one row per state change.
|
|
45
|
+
|
|
46
|
+
``exit_code`` is FFmpeg's exit code from the remux stage
|
|
47
|
+
(:attr:`~collapsarr.downmix.remux.RemuxResult.returncode`) when the
|
|
48
|
+
pipeline reached that stage, else ``None`` (e.g. a probe failure, or
|
|
49
|
+
"nothing to do"). ``error_text`` is populated for a failed run: either
|
|
50
|
+
the unexpected exception's message, or the pipeline result's ``detail``
|
|
51
|
+
when the pipeline itself reported a failure outcome.
|
|
52
|
+
|
|
53
|
+
``target``/``language`` capture what the job was configured to do --
|
|
54
|
+
the comma-joined enabled downmix targets and language allow-list from
|
|
55
|
+
the job's :class:`~collapsarr.downmix.targets.DownmixSettings` -- so
|
|
56
|
+
they're always present regardless of which stage the run reached
|
|
57
|
+
(unlike the pipeline's ``tracks_added``, which is only populated on
|
|
58
|
+
success). ``language`` is ``None`` when the job had no allow-list
|
|
59
|
+
(evaluates every language present on the file).
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
__tablename__ = "job_history"
|
|
63
|
+
|
|
64
|
+
id: Mapped[int] = mapped_column(primary_key=True)
|
|
65
|
+
job_id: Mapped[str] = mapped_column(String(36), nullable=False, unique=True, index=True)
|
|
66
|
+
file_path: Mapped[str] = mapped_column(String(1000), nullable=False, index=True)
|
|
67
|
+
status: Mapped[JobStatus] = mapped_column(
|
|
68
|
+
SAEnum(
|
|
69
|
+
JobStatus,
|
|
70
|
+
values_callable=lambda enum_cls: [member.value for member in enum_cls],
|
|
71
|
+
),
|
|
72
|
+
nullable=False,
|
|
73
|
+
default=JobStatus.PENDING,
|
|
74
|
+
index=True,
|
|
75
|
+
)
|
|
76
|
+
started_at: Mapped[datetime | None] = mapped_column(nullable=True, default=None)
|
|
77
|
+
ended_at: Mapped[datetime | None] = mapped_column(nullable=True, default=None)
|
|
78
|
+
exit_code: Mapped[int | None] = mapped_column(Integer, nullable=True, default=None)
|
|
79
|
+
error_text: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
|
|
80
|
+
target: Mapped[str | None] = mapped_column(String(100), nullable=True, default=None)
|
|
81
|
+
language: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
|
82
|
+
|
|
83
|
+
created_at: Mapped[datetime] = mapped_column(default=_utcnow)
|
|
84
|
+
updated_at: Mapped[datetime] = mapped_column(default=_utcnow, onupdate=_utcnow)
|
|
85
|
+
|
|
86
|
+
def __repr__(self) -> str:
|
|
87
|
+
return (
|
|
88
|
+
f"JobHistory(id={self.id!r}, job_id={self.job_id!r}, "
|
|
89
|
+
f"file_path={self.file_path!r}, status={self.status!r})"
|
|
90
|
+
)
|
collapsarr/jobs/queue.py
ADDED
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
"""Job queue and bounded-concurrency worker pool (COL-20).
|
|
2
|
+
|
|
3
|
+
Wires the Downmix Engine's end-to-end pipeline
|
|
4
|
+
(:func:`~collapsarr.downmix.pipeline.run_downmix_pipeline`, COL-19) to a job
|
|
5
|
+
queue: :meth:`JobQueue.enqueue` a file plus its target/language context (a
|
|
6
|
+
:class:`~collapsarr.downmix.targets.DownmixSettings`), then
|
|
7
|
+
:meth:`JobQueue.run_pending` drains the queue, running at most
|
|
8
|
+
``max_concurrency`` jobs at once (default 1).
|
|
9
|
+
|
|
10
|
+
Each job's execution invokes the pipeline synchronously in a worker thread
|
|
11
|
+
and captures whatever it returns (or, as a safety net, whatever it raises)
|
|
12
|
+
onto the :class:`Job` itself -- ``status`` plus ``result``/``error``. When a
|
|
13
|
+
``history_recorder`` is configured (see :class:`JobQueue`), that same
|
|
14
|
+
worker thread calls it once the job reaches a terminal status, so every job
|
|
15
|
+
run is persisted (COL-21, :mod:`collapsarr.jobs.history`) without the
|
|
16
|
+
caller having to remember to do it. Likewise, when a ``failure_notifier`` is
|
|
17
|
+
configured, that same worker thread calls it -- but only for a job that
|
|
18
|
+
reached ``FAILED`` -- so a downmix failure fans out to the configured
|
|
19
|
+
notifiers (COL-37, :mod:`collapsarr.jobs.failure_notify`) without the
|
|
20
|
+
caller having to remember to do it either.
|
|
21
|
+
|
|
22
|
+
This module deliberately does not import :mod:`collapsarr.jobs.history` or
|
|
23
|
+
:mod:`collapsarr.jobs.failure_notify` itself (those modules import *this*
|
|
24
|
+
one, for :class:`Job`/:class:`JobStatus` -- importing them back here would
|
|
25
|
+
be circular). Instead ``history_recorder``/``failure_notifier`` are plain
|
|
26
|
+
injected callables, the same seam ``pipeline_runner`` already uses;
|
|
27
|
+
:func:`collapsarr.jobs.history.make_history_recorder` and
|
|
28
|
+
:func:`collapsarr.jobs.failure_notify.make_failure_notifier` build ones
|
|
29
|
+
bound to a session factory.
|
|
30
|
+
|
|
31
|
+
Threads, not asyncio: every stage of the downmix pipeline shells out to
|
|
32
|
+
``ffprobe``/``ffmpeg`` via blocking :mod:`subprocess` calls, so a small
|
|
33
|
+
:class:`~concurrent.futures.ThreadPoolExecutor` sized to ``max_concurrency``
|
|
34
|
+
gives genuine bounded parallelism (the GIL is released for the whole
|
|
35
|
+
``subprocess.run`` call) without pulling the rest of this synchronous
|
|
36
|
+
codebase onto an event loop.
|
|
37
|
+
|
|
38
|
+
``max_concurrency`` is a plain constructor argument, the same "Settings-
|
|
39
|
+
shaped stand-in" pattern :class:`~collapsarr.downmix.targets.DownmixSettings`
|
|
40
|
+
already uses -- there is no persisted Settings model yet. It defaults to 1,
|
|
41
|
+
and :meth:`JobQueue.from_settings` sources it from
|
|
42
|
+
:class:`~collapsarr.config.Settings`'s ``job_max_concurrency`` (env
|
|
43
|
+
``COLLAPSARR_JOB_MAX_CONCURRENCY``), which is the closest thing this repo has
|
|
44
|
+
to a Settings store today.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
from __future__ import annotations
|
|
48
|
+
|
|
49
|
+
import threading
|
|
50
|
+
from collections.abc import Callable, Mapping
|
|
51
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
52
|
+
from dataclasses import dataclass, field
|
|
53
|
+
from datetime import UTC, datetime
|
|
54
|
+
from enum import Enum
|
|
55
|
+
from pathlib import Path
|
|
56
|
+
from typing import Any
|
|
57
|
+
from uuid import UUID, uuid4
|
|
58
|
+
|
|
59
|
+
from collapsarr.config import Settings, get_settings
|
|
60
|
+
from collapsarr.downmix.pipeline import PipelineResult, run_downmix_pipeline
|
|
61
|
+
from collapsarr.downmix.targets import DownmixSettings
|
|
62
|
+
|
|
63
|
+
DEFAULT_MAX_CONCURRENCY = 1
|
|
64
|
+
|
|
65
|
+
#: Signature every pipeline runner (real or injected-for-tests) must match:
|
|
66
|
+
#: ``(file_path, settings, **pipeline_kwargs) -> PipelineResult``.
|
|
67
|
+
PipelineRunner = Callable[..., PipelineResult]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class JobStatus(Enum):
|
|
71
|
+
"""Lifecycle state of a single :class:`Job`."""
|
|
72
|
+
|
|
73
|
+
PENDING = "pending"
|
|
74
|
+
RUNNING = "running"
|
|
75
|
+
SUCCEEDED = "succeeded"
|
|
76
|
+
FAILED = "failed"
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass(slots=True)
|
|
80
|
+
class Job:
|
|
81
|
+
"""One enqueued unit of work: a file plus its downmix target/language context.
|
|
82
|
+
|
|
83
|
+
``id`` uniquely identifies the job -- COL-21's job-history layer
|
|
84
|
+
(:mod:`collapsarr.jobs.history`) persists against it as ``job_id``.
|
|
85
|
+
``status``, ``result``/``error``, and ``started_at``/``ended_at`` start
|
|
86
|
+
empty and are filled in by the queue as the job runs -- never mutate
|
|
87
|
+
them directly.
|
|
88
|
+
|
|
89
|
+
``result`` carries the pipeline's :class:`~collapsarr.downmix.pipeline.PipelineResult`
|
|
90
|
+
when the pipeline ran (success, no-op, or a captured failure at any
|
|
91
|
+
stage). ``error`` is populated instead only in the unexpected case where
|
|
92
|
+
the pipeline runner itself raised rather than returning a result (the
|
|
93
|
+
real pipeline never does this -- see its own docstring -- but an
|
|
94
|
+
injected runner in a test, or a future alternate runner, might).
|
|
95
|
+
|
|
96
|
+
``started_at``/``ended_at`` are stamped (UTC) by :meth:`JobQueue._run_job`
|
|
97
|
+
when the job transitions to ``RUNNING`` and when it reaches a terminal
|
|
98
|
+
status, respectively -- the start/end timestamps COL-21's job history
|
|
99
|
+
persists. Both stay ``None`` for a job that has never run.
|
|
100
|
+
"""
|
|
101
|
+
|
|
102
|
+
file_path: Path
|
|
103
|
+
settings: DownmixSettings
|
|
104
|
+
id: UUID = field(default_factory=uuid4)
|
|
105
|
+
status: JobStatus = JobStatus.PENDING
|
|
106
|
+
result: PipelineResult | None = None
|
|
107
|
+
error: BaseException | None = None
|
|
108
|
+
started_at: datetime | None = None
|
|
109
|
+
ended_at: datetime | None = None
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
#: Signature a ``history_recorder`` must match: takes the just-terminated
|
|
113
|
+
#: ``Job`` and persists it (see :func:`collapsarr.jobs.history.make_history_recorder`).
|
|
114
|
+
HistoryRecorder = Callable[[Job], None]
|
|
115
|
+
|
|
116
|
+
#: Signature a ``failure_notifier`` must match: takes the just-terminated
|
|
117
|
+
#: ``Job`` (only ever called for one with ``status is JobStatus.FAILED``) and
|
|
118
|
+
#: dispatches a notification for it (see :func:`collapsarr.jobs.
|
|
119
|
+
#: failure_notify.make_failure_notifier`). Must never raise -- see
|
|
120
|
+
#: :meth:`JobQueue._notify_failure`.
|
|
121
|
+
FailureNotifier = Callable[[Job], None]
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class JobQueue:
|
|
125
|
+
"""Bounded-concurrency queue that runs the downmix pipeline per enqueued file.
|
|
126
|
+
|
|
127
|
+
Usage::
|
|
128
|
+
|
|
129
|
+
queue = JobQueue(max_concurrency=2)
|
|
130
|
+
queue.enqueue("/media/movie.mkv", DownmixSettings())
|
|
131
|
+
queue.enqueue("/media/episode.mkv", DownmixSettings())
|
|
132
|
+
jobs = queue.run_pending() # blocks until both have run
|
|
133
|
+
|
|
134
|
+
:meth:`run_pending` snapshots whatever is pending at the moment it is
|
|
135
|
+
called and runs exactly that batch, respecting ``max_concurrency``, then
|
|
136
|
+
returns those jobs (each updated in place with its final ``status`` and
|
|
137
|
+
``result``). Jobs enqueued *during* a call are not picked up by it --
|
|
138
|
+
call :meth:`run_pending` again for a later batch. This keeps behaviour
|
|
139
|
+
simple and fully deterministic for tests; a long-running background
|
|
140
|
+
worker loop is left for a future scheduler ticket to build on top of
|
|
141
|
+
this primitive.
|
|
142
|
+
|
|
143
|
+
``history_recorder``, when set, is called with each :class:`Job` from
|
|
144
|
+
the same worker thread that ran it, right after it reaches a terminal
|
|
145
|
+
status (``SUCCEEDED``/``FAILED``) -- see :func:`collapsarr.jobs.history.
|
|
146
|
+
make_history_recorder` for the constructor that builds one bound to a
|
|
147
|
+
real DB session factory. Since :meth:`run_pending` runs jobs across a
|
|
148
|
+
:class:`~concurrent.futures.ThreadPoolExecutor` (up to ``max_concurrency``
|
|
149
|
+
at once), ``history_recorder`` must itself be safe to call concurrently
|
|
150
|
+
from multiple threads; :func:`~collapsarr.jobs.history.
|
|
151
|
+
make_history_recorder` satisfies this by opening a fresh
|
|
152
|
+
:class:`~sqlalchemy.orm.Session` per call rather than sharing one --
|
|
153
|
+
SQLAlchemy sessions aren't thread-safe, but a ``sessionmaker`` safely
|
|
154
|
+
creates independent sessions from any thread.
|
|
155
|
+
|
|
156
|
+
``failure_notifier``, when set, is called the same way -- same worker
|
|
157
|
+
thread, right after the job reaches its terminal status -- but only for
|
|
158
|
+
a job whose terminal status is ``FAILED`` (a ``SUCCEEDED`` job never
|
|
159
|
+
triggers it). See :func:`collapsarr.jobs.failure_notify.
|
|
160
|
+
make_failure_notifier` for the constructor that dispatches to the
|
|
161
|
+
configured notifiers (COL-37); it must be safe to call concurrently for
|
|
162
|
+
the same reason ``history_recorder`` must, and -- like
|
|
163
|
+
``history_recorder`` -- any exception it raises is swallowed by
|
|
164
|
+
:meth:`_notify_failure` so a notifier problem can never fail the job it
|
|
165
|
+
is reporting on.
|
|
166
|
+
"""
|
|
167
|
+
|
|
168
|
+
def __init__(
|
|
169
|
+
self,
|
|
170
|
+
*,
|
|
171
|
+
max_concurrency: int = DEFAULT_MAX_CONCURRENCY,
|
|
172
|
+
pipeline_runner: PipelineRunner = run_downmix_pipeline,
|
|
173
|
+
pipeline_kwargs: Mapping[str, Any] | None = None,
|
|
174
|
+
history_recorder: HistoryRecorder | None = None,
|
|
175
|
+
failure_notifier: FailureNotifier | None = None,
|
|
176
|
+
) -> None:
|
|
177
|
+
if max_concurrency < 1:
|
|
178
|
+
raise ValueError(f"max_concurrency must be >= 1, got {max_concurrency}")
|
|
179
|
+
self._max_concurrency = max_concurrency
|
|
180
|
+
self._pipeline_runner = pipeline_runner
|
|
181
|
+
self._pipeline_kwargs = dict(pipeline_kwargs or {})
|
|
182
|
+
self._history_recorder = history_recorder
|
|
183
|
+
self._failure_notifier = failure_notifier
|
|
184
|
+
self._lock = threading.Lock()
|
|
185
|
+
self._jobs: dict[UUID, Job] = {}
|
|
186
|
+
|
|
187
|
+
@classmethod
|
|
188
|
+
def from_settings(
|
|
189
|
+
cls,
|
|
190
|
+
settings: Settings | None = None,
|
|
191
|
+
*,
|
|
192
|
+
pipeline_runner: PipelineRunner = run_downmix_pipeline,
|
|
193
|
+
pipeline_kwargs: Mapping[str, Any] | None = None,
|
|
194
|
+
history_recorder: HistoryRecorder | None = None,
|
|
195
|
+
failure_notifier: FailureNotifier | None = None,
|
|
196
|
+
) -> JobQueue:
|
|
197
|
+
"""Build a :class:`JobQueue` whose concurrency cap comes from Settings.
|
|
198
|
+
|
|
199
|
+
``settings`` defaults to the process-wide cached
|
|
200
|
+
:func:`~collapsarr.config.get_settings`. Its ``job_max_concurrency``
|
|
201
|
+
(default 1, env ``COLLAPSARR_JOB_MAX_CONCURRENCY``) becomes
|
|
202
|
+
``max_concurrency``.
|
|
203
|
+
|
|
204
|
+
Unlike the raw :meth:`__init__` (where ``history_recorder``/
|
|
205
|
+
``failure_notifier`` default to ``None`` -- the right default for
|
|
206
|
+
lightweight unit construction that doesn't want DB writes, e.g.
|
|
207
|
+
COL-20's concurrency tests), this factory is the production path:
|
|
208
|
+
when either isn't passed explicitly, it defaults to a *real* one --
|
|
209
|
+
``history_recorder`` via :func:`collapsarr.jobs.history.
|
|
210
|
+
make_history_recorder` and ``failure_notifier`` via
|
|
211
|
+
:func:`collapsarr.jobs.failure_notify.make_failure_notifier`, both
|
|
212
|
+
bound to the same session factory for ``resolved``'s database
|
|
213
|
+
(schema created via :func:`~collapsarr.database.init_db` if not
|
|
214
|
+
already present) -- rather than staying ``None``. This mirrors how
|
|
215
|
+
``pipeline_runner`` already defaults to the real
|
|
216
|
+
:func:`~collapsarr.downmix.pipeline.run_downmix_pipeline` in the raw
|
|
217
|
+
``__init__``: a bare ``JobQueue.from_settings()`` call, with no extra
|
|
218
|
+
plumbing, persists history and dispatches failure notifications for
|
|
219
|
+
real. Pass either explicitly (or ``None`` isn't obtainable here --
|
|
220
|
+
construct via :meth:`__init__` directly instead) to opt out.
|
|
221
|
+
|
|
222
|
+
The database engine backing those defaults is created at most once,
|
|
223
|
+
here, for this :class:`JobQueue` instance -- shared between
|
|
224
|
+
``history_recorder`` and ``failure_notifier`` when both need
|
|
225
|
+
defaulting, but not shared with the FastAPI app's own request-scoped
|
|
226
|
+
engine (see :mod:`collapsarr.main`). For SQLite (this project's only
|
|
227
|
+
supported backend today) that's safe -- both point at the same
|
|
228
|
+
on-disk file -- but it does mean calling this factory repeatedly
|
|
229
|
+
opens a new engine each time, so production code should call it once
|
|
230
|
+
and hold onto the resulting :class:`JobQueue` (e.g. on
|
|
231
|
+
``app.state``), the same way it already holds onto one session
|
|
232
|
+
factory.
|
|
233
|
+
|
|
234
|
+
The imports of :mod:`collapsarr.jobs.history` and
|
|
235
|
+
:mod:`collapsarr.jobs.failure_notify` below are deferred (inside this
|
|
236
|
+
method, not at module scope) because those modules import *this* one
|
|
237
|
+
(for :class:`Job`/:class:`JobStatus`) -- the same defer-to-break-a-
|
|
238
|
+
cycle trick :func:`collapsarr.database.init_db` already uses for its
|
|
239
|
+
own model-registration imports.
|
|
240
|
+
"""
|
|
241
|
+
resolved = settings or get_settings()
|
|
242
|
+
|
|
243
|
+
resolved_history_recorder = history_recorder
|
|
244
|
+
resolved_failure_notifier = failure_notifier
|
|
245
|
+
if resolved_history_recorder is None or resolved_failure_notifier is None:
|
|
246
|
+
from collapsarr.database import (
|
|
247
|
+
create_engine_from_settings,
|
|
248
|
+
create_session_factory,
|
|
249
|
+
init_db,
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
engine = create_engine_from_settings(resolved)
|
|
253
|
+
init_db(engine)
|
|
254
|
+
session_factory = create_session_factory(engine)
|
|
255
|
+
|
|
256
|
+
if resolved_history_recorder is None:
|
|
257
|
+
from collapsarr.jobs.history import make_history_recorder
|
|
258
|
+
|
|
259
|
+
resolved_history_recorder = make_history_recorder(session_factory)
|
|
260
|
+
|
|
261
|
+
if resolved_failure_notifier is None:
|
|
262
|
+
from collapsarr.jobs.failure_notify import make_failure_notifier
|
|
263
|
+
|
|
264
|
+
resolved_failure_notifier = make_failure_notifier(session_factory)
|
|
265
|
+
|
|
266
|
+
return cls(
|
|
267
|
+
max_concurrency=resolved.job_max_concurrency,
|
|
268
|
+
pipeline_runner=pipeline_runner,
|
|
269
|
+
pipeline_kwargs=pipeline_kwargs,
|
|
270
|
+
history_recorder=resolved_history_recorder,
|
|
271
|
+
failure_notifier=resolved_failure_notifier,
|
|
272
|
+
)
|
|
273
|
+
|
|
274
|
+
@property
|
|
275
|
+
def max_concurrency(self) -> int:
|
|
276
|
+
"""The configured cap on simultaneously running jobs."""
|
|
277
|
+
return self._max_concurrency
|
|
278
|
+
|
|
279
|
+
def enqueue(self, file_path: str | Path, settings: DownmixSettings) -> Job:
|
|
280
|
+
"""Add a file + its target/language context to the queue as a new job.
|
|
281
|
+
|
|
282
|
+
Returns the created :class:`Job` (status ``PENDING``) immediately;
|
|
283
|
+
it is not run until a subsequent :meth:`run_pending` call.
|
|
284
|
+
"""
|
|
285
|
+
job = Job(file_path=Path(file_path), settings=settings)
|
|
286
|
+
with self._lock:
|
|
287
|
+
self._jobs[job.id] = job
|
|
288
|
+
return job
|
|
289
|
+
|
|
290
|
+
def get_job(self, job_id: UUID) -> Job | None:
|
|
291
|
+
"""Return the job with ``job_id``, or ``None`` if no such job exists."""
|
|
292
|
+
with self._lock:
|
|
293
|
+
return self._jobs.get(job_id)
|
|
294
|
+
|
|
295
|
+
def list_jobs(self) -> list[Job]:
|
|
296
|
+
"""Return every job ever enqueued on this queue, in enqueue order."""
|
|
297
|
+
with self._lock:
|
|
298
|
+
return list(self._jobs.values())
|
|
299
|
+
|
|
300
|
+
def run_pending(self) -> list[Job]:
|
|
301
|
+
"""Run every currently-``PENDING`` job to completion, then return them.
|
|
302
|
+
|
|
303
|
+
Runs the batch through a :class:`~concurrent.futures.ThreadPoolExecutor`
|
|
304
|
+
sized to ``max_concurrency``, so at most that many jobs execute the
|
|
305
|
+
pipeline at once; with ``max_concurrency=1`` the executor has a
|
|
306
|
+
single worker, so jobs run strictly one at a time, in the order they
|
|
307
|
+
were enqueued.
|
|
308
|
+
|
|
309
|
+
Blocks until the whole batch has finished. Returns an empty list if
|
|
310
|
+
nothing was pending. Each returned :class:`Job` has been updated in
|
|
311
|
+
place with its final ``status`` and ``result``/``error``, and -- if
|
|
312
|
+
this queue was built with a ``history_recorder`` -- already
|
|
313
|
+
persisted via it.
|
|
314
|
+
"""
|
|
315
|
+
with self._lock:
|
|
316
|
+
batch = [job for job in self._jobs.values() if job.status is JobStatus.PENDING]
|
|
317
|
+
if not batch:
|
|
318
|
+
return []
|
|
319
|
+
|
|
320
|
+
with ThreadPoolExecutor(max_workers=self._max_concurrency) as executor:
|
|
321
|
+
futures = [executor.submit(self._run_job, job) for job in batch]
|
|
322
|
+
for future in futures:
|
|
323
|
+
future.result() # re-raise any unexpected executor-level error
|
|
324
|
+
|
|
325
|
+
return batch
|
|
326
|
+
|
|
327
|
+
def _run_job(self, job: Job) -> None:
|
|
328
|
+
"""Execute one job's pipeline run, record its outcome, and persist/notify.
|
|
329
|
+
|
|
330
|
+
Runs entirely on the calling (worker) thread. Once ``job`` reaches
|
|
331
|
+
its terminal status (``SUCCEEDED``/``FAILED``), ``self._history_recorder``
|
|
332
|
+
(if configured) is called with it, followed by ``self._notify_failure``
|
|
333
|
+
(a no-op unless the job actually ``FAILED`` and a ``failure_notifier``
|
|
334
|
+
is configured) -- both outside ``self._lock``, since by that point
|
|
335
|
+
only this thread ever touches this particular ``job`` (each job is
|
|
336
|
+
submitted to the executor exactly once), so there is nothing left to
|
|
337
|
+
race against.
|
|
338
|
+
"""
|
|
339
|
+
with self._lock:
|
|
340
|
+
job.status = JobStatus.RUNNING
|
|
341
|
+
job.started_at = datetime.now(UTC)
|
|
342
|
+
|
|
343
|
+
try:
|
|
344
|
+
result = self._pipeline_runner(job.file_path, job.settings, **self._pipeline_kwargs)
|
|
345
|
+
except Exception as exc: # noqa: BLE001 - captured as the job's outcome, not re-raised
|
|
346
|
+
with self._lock:
|
|
347
|
+
job.error = exc
|
|
348
|
+
job.status = JobStatus.FAILED
|
|
349
|
+
job.ended_at = datetime.now(UTC)
|
|
350
|
+
self._record_history(job)
|
|
351
|
+
self._notify_failure(job)
|
|
352
|
+
return
|
|
353
|
+
|
|
354
|
+
with self._lock:
|
|
355
|
+
job.result = result
|
|
356
|
+
job.status = JobStatus.SUCCEEDED if result.success else JobStatus.FAILED
|
|
357
|
+
job.ended_at = datetime.now(UTC)
|
|
358
|
+
self._record_history(job)
|
|
359
|
+
self._notify_failure(job)
|
|
360
|
+
|
|
361
|
+
def _record_history(self, job: Job) -> None:
|
|
362
|
+
"""Persist ``job``'s just-reached terminal state, if configured to."""
|
|
363
|
+
if self._history_recorder is not None:
|
|
364
|
+
self._history_recorder(job)
|
|
365
|
+
|
|
366
|
+
def _notify_failure(self, job: Job) -> None:
|
|
367
|
+
"""Dispatch a failure notification for ``job``, if configured and it failed.
|
|
368
|
+
|
|
369
|
+
A no-op for a job that reached ``SUCCEEDED``, or when no
|
|
370
|
+
``failure_notifier`` was configured. ``self._failure_notifier`` is
|
|
371
|
+
expected to never raise on its own (COL-37's
|
|
372
|
+
:func:`~collapsarr.jobs.failure_notify.notify_job_failure` guarantees
|
|
373
|
+
this), but it is called inside a defensive ``try``/``except`` anyway
|
|
374
|
+
-- a notification problem must never be able to fail the job it is
|
|
375
|
+
reporting on, or the worker thread running it.
|
|
376
|
+
"""
|
|
377
|
+
if self._failure_notifier is None or job.status is not JobStatus.FAILED:
|
|
378
|
+
return
|
|
379
|
+
try:
|
|
380
|
+
self._failure_notifier(job)
|
|
381
|
+
except Exception: # noqa: BLE001 - a notifier failure must never fail the job
|
|
382
|
+
pass
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
"""HTTP REST endpoints for job history & on-demand triggers (COL-29).
|
|
2
|
+
|
|
3
|
+
Thin layer wrapping the Job Queue & Scheduler epic's service surface, exposed as
|
|
4
|
+
a FastAPI :class:`~fastapi.APIRouter` mounted under ``/api`` by
|
|
5
|
+
:func:`collapsarr.main.create_app`. Because everything under ``/api`` is gated by
|
|
6
|
+
the API-key middleware (COL-26), every route here inherits key-based auth -- no
|
|
7
|
+
per-route auth wiring is needed.
|
|
8
|
+
|
|
9
|
+
Three endpoints, each wrapping an existing service without adding new job logic:
|
|
10
|
+
|
|
11
|
+
- ``GET /api/jobs/history`` -- lists persisted job history (COL-21,
|
|
12
|
+
:func:`collapsarr.jobs.history.list_job_history`), optionally filtered by
|
|
13
|
+
``file`` (exact file path) and/or ``status`` (a :class:`~collapsarr.jobs.queue.
|
|
14
|
+
JobStatus` value). Mirrors Sonarr/Radarr's ``/history`` view.
|
|
15
|
+
- ``POST /api/jobs/scan`` -- triggers an immediate full-library scan
|
|
16
|
+
(:meth:`collapsarr.jobs.scheduler.JobScheduler.scan_now`, COL-23), enqueuing a
|
|
17
|
+
downmix job for every monitored file that has a qualifying missing target. The
|
|
18
|
+
Sonarr/Radarr analogue is the ``RescanSeries``/``RefreshMovie`` command.
|
|
19
|
+
- ``POST /api/jobs/trigger`` -- manually enqueues a downmix job for one specific
|
|
20
|
+
file (:meth:`collapsarr.jobs.scheduler.JobScheduler.trigger_file`, COL-23). The
|
|
21
|
+
optional ``extra_languages`` list is the allow-list-bypass option: those
|
|
22
|
+
languages are forced past the scheduler's ``language_allow_list`` for this one
|
|
23
|
+
call, letting a user downmix a language they normally don't auto-process.
|
|
24
|
+
|
|
25
|
+
The scan/trigger endpoints operate on the live
|
|
26
|
+
:class:`~collapsarr.jobs.scheduler.JobScheduler` the app wired onto
|
|
27
|
+
``app.state.job_scheduler`` (see :func:`collapsarr.main.create_app`,
|
|
28
|
+
``enable_scheduler=True``) so a manual trigger and the background loop share one
|
|
29
|
+
queue. When no scheduler is wired (e.g. an app built without the scheduler), the
|
|
30
|
+
dependency raises ``503`` rather than silently doing nothing.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
from datetime import datetime
|
|
36
|
+
|
|
37
|
+
from fastapi import APIRouter, Depends, HTTPException, Request
|
|
38
|
+
from pydantic import BaseModel, ConfigDict
|
|
39
|
+
from sqlalchemy.orm import Session
|
|
40
|
+
|
|
41
|
+
from ..database import get_session
|
|
42
|
+
from .history import list_job_history
|
|
43
|
+
from .models import JobHistory
|
|
44
|
+
from .queue import Job, JobStatus
|
|
45
|
+
from .scheduler import JobScheduler
|
|
46
|
+
|
|
47
|
+
router = APIRouter(prefix="/api", tags=["jobs"])
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
# --- dependencies ------------------------------------------------------------
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def get_job_scheduler(request: Request) -> JobScheduler:
|
|
54
|
+
"""Return the app's live :class:`JobScheduler`, or ``503`` if none is wired.
|
|
55
|
+
|
|
56
|
+
The scan/trigger endpoints act on the same scheduler (and its shared queue)
|
|
57
|
+
the background loop drains, exposed on ``app.state.job_scheduler`` by
|
|
58
|
+
:func:`collapsarr.main.create_app` when built with ``enable_scheduler=True``
|
|
59
|
+
(the production path). An app built without it has no on-demand trigger
|
|
60
|
+
surface, so we fail loudly with ``503`` instead of pretending to enqueue.
|
|
61
|
+
"""
|
|
62
|
+
scheduler: JobScheduler | None = getattr(request.app.state, "job_scheduler", None)
|
|
63
|
+
if scheduler is None:
|
|
64
|
+
raise HTTPException(
|
|
65
|
+
status_code=503,
|
|
66
|
+
detail="Job scheduler is not available.",
|
|
67
|
+
)
|
|
68
|
+
return scheduler
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
# --- schemas -----------------------------------------------------------------
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class JobHistoryRead(BaseModel):
|
|
75
|
+
"""Response shape for one persisted job-history row (COL-21)."""
|
|
76
|
+
|
|
77
|
+
model_config = ConfigDict(from_attributes=True)
|
|
78
|
+
|
|
79
|
+
id: int
|
|
80
|
+
job_id: str
|
|
81
|
+
file_path: str
|
|
82
|
+
status: JobStatus
|
|
83
|
+
started_at: datetime | None
|
|
84
|
+
ended_at: datetime | None
|
|
85
|
+
exit_code: int | None
|
|
86
|
+
error_text: str | None
|
|
87
|
+
target: str | None
|
|
88
|
+
language: str | None
|
|
89
|
+
created_at: datetime
|
|
90
|
+
updated_at: datetime
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class EnqueuedJob(BaseModel):
|
|
94
|
+
"""The identifying summary of a job the scheduler just enqueued in-memory."""
|
|
95
|
+
|
|
96
|
+
id: str
|
|
97
|
+
file_path: str
|
|
98
|
+
status: JobStatus
|
|
99
|
+
|
|
100
|
+
@classmethod
|
|
101
|
+
def from_job(cls, job: Job) -> EnqueuedJob:
|
|
102
|
+
return cls(id=str(job.id), file_path=str(job.file_path), status=job.status)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class ScanResult(BaseModel):
|
|
106
|
+
"""Response for ``POST /api/jobs/scan``: the jobs the scan pass enqueued."""
|
|
107
|
+
|
|
108
|
+
enqueued: list[EnqueuedJob]
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class ManualTriggerRequest(BaseModel):
|
|
112
|
+
"""Request body for ``POST /api/jobs/trigger``.
|
|
113
|
+
|
|
114
|
+
``file_path`` names the (host-local) file to downmix. ``extra_languages`` is
|
|
115
|
+
the allow-list-bypass option (COL-23): languages listed here are unioned onto
|
|
116
|
+
the scheduler's ``language_allow_list`` for this one call, so a language the
|
|
117
|
+
global allow-list would otherwise exclude still gets a downmix job. When the
|
|
118
|
+
scheduler has no allow-list (every language is already eligible) it has no
|
|
119
|
+
effect. Omit it (or send ``[]``) for a plain trigger honouring the allow-list.
|
|
120
|
+
"""
|
|
121
|
+
|
|
122
|
+
model_config = ConfigDict(extra="forbid")
|
|
123
|
+
|
|
124
|
+
file_path: str
|
|
125
|
+
extra_languages: list[str] = []
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
class ManualTriggerResult(BaseModel):
|
|
129
|
+
"""Response for ``POST /api/jobs/trigger``.
|
|
130
|
+
|
|
131
|
+
``enqueued`` is ``True`` with the created ``job`` when a downmix job was
|
|
132
|
+
queued. It is ``False`` with ``job`` ``null`` when the file was skipped -- a
|
|
133
|
+
duplicate (already queued / recently processed), unprobeable, or with no
|
|
134
|
+
qualifying target even after ``extra_languages`` -- mirroring
|
|
135
|
+
:meth:`collapsarr.jobs.scheduler.JobScheduler.trigger_file` returning ``None``.
|
|
136
|
+
"""
|
|
137
|
+
|
|
138
|
+
enqueued: bool
|
|
139
|
+
job: EnqueuedJob | None
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
# --- endpoints ---------------------------------------------------------------
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@router.get("/jobs/history", response_model=list[JobHistoryRead])
|
|
146
|
+
def list_job_history_endpoint(
|
|
147
|
+
file: str | None = None,
|
|
148
|
+
status: JobStatus | None = None,
|
|
149
|
+
session: Session = Depends(get_session),
|
|
150
|
+
) -> list[JobHistory]:
|
|
151
|
+
"""List persisted job history, optionally filtered by file and/or status.
|
|
152
|
+
|
|
153
|
+
``file`` matches a file path exactly (the form job history stores);
|
|
154
|
+
``status`` matches a single :class:`~collapsarr.jobs.queue.JobStatus`
|
|
155
|
+
(``pending``/``running``/``succeeded``/``failed``). Both may be combined;
|
|
156
|
+
omitting both returns every row, ordered by insertion.
|
|
157
|
+
"""
|
|
158
|
+
return list_job_history(session, file_path=file, status=status)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
@router.post("/jobs/scan", response_model=ScanResult, status_code=202)
|
|
162
|
+
def scan_now_endpoint(scheduler: JobScheduler = Depends(get_job_scheduler)) -> ScanResult:
|
|
163
|
+
"""Trigger an immediate full-library scan, returning the jobs it enqueued.
|
|
164
|
+
|
|
165
|
+
Runs the same scan the periodic loop runs (COL-22/COL-23), synchronously:
|
|
166
|
+
every configured instance's monitored files are re-probed and a downmix job
|
|
167
|
+
is enqueued for each with a qualifying missing target (skipped/no-op files
|
|
168
|
+
excluded). ``202 Accepted`` -- the jobs are queued, not yet run.
|
|
169
|
+
"""
|
|
170
|
+
jobs = scheduler.scan_now()
|
|
171
|
+
return ScanResult(enqueued=[EnqueuedJob.from_job(job) for job in jobs])
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
@router.post("/jobs/trigger", response_model=ManualTriggerResult, status_code=202)
|
|
175
|
+
def manual_trigger_endpoint(
|
|
176
|
+
body: ManualTriggerRequest,
|
|
177
|
+
scheduler: JobScheduler = Depends(get_job_scheduler),
|
|
178
|
+
) -> ManualTriggerResult:
|
|
179
|
+
"""Manually enqueue a downmix job for one file, honouring the bypass option.
|
|
180
|
+
|
|
181
|
+
Wraps :meth:`collapsarr.jobs.scheduler.JobScheduler.trigger_file`: probes the
|
|
182
|
+
file, enqueues a job when a target qualifies, and threads ``extra_languages``
|
|
183
|
+
through as the allow-list-bypass. A ``202`` is returned whether or not a job
|
|
184
|
+
was enqueued; the ``enqueued`` flag distinguishes the two (a skipped file --
|
|
185
|
+
duplicate/unprobeable/nothing to do -- is not an error).
|
|
186
|
+
"""
|
|
187
|
+
job = scheduler.trigger_file(
|
|
188
|
+
body.file_path,
|
|
189
|
+
extra_languages=body.extra_languages or None,
|
|
190
|
+
)
|
|
191
|
+
if job is None:
|
|
192
|
+
return ManualTriggerResult(enqueued=False, job=None)
|
|
193
|
+
return ManualTriggerResult(enqueued=True, job=EnqueuedJob.from_job(job))
|