lorebase-scribe 0.2.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lorebase_scribe/__init__.py +3 -0
- lorebase_scribe/__main__.py +156 -0
- lorebase_scribe/checkpoints.py +324 -0
- lorebase_scribe/cron.py +117 -0
- lorebase_scribe/embeddings.py +570 -0
- lorebase_scribe/extraction/__init__.py +17 -0
- lorebase_scribe/extraction/assemble.py +339 -0
- lorebase_scribe/extraction/candidates.py +136 -0
- lorebase_scribe/extraction/discovery.py +567 -0
- lorebase_scribe/extraction/llm_discovery.py +271 -0
- lorebase_scribe/extraction/matching.py +285 -0
- lorebase_scribe/extraction/ontology.py +432 -0
- lorebase_scribe/extraction/parsing.py +114 -0
- lorebase_scribe/extraction/pipeline.py +1091 -0
- lorebase_scribe/extraction/relations.py +194 -0
- lorebase_scribe/extraction/sandbox.py +269 -0
- lorebase_scribe/extraction/settings.py +293 -0
- lorebase_scribe/extraction/text.py +179 -0
- lorebase_scribe/extraction/tier2.py +573 -0
- lorebase_scribe/extraction/values.py +282 -0
- lorebase_scribe/jobs.py +418 -0
- lorebase_scribe/scheduler.py +318 -0
- lorebase_scribe/service.py +146 -0
- lorebase_scribe/settings.py +521 -0
- lorebase_scribe/sources/__init__.py +60 -0
- lorebase_scribe/sources/access.py +241 -0
- lorebase_scribe/sources/connectors.py +320 -0
- lorebase_scribe/sources/delta.py +175 -0
- lorebase_scribe/sources/http_json.py +283 -0
- lorebase_scribe/sources/mapping.py +429 -0
- lorebase_scribe/sources/profile.py +36 -0
- lorebase_scribe/sources/records.py +298 -0
- lorebase_scribe/sources/sql.py +332 -0
- lorebase_scribe/sources/sync.py +730 -0
- lorebase_scribe/telemetry.py +113 -0
- lorebase_scribe/webhooks.py +99 -0
- lorebase_scribe/worker.py +254 -0
- lorebase_scribe-0.2.3.dist-info/METADATA +23 -0
- lorebase_scribe-0.2.3.dist-info/RECORD +41 -0
- lorebase_scribe-0.2.3.dist-info/WHEEL +4 -0
- lorebase_scribe-0.2.3.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""OpenTelemetry instruments of Scribe. They are no-ops until `setup_telemetry` installs
|
|
2
|
+
providers (`telemetry.otlp_endpoint` set)."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from collections.abc import Iterable, Mapping
|
|
7
|
+
from threading import Lock
|
|
8
|
+
|
|
9
|
+
from opentelemetry import metrics, trace
|
|
10
|
+
from opentelemetry.metrics import CallbackOptions, Observation
|
|
11
|
+
|
|
12
|
+
from lorebase_platform.telemetry import LATENCY_BUCKETS_S
|
|
13
|
+
|
|
14
|
+
INSTRUMENTATION = "lorebase.scribe"
|
|
15
|
+
|
|
16
|
+
tracer = trace.get_tracer(INSTRUMENTATION)
|
|
17
|
+
meter = metrics.get_meter(INSTRUMENTATION)
|
|
18
|
+
|
|
19
|
+
JOBS = meter.create_counter(
|
|
20
|
+
"lorebase.scribe.jobs", unit="{job}", description="Jobs run, by type and outcome."
|
|
21
|
+
)
|
|
22
|
+
JOB_DURATION = meter.create_histogram(
|
|
23
|
+
"lorebase.scribe.job.duration",
|
|
24
|
+
unit="s",
|
|
25
|
+
description="Duration of one job run.",
|
|
26
|
+
explicit_bucket_boundaries_advisory=LATENCY_BUCKETS_S,
|
|
27
|
+
)
|
|
28
|
+
SCHEDULED = meter.create_counter(
|
|
29
|
+
"lorebase.scribe.scheduled", unit="{job}", description="Jobs queued by the scheduler."
|
|
30
|
+
)
|
|
31
|
+
GC_SCHEDULED = meter.create_counter(
|
|
32
|
+
"lorebase.gc.scheduled",
|
|
33
|
+
unit="{job}",
|
|
34
|
+
description="gc jobs queued by the scheduler at gc_schedule, by dry_run. Starts at 0 when "
|
|
35
|
+
"the schedule is set, so a schedule that queues nothing still reports its mode.",
|
|
36
|
+
)
|
|
37
|
+
SOURCE_ROWS = meter.create_counter(
|
|
38
|
+
"lorebase.scribe.source.rows", unit="{row}", description="Source rows read."
|
|
39
|
+
)
|
|
40
|
+
SOURCE_CHANGES = meter.create_counter(
|
|
41
|
+
"lorebase.scribe.source.changes",
|
|
42
|
+
unit="{change}",
|
|
43
|
+
description="Data changes computed by Source runs (committed unless dry_run).",
|
|
44
|
+
)
|
|
45
|
+
EXTRACTION_ITEMS = meter.create_counter(
|
|
46
|
+
"lorebase.extraction.items",
|
|
47
|
+
unit="{item}",
|
|
48
|
+
description="Items proposed by extraction jobs, by tier and initial decision.",
|
|
49
|
+
)
|
|
50
|
+
EVIDENCE_WITHHELD = meter.create_counter(
|
|
51
|
+
"lorebase.extraction.evidence_withheld",
|
|
52
|
+
unit="{observation}",
|
|
53
|
+
description="Schema suggestion observations kept without evidence because it held a "
|
|
54
|
+
"sensitive value. For operators only, without labels.",
|
|
55
|
+
)
|
|
56
|
+
EMBEDDING_RECORDS = meter.create_counter(
|
|
57
|
+
"lorebase.scribe.embeddings.records",
|
|
58
|
+
unit="{record}",
|
|
59
|
+
description="Records an embedding pass rendered, by outcome.",
|
|
60
|
+
)
|
|
61
|
+
EMBEDDING_CALL_DURATION = meter.create_histogram(
|
|
62
|
+
"lorebase.scribe.embeddings.call.duration",
|
|
63
|
+
unit="s",
|
|
64
|
+
description="Duration of one call of the embedding model by an embedding job.",
|
|
65
|
+
explicit_bucket_boundaries_advisory=LATENCY_BUCKETS_S,
|
|
66
|
+
)
|
|
67
|
+
AUDIT_VERIFICATIONS = meter.create_counter(
|
|
68
|
+
"lorebase.audit.verifications",
|
|
69
|
+
unit="{verification}",
|
|
70
|
+
description="Verifications of the audit hash chains by audit_verify jobs, by outcome and "
|
|
71
|
+
"stream.",
|
|
72
|
+
)
|
|
73
|
+
AUDIT_CHECKPOINTS = meter.create_counter(
|
|
74
|
+
"lorebase.audit.checkpoints",
|
|
75
|
+
unit="{checkpoint}",
|
|
76
|
+
description="Audit checkpoints published by audit_checkpoint jobs, by outcome.",
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class QueueDepth:
|
|
81
|
+
"""The latest count of waiting and running jobs per (type, status), as the scheduler
|
|
82
|
+
leader saw it. A pair that empties reports 0; after `clear` (leadership lost) nothing is
|
|
83
|
+
reported, so only the leader's view is exported."""
|
|
84
|
+
|
|
85
|
+
def __init__(self) -> None:
|
|
86
|
+
self._lock = Lock()
|
|
87
|
+
self._counts: dict[tuple[str, str], int] = {}
|
|
88
|
+
|
|
89
|
+
def update(self, counts: Mapping[tuple[str, str], int]) -> None:
|
|
90
|
+
with self._lock:
|
|
91
|
+
self._counts = dict.fromkeys(self._counts, 0) | dict(counts)
|
|
92
|
+
|
|
93
|
+
def clear(self) -> None:
|
|
94
|
+
with self._lock:
|
|
95
|
+
self._counts = {}
|
|
96
|
+
|
|
97
|
+
def observe(self, _options: CallbackOptions | None = None) -> Iterable[Observation]:
|
|
98
|
+
with self._lock:
|
|
99
|
+
counts = dict(self._counts)
|
|
100
|
+
return [
|
|
101
|
+
Observation(count, {"type": job_type, "status": status})
|
|
102
|
+
for (job_type, status), count in sorted(counts.items())
|
|
103
|
+
]
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
QUEUE_DEPTH = QueueDepth()
|
|
107
|
+
meter.create_observable_gauge(
|
|
108
|
+
"lorebase.jobs.queue_depth",
|
|
109
|
+
callbacks=[QUEUE_DEPTH.observe],
|
|
110
|
+
unit="{job}",
|
|
111
|
+
description="Jobs waiting or running, by type and status: queued (claimable), retrying "
|
|
112
|
+
"(waiting for run_after) or running (scheduler leader only).",
|
|
113
|
+
)
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""HTTP sender of webhook deliveries, for `lorebase_store.webhooks.deliver_pending`.
|
|
2
|
+
|
|
3
|
+
The store makes no HTTP calls itself; Scribe's `webhook_delivery` jobs pass this `Sender`. It
|
|
4
|
+
uses an `httpx.AsyncClient` owned by the caller (build it with `trust_env=False`, so no proxy from
|
|
5
|
+
the environment sits between the checks and the target) and:
|
|
6
|
+
|
|
7
|
+
- connects to the addresses the worker checked, never to a fresh lookup of the host: the URL
|
|
8
|
+
is rewritten to the address while the `Host` header and the TLS server name (SNI and
|
|
9
|
+
certificate verification) keep the host name, so a DNS answer that changed after the check
|
|
10
|
+
cannot reach an internal service;
|
|
11
|
+
- never follows redirects (a redirect could lead to a target the checks did not see);
|
|
12
|
+
- never reads the response body, and bounds each exchange by the timeout as a whole, so a
|
|
13
|
+
target that streams or trickles cannot hold the worker;
|
|
14
|
+
- gives up a connection after `connect_timeout_s` (at most the timeout) and tries the next
|
|
15
|
+
checked address, so one address that drops packets does not use up the whole budget.
|
|
16
|
+
`deliver_pending` bounds the attempt over all addresses by the same timeout.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import asyncio
|
|
22
|
+
from collections.abc import Mapping, Sequence
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
import httpx
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def pin(url: httpx.URL, address: str) -> tuple[httpx.URL, dict[str, str], dict[str, Any]]:
|
|
29
|
+
"""`url` rewritten to connect to the checked `address`, with the `Host` header and request
|
|
30
|
+
extensions that keep the host name for the server and for TLS (SNI and certificate
|
|
31
|
+
verification)."""
|
|
32
|
+
return (
|
|
33
|
+
url.copy_with(host=address.split("%", 1)[0]),
|
|
34
|
+
{"Host": url.netloc.decode("ascii")},
|
|
35
|
+
{"sni_hostname": url.host} if url.scheme == "https" else {},
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class HttpxSender:
|
|
40
|
+
"""`Sender` over an `httpx.AsyncClient`."""
|
|
41
|
+
|
|
42
|
+
def __init__(
|
|
43
|
+
self, client: httpx.AsyncClient, *, connect_timeout_s: float | None = None
|
|
44
|
+
) -> None:
|
|
45
|
+
self._client = client
|
|
46
|
+
self._connect_timeout_s = connect_timeout_s
|
|
47
|
+
|
|
48
|
+
async def __call__(
|
|
49
|
+
self,
|
|
50
|
+
url: str,
|
|
51
|
+
body: bytes,
|
|
52
|
+
headers: Mapping[str, str],
|
|
53
|
+
timeout_s: float,
|
|
54
|
+
addresses: Sequence[str],
|
|
55
|
+
) -> int:
|
|
56
|
+
target = httpx.URL(url)
|
|
57
|
+
if not addresses:
|
|
58
|
+
return await self._post(target, body, dict(headers), timeout_s, {})
|
|
59
|
+
failure: httpx.TransportError | None = None
|
|
60
|
+
for address in addresses:
|
|
61
|
+
pinned, host, extensions = pin(target, address)
|
|
62
|
+
try:
|
|
63
|
+
return await self._post(pinned, body, {**headers, **host}, timeout_s, extensions)
|
|
64
|
+
except (httpx.ConnectError, httpx.ConnectTimeout) as error:
|
|
65
|
+
failure = error # the next checked address, never a new lookup
|
|
66
|
+
assert failure is not None
|
|
67
|
+
raise failure
|
|
68
|
+
|
|
69
|
+
def _timeout(self, timeout_s: float) -> httpx.Timeout:
|
|
70
|
+
"""Every phase bounded: connect by `connect_timeout_s` (capped by `timeout_s`), each
|
|
71
|
+
write and read and the wait for a pooled connection by `timeout_s`."""
|
|
72
|
+
connect = timeout_s
|
|
73
|
+
if self._connect_timeout_s is not None:
|
|
74
|
+
connect = min(self._connect_timeout_s, timeout_s)
|
|
75
|
+
return httpx.Timeout(timeout_s, connect=connect)
|
|
76
|
+
|
|
77
|
+
async def _post(
|
|
78
|
+
self,
|
|
79
|
+
url: httpx.URL,
|
|
80
|
+
body: bytes,
|
|
81
|
+
headers: dict[str, str],
|
|
82
|
+
timeout_s: float,
|
|
83
|
+
extensions: dict[str, Any],
|
|
84
|
+
) -> int:
|
|
85
|
+
# httpx applies `timeout` to each connect, write and read, not to the whole exchange,
|
|
86
|
+
# and `post` reads the whole body: bound the exchange and close without reading it.
|
|
87
|
+
async with (
|
|
88
|
+
asyncio.timeout(timeout_s),
|
|
89
|
+
self._client.stream(
|
|
90
|
+
"POST",
|
|
91
|
+
url,
|
|
92
|
+
content=body,
|
|
93
|
+
headers=headers,
|
|
94
|
+
timeout=self._timeout(timeout_s),
|
|
95
|
+
follow_redirects=False,
|
|
96
|
+
extensions=extensions,
|
|
97
|
+
) as response,
|
|
98
|
+
):
|
|
99
|
+
return response.status_code
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
"""The job loop (design 9, AC-18): claim jobs from the store queue and run them.
|
|
2
|
+
|
|
3
|
+
`concurrency` slots each claim one due job at a time (`FOR UPDATE SKIP LOCKED`, so replicas
|
|
4
|
+
never get the same job), run its handler outside any transaction and record the outcome with
|
|
5
|
+
`lorebase_store.jobs.finish_job`: a failure is retried after `store.jobs.retry_delay_seconds`
|
|
6
|
+
until `store.jobs.max_attempts`, except permanent failures (`PermanentJobError` and the types
|
|
7
|
+
passed as `permanent`), which fail at once. Every `heartbeat_interval_s` the worker refreshes
|
|
8
|
+
the lock time of its running jobs; a worker that stops sending heartbeats (killed, frozen, cut
|
|
9
|
+
off from the database) loses its jobs to `requeue_stale` after `store.jobs.stale_after_seconds`,
|
|
10
|
+
so jobs run at least once and handlers must tolerate a second run.
|
|
11
|
+
|
|
12
|
+
On shutdown (SIGTERM), slots stop claiming and running jobs get `shutdown_grace_s` to finish.
|
|
13
|
+
The rest are cancelled; whatever has not stopped `shutdown_cancel_s` later is put back in the
|
|
14
|
+
queue anyway (`release_jobs`), so a stopping worker never leaves jobs `running`. Released jobs
|
|
15
|
+
do not count the attempt.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import asyncio
|
|
21
|
+
import contextlib
|
|
22
|
+
import logging
|
|
23
|
+
import time
|
|
24
|
+
from collections.abc import Awaitable, Callable, Mapping
|
|
25
|
+
from typing import Any
|
|
26
|
+
|
|
27
|
+
from opentelemetry.trace import Status, StatusCode
|
|
28
|
+
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
|
29
|
+
|
|
30
|
+
from lorebase_scribe.settings import WorkerSettings
|
|
31
|
+
from lorebase_scribe.telemetry import JOB_DURATION, JOBS, tracer
|
|
32
|
+
from lorebase_store.jobs import (
|
|
33
|
+
JobRow,
|
|
34
|
+
claim_jobs,
|
|
35
|
+
default_worker_id,
|
|
36
|
+
finish_job,
|
|
37
|
+
release_jobs,
|
|
38
|
+
touch_jobs,
|
|
39
|
+
)
|
|
40
|
+
from lorebase_store.settings import JobSettings
|
|
41
|
+
|
|
42
|
+
log = logging.getLogger("lorebase.scribe.worker")
|
|
43
|
+
|
|
44
|
+
JobHandler = Callable[[JobRow], Awaitable[Mapping[str, Any]]]
|
|
45
|
+
"""Runs a job and returns its JSON result; an exception fails the attempt."""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class PermanentJobError(Exception):
|
|
49
|
+
"""The job can never succeed as queued: it is failed without retries."""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class Worker:
|
|
53
|
+
"""Claims and runs jobs of the handled types."""
|
|
54
|
+
|
|
55
|
+
def __init__(
|
|
56
|
+
self,
|
|
57
|
+
sessions: async_sessionmaker[AsyncSession],
|
|
58
|
+
handlers: Mapping[str, JobHandler],
|
|
59
|
+
*,
|
|
60
|
+
settings: WorkerSettings,
|
|
61
|
+
jobs: JobSettings,
|
|
62
|
+
worker_id: str | None = None,
|
|
63
|
+
permanent: tuple[type[Exception], ...] = (),
|
|
64
|
+
) -> None:
|
|
65
|
+
if not handlers:
|
|
66
|
+
raise ValueError("a worker needs at least one job type")
|
|
67
|
+
if settings.heartbeat_interval_s * 2 > jobs.stale_after_seconds:
|
|
68
|
+
raise ValueError(
|
|
69
|
+
f"scribe.worker.heartbeat_interval_s ({settings.heartbeat_interval_s:g}) must be "
|
|
70
|
+
f"at most half of store.jobs.stale_after_seconds ({jobs.stale_after_seconds:g}), "
|
|
71
|
+
"or running jobs are reclaimed from a live worker"
|
|
72
|
+
)
|
|
73
|
+
self._sessions = sessions
|
|
74
|
+
self._handlers = dict(handlers)
|
|
75
|
+
self._settings = settings
|
|
76
|
+
self._jobs = jobs
|
|
77
|
+
self._final = jobs.model_copy(update={"max_attempts": 1})
|
|
78
|
+
self._permanent: tuple[type[Exception], ...] = (PermanentJobError, *permanent)
|
|
79
|
+
self.worker_id = worker_id or default_worker_id()
|
|
80
|
+
self._running: dict[int, JobRow] = {}
|
|
81
|
+
|
|
82
|
+
@property
|
|
83
|
+
def types(self) -> tuple[str, ...]:
|
|
84
|
+
return tuple(self._handlers)
|
|
85
|
+
|
|
86
|
+
@property
|
|
87
|
+
def running(self) -> tuple[int, ...]:
|
|
88
|
+
"""Ids of the jobs running now."""
|
|
89
|
+
return tuple(self._running)
|
|
90
|
+
|
|
91
|
+
async def claim(self) -> JobRow | None:
|
|
92
|
+
async with self._sessions() as session, session.begin():
|
|
93
|
+
claimed = await claim_jobs(session, self.worker_id, types=self.types, limit=1)
|
|
94
|
+
return claimed[0] if claimed else None
|
|
95
|
+
|
|
96
|
+
async def run_once(self, *, limit: int | None = None) -> int:
|
|
97
|
+
"""Run due jobs one by one until none is due (or `limit` ran); returns how many ran."""
|
|
98
|
+
ran = 0
|
|
99
|
+
while limit is None or ran < limit:
|
|
100
|
+
job = await self.claim()
|
|
101
|
+
if job is None:
|
|
102
|
+
break
|
|
103
|
+
await self.execute(job)
|
|
104
|
+
ran += 1
|
|
105
|
+
return ran
|
|
106
|
+
|
|
107
|
+
async def execute(self, job: JobRow) -> str:
|
|
108
|
+
"""Run a claimed job and record its outcome; returns the job's new status."""
|
|
109
|
+
self._running[job.id] = job
|
|
110
|
+
started = time.monotonic()
|
|
111
|
+
attributes: dict[str, str | int] = {
|
|
112
|
+
"lorebase.job.type": job.type,
|
|
113
|
+
"lorebase.job.id": job.id,
|
|
114
|
+
}
|
|
115
|
+
if job.domain_id is not None:
|
|
116
|
+
attributes["lorebase.domain"] = job.domain_id
|
|
117
|
+
cancelled = False
|
|
118
|
+
try:
|
|
119
|
+
with tracer.start_as_current_span(
|
|
120
|
+
f"scribe.job {job.type}", attributes=attributes
|
|
121
|
+
) as span:
|
|
122
|
+
result: Mapping[str, Any] | None = None
|
|
123
|
+
error: str | None = None
|
|
124
|
+
settings = self._jobs
|
|
125
|
+
try:
|
|
126
|
+
result = await self._handlers[job.type](job)
|
|
127
|
+
except self._permanent as exc:
|
|
128
|
+
error, settings = f"{type(exc).__name__}: {exc}", self._final
|
|
129
|
+
span.record_exception(exc)
|
|
130
|
+
except Exception as exc:
|
|
131
|
+
log.exception("job failed", extra={"job": job.id, "type": job.type})
|
|
132
|
+
error = f"{type(exc).__name__}: {exc}"
|
|
133
|
+
span.record_exception(exc)
|
|
134
|
+
async with self._sessions() as session, session.begin():
|
|
135
|
+
status = await finish_job(
|
|
136
|
+
session, job, result=result, error=error, settings=settings
|
|
137
|
+
)
|
|
138
|
+
if error is not None:
|
|
139
|
+
span.set_status(Status(StatusCode.ERROR, error))
|
|
140
|
+
span.set_attribute("lorebase.job.status", status)
|
|
141
|
+
except asyncio.CancelledError:
|
|
142
|
+
# Shutdown: the job goes back to the queue. It stays tracked (and heartbeated)
|
|
143
|
+
# until released, so `run` releases it if this release does not finish.
|
|
144
|
+
cancelled = True
|
|
145
|
+
await asyncio.shield(self.release(job))
|
|
146
|
+
raise
|
|
147
|
+
finally:
|
|
148
|
+
if not cancelled:
|
|
149
|
+
self._running.pop(job.id, None)
|
|
150
|
+
duration = time.monotonic() - started
|
|
151
|
+
JOBS.add(1, {"type": job.type, "status": status})
|
|
152
|
+
JOB_DURATION.record(duration, {"type": job.type})
|
|
153
|
+
log.log(
|
|
154
|
+
logging.WARNING if error is not None else logging.INFO,
|
|
155
|
+
"job finished",
|
|
156
|
+
extra={
|
|
157
|
+
"job": job.id,
|
|
158
|
+
"type": job.type,
|
|
159
|
+
"domain": job.domain_id,
|
|
160
|
+
"attempt": job.attempts,
|
|
161
|
+
"status": status,
|
|
162
|
+
"error": error,
|
|
163
|
+
"duration_ms": int(duration * 1000),
|
|
164
|
+
},
|
|
165
|
+
)
|
|
166
|
+
return status
|
|
167
|
+
|
|
168
|
+
async def release(self, *jobs: JobRow) -> int:
|
|
169
|
+
"""Put `jobs` (default: every running job) back in the queue without counting the
|
|
170
|
+
attempt and stop tracking them; returns how many were still held by this worker."""
|
|
171
|
+
chosen = list(jobs) or list(self._running.values())
|
|
172
|
+
if not chosen:
|
|
173
|
+
return 0
|
|
174
|
+
try:
|
|
175
|
+
async with self._sessions() as session, session.begin():
|
|
176
|
+
released = await release_jobs(session, self.worker_id, [j.id for j in chosen])
|
|
177
|
+
except Exception:
|
|
178
|
+
log.exception(
|
|
179
|
+
"job release failed; the jobs are reclaimed once their heartbeat goes stale",
|
|
180
|
+
extra={"jobs": [j.id for j in chosen]},
|
|
181
|
+
)
|
|
182
|
+
return 0
|
|
183
|
+
for job in chosen:
|
|
184
|
+
self._running.pop(job.id, None)
|
|
185
|
+
log.info("job released on shutdown", extra={"job": job.id, "type": job.type})
|
|
186
|
+
return released
|
|
187
|
+
|
|
188
|
+
async def heartbeat(self) -> int:
|
|
189
|
+
"""Refresh the lock time of the running jobs."""
|
|
190
|
+
if not self._running:
|
|
191
|
+
return 0
|
|
192
|
+
ids = list(self._running)
|
|
193
|
+
async with self._sessions() as session, session.begin():
|
|
194
|
+
touched = await touch_jobs(session, self.worker_id, ids)
|
|
195
|
+
if touched < sum(1 for i in ids if i in self._running): # not just finished meanwhile
|
|
196
|
+
log.warning(
|
|
197
|
+
"running jobs were reclaimed from this worker (heartbeat went stale); their "
|
|
198
|
+
"outcome will not be recorded",
|
|
199
|
+
extra={"worker": self.worker_id, "running": len(ids), "held": touched},
|
|
200
|
+
)
|
|
201
|
+
return touched
|
|
202
|
+
|
|
203
|
+
async def _slot(self, stop: asyncio.Event) -> None:
|
|
204
|
+
while not stop.is_set():
|
|
205
|
+
try:
|
|
206
|
+
job = await self.claim()
|
|
207
|
+
except Exception:
|
|
208
|
+
log.exception("claiming jobs failed")
|
|
209
|
+
job = None
|
|
210
|
+
if job is None:
|
|
211
|
+
with contextlib.suppress(TimeoutError):
|
|
212
|
+
await asyncio.wait_for(stop.wait(), timeout=self._settings.poll_interval_s)
|
|
213
|
+
continue
|
|
214
|
+
await self.execute(job)
|
|
215
|
+
|
|
216
|
+
async def _heartbeats(self, stop: asyncio.Event) -> None:
|
|
217
|
+
while not stop.is_set():
|
|
218
|
+
with contextlib.suppress(TimeoutError):
|
|
219
|
+
await asyncio.wait_for(stop.wait(), timeout=self._settings.heartbeat_interval_s)
|
|
220
|
+
try:
|
|
221
|
+
await self.heartbeat()
|
|
222
|
+
except Exception:
|
|
223
|
+
log.exception("job heartbeat failed")
|
|
224
|
+
|
|
225
|
+
async def run(self, stop: asyncio.Event) -> None:
|
|
226
|
+
"""Run until `stop` is set, then shut down gracefully."""
|
|
227
|
+
log.info(
|
|
228
|
+
"worker started",
|
|
229
|
+
extra={
|
|
230
|
+
"worker": self.worker_id,
|
|
231
|
+
"types": list(self.types),
|
|
232
|
+
"concurrency": self._settings.concurrency,
|
|
233
|
+
},
|
|
234
|
+
)
|
|
235
|
+
slots = [
|
|
236
|
+
asyncio.create_task(self._slot(stop), name=f"scribe-slot-{i}")
|
|
237
|
+
for i in range(self._settings.concurrency)
|
|
238
|
+
]
|
|
239
|
+
beat = asyncio.create_task(self._heartbeats(stop), name="scribe-heartbeat")
|
|
240
|
+
await stop.wait()
|
|
241
|
+
log.info(
|
|
242
|
+
"worker stopping",
|
|
243
|
+
extra={"worker": self.worker_id, "running": len(self._running)},
|
|
244
|
+
)
|
|
245
|
+
_, pending = await asyncio.wait(slots, timeout=self._settings.shutdown_grace_s)
|
|
246
|
+
for task in pending:
|
|
247
|
+
task.cancel()
|
|
248
|
+
if pending:
|
|
249
|
+
await asyncio.wait(pending, timeout=self._settings.shutdown_cancel_s)
|
|
250
|
+
# Jobs whose cancellation has not released them yet (a handler that does not stop).
|
|
251
|
+
await self.release()
|
|
252
|
+
beat.cancel()
|
|
253
|
+
await asyncio.gather(beat, return_exceptions=True)
|
|
254
|
+
log.info("worker stopped", extra={"worker": self.worker_id})
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: lorebase-scribe
|
|
3
|
+
Version: 0.2.3
|
|
4
|
+
Summary: Lorebase Scribe: job worker, scheduler and Source synchronization
|
|
5
|
+
Requires-Python: >=3.12
|
|
6
|
+
Requires-Dist: asyncpg>=0.29
|
|
7
|
+
Requires-Dist: httpx>=0.27
|
|
8
|
+
Requires-Dist: lorebase-core<0.3,>=0.2
|
|
9
|
+
Requires-Dist: lorebase-inference<0.3,>=0.2
|
|
10
|
+
Requires-Dist: lorebase-platform<0.3,>=0.2
|
|
11
|
+
Requires-Dist: lorebase-store<0.3,>=0.2
|
|
12
|
+
Requires-Dist: opentelemetry-api>=1.25
|
|
13
|
+
Requires-Dist: pydantic>=2.7
|
|
14
|
+
Requires-Dist: sqlalchemy[asyncio]>=2.0.30
|
|
15
|
+
Provides-Extra: documents
|
|
16
|
+
Requires-Dist: pypdf>=4.2; extra == 'documents'
|
|
17
|
+
Requires-Dist: python-docx>=1.1; extra == 'documents'
|
|
18
|
+
Provides-Extra: mssql
|
|
19
|
+
Requires-Dist: pymssql>=2.3; extra == 'mssql'
|
|
20
|
+
Provides-Extra: mysql
|
|
21
|
+
Requires-Dist: asyncmy>=0.2.10; extra == 'mysql'
|
|
22
|
+
Provides-Extra: parquet
|
|
23
|
+
Requires-Dist: pyarrow>=15; extra == 'parquet'
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
lorebase_scribe/__init__.py,sha256=KDhMcdNTYzjgVkKMSNRdQeNLCVHSqWO5DqtgwSDcfWY,96
|
|
2
|
+
lorebase_scribe/__main__.py,sha256=Zf_ySOK8Ew1vzB3jTG13c8jJqP3CJEs9BLmhSLdL2z4,6098
|
|
3
|
+
lorebase_scribe/checkpoints.py,sha256=vD2yypjIQFjdFn4SXnY1hhJlqjE3mbPlXGEFTXZiDak,11969
|
|
4
|
+
lorebase_scribe/cron.py,sha256=bZijerwzdg9lANViJzi9kxgdBehcAcmLPgaM2k8pfz4,4336
|
|
5
|
+
lorebase_scribe/embeddings.py,sha256=a17XQrKhIEkaiy2_kQrXhfwSrulhrATp3TQOXMRUt2Y,22335
|
|
6
|
+
lorebase_scribe/jobs.py,sha256=kn24YYtFTazNLmzNuY6Jg_JbnTupLt5CPAnJdgWCmEQ,17660
|
|
7
|
+
lorebase_scribe/scheduler.py,sha256=dZUDxFFPgLyHJfaPO0yvYLnDEZGGvK0cg37VRKlmCI8,12917
|
|
8
|
+
lorebase_scribe/service.py,sha256=Hn6Dp9kSOIf28IugAWReTGQ4pM61BxZoN_NS8VzRutw,6006
|
|
9
|
+
lorebase_scribe/settings.py,sha256=bfPLPfXHNLfOIR03LSicxAAYfzgwMW-fp3BmgAKa2Vs,21856
|
|
10
|
+
lorebase_scribe/telemetry.py,sha256=MWpanvCeAfWdurUTUXQTkmv3mlzN382Z4UM95mzDgH8,4081
|
|
11
|
+
lorebase_scribe/webhooks.py,sha256=knWCeJv-UpYuJ0lqmX23tl-QjLDWaPYXu31OEukbKIs,4041
|
|
12
|
+
lorebase_scribe/worker.py,sha256=SW-f8KVJD5w3PqA0A6QBAMombQcevRpKIFjpD1z1aMw,10598
|
|
13
|
+
lorebase_scribe/extraction/__init__.py,sha256=6PbQBF3WeJm5YWvLQ9oDRJbqsZMB0tfA64DzOxCDuHY,888
|
|
14
|
+
lorebase_scribe/extraction/assemble.py,sha256=sAjtM4sphQrSMwGz_j19Yby6-K-C4y1FolyeS0aPIfM,13842
|
|
15
|
+
lorebase_scribe/extraction/candidates.py,sha256=L7HDJUHv1XjlKH63QIiQ-RdoH95ZcB8tOjXmymqLdWg,3147
|
|
16
|
+
lorebase_scribe/extraction/discovery.py,sha256=NJlXZMqjTPMox7jKBYAdFrLb2ro5MrXBk9elwhaMlbg,23327
|
|
17
|
+
lorebase_scribe/extraction/llm_discovery.py,sha256=fxc6-hrameitw9YT2PI8zcvTsfjhHWzamfMakIEeu5E,11395
|
|
18
|
+
lorebase_scribe/extraction/matching.py,sha256=v5N4ZyaoeHArfjvajTp4RVf-JSFKNjaTnwjPHHC1EWQ,11383
|
|
19
|
+
lorebase_scribe/extraction/ontology.py,sha256=r5xzbTzDQpEfSZ0Jjxw63wvuUVNNJ6kGLq4R2QCe2uo,15493
|
|
20
|
+
lorebase_scribe/extraction/parsing.py,sha256=youEZgzw6LuRWwGtgYlP7BHGSvBUGlBYCHepbyO9g5o,4182
|
|
21
|
+
lorebase_scribe/extraction/pipeline.py,sha256=pVSwFX9ZR14fABNt-gKct8FYG_AYP7aygs559pGIlhE,44110
|
|
22
|
+
lorebase_scribe/extraction/relations.py,sha256=ZHbxz0fWtz0Nt8QCWkVB0WUjawi0XLTC9w8zD2JQqUc,7839
|
|
23
|
+
lorebase_scribe/extraction/sandbox.py,sha256=__7VsnfQpbtXZQU5_hXcg0-3Q2oIdFDSZKqM9npX8_I,9931
|
|
24
|
+
lorebase_scribe/extraction/settings.py,sha256=_cw2qDRPHi-puhkMEhEvZkS3TGrr2gfoQqrvJ-NcP_8,11986
|
|
25
|
+
lorebase_scribe/extraction/text.py,sha256=nqQ7x2wz_ch1RPYnGdROPqsWRQsbJe8j6yuUjrnHzB8,6653
|
|
26
|
+
lorebase_scribe/extraction/tier2.py,sha256=qCBg_w2UdFWnfKUxKGZo8I95aAaIV6bsXLops_jQknE,21258
|
|
27
|
+
lorebase_scribe/extraction/values.py,sha256=F5Txs5XtP4TMICD9wnfWPHeFkvw2xdA-jyJxiGAtJNQ,11171
|
|
28
|
+
lorebase_scribe/sources/__init__.py,sha256=mllXarXxoPKG-X7YA18_HpYCmm7QJAOijZZ0EfwBIaw,1389
|
|
29
|
+
lorebase_scribe/sources/access.py,sha256=W3_hu7pCiPmHx0K3OS0ZcDOae05FhpGZBogkFQ8y7Zc,10007
|
|
30
|
+
lorebase_scribe/sources/connectors.py,sha256=XD4kJQ47fkz0-u9rYAuw53nDhyRfRPi0LsNlIM01pnA,13451
|
|
31
|
+
lorebase_scribe/sources/delta.py,sha256=ex-JNj5Q9dXut3Ojj3RnSiSzfZ6CY-eXtC0RlQmTtd8,6378
|
|
32
|
+
lorebase_scribe/sources/http_json.py,sha256=py1piR4yleqylTxfsCNeqmV2pPFmpwtqmUqj3dkS8CU,11964
|
|
33
|
+
lorebase_scribe/sources/mapping.py,sha256=2I95jTUuqByNFfkfxGu_UgDAuwygz-oWZtNJnbRAfwM,18661
|
|
34
|
+
lorebase_scribe/sources/profile.py,sha256=CzxV24NCtxSDiGVDxeVm2fYZ_TUroQ-9tpkP8CrEtA0,1599
|
|
35
|
+
lorebase_scribe/sources/records.py,sha256=neyWkorC83n4Ec_scgBFaundqJ6-42viDJTOC4rr2Sg,11963
|
|
36
|
+
lorebase_scribe/sources/sql.py,sha256=Atias2x0ai1CQv0QV2okj33D_Hwz43-ESWI-2LNg6Fw,14093
|
|
37
|
+
lorebase_scribe/sources/sync.py,sha256=btGdJY2f-fZFUmr1vUX1nVNizA-Tjxc89utG3RR1jcs,29671
|
|
38
|
+
lorebase_scribe-0.2.3.dist-info/METADATA,sha256=N90d6j48RbDcNQB8K2ZkU_3E_w5ftMEgAy125ni8O0U,829
|
|
39
|
+
lorebase_scribe-0.2.3.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
40
|
+
lorebase_scribe-0.2.3.dist-info/entry_points.txt,sha256=Vbjv48Bhe4C8BL-aI0ffvmrnzf_6IrJMlWdYm711Pno,66
|
|
41
|
+
lorebase_scribe-0.2.3.dist-info/RECORD,,
|