flowocr 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. flowocr/__init__.py +15 -0
  2. flowocr/analyze/__init__.py +8 -0
  3. flowocr/analyze/align.py +151 -0
  4. flowocr/analyze/build_tracks.py +2886 -0
  5. flowocr/analyze/cluster_layers.py +155 -0
  6. flowocr/analyze/game_align.py +652 -0
  7. flowocr/analyze/gamescript.py +1220 -0
  8. flowocr/analyze/gtdbundle.py +306 -0
  9. flowocr/analyze/match.py +67 -0
  10. flowocr/analyze/matchers/__init__.py +5 -0
  11. flowocr/analyze/matchers/gametext.py +70 -0
  12. flowocr/analyze/merge_nameplate.py +178 -0
  13. flowocr/analyze/models/slot_pair.json +148 -0
  14. flowocr/analyze/nameplate.py +148 -0
  15. flowocr/analyze/pair_features.py +168 -0
  16. flowocr/analyze/pair_model.py +127 -0
  17. flowocr/analyze/refine_boundaries.py +409 -0
  18. flowocr/analyze/script_align.py +641 -0
  19. flowocr/analyze/scriptmatch.py +1018 -0
  20. flowocr/analyze/slot_learned.py +306 -0
  21. flowocr/analyze/slot_lines.py +251 -0
  22. flowocr/analyze/slot_modes.py +143 -0
  23. flowocr/analyze/slot_pairs.py +562 -0
  24. flowocr/analyze/slot_veto.py +104 -0
  25. flowocr/analyze/uigate.py +351 -0
  26. flowocr/artifacts/__init__.py +5 -0
  27. flowocr/artifacts/evalkit.py +123 -0
  28. flowocr/artifacts/matchedio.py +75 -0
  29. flowocr/artifacts/srtio.py +166 -0
  30. flowocr/artifacts/tracksio.py +345 -0
  31. flowocr/extensions.py +77 -0
  32. flowocr/extract/__init__.py +8 -0
  33. flowocr/extract/childproc.py +118 -0
  34. flowocr/extract/decode_proc.py +511 -0
  35. flowocr/extract/decode_shards.py +574 -0
  36. flowocr/extract/detpost.py +215 -0
  37. flowocr/extract/edge_proc.py +314 -0
  38. flowocr/extract/edge_refine.py +598 -0
  39. flowocr/extract/fast_det.py +214 -0
  40. flowocr/extract/ffcheck.py +87 -0
  41. flowocr/extract/framegrid.py +265 -0
  42. flowocr/extract/framesource.py +1264 -0
  43. flowocr/extract/ocr_args.py +754 -0
  44. flowocr/extract/ocr_complete.py +231 -0
  45. flowocr/extract/ocr_parallel.py +208 -0
  46. flowocr/extract/ort_server.py +632 -0
  47. flowocr/extract/ortclient.py +363 -0
  48. flowocr/extract/ptsclock.py +150 -0
  49. flowocr/extract/recdecode.py +78 -0
  50. flowocr/extract/recort.py +162 -0
  51. flowocr/extract/recpack.py +94 -0
  52. flowocr/extract/recpool.py +206 -0
  53. flowocr/extract/recprep.py +71 -0
  54. flowocr/extract/refine_video.py +271 -0
  55. flowocr/extract/regions.py +387 -0
  56. flowocr/extract/reuse_v2.py +593 -0
  57. flowocr/extract/run_groups.py +247 -0
  58. flowocr/extract/run_ocr2.py +1605 -0
  59. flowocr/extract/supervisor.py +261 -0
  60. flowocr/extract/timeline.py +84 -0
  61. flowocr/extract/typewriter_fuse.py +394 -0
  62. flowocr/models.py +263 -0
  63. flowocr/output/__init__.py +3 -0
  64. flowocr/output/export.py +205 -0
  65. flowocr/output/layout.py +210 -0
  66. flowocr/output/presets/__init__.py +6 -0
  67. flowocr/output/presets/_overlay.py +40 -0
  68. flowocr/output/presets/default.py +21 -0
  69. flowocr/output/presets/default_all.py +20 -0
  70. flowocr/output/presets/dev.py +21 -0
  71. flowocr/output/presets/matched_srt.py +32 -0
  72. flowocr/output/presets/script.py +20 -0
  73. flowocr/output/presets/srt_main.py +33 -0
  74. flowocr/output/render.py +82 -0
  75. flowocr/output/run_srt.py +83 -0
  76. flowocr/output/script.py +541 -0
  77. flowocr/paths.py +168 -0
  78. flowocr/provenance.py +119 -0
  79. flowocr/typeset/__init__.py +9 -0
  80. flowocr/typeset/__main__.py +38 -0
  81. flowocr/typeset/assfile.py +216 -0
  82. flowocr/typeset/core.py +821 -0
  83. flowocr/typeset/fx/__init__.py +11 -0
  84. flowocr-0.1.0.dist-info/METADATA +109 -0
  85. flowocr-0.1.0.dist-info/RECORD +91 -0
  86. flowocr-0.1.0.dist-info/WHEEL +5 -0
  87. flowocr-0.1.0.dist-info/entry_points.txt +10 -0
  88. flowocr-0.1.0.dist-info/licenses/LICENSE +674 -0
  89. flowocr-0.1.0.dist-info/licenses/LICENSES/Apache-2.0.txt +201 -0
  90. flowocr-0.1.0.dist-info/licenses/LICENSES/PP-OCRv6-NOTICE.md +18 -0
  91. flowocr-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,118 @@
1
+ """拆进程的共用胶水(2026-09-24 审计:原来 `EdgeProxy` / `FrontProxy` / `RemoteDet` 和两个 `serve` 各写一份)。
2
+
3
+ 管线进程这边(`Child`):本机端口上 `Listener` 等、`subprocess` 起 `python -m <模块> <端口>`(不用 multiprocessing 的 spawn——
4
+ 那会在子进程里重新 import 管线主模块、把推理库拉起来);口令走环境变量;**限时**等它连上(`accept` 没有超时参数,放线程里等);
5
+ 发送加锁;收尾一律"发告别 -> 限时等它退 -> 杀"。子进程那边(`connect`)按口令连回来,拿到连接和加了锁的 `send`。
6
+
7
+ 各进程只写自己的消息协议(回抠:`edge_proc`;只解码:`decode_proc`)。子进程出错的约定:发 `("hw", 原因)`(硬解交付不对,
8
+ 管线进程原样抛 `HwaccelMismatch` -> 外层监督者整段改软解)或 `("err", 回溯)`;`Child.recv_checked` 把这两种换成异常。
9
+ """
10
+ from __future__ import annotations
11
+
12
+ import os
13
+ import sys
14
+ import threading
15
+ import time
16
+
17
+ AUTH_ENV = "FLOWOCR_CHILD_KEY"
18
+
19
+
20
+ class Child:
21
+ """起一个子进程、等它连回来。`timeout` 是连上的时限(子进程 import 的时间在里面)。
22
+ 构造不等连接(起动可以和管线进程自己的起动并行);第一次用 `conn` 时才等。"""
23
+
24
+ def __init__(self, module: str, what: str, timeout: float = 120.0) -> None:
25
+ import secrets
26
+ import subprocess
27
+ from multiprocessing.connection import Listener
28
+
29
+ from flowocr.paths import child_env
30
+
31
+ key = secrets.token_bytes(16)
32
+ self.what, self.timeout = what, timeout
33
+ self.lst = Listener(("127.0.0.1", 0), authkey=key)
34
+ self.proc = subprocess.Popen([sys.executable, "-m", module, str(self.lst.address[1])],
35
+ env=child_env(dict(os.environ, **{AUTH_ENV: key.hex()})))
36
+ self._got: list = []
37
+ self._accept = threading.Thread(target=lambda: self._got.append(self.lst.accept()), daemon=True,
38
+ name=f"{module.rsplit('.', 1)[-1]}-accept")
39
+ self._accept.start()
40
+ self._conn = None
41
+ self._lock = threading.Lock()
42
+ self._closed = False
43
+
44
+ @property
45
+ def conn(self):
46
+ if self._conn is None:
47
+ # 分段等、每段看一眼子进程还在不在:import 阶段就崩了的,不必干等满 timeout 才报
48
+ deadline = time.monotonic() + self.timeout
49
+ while not self._got and self.proc.poll() is None and time.monotonic() < deadline:
50
+ self._accept.join(0.5)
51
+ if not self._got:
52
+ self.proc.kill()
53
+ rc = self.proc.poll()
54
+ raise RuntimeError(f"{self.what} 没连上:" + (f"子进程已退出(退出码 {rc})" if rc is not None
55
+ else f"{self.timeout:.0f} s 内没连上"))
56
+ self._conn = self._got[0]
57
+ return self._conn
58
+
59
+ def send(self, msg) -> None:
60
+ conn = self.conn
61
+ with self._lock:
62
+ conn.send(msg)
63
+
64
+ def recv(self):
65
+ return self.conn.recv()
66
+
67
+ def recv_checked(self):
68
+ """收一条;`("hw", …)` / `("err", …)` 换成异常。"""
69
+ return check(self.recv(), self.what)
70
+
71
+ def close(self, wait: float = 60.0) -> None:
72
+ """收摊(连上前后都能调):发 `("x",)` 让它退、限时等、不退就杀。连都没连上的直接杀。"""
73
+ if self._closed:
74
+ return
75
+ self._closed = True
76
+ if self._conn is None and self._got:
77
+ self._conn = self._got[0]
78
+ if self._conn is not None:
79
+ try:
80
+ with self._lock:
81
+ self._conn.send(("x",))
82
+ except OSError:
83
+ pass
84
+ try:
85
+ self.proc.wait(wait if self._conn is not None else 5)
86
+ except Exception: # noqa: BLE001
87
+ self.proc.kill()
88
+ for h in (self._conn, self.lst):
89
+ try:
90
+ if h is not None:
91
+ h.close()
92
+ except OSError:
93
+ pass
94
+
95
+
96
+ def check(msg, what: str):
97
+ """子进程发来的一条消息:`("hw", 原因)` -> `HwaccelMismatch`(外层监督者整段改软解)、`("err", 回溯)` -> RuntimeError,其余原样返回。"""
98
+ if msg[0] == "hw":
99
+ from flowocr.extract.framesource import HwaccelMismatch
100
+ raise HwaccelMismatch(msg[1])
101
+ if msg[0] == "err":
102
+ raise RuntimeError(f"{what}出错:\n{msg[1]}")
103
+ return msg
104
+
105
+
106
+ def connect():
107
+ """子进程那边:按 argv 的端口、环境变量的口令连回管线进程。返回 `(conn, send)`,`send` 加了锁(几个线程都会发)。"""
108
+ from multiprocessing.connection import Client
109
+
110
+ key = bytes.fromhex(os.environ.pop(AUTH_ENV))
111
+ conn = Client(("127.0.0.1", int(sys.argv[1])), authkey=key)
112
+ lock = threading.Lock()
113
+
114
+ def send(msg) -> None:
115
+ with lock:
116
+ conn.send(msg)
117
+
118
+ return conn, send
@@ -0,0 +1,511 @@
1
+ """只解码的子进程(`--decode-shards`,decode-buffer):`decode_shards.ShardedSource` 跑在这里,采样帧经共享内存槽交给管线进程
2
+ (det 在那边),辅助流按块序直接转给回抠进程。**不 import 推理库**(起动约 1 s,和管线进程建模型并行)。
3
+
4
+ 帧怎么过去:这边建一块共享内存、切成若干槽(BGR 整帧),写进空槽后发 `("f", 帧号, 时刻, 槽)`;管线进程**拷出来**(整帧 6 MB,约 1 ms)
5
+ 就发 `("r", 槽)` 还回去——那边没有任何东西会指着槽("以后谁存了一个切片"会静默写坏,所以先拷;零拷贝没量过)。
6
+ 槽满了这边就停(背压一路传回各路 ffmpeg),槽数按 det 的前瞻给(`slots_for`),帧环容量也要把它算进在途(`edge_refine.ring_in_flight`)。
7
+
8
+ 起动分两段:管线进程一解析完参数就 `DecodeProxy()`(只起进程、不等);算好取帧参数后 `start(cfg)`,这边起各路 ffmpeg、回 `("hello", …)`。
9
+ 流结束发 `("end", 取帧层的属性)`。故障按 `childproc` 的约定:硬解交付不对 `("hw", 原因)`,别的 `("err", 回溯)`;管线进程断开(EOF)就收摊退出。
10
+
11
+ 2026-09-24 以前这里是 `front_proc`,还有"解码 + det"(`--front-proc`)和"只做 det"(`--det-proc`)两种模式——量过稳态更慢 / ±0%,
12
+ owner 定删掉(decode-buffer)。
13
+ """
14
+ from __future__ import annotations
15
+
16
+ import json
17
+ import os
18
+ import sys
19
+ import threading
20
+ import time
21
+ import traceback
22
+ from pathlib import Path
23
+ from queue import Queue
24
+
25
+ from flowocr.extract import childproc
26
+
27
+
28
+ def slots_for(det_batch: int, det_prefetch: int) -> int:
29
+ """共享内存槽几个:盖住 det 一次要的批和它的前瞻,再留几个给管线进程慢半拍。"""
30
+ return max(8, det_batch * (1 + det_prefetch) + 4)
31
+
32
+
33
+ def aux_remote(acfg: dict):
34
+ """一个回抠进程的辅助流去处:像素连 `url`、逐帧 pts 经它的 `pts_addr` 直接送过去(不绕管线进程)。"""
35
+ from multiprocessing.connection import Client
36
+ from types import SimpleNamespace
37
+
38
+ from flowocr.extract import framesource
39
+
40
+ port2, key2 = acfg["pts_addr"]
41
+ feed = Client(("127.0.0.1", int(port2)), authkey=bytes.fromhex(key2))
42
+ flock = threading.Lock()
43
+
44
+ def put(item) -> None:
45
+ with flock:
46
+ feed.send(("pe",) if item is framesource.QUEUE_SENTINEL else ("p", item))
47
+
48
+ return SimpleNamespace(url=acfg["url"], pts_put=put)
49
+
50
+
51
+ def make_source(cfg: dict, remote):
52
+ """按管线进程发来的取帧参数建分片取帧层;`remote` 是辅助流去处(`aux_remote` / `AuxFanout` / None)。"""
53
+ from flowocr.extract import decode_shards
54
+
55
+ kw = cfg["source_kw"]
56
+ a = cfg.get("aux") or {}
57
+ return decode_shards.ShardedSource(
58
+ cfg["video"], lanes=cfg["lanes"], grid=kw["grid"], src_fps=kw["src_fps"], width=kw["width"],
59
+ height=kw["height"], pix=kw["pix"], select_by=kw["select_by"], hwaccel=kw["hwaccel"],
60
+ start_idx=kw["start_idx"], end_idx=kw["end_idx"],
61
+ blocks=decode_shards.BlockPlan(cfg["video"], kw["start_idx"], kw["end_idx"], kw["src_fps"], cfg["block_sec"]),
62
+ aux_remote=remote, aux_grid=a.get("grid"), aux_scale=a.get("scale", 0.35))
63
+
64
+
65
+ def end_stats(source, slot_wait: float) -> dict:
66
+ return {"stopped_early": source.stopped_early, "advanced": source.advanced,
67
+ "grid_off": dict(source.grid_off), "grid_skipped": source.grid_skipped,
68
+ "stderr_tail": list(source.stderr_tail), "preroll": source.preroll_ticks,
69
+ "slot_wait": round(slot_wait, 2),
70
+ "shards": {"lanes": source.lanes, "blocks": len(source.blocks), "lane_blocks": source.lane_blocks,
71
+ "lane_sec": [round(x, 2) for x in source.lane_sec], "peak_mb": round(source.peak_mb),
72
+ **({"probe_unknown": source.probe_unknown} if source.probe_unknown else {}),
73
+ **({"plan_error": source.plan_error} if source.plan_error else {})}}
74
+
75
+
76
+ def serve() -> int:
77
+ from multiprocessing import shared_memory
78
+
79
+ import numpy as np
80
+
81
+ from flowocr.extract import framesource
82
+ from flowocr.extract import timeline as tl
83
+
84
+ conn, send = childproc.connect()
85
+ tl.start_from_env("decode") # --timeline:这个进程写 `<路径>.decode.jsonl`
86
+ try:
87
+ msg = conn.recv()
88
+ except (EOFError, OSError):
89
+ return 0
90
+ if msg[0] != "go": # "x":管线进程起不来 / 不用了
91
+ return 0
92
+ cfg = msg[1]
93
+ shm = None
94
+ stop = threading.Event()
95
+ free: Queue = Queue()
96
+ listening = False
97
+ try:
98
+ source = make_source(cfg, aux_remote(cfg["aux"]) if cfg.get("aux") else None)
99
+ H, W, n = cfg["H"], cfg["W"], cfg["slots"]
100
+ nb = H * W * 3
101
+ shm = shared_memory.SharedMemory(create=True, size=nb * n)
102
+ for i in range(n):
103
+ free.put(i)
104
+
105
+ def recv_loop() -> None:
106
+ try:
107
+ while True:
108
+ m = conn.recv()
109
+ if m[0] == "r":
110
+ free.put(m[1])
111
+ else: # "x":管线进程要收摊
112
+ break
113
+ except (EOFError, OSError):
114
+ pass
115
+ stop.set()
116
+ free.put(None)
117
+
118
+ threading.Thread(target=recv_loop, daemon=True, name="decode-recv").start()
119
+ listening = True
120
+ send(("hello", {"shm": shm.name, "slots": n, "describe": source.describe()}))
121
+ slot_wait = 0.0
122
+ for idx, t_sec, frame in source:
123
+ t0 = time.perf_counter()
124
+ slot = free.get()
125
+ slot_wait += time.perf_counter() - t0
126
+ if slot is None or stop.is_set():
127
+ break
128
+ np.ndarray((H, W, 3), np.uint8, buffer=shm.buf[slot * nb:(slot + 1) * nb])[...] = frame
129
+ send(("f", idx, t_sec, slot))
130
+ source.close()
131
+ tl.dump() # 在回 end 之前:管线进程收到就收摊了
132
+ send(("end", end_stats(source, slot_wait)))
133
+ except BaseException as exc: # noqa: BLE001 一律交回管线进程,由它决定怎么退
134
+ try:
135
+ if isinstance(exc, framesource.HwaccelMismatch):
136
+ send(("hw", str(exc)))
137
+ else:
138
+ send(("err", traceback.format_exc()))
139
+ except OSError:
140
+ pass
141
+ if not listening: # 收消息的线程还没起:没人会置 stop,等就是白等
142
+ stop.set()
143
+ # 管线进程拷完最后一帧、发 "x"(或断开)之前不能拆共享内存。**出错时也要等它**(2026-09-24 审核):
144
+ # hello 已发、它还没附上共享内存时这边先拆,Windows 上段就没了,它附时 FileNotFoundError 会盖掉管道里的 "hw"——软解重跑就丢了
145
+ if shm is not None:
146
+ stop.wait(120)
147
+ shm.close()
148
+ shm.unlink()
149
+ try:
150
+ conn.close()
151
+ except OSError:
152
+ pass
153
+ return 0
154
+
155
+
156
+ class DecodeProxy:
157
+ """管线进程这边:取帧层的接口——迭代吐 `(帧号, 时刻, BGR 帧)`;流结束后 `stopped_early` / `advanced` / `grid_off` /
158
+ `grid_skipped` / `stderr_tail` / `preroll_ticks` / `shards` 从子进程带回来。构造只起进程、不等;`start(cfg)` 开流。"""
159
+
160
+ def __init__(self, link=None) -> None:
161
+ self.child = link or childproc.Child("flowocr.extract.decode_proc", "解码进程")
162
+ self.shm = None
163
+ self.aux = None # 辅助流收在回抠进程里(--edge-proc)
164
+ self.stopped_early = False
165
+ self.advanced = 0
166
+ self.grid_off: dict = {}
167
+ self.grid_skipped = 0
168
+ self.stderr_tail: list = []
169
+ self.preroll_ticks = 0
170
+ self.shards: dict | None = None # 各路领了几块、各花多久、预取峰值
171
+ self.wait_start_sec = 0.0
172
+ self.wait_sec = 0.0
173
+ self._describe = ""
174
+
175
+ def start(self, cfg: dict) -> None:
176
+ """开流:`cfg` 是取帧参数。各路 ffmpeg 在子进程里起,回 hello 之后才返回。"""
177
+ from multiprocessing import shared_memory
178
+
179
+ t0 = time.perf_counter()
180
+ self.child.send(("go", cfg))
181
+ msg = self.child.recv_checked()
182
+ if msg[0] != "hello":
183
+ raise RuntimeError(f"解码进程起不来:{msg}")
184
+ h = msg[1]
185
+ self.wait_start_sec = time.perf_counter() - t0
186
+ self.shm = shared_memory.SharedMemory(name=h["shm"])
187
+ self.H, self.W = cfg["H"], cfg["W"]
188
+ self.nb = self.H * self.W * 3
189
+ self._describe = h["describe"]
190
+
191
+ def describe(self) -> str:
192
+ return f"解码在独立进程(--decode-shards):{self._describe}"
193
+
194
+ def __iter__(self):
195
+ import numpy as np
196
+
197
+ while True:
198
+ t0 = time.perf_counter()
199
+ msg = self.child.recv_checked() # "hw" / "err" 在这里抛
200
+ self.wait_sec += time.perf_counter() - t0
201
+ if msg[0] == "f":
202
+ _, idx, t_sec, slot = msg
203
+ frame = np.ndarray((self.H, self.W, 3), np.uint8,
204
+ buffer=self.shm.buf[slot * self.nb:(slot + 1) * self.nb]).copy()
205
+ self.child.send(("r", slot))
206
+ yield idx, t_sec, frame
207
+ elif msg[0] == "end":
208
+ a = msg[1]
209
+ self.stopped_early, self.advanced = a["stopped_early"], a["advanced"]
210
+ self.grid_off = {int(k): v for k, v in a["grid_off"].items()}
211
+ self.grid_skipped, self.stderr_tail = a["grid_skipped"], a["stderr_tail"]
212
+ self.preroll_ticks, self.shards = a["preroll"], a["shards"]
213
+ return
214
+ else:
215
+ raise RuntimeError(f"解码进程发来不认识的消息:{msg}")
216
+
217
+ def close(self) -> None:
218
+ """收摊(开流前后都能调)。"""
219
+ self.child.close()
220
+ if self.shm is not None:
221
+ try:
222
+ self.shm.close()
223
+ except (OSError, BufferError):
224
+ pass
225
+ self.shm = None
226
+
227
+
228
+ # ---------------------------------------------------------------------------------------------------------------
229
+ # 广播模式(`run_groups --share-decode`,ocr-regions 计划):**一个**解码进程给多组供帧
230
+ # ---------------------------------------------------------------------------------------------------------------
231
+
232
+ HUB_ENV = "FLOWOCR_DECODE_HUB"
233
+ """子进程(各组的管线进程)从这里认出"去接广播解码":`端口:口令hex`。"""
234
+ HUB_KEY_ENV = "FLOWOCR_DECODE_HUB_KEY"
235
+ HUB_REASON_ENV = "FLOWOCR_DECODE_HUB_REASON"
236
+ """广播解码进程硬解交付不对时把原因写到这个文件(`run_groups` 读了传给整批软解重跑,重跑的 obs 记 `_meta.hwaccel_fallback`)。"""
237
+
238
+
239
+ def connect_decoder() -> "DecodeProxy":
240
+ """管线进程要一个只解码的来源:有广播解码进程(`HUB_ENV`)就接上去当一个订阅者,否则自己起一个子进程。"""
241
+ hub = os.environ.get(HUB_ENV)
242
+ return DecodeProxy(HubLink(hub) if hub else None)
243
+
244
+
245
+ class HubLink:
246
+ """管线进程接广播解码进程的一条连接(`DecodeProxy` 用的 `send` / `recv_checked` / `close` 三件套,同 `childproc.Child`)。"""
247
+
248
+ def __init__(self, addr: str) -> None:
249
+ from multiprocessing.connection import Client
250
+
251
+ port, key = addr.split(":")
252
+ try:
253
+ self.conn = Client(("127.0.0.1", int(port)), authkey=bytes.fromhex(key))
254
+ except OSError as exc:
255
+ raise RuntimeError(f"接不上广播解码进程({exc!r}):多半是别的组没起来、run_groups 已经把它收了") from None
256
+ self._lock = threading.Lock()
257
+ self.proc = None
258
+ self._closed = False
259
+
260
+ def send(self, msg) -> None:
261
+ with self._lock:
262
+ self.conn.send(msg)
263
+
264
+ def recv_checked(self):
265
+ try:
266
+ msg = self.conn.recv()
267
+ except (EOFError, OSError) as exc:
268
+ raise RuntimeError(f"广播解码进程断开了({exc!r}):它自己出错、或 run_groups 因为有一组没起来把它收了") from None
269
+ return childproc.check(msg, "广播解码进程")
270
+
271
+ def close(self, wait: float = 0.0) -> None:
272
+ if self._closed:
273
+ return
274
+ self._closed = True
275
+ try:
276
+ self.send(("x",))
277
+ except OSError:
278
+ pass
279
+ try:
280
+ self.conn.close()
281
+ except OSError:
282
+ pass
283
+
284
+
285
+ class AuxFanout:
286
+ """一份辅助流转给**多个**回抠进程(各组一个):每个订阅者一条**独立的有界队列** + 一个发送线程(审核:别让一个发送线程按顺序
287
+ sendall 给 N 个——一个回抠进程环满会把别的组一起堵住)。队列满时 `send` 阻塞(背压回到分片的转发线程)——有界、不丢;
288
+ 慢的那组仍会拖住大家(它的帧总得送到),但多了 `depth` 帧的余量。某个订阅者的连接断了:它的队列不再收,别的组照常。
289
+ 内存上限约 depth × 帧字节 × 订阅者数(0.35 辅助流 672×378 约 0.25 MB / 帧:512 帧 ≈ 每组 +128 MB),
290
+ **在分片的 1 GB 预取预算之外**。"""
291
+
292
+ def __init__(self, remotes: list, depth: int = 512) -> None:
293
+ from queue import Queue
294
+
295
+ self.remotes = remotes
296
+ self.qs = [Queue(depth) for _ in remotes]
297
+ self.dead = [False] * len(remotes)
298
+ self.errors: list[str] = []
299
+ self.ths = [threading.Thread(target=self._pump, args=(i,), daemon=True, name=f"aux-fan-{i}") for i in range(len(remotes))]
300
+
301
+ def open(self) -> "AuxFanout":
302
+ for th in self.ths:
303
+ th.start()
304
+ return self
305
+
306
+ def _pump(self, i: int) -> None:
307
+ from flowocr.extract import decode_shards, framesource
308
+
309
+ w = None
310
+ try:
311
+ w = decode_shards.TcpAux(self.remotes[i])
312
+ while True:
313
+ item = self.qs[i].get()
314
+ if item is None:
315
+ return
316
+ w.send(*item)
317
+ except Exception as exc: # noqa: BLE001 这一组的回抠进程没了:它自己的管线会报,别的组照常
318
+ self.dead[i] = True
319
+ self.errors.append(f"订阅者 {i}:{exc!r}")
320
+ while self.qs[i].get() is not None: # 排空,免得 send 卡在这条满队列上
321
+ pass
322
+ finally:
323
+ if w is not None:
324
+ w.close()
325
+ else:
326
+ try:
327
+ self.remotes[i].pts_put(framesource.QUEUE_SENTINEL)
328
+ except Exception: # noqa: BLE001
329
+ pass
330
+
331
+ def send(self, frame, n: int, t_sec: float) -> None:
332
+ buf = bytes(memoryview(frame).cast("B")) # 一份字节,几条队列共用(不可变)
333
+ for i, q in enumerate(self.qs):
334
+ if not self.dead[i]:
335
+ q.put((buf, n, t_sec))
336
+
337
+ def close(self) -> None:
338
+ for q in self.qs:
339
+ q.put(None)
340
+ for th in self.ths:
341
+ th.join(60)
342
+
343
+
344
+ def hub_serve(n_subs: int, accept_timeout: float = 120.0) -> int:
345
+ """广播解码:等 `n_subs` 个订阅者连上、各发 `("go", cfg)`(或 `("x",)` = 这一组不用了),核对取帧参数一致,
346
+ 然后只起**一个**分片取帧层;采样帧写进一个共享内存环、同一个槽号发给每个活着的订阅者,**每个订阅者都还了**这个槽才回收
347
+ (订阅者断开 = 它手里的槽全还、之后不再算它);辅助流经 `AuxFanout` 转给各组的回抠进程。出错按 `childproc` 的约定发给所有订阅者。"""
348
+ from multiprocessing import shared_memory
349
+ from multiprocessing.connection import Listener
350
+ from queue import Empty, Queue
351
+
352
+ import numpy as np
353
+
354
+ from flowocr.extract import framesource
355
+
356
+ key = bytes.fromhex(os.environ.pop(HUB_KEY_ENV))
357
+ lst = Listener(("127.0.0.1", 0), authkey=key)
358
+ print(json.dumps({"port": lst.address[1]}), flush=True)
359
+ subs: list = [] # [(conn, cfg)]
360
+ got = threading.Condition()
361
+
362
+ def hear(c) -> None: # 每个连接一个线程收 go / x:一个连上却迟迟不发的组不挡后面的连接
363
+ try:
364
+ m = c.recv()
365
+ except (EOFError, OSError):
366
+ m = ("x",)
367
+ with got:
368
+ subs.append((c, m[1] if m[0] == "go" else None))
369
+ got.notify_all()
370
+
371
+ def accept_all() -> None:
372
+ for _ in range(n_subs):
373
+ try:
374
+ c = lst.accept()
375
+ except OSError:
376
+ return
377
+ threading.Thread(target=hear, args=(c,), daemon=True).start()
378
+
379
+ threading.Thread(target=accept_all, daemon=True).start()
380
+ # 等满 n_subs 个"go / x"。**有组起不来**由 run_groups 发现(它看着子进程)并收掉这个进程;这里的超时只是兜底
381
+ deadline = time.perf_counter() + accept_timeout
382
+ with got:
383
+ while len(subs) < n_subs and time.perf_counter() < deadline:
384
+ got.wait(1.0)
385
+ subs = list(subs)
386
+ live = [(c, cfg) for c, cfg in subs if cfg is not None]
387
+ if not live:
388
+ return 0
389
+
390
+ def tell_all(msg) -> None:
391
+ for c, _ in live:
392
+ try:
393
+ c.send(msg)
394
+ except OSError:
395
+ pass
396
+
397
+ shm = None
398
+ stop = threading.Event()
399
+ try:
400
+ if len(subs) < n_subs:
401
+ raise RuntimeError(f"广播解码:{accept_timeout:.0f} s 里只等到 {len(subs)} / {n_subs} 个订阅者")
402
+ cfg0 = live[0][1]
403
+ same = ("video", "source_kw", "lanes", "block_sec", "H", "W")
404
+ bad = [k for k in same for _, c in live if c.get(k) != cfg0.get(k)]
405
+ # 辅助流也只有一路:各组的取帧网格 / 缩放要相同(广播的是同一个 ffmpeg 的辅路)
406
+ bad += [f"aux.{k}" for k in ("grid", "scale") for _, c in live
407
+ if (c.get("aux") or {}).get(k) != (cfg0.get("aux") or {}).get(k)]
408
+ if bad or any(bool(c.get("aux")) != bool(cfg0.get("aux")) for _, c in live):
409
+ raise RuntimeError(f"广播解码:各组的取帧参数不一致({sorted(set(bad))})——同一个视频、同一个窗口、同一组解码旋钮才能共用")
410
+ fan = AuxFanout([aux_remote(c["aux"]) for _, c in live]) if cfg0.get("aux") else None
411
+ source = make_source(cfg0, fan)
412
+ H, W = cfg0["H"], cfg0["W"]
413
+ n = max(c["slots"] for _, c in live)
414
+ nb = H * W * 3
415
+ shm = shared_memory.SharedMemory(create=True, size=nb * n)
416
+ free: Queue = Queue()
417
+ for i in range(n):
418
+ free.put(i)
419
+ lock = threading.Lock()
420
+ ref = [0] * n
421
+ held: list[set] = [set() for _ in live]
422
+ alive = [True] * len(live)
423
+
424
+ def release(i: int, slot: int) -> None:
425
+ with lock:
426
+ if slot in held[i]:
427
+ held[i].discard(slot)
428
+ ref[slot] -= 1
429
+ if ref[slot] == 0:
430
+ free.put(slot)
431
+
432
+ def recv_loop(i: int) -> None:
433
+ c = live[i][0]
434
+ try:
435
+ while True:
436
+ m = c.recv()
437
+ if m[0] == "r":
438
+ release(i, m[1])
439
+ else: # "x":这一组收摊了
440
+ break
441
+ except (EOFError, OSError):
442
+ pass
443
+ with lock:
444
+ alive[i] = False
445
+ mine = list(held[i])
446
+ for slot in mine:
447
+ release(i, slot)
448
+ if not any(alive):
449
+ stop.set()
450
+ free.put(None)
451
+
452
+ for i in range(len(live)):
453
+ threading.Thread(target=recv_loop, args=(i,), daemon=True, name=f"hub-recv-{i}").start()
454
+ tell_all(("hello", {"shm": shm.name, "slots": n, "describe": source.describe() + f"、广播给 {len(live)} 组"}))
455
+ print(json.dumps({"started": len(live)}), flush=True) # run_groups 看到它才不再"有组退出就收掉广播进程"
456
+ slot_wait = 0.0
457
+ for idx, t_sec, frame in source:
458
+ t0 = time.perf_counter()
459
+ while True:
460
+ try:
461
+ slot = free.get(timeout=1.0)
462
+ break
463
+ except Empty:
464
+ if stop.is_set():
465
+ slot = None
466
+ break
467
+ slot_wait += time.perf_counter() - t0
468
+ if slot is None or stop.is_set():
469
+ break
470
+ np.ndarray((H, W, 3), np.uint8, buffer=shm.buf[slot * nb:(slot + 1) * nb])[...] = frame
471
+ with lock:
472
+ who = [i for i in range(len(live)) if alive[i]]
473
+ ref[slot] = len(who)
474
+ for i in who:
475
+ held[i].add(slot)
476
+ for i in who:
477
+ try:
478
+ live[i][0].send(("f", idx, t_sec, slot))
479
+ except OSError:
480
+ release(i, slot)
481
+ source.close()
482
+ st = end_stats(source, slot_wait)
483
+ st["shards"]["hub"] = len(live)
484
+ if fan is not None and fan.errors:
485
+ st["shards"]["fanout_errors"] = fan.errors[:4]
486
+ tell_all(("end", st))
487
+ except BaseException as exc: # noqa: BLE001
488
+ if isinstance(exc, framesource.HwaccelMismatch) and os.environ.get(HUB_REASON_ENV):
489
+ try:
490
+ Path(os.environ[HUB_REASON_ENV]).write_text(str(exc), encoding="utf-8")
491
+ except OSError:
492
+ pass
493
+ tell_all(("hw", str(exc)) if isinstance(exc, framesource.HwaccelMismatch) else ("err", traceback.format_exc()))
494
+ # 各组拷完最后一帧、告别(或断开)之前不能拆共享内存(300 s 只是兜底:正常路径是各组 close -> "x" -> 全走了 -> stop;
495
+ # Windows 上 unlink 是空操作、各组还开着的映射照样有效,所以这不是正确性上限)
496
+ if shm is not None:
497
+ stop.wait(300)
498
+ shm.close()
499
+ shm.unlink()
500
+ for c, _ in subs:
501
+ try:
502
+ c.close()
503
+ except OSError:
504
+ pass
505
+ return 0
506
+
507
+
508
+ if __name__ == "__main__":
509
+ if len(sys.argv) > 2 and sys.argv[1] == "--hub":
510
+ sys.exit(hub_serve(int(sys.argv[2])))
511
+ sys.exit(serve())