func2stream 0.0.1.dev2405190643__tar.gz → 0.0.1.dev2405191627__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {func2stream-0.0.1.dev2405190643 → func2stream-0.0.1.dev2405191627}/PKG-INFO +1 -1
- func2stream-0.0.1.dev2405191627/func2stream/__init__.py +11 -0
- func2stream-0.0.1.dev2405191627/func2stream/core.py +370 -0
- func2stream-0.0.1.dev2405191627/func2stream/utils.py +36 -0
- func2stream-0.0.1.dev2405191627/func2stream/video.py +143 -0
- {func2stream-0.0.1.dev2405190643 → func2stream-0.0.1.dev2405191627}/func2stream.egg-info/PKG-INFO +1 -1
- {func2stream-0.0.1.dev2405190643 → func2stream-0.0.1.dev2405191627}/func2stream.egg-info/SOURCES.txt +4 -0
- func2stream-0.0.1.dev2405191627/func2stream.egg-info/top_level.txt +1 -0
- {func2stream-0.0.1.dev2405190643 → func2stream-0.0.1.dev2405191627}/setup.py +7 -2
- func2stream-0.0.1.dev2405190643/func2stream.egg-info/top_level.txt +0 -1
- {func2stream-0.0.1.dev2405190643 → func2stream-0.0.1.dev2405191627}/LICENSE +0 -0
- {func2stream-0.0.1.dev2405190643 → func2stream-0.0.1.dev2405191627}/README.md +0 -0
- {func2stream-0.0.1.dev2405190643 → func2stream-0.0.1.dev2405191627}/func2stream.egg-info/dependency_links.txt +0 -0
- {func2stream-0.0.1.dev2405190643 → func2stream-0.0.1.dev2405191627}/func2stream.egg-info/requires.txt +0 -0
- {func2stream-0.0.1.dev2405190643 → func2stream-0.0.1.dev2405191627}/pyproject.toml +0 -0
- {func2stream-0.0.1.dev2405190643 → func2stream-0.0.1.dev2405191627}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: func2stream
|
|
3
|
-
Version: 0.0.1.
|
|
3
|
+
Version: 0.0.1.dev2405191627
|
|
4
4
|
Summary: Effortlessly transform functions into asynchronous elements for building high-performance pipelines
|
|
5
5
|
Home-page: https://github.com/BICHENG/func2stream
|
|
6
6
|
Author: BI CHENG
|
|
@@ -0,0 +1,370 @@
|
|
|
1
|
+
"""
|
|
2
|
+
func2stream.py
|
|
3
|
+
|
|
4
|
+
Effortlessly transform functions into asynchronous elements for building high-performance pipelines.
|
|
5
|
+
|
|
6
|
+
Author: BI CHENG
|
|
7
|
+
GitHub: https://github.com/BICHENG/func2stream
|
|
8
|
+
License: MPL2.0
|
|
9
|
+
Created: 2024/5/1
|
|
10
|
+
|
|
11
|
+
For Usage, please refer to https://github.com/BICHENG/func2stream/samples or README.md
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
__author__ = "BI CHENG"
|
|
15
|
+
__version__ = "0.0.0"
|
|
16
|
+
__license__ = "MPL2.0"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
import time,threading,inspect,traceback,queue
|
|
20
|
+
from collections import deque
|
|
21
|
+
from concurrent.futures import ThreadPoolExecutor,wait
|
|
22
|
+
import numpy as np
|
|
23
|
+
|
|
24
|
+
class _queue:
|
|
25
|
+
def __init__(self,depth,leaky=False):
|
|
26
|
+
self.depth = depth
|
|
27
|
+
self.queue = queue.Queue(depth)
|
|
28
|
+
self.leaky = leaky
|
|
29
|
+
def put(self,item):
|
|
30
|
+
if self.queue.full() and self.leaky:
|
|
31
|
+
self.queue.get()
|
|
32
|
+
self.queue.put(item)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def get(self): return self.queue.get()
|
|
36
|
+
def qsize(self): return self.queue.qsize()
|
|
37
|
+
def empty(self): return self.queue.empty()
|
|
38
|
+
def full(self): return self.queue.full()
|
|
39
|
+
def clear(self):
|
|
40
|
+
while not self.queue.empty():
|
|
41
|
+
self.queue.get()
|
|
42
|
+
|
|
43
|
+
class Element:
|
|
44
|
+
def __init__(self, friendly_name, fn, kwargs={}, source=None, sink=None):
|
|
45
|
+
if fn is None:
|
|
46
|
+
fn = lambda x: x
|
|
47
|
+
kwargs = {}
|
|
48
|
+
|
|
49
|
+
assert callable(fn), f"Element {friendly_name} cannot be created, {fn.__name__} is not callable"
|
|
50
|
+
assert isinstance(kwargs, dict), f"Element {friendly_name} cannot be c。/reated, {kwargs} is not a dictionary"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
sig = inspect.signature(fn)
|
|
54
|
+
params = list(sig.parameters.values())
|
|
55
|
+
|
|
56
|
+
fn_params = [param.name for param in params[1:]]
|
|
57
|
+
missing_params = [param.name for param in params[1:] if all([
|
|
58
|
+
param.kind in [inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD],
|
|
59
|
+
param.default == inspect.Parameter.empty,
|
|
60
|
+
param.name not in kwargs])]
|
|
61
|
+
extra_kwargs = set(kwargs.keys()) - set(fn_params)
|
|
62
|
+
|
|
63
|
+
assert params, f"{friendly_name}: The processing function needs at least one positional parameter, e.g. def {fn.__name__}(item, ...)"
|
|
64
|
+
assert not params[0].default != inspect.Parameter.empty, f"{friendly_name}: The first positional parameter {params[0].name} cannot have a default value"
|
|
65
|
+
assert not missing_params, f"{friendly_name}: Missing {len(missing_params)} required parameters: {missing_params}, valid parameters are: {fn_params}"
|
|
66
|
+
assert not extra_kwargs, f"{friendly_name}: Provided {len(extra_kwargs)} extra parameters: {extra_kwargs}"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
self.friendly_name = friendly_name
|
|
71
|
+
self.fn = fn
|
|
72
|
+
self.kwargs = kwargs
|
|
73
|
+
self.source = source
|
|
74
|
+
self.sink = sink
|
|
75
|
+
|
|
76
|
+
self.cnt = 0
|
|
77
|
+
self.thread = None
|
|
78
|
+
self.stop_flag = threading.Event()
|
|
79
|
+
|
|
80
|
+
self.exec_times = deque(maxlen=50)
|
|
81
|
+
self.exec_times.append(0)
|
|
82
|
+
|
|
83
|
+
def set_source(self, source: _queue):
|
|
84
|
+
self.source = source
|
|
85
|
+
return self
|
|
86
|
+
def set_sink(self, sink: _queue):
|
|
87
|
+
self.sink = sink
|
|
88
|
+
return self
|
|
89
|
+
def get_source(self):
|
|
90
|
+
return self.source
|
|
91
|
+
def get_sink(self):
|
|
92
|
+
return self.sink
|
|
93
|
+
|
|
94
|
+
def _worker(self):
|
|
95
|
+
while not self.stop_flag.is_set():
|
|
96
|
+
print(f"{self.friendly_name} has started")
|
|
97
|
+
try:
|
|
98
|
+
while not self.stop_flag.is_set():
|
|
99
|
+
if self.source.empty():
|
|
100
|
+
time.sleep(0.0001)
|
|
101
|
+
continue
|
|
102
|
+
item = self.source.get()
|
|
103
|
+
t0 = time.time()
|
|
104
|
+
result = self.fn(item, **self.kwargs)
|
|
105
|
+
self.exec_times.append(time.time()-t0)
|
|
106
|
+
self.sink.put(result)
|
|
107
|
+
self.cnt += 1
|
|
108
|
+
|
|
109
|
+
except Exception as e:
|
|
110
|
+
traceback_info = '\t'.join(traceback.format_exception(None, e, e.__traceback__))
|
|
111
|
+
print(f"{self.friendly_name} element has encountered an exception:",
|
|
112
|
+
f"\t{e} occurred in {self.fn.__name__}, with arguments:{self.kwargs}",
|
|
113
|
+
f"\ttraceback: {traceback_info}")
|
|
114
|
+
|
|
115
|
+
time.sleep(1)
|
|
116
|
+
print(f"{self.friendly_name} has stopped")
|
|
117
|
+
|
|
118
|
+
def _link_to(self, other):
|
|
119
|
+
assert isinstance(other, Element), f"{other} is not an instance of Element"
|
|
120
|
+
|
|
121
|
+
if all([self.sink is None, other.source is None]): self.sink = _queue(1, leaky=False); other.set_source(self.sink)
|
|
122
|
+
if all([self.sink is None, other.source is not None]): self.set_sink(other.source)
|
|
123
|
+
if all([self.sink is not None, other.source is None]): other.set_source(self.sink)
|
|
124
|
+
return other
|
|
125
|
+
|
|
126
|
+
def __call__(self, item):
|
|
127
|
+
assert self.source is not None, f"{self.friendly_name} element has no input queue, cannot process item"
|
|
128
|
+
self.source.put(item)
|
|
129
|
+
return self.sink.get()
|
|
130
|
+
|
|
131
|
+
def start(self):
|
|
132
|
+
assert self.source is not None, f"{self.friendly_name} element has no input queue, cannot start"
|
|
133
|
+
assert self.sink is not None, f"{self.friendly_name} element has no output queue, cannot start"
|
|
134
|
+
assert self.thread is None, f"{self.friendly_name} element has already started, cannot start again"
|
|
135
|
+
|
|
136
|
+
self.thread = threading.Thread(target=self._worker, name=self.friendly_name, daemon=True)
|
|
137
|
+
self.thread.start()
|
|
138
|
+
return self
|
|
139
|
+
|
|
140
|
+
def stop(self):
|
|
141
|
+
self.stop_flag.set()
|
|
142
|
+
if self.thread is not None: self.thread.join()
|
|
143
|
+
return self
|
|
144
|
+
|
|
145
|
+
def time_per_item(self):
|
|
146
|
+
return np.mean(self.exec_times) if len(self.exec_times) > 0 else 0
|
|
147
|
+
|
|
148
|
+
def exec_time_summary(self,print_summary=True):
|
|
149
|
+
# 最近执行的平均时间、最大时间、最小时间、top 5% 和 bottom 5%
|
|
150
|
+
exec_times = np.array(self.exec_times)
|
|
151
|
+
t_avg, t_max, t_min, t_95, t_5 = np.mean(exec_times)*1000, np.max(exec_times)*1000, np.min(exec_times)*1000, np.percentile(exec_times, 95)*1000, np.percentile(exec_times, 5)*1000
|
|
152
|
+
if print_summary:
|
|
153
|
+
print("".join([
|
|
154
|
+
f"{self.friendly_name} Execution Time Summary:",
|
|
155
|
+
f"\tAverage Processing Time:{t_avg:.2f} ms",
|
|
156
|
+
f"\tMaximum Processing Time:{t_max:.2f} ms",
|
|
157
|
+
f"\tMinimum Processing Time:{t_min:.2f} ms",
|
|
158
|
+
f"\tTop 5% Processing Time:{t_95:.2f} ms",
|
|
159
|
+
f"\tBottom 5% Processing Time:{t_5:.2f} ms"
|
|
160
|
+
]))
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
return t_avg, t_max, t_min, t_95, t_5
|
|
165
|
+
|
|
166
|
+
class DataSource(Element):
|
|
167
|
+
def __init__(self, reader_call,
|
|
168
|
+
friendly_name=""
|
|
169
|
+
):
|
|
170
|
+
super().__init__(reader_call.__name__ if friendly_name == "" else friendly_name, fn=None, kwargs={}, source=None, sink=None)
|
|
171
|
+
self.reader_call = reader_call
|
|
172
|
+
|
|
173
|
+
def _worker(self):
|
|
174
|
+
while not self.stop_flag.is_set():
|
|
175
|
+
try:
|
|
176
|
+
print(f"{self.friendly_name} has started")
|
|
177
|
+
while not self.stop_flag.is_set(): self.sink.put(self.reader_call())
|
|
178
|
+
except Exception as e:
|
|
179
|
+
traceback_info = '\t'.join(traceback.format_exception(None, e, e.__traceback__))
|
|
180
|
+
print(f"{self.friendly_name} source has encountered an exception:",
|
|
181
|
+
f"\t{e} occurred in {self.reader_call.__name__}, with arguments:{self.kwargs}",
|
|
182
|
+
f"\ttraceback: {traceback_info}")
|
|
183
|
+
time.sleep(1)
|
|
184
|
+
print(f"{self.friendly_name} has stopped")
|
|
185
|
+
def start(self):
|
|
186
|
+
self.source = _queue(1, leaky=False)
|
|
187
|
+
return super().start()
|
|
188
|
+
|
|
189
|
+
class Pipeline(Element):
|
|
190
|
+
def __init__(self, elements: list, friendly_name="Pipeline"):
|
|
191
|
+
super().__init__(friendly_name, fn=None, kwargs={}, source=None, sink=None)
|
|
192
|
+
assert len(elements) > 1, f"Pipeline needs at least 2 elements, but only {len(elements)} found"
|
|
193
|
+
self.elements = elements
|
|
194
|
+
# 针对elements中每个item, 检查是否是Element的实例, 并尝试转换为Element的实例
|
|
195
|
+
for i, elm in enumerate(self.elements):
|
|
196
|
+
if not isinstance(elm, Element):
|
|
197
|
+
if callable(elm):
|
|
198
|
+
self.elements[i] = Element(elm.__name__, elm)
|
|
199
|
+
if isinstance(elm, tuple) and len(elm) == 2 and callable(elm[0]) and isinstance(elm[1], dict):
|
|
200
|
+
self.elements[i] = Element(elm[0].__name__, elm[0], elm[1])
|
|
201
|
+
# 针对elements中每对相邻元素, 创建连接队列
|
|
202
|
+
print(f"Building\t┌{self.friendly_name} with {len(self.elements)} elements:┐")
|
|
203
|
+
|
|
204
|
+
for i in range(len(self.elements) - 1):
|
|
205
|
+
self.elements[i]._link_to(self.elements[i + 1])
|
|
206
|
+
print(f"\t│Linked {self.elements[i].friendly_name} -> {self.elements[i + 1].friendly_name} [{i+1}/{len(self.elements)-1}]")
|
|
207
|
+
print("\t└───────────────────")
|
|
208
|
+
for i, elm in enumerate(self.elements):
|
|
209
|
+
elm.friendly_name = f"{self.friendly_name}/{elm.friendly_name} [{i+1}/{len(self.elements)}]"
|
|
210
|
+
if i > 0 and i < len(self.elements) - 1: elm.start()
|
|
211
|
+
|
|
212
|
+
# Pipeline 自身的源头(source)和汇点(sink)委托给了首个和末尾元素以实现与外部世界的接口, 实际上是与 Pipeline 中的这两个特定元素的交互。
|
|
213
|
+
self.source = self.elements[0].source
|
|
214
|
+
self.sink = self.elements[-1].sink
|
|
215
|
+
|
|
216
|
+
def _set_source(source):
|
|
217
|
+
print(f"Setting the input queue of {self.friendly_name}")
|
|
218
|
+
self.elements[0].source = source;return self
|
|
219
|
+
def _set_sink(sink):
|
|
220
|
+
print(f"Setting the output queue of {self.friendly_name}")
|
|
221
|
+
self.elements[-1].sink = sink;return self
|
|
222
|
+
|
|
223
|
+
self.set_source=_set_source
|
|
224
|
+
self.set_sink=_set_sink
|
|
225
|
+
|
|
226
|
+
def start(self):
|
|
227
|
+
assert any([
|
|
228
|
+
self.elements[-1].sink is not None,
|
|
229
|
+
isinstance(self.elements[-1], DataSource)
|
|
230
|
+
]), f"{self.elements[-1].friendly_name}@{self.friendly_name} has no output queue, cannot start"
|
|
231
|
+
|
|
232
|
+
if self.elements[-1].sink is None:
|
|
233
|
+
self.elements[-1].sink = _queue(1, leaky=True)
|
|
234
|
+
print(f"SINK {self.elements[-1].friendly_name} will be a pipe that automatically discards old items when full")
|
|
235
|
+
for i in [0, -1]: self.elements[i].start()
|
|
236
|
+
self.source = self.elements[0].source
|
|
237
|
+
self.sink = self.elements[-1].sink
|
|
238
|
+
return self
|
|
239
|
+
|
|
240
|
+
def stop(self):
|
|
241
|
+
for element in self.elements: element.stop()
|
|
242
|
+
return self
|
|
243
|
+
|
|
244
|
+
def nodrop(self):
|
|
245
|
+
self.elements[-1].sink = _queue(1, leaky=False)
|
|
246
|
+
return self
|
|
247
|
+
|
|
248
|
+
def exec_time_summary(self,print_summary=True):
|
|
249
|
+
exec_times = [element.exec_time_summary(print_summary=False) for element in self.elements]
|
|
250
|
+
msg = [f"{self.friendly_name} has {len(self.elements)} elements, execution time summary:",]
|
|
251
|
+
for i, (t_avg, t_max, t_min, t_95, t_5) in enumerate(exec_times):
|
|
252
|
+
msg.append(f"\t{self.elements[i].friendly_name}:")
|
|
253
|
+
msg.append(f"\t\tAverage Processing Time:{t_avg:.2f} ms")
|
|
254
|
+
msg.append(f"\t\tMaximum Processing Time:{t_max:.2f} ms")
|
|
255
|
+
msg.append(f"\t\tMinimum Processing Time:{t_min:.2f} ms")
|
|
256
|
+
msg.append(f"\t\tTop 5% Processing Time:{t_95:.2f} ms")
|
|
257
|
+
msg.append(f"\t\tBottom 5% Processing Time:{t_5:.2f} ms")
|
|
258
|
+
|
|
259
|
+
if print_summary: print("\n".join(msg))
|
|
260
|
+
return exec_times
|
|
261
|
+
|
|
262
|
+
def exec_time_summary_lite(self,print_summary=True):
|
|
263
|
+
exec_times = [element.exec_time_summary(print_summary=False) for element in self.elements]
|
|
264
|
+
msg = [f"{self.friendly_name} has {len(self.elements)} elements, execution time summary:",]
|
|
265
|
+
|
|
266
|
+
for i, (t_avg, t_max, t_min, t_95, t_5) in enumerate(exec_times):
|
|
267
|
+
msg.append(f"\t{self.elements[i].friendly_name}:top 5% processing time:{t_95:.2f} ms")
|
|
268
|
+
|
|
269
|
+
most_time_consuming = np.argmax([t[0] for t in exec_times])
|
|
270
|
+
msg.append(f"\n\tThe most time-consuming element is {self.elements[most_time_consuming].friendly_name}:")
|
|
271
|
+
msg.append(f"\t\Average Processing Time:{exec_times[most_time_consuming][0]:.2f} ms")
|
|
272
|
+
msg.append(f"\t\Maximum Processing Time:{exec_times[most_time_consuming][1]:.2f} ms")
|
|
273
|
+
msg.append(f"\t\Minimum Processing Time:{exec_times[most_time_consuming][2]:.2f} ms")
|
|
274
|
+
msg.append(f"\t\Top 5% Processing Time:{exec_times[most_time_consuming][3]:.2f} ms")
|
|
275
|
+
msg.append(f"\t\Bottom 5% Processing Time:{exec_times[most_time_consuming][4]:.2f} ms")
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
if print_summary: print("\n".join(msg))
|
|
279
|
+
return exec_times
|
|
280
|
+
|
|
281
|
+
class MapReduce(Element):
|
|
282
|
+
def __init__(self, fn_with_kwargs, friendly_name="", source=None, sink=None,nocopy=True):
|
|
283
|
+
super().__init__(friendly_name, fn=None, kwargs={}, source=source, sink=sink)
|
|
284
|
+
self.fn_list = []
|
|
285
|
+
self.kwargs_list = []
|
|
286
|
+
for v in fn_with_kwargs:
|
|
287
|
+
fn, kwargs = v if isinstance(v, tuple) else (v, {})
|
|
288
|
+
self.fn_list.append(fn)
|
|
289
|
+
self.kwargs_list.append(kwargs)
|
|
290
|
+
|
|
291
|
+
self.exec = ThreadPoolExecutor(max_workers=len(self.fn_list))
|
|
292
|
+
def _fn_readonly(item):
|
|
293
|
+
return list(self.exec.map(lambda fn, kwargs: fn(item, **kwargs), self.fn_list, self.kwargs_list))
|
|
294
|
+
def _fn_copied_items(item):
|
|
295
|
+
items = [item]+[item.copy() for _ in range(len(self.fn_list)-1)] if hasattr(item, "copy") else [item for _ in range(len(self.fn_list))]
|
|
296
|
+
return list(self.exec.map(lambda item, fn, kwargs: fn(item, **kwargs), items, self.fn_list, self.kwargs_list))
|
|
297
|
+
|
|
298
|
+
self.fn = _fn_readonly if nocopy else _fn_copied_items
|
|
299
|
+
fn_names = [fn.__name__ for fn in self.fn_list]
|
|
300
|
+
self.friendly_name = f"{friendly_name if friendly_name else 'MapReduce'}[{'ReadOnly' if nocopy else 'Copied'}]━┓"
|
|
301
|
+
for fn_name, kwargs in zip(fn_names, self.kwargs_list):
|
|
302
|
+
self.friendly_name += f"\n\t┣━{fn_name}({kwargs})"
|
|
303
|
+
self.friendly_name += f"\n\t┗━T{len(self.fn_list)}"
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def ctx_mode(self,get=None, ret=None):
|
|
307
|
+
self.get = get or [[] for _ in self.fn_list] # 默认为空列表的列表
|
|
308
|
+
self.ret = ret or [[] for _ in self.fn_list] # 默认为空列表的列表
|
|
309
|
+
def _fn_ctx(item):
|
|
310
|
+
futures = []
|
|
311
|
+
for index, fn in enumerate(self.fn_list):
|
|
312
|
+
# 根据get列表解构item
|
|
313
|
+
args = [item[key] for key in self.get[index]] if self.get[index] else [item]
|
|
314
|
+
# 提交任务时,将解构的参数作为fn的输入
|
|
315
|
+
future = self.exec.submit(fn, *args)
|
|
316
|
+
futures.append(future)
|
|
317
|
+
|
|
318
|
+
# 等待所有任务完成
|
|
319
|
+
wait(futures)
|
|
320
|
+
|
|
321
|
+
# 处理返回结果,根据ret列表回填到item
|
|
322
|
+
for index, future in enumerate(futures):
|
|
323
|
+
result = future.result()
|
|
324
|
+
if self.ret[index]:
|
|
325
|
+
for key, value in zip(self.ret[index], result if isinstance(result, tuple) else [result]):
|
|
326
|
+
item[key] = value
|
|
327
|
+
|
|
328
|
+
return item
|
|
329
|
+
|
|
330
|
+
self.fn = _fn_ctx
|
|
331
|
+
return self
|
|
332
|
+
|
|
333
|
+
import functools
|
|
334
|
+
def from_ctx(get=None, ret=None):
|
|
335
|
+
if get is None:
|
|
336
|
+
get = []
|
|
337
|
+
if ret is None:
|
|
338
|
+
ret = []
|
|
339
|
+
def decorator(func):
|
|
340
|
+
@functools.wraps(func) # 保持原函数的名字和文档字符串
|
|
341
|
+
def wrapper(ctx):
|
|
342
|
+
assert isinstance(ctx, dict), f"{func.__name__} expects a dictionary type context, but received a {type(ctx).__name__}, please check the output of the upstream."
|
|
343
|
+
|
|
344
|
+
missing_keys = [k for k in get if k not in ctx]
|
|
345
|
+
assert not missing_keys, f"{func.__name__} requires keys: {missing_keys} that are not in the context, please check the output of the upstream."
|
|
346
|
+
|
|
347
|
+
if len(get): result = func([ctx[g] for g in get] if len(get) > 1 else ctx[get[0]])
|
|
348
|
+
else: result = func()
|
|
349
|
+
|
|
350
|
+
if not ret: return ctx
|
|
351
|
+
if not isinstance(result, tuple): result = (result,)
|
|
352
|
+
|
|
353
|
+
assert len(ret) == len(result), f"The number of results returned by {func.__name__}: {len(result)} does not match the number of keys set ({ret}), please check the return value of the function."
|
|
354
|
+
|
|
355
|
+
for key, value in zip(ret, result): ctx[key] = value
|
|
356
|
+
|
|
357
|
+
return ctx
|
|
358
|
+
wrapper.fn = func
|
|
359
|
+
wrapper.get = get
|
|
360
|
+
wrapper.ret = ret
|
|
361
|
+
return wrapper
|
|
362
|
+
return decorator
|
|
363
|
+
|
|
364
|
+
def build_ctx(key,constants={}):
|
|
365
|
+
def ctx_fn(x):
|
|
366
|
+
d = {key: x}
|
|
367
|
+
for k, v in constants.items(): d[k] = v
|
|
368
|
+
return d
|
|
369
|
+
return ctx_fn
|
|
370
|
+
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""
|
|
2
|
+
utils.py
|
|
3
|
+
|
|
4
|
+
This file is part of func2stream: Utilities for user environment.
|
|
5
|
+
|
|
6
|
+
Author: BI CHENG
|
|
7
|
+
GitHub: https://github.com/BICHENG/func2stream
|
|
8
|
+
License: MPL2.0
|
|
9
|
+
Created: 2024/5/1
|
|
10
|
+
|
|
11
|
+
For Usage, please refer to https://github.com/BICHENG/func2stream/samples or README.md
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
__author__ = "BI CHENG"
|
|
15
|
+
__version__ = "0.0.0"
|
|
16
|
+
__license__ = "MPL2.0"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def find_gstreamer():
|
|
20
|
+
import cv2
|
|
21
|
+
build_info = cv2.getBuildInformation().split('\n')
|
|
22
|
+
gstreamer_info = None
|
|
23
|
+
for line in build_info:
|
|
24
|
+
if 'GStreamer' in line:
|
|
25
|
+
gstreamer_info = line
|
|
26
|
+
break
|
|
27
|
+
if gstreamer_info is None:
|
|
28
|
+
print("GStreamer information not found in OpenCV build information.")
|
|
29
|
+
return False, "Unknown"
|
|
30
|
+
|
|
31
|
+
tokens = gstreamer_info.split()
|
|
32
|
+
# Typically, tokens[1] is 'YES' and the version follows.
|
|
33
|
+
# The structure might look like: ['GStreamer:', 'YES', '(1.16.2)']
|
|
34
|
+
gstreamer_found = True if tokens[1] == "YES" else False
|
|
35
|
+
gstreamer_version = "Unknown" if len(tokens) < 3 else tokens[2]
|
|
36
|
+
return gstreamer_found,gstreamer_version
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"""
|
|
2
|
+
video.py
|
|
3
|
+
|
|
4
|
+
This file is part of func2stream: DataSources for video stream processing.
|
|
5
|
+
|
|
6
|
+
Author: BI CHENG
|
|
7
|
+
GitHub: https://github.com/BICHENG/func2stream
|
|
8
|
+
License: MPL2.0
|
|
9
|
+
Created: 2024/5/1
|
|
10
|
+
|
|
11
|
+
For Usage, please refer to https://github.com/BICHENG/func2stream/samples or README.md
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
__author__ = "BI CHENG"
|
|
15
|
+
__version__ = "0.0.0"
|
|
16
|
+
__license__ = "MPL2.0"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
import os,time,threading,traceback,queue
|
|
20
|
+
import cv2
|
|
21
|
+
|
|
22
|
+
from core import DataSource
|
|
23
|
+
from utils import find_gstreamer
|
|
24
|
+
|
|
25
|
+
class _VideoCapture:
|
|
26
|
+
def __init__(self, uri, cap_options={}, use_umat=False):
|
|
27
|
+
self.uri = uri
|
|
28
|
+
self.cap_options = cap_options if len(cap_options) > 0 else self.get_capture_params(uri)
|
|
29
|
+
self._swap = queue.Queue(1)
|
|
30
|
+
self.stop_flag = threading.Event()
|
|
31
|
+
self.thread = threading.Thread(target=self._worker, name="VideoCapture", daemon=True)
|
|
32
|
+
self.thread.start()
|
|
33
|
+
self.use_umat = use_umat
|
|
34
|
+
|
|
35
|
+
def get_capture_params(self, video_uri):
|
|
36
|
+
import sys
|
|
37
|
+
"""
|
|
38
|
+
Automatically recognize the mode based on the input video_uri and return the parameter list required for video capture.
|
|
39
|
+
|
|
40
|
+
:param video_uri: Identifier of the resource, such as the path of the video file or the URL of the network stream
|
|
41
|
+
:return: Parameter list or configuration for initializing cv2.VideoCapture
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
mode = ""
|
|
45
|
+
if sys.platform == "win32" and video_uri.isdigit():
|
|
46
|
+
mode = "uvc"
|
|
47
|
+
elif sys.platform == "linux" and video_uri.startswith("/dev/video"):
|
|
48
|
+
mode = "uvc"
|
|
49
|
+
elif video_uri.startswith("rtsp://"):
|
|
50
|
+
mode = "rtsp"
|
|
51
|
+
elif video_uri.startswith("rtmp://"):
|
|
52
|
+
mode = "rtmp"
|
|
53
|
+
elif video_uri.startswith("gst-launch-1.0 "):
|
|
54
|
+
mode = "gst"
|
|
55
|
+
else:
|
|
56
|
+
# 检查是否是视频文件URI
|
|
57
|
+
uri_mode_map = {
|
|
58
|
+
".mp4": "video", ".avi": "video", ".mkv": "video"
|
|
59
|
+
}
|
|
60
|
+
for ext, possible_mode in uri_mode_map.items():
|
|
61
|
+
if video_uri.endswith(ext):
|
|
62
|
+
mode = possible_mode
|
|
63
|
+
video_uri=os.path.abspath(video_uri)
|
|
64
|
+
break
|
|
65
|
+
print(mode)
|
|
66
|
+
assert mode, f"Unrecognized video resource: {video_uri}, available modes include: uvc, rtsp, rtmp, video file path"
|
|
67
|
+
|
|
68
|
+
# 依据模式返回不同的参数
|
|
69
|
+
if mode == "uvc":
|
|
70
|
+
if sys.platform == "win32":
|
|
71
|
+
return [int(video_uri)]
|
|
72
|
+
elif sys.platform == "linux":
|
|
73
|
+
return [video_uri, cv2.CAP_V4L]
|
|
74
|
+
|
|
75
|
+
elif mode in ["rtsp", "rtmp"]:
|
|
76
|
+
pipeline_base = {
|
|
77
|
+
"rtsp": f"rtspsrc location={video_uri} latency=50 ! queue ! parsebin ! decodebin ! videoconvert ! appsink max-buffers=1 drop=true sync=false",
|
|
78
|
+
"rtmp": f"rtmpsrc location={video_uri} ! queue ! parsebin ! decodebin ! videoconvert ! appsink max-buffers=1 drop=true sync=false"
|
|
79
|
+
}
|
|
80
|
+
gst_found, gst_version = find_gstreamer()
|
|
81
|
+
if not gst_found:
|
|
82
|
+
print(f"Warning: OpenCV is built without GStreamer support, {mode} will try to use FFMPEG backend")
|
|
83
|
+
print(f"\tYOUR {mode.upper()} MAY SUFFER LATENCY ISSUES!")
|
|
84
|
+
return [video_uri]
|
|
85
|
+
return [pipeline_base[mode], cv2.CAP_GSTREAMER]
|
|
86
|
+
|
|
87
|
+
elif mode == "video":
|
|
88
|
+
appsink_config = "appsink max-buffers=1 drop=false"
|
|
89
|
+
pipeline = f"filesrc location={video_uri} ! decodebin ! videoconvert ! {appsink_config} sync=false"
|
|
90
|
+
|
|
91
|
+
return [
|
|
92
|
+
video_uri,cv2.CAP_FFMPEG,
|
|
93
|
+
[cv2.CAP_PROP_HW_ACCELERATION, cv2.VIDEO_ACCELERATION_ANY]
|
|
94
|
+
]
|
|
95
|
+
|
|
96
|
+
def _worker(self):
|
|
97
|
+
while not self.stop_flag.is_set():
|
|
98
|
+
try:
|
|
99
|
+
if isinstance(self.cap_options, dict):
|
|
100
|
+
cap = cv2.VideoCapture(self.uri, **self.cap_options)
|
|
101
|
+
elif isinstance(self.cap_options, list):
|
|
102
|
+
cap = cv2.VideoCapture(*self.cap_options)
|
|
103
|
+
else:
|
|
104
|
+
raise Exception(f"Unrecognized cap_options type: {type(self.cap_options)}")
|
|
105
|
+
|
|
106
|
+
if not cap.isOpened():
|
|
107
|
+
raise Exception(f"cap.isOpened() returns False")
|
|
108
|
+
if self.use_umat: buf = cv2.UMat(cap.read()[1])
|
|
109
|
+
else: buf = cap.read()[1]
|
|
110
|
+
self._swap.put(buf)
|
|
111
|
+
print(f"{self.uri} opened")
|
|
112
|
+
while buf is not None and not self.stop_flag.is_set():
|
|
113
|
+
if self._swap.full():
|
|
114
|
+
time.sleep(0.0001)
|
|
115
|
+
continue
|
|
116
|
+
# self._swap.get()
|
|
117
|
+
self._swap.put(buf)
|
|
118
|
+
good = cap.grab()
|
|
119
|
+
good, buf = cap.retrieve(buf)
|
|
120
|
+
cap.release()
|
|
121
|
+
except Exception as e:
|
|
122
|
+
traceback_info = '\t'.join(traceback.format_exception(None, e, e.__traceback__))
|
|
123
|
+
print(f"VideoCapture@{self.uri} will try to reopen, reason:{e}, traceback: {traceback_info}")
|
|
124
|
+
time.sleep(1)
|
|
125
|
+
print(f"{self.uri} closed")
|
|
126
|
+
def read(self):
|
|
127
|
+
return self._swap.get().copy()
|
|
128
|
+
|
|
129
|
+
def stop(self):
|
|
130
|
+
self.stop_flag.set()
|
|
131
|
+
self.thread.join()
|
|
132
|
+
return self
|
|
133
|
+
|
|
134
|
+
class VideoSource(DataSource):
|
|
135
|
+
def __init__(self, uri, cap_options={}, use_umat=False,friendly_name=""):
|
|
136
|
+
self.video_capture = _VideoCapture(uri, cap_options, use_umat)
|
|
137
|
+
super().__init__(reader_call=self.video_capture.read,
|
|
138
|
+
friendly_name=uri if friendly_name == "" else friendly_name)
|
|
139
|
+
|
|
140
|
+
def stop(self):
|
|
141
|
+
super().stop()
|
|
142
|
+
self.video_capture.stop()
|
|
143
|
+
return self
|
{func2stream-0.0.1.dev2405190643 → func2stream-0.0.1.dev2405191627}/func2stream.egg-info/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: func2stream
|
|
3
|
-
Version: 0.0.1.
|
|
3
|
+
Version: 0.0.1.dev2405191627
|
|
4
4
|
Summary: Effortlessly transform functions into asynchronous elements for building high-performance pipelines
|
|
5
5
|
Home-page: https://github.com/BICHENG/func2stream
|
|
6
6
|
Author: BI CHENG
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
func2stream
|
|
@@ -3,13 +3,18 @@ from setuptools import setup, find_packages
|
|
|
3
3
|
from datetime import datetime
|
|
4
4
|
|
|
5
5
|
date_suffix = datetime.now().strftime("%y%m%d%H%M")
|
|
6
|
-
|
|
6
|
+
|
|
7
|
+
major_version = 0
|
|
8
|
+
minor_version = 0
|
|
9
|
+
patch_version = 0
|
|
10
|
+
base_version = f"{major_version}.{minor_version}.{patch_version}"
|
|
11
|
+
base_version_next = f"{major_version}.{minor_version}.{patch_version+1}"
|
|
7
12
|
|
|
8
13
|
# Determine the version based on environment variable
|
|
9
14
|
if os.getenv('RELEASE_VERSION'):
|
|
10
15
|
full_version = base_version
|
|
11
16
|
else:
|
|
12
|
-
full_version = f"{
|
|
17
|
+
full_version = f"{base_version_next}.dev{date_suffix}"
|
|
13
18
|
|
|
14
19
|
setup(
|
|
15
20
|
name='func2stream',
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|