funboost 49.7__py3-none-any.whl → 49.9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of funboost might be problematic. Click here for more details.

Files changed (36) hide show
  1. funboost/__init__.py +1 -1
  2. funboost/assist/celery_helper.py +1 -1
  3. funboost/concurrent_pool/async_pool_executor.py +23 -17
  4. funboost/constant.py +23 -0
  5. funboost/consumers/base_consumer.py +15 -8
  6. funboost/consumers/grpc_consumer.py +102 -0
  7. funboost/consumers/kafka_consumer.py +4 -2
  8. funboost/consumers/kafka_consumer_manually_commit.py +7 -2
  9. funboost/consumers/mysql_cdc_consumer.py +95 -0
  10. funboost/contrib/cdc/__init__.py +0 -0
  11. funboost/contrib/cdc/mysql2mysql.py +44 -0
  12. funboost/core/booster.py +25 -2
  13. funboost/core/exceptions.py +3 -0
  14. funboost/core/func_params_model.py +7 -4
  15. funboost/core/msg_result_getter.py +8 -7
  16. funboost/factories/broker_kind__publsiher_consumer_type_map.py +10 -1
  17. funboost/publishers/base_publisher.py +5 -6
  18. funboost/publishers/grpc_publisher.py +53 -0
  19. funboost/publishers/kafka_publisher.py +3 -1
  20. funboost/publishers/mysql_cdc_publisher.py +24 -0
  21. funboost/timing_job/timing_push.py +3 -1
  22. {funboost-49.7.dist-info → funboost-49.9.dist-info}/METADATA +69 -33
  23. {funboost-49.7.dist-info → funboost-49.9.dist-info}/RECORD +27 -30
  24. funboost/utils/class_utils2.py +0 -94
  25. funboost/utils/custom_pysnooper.py +0 -149
  26. funboost/utils/pysnooper_ydf/__init__.py +0 -32
  27. funboost/utils/pysnooper_ydf/pycompat.py +0 -82
  28. funboost/utils/pysnooper_ydf/tracer.py +0 -479
  29. funboost/utils/pysnooper_ydf/utils.py +0 -101
  30. funboost/utils/pysnooper_ydf/variables.py +0 -133
  31. funboost/utils/times/__init__.py +0 -85
  32. funboost/utils/times/version.py +0 -1
  33. {funboost-49.7.dist-info → funboost-49.9.dist-info}/LICENSE +0 -0
  34. {funboost-49.7.dist-info → funboost-49.9.dist-info}/WHEEL +0 -0
  35. {funboost-49.7.dist-info → funboost-49.9.dist-info}/entry_points.txt +0 -0
  36. {funboost-49.7.dist-info → funboost-49.9.dist-info}/top_level.txt +0 -0
@@ -1,479 +0,0 @@
1
- # Copyright 2019 Ram Rachum and collaborators.
2
- # This program is distributed under the MIT license.
3
-
4
- import functools
5
- import inspect
6
- import opcode
7
- import os
8
- import sys
9
- import re
10
- import collections
11
- import datetime as datetime_module
12
- import itertools
13
- import threading
14
- import traceback
15
-
16
- from .variables import CommonVariable, Exploding, BaseVariable
17
- from . import utils, pycompat
18
- if pycompat.PY2:
19
- from io import open
20
-
21
-
22
- ipython_filename_pattern = re.compile('^<ipython-input-([0-9]+)-.*>$')
23
-
24
-
25
- def get_local_reprs(frame, watch=(), custom_repr=(), max_length=None):
26
- code = frame.f_code
27
- vars_order = (code.co_varnames + code.co_cellvars + code.co_freevars +
28
- tuple(frame.f_locals.keys()))
29
-
30
- result_items = [(key, utils.get_shortish_repr(value, custom_repr,
31
- max_length))
32
- for key, value in frame.f_locals.items()]
33
- result_items.sort(key=lambda key_value: vars_order.index(key_value[0]))
34
- result = collections.OrderedDict(result_items)
35
-
36
- for variable in watch:
37
- result.update(sorted(variable.items(frame)))
38
- return result
39
-
40
-
41
- class UnavailableSource(object):
42
- def __getitem__(self, i):
43
- return u'SOURCE IS UNAVAILABLE'
44
-
45
-
46
- source_and_path_cache = {}
47
-
48
-
49
- def get_path_and_source_from_frame(frame):
50
- globs = frame.f_globals or {}
51
- module_name = globs.get('__name__')
52
- file_name = frame.f_code.co_filename
53
- cache_key = (module_name, file_name)
54
- try:
55
- return source_and_path_cache[cache_key]
56
- except KeyError:
57
- pass
58
- loader = globs.get('__loader__')
59
-
60
- source = None
61
- if hasattr(loader, 'get_source'):
62
- try:
63
- source = loader.get_source(module_name)
64
- except ImportError:
65
- pass
66
- if source is not None:
67
- source = source.splitlines()
68
- if source is None:
69
- ipython_filename_match = ipython_filename_pattern.match(file_name)
70
- if ipython_filename_match:
71
- entry_number = int(ipython_filename_match.group(1))
72
- try:
73
- import IPython
74
- ipython_shell = IPython.get_ipython()
75
- ((_, _, source_chunk),) = ipython_shell.history_manager. \
76
- get_range(0, entry_number, entry_number + 1)
77
- source = source_chunk.splitlines()
78
- except BaseException :
79
- pass
80
- else:
81
- try:
82
- with open(file_name, 'rb') as fp:
83
- source = fp.read().splitlines()
84
- except utils.file_reading_errors:
85
- pass
86
- if not source:
87
- # We used to check `if source is None` but I found a rare bug where it
88
- # was empty, but not `None`, so now we check `if not source`.
89
- source = UnavailableSource()
90
-
91
- # If we just read the source from a file, or if the loader did not
92
- # apply tokenize.detect_encoding to decode the source into a
93
- # string, then we should do that ourselves.
94
- if isinstance(source[0], bytes):
95
- encoding = 'utf-8'
96
- for line in source[:2]:
97
- # File coding may be specified. Match pattern from PEP-263
98
- # (https://www.python.org/dev/peps/pep-0263/)
99
- match = re.search(br'coding[:=]\s*([-\w.]+)', line)
100
- if match:
101
- encoding = match.group(1).decode('ascii')
102
- break
103
- source = [pycompat.text_type(sline, encoding, 'replace') for sline in
104
- source]
105
-
106
- result = (file_name, source)
107
- source_and_path_cache[cache_key] = result
108
- return result
109
-
110
-
111
- def get_write_function(output, overwrite):
112
- is_path = isinstance(output, (pycompat.PathLike, str))
113
- if overwrite and not is_path:
114
- raise Exception('`overwrite=True` can only be used when writing '
115
- 'content to file.')
116
- if output is None:
117
- def write(s):
118
- # stderr = sys.stderr
119
- stderr = sys.stdout # REMIND 这行改了,out才能自定义颜色。
120
- try:
121
- stderr.write(s)
122
- except UnicodeEncodeError:
123
- # God damn Python 2
124
- stderr.write(utils.shitcode(s))
125
- elif is_path:
126
- return FileWriter(output, overwrite).write
127
- elif callable(output):
128
- write = output
129
- else:
130
- assert isinstance(output, utils.WritableStream)
131
-
132
- def write(s):
133
- output.write(s)
134
- return write
135
-
136
-
137
- class FileWriter(object):
138
- def __init__(self, path, overwrite):
139
- self.path = pycompat.text_type(path)
140
- self.overwrite = overwrite
141
-
142
- def write(self, s):
143
- with open(self.path, 'w' if self.overwrite else 'a',
144
- encoding='utf-8') as output_file:
145
- output_file.write(s)
146
- self.overwrite = False
147
-
148
-
149
- thread_global = threading.local()
150
- DISABLED = bool(os.getenv('PYSNOOPER_DISABLED', ''))
151
-
152
- class Tracer:
153
- '''
154
- Snoop on the function, writing everything it's doing to stderr.
155
-
156
- This is useful for debugging.
157
-
158
- When you decorate a function with `@pysnooper.snoop()`
159
- or wrap a block of code in `with pysnooper.snoop():`, you'll get a log of
160
- every line that ran in the function and a play-by-play of every local
161
- variable that changed.
162
-
163
- If stderr is not easily accessible for you, you can redirect the output to
164
- a file::
165
-
166
- @pysnooper.snoop('/my/log/file.log')
167
-
168
- See values of some expressions that aren't local variables::
169
-
170
- @pysnooper.snoop(watch=('foo.bar', 'self.x["whatever"]'))
171
-
172
- Expand values to see all their attributes or items of lists/dictionaries:
173
-
174
- @pysnooper.snoop(watch_explode=('foo', 'self'))
175
-
176
- (see Advanced Usage in the README for more control)
177
-
178
- Show snoop lines for functions that your function calls::
179
-
180
- @pysnooper.snoop(depth=2)
181
-
182
- Start all snoop lines with a prefix, to grep for them easily::
183
-
184
- @pysnooper.snoop(prefix='ZZZ ')
185
-
186
- On multi-threaded apps identify which thread are snooped in output::
187
-
188
- @pysnooper.snoop(thread_info=True)
189
-
190
- Customize how values are represented as strings::
191
-
192
- @pysnooper.snoop(custom_repr=((type1, custom_repr_func1),
193
- (condition2, custom_repr_func2), ...))
194
-
195
- Variables and exceptions get truncated to 100 characters by default. You
196
- can customize that:
197
-
198
- @pysnooper.snoop(max_variable_length=200)
199
-
200
- You can also use `max_variable_length=None` to never truncate them.
201
-
202
- '''
203
- def __init__(self, output=None, watch=(), watch_explode=(), depth=1,
204
- prefix='', overwrite=False, thread_info=False, custom_repr=(),
205
- max_variable_length=100,dont_effect_on_linux=True):
206
- self._write = get_write_function(output, overwrite)
207
-
208
- self.watch = [
209
- v if isinstance(v, BaseVariable) else CommonVariable(v)
210
- for v in utils.ensure_tuple(watch)
211
- ] + [
212
- v if isinstance(v, BaseVariable) else Exploding(v)
213
- for v in utils.ensure_tuple(watch_explode)
214
- ]
215
- self.frame_to_local_reprs = {}
216
- self.depth = depth
217
-
218
- self.prefix = prefix
219
- self.thread_info = thread_info
220
- self.thread_info_padding = 0
221
- assert self.depth >= 1
222
- self.target_codes = set()
223
- self.target_frames = set()
224
- self.thread_local = threading.local()
225
- if len(custom_repr) == 2 and not all(isinstance(x,
226
- pycompat.collections_abc.Iterable) for x in custom_repr):
227
- custom_repr = (custom_repr,)
228
- self.custom_repr = custom_repr
229
- self.last_source_path = None
230
- self.max_variable_length = max_variable_length
231
- if dont_effect_on_linux and os.name == 'posix':
232
- global DISABLED
233
- DISABLED = True # linux上装饰器自动失效,避免发到生产。
234
- self.total_run_line = 0
235
-
236
- def __call__(self, function_or_class):
237
- if DISABLED:
238
- return function_or_class
239
- if self.prefix == '':
240
- self.prefix = f'调试 [{function_or_class.__name__}] 函数 --> '
241
- if inspect.isclass(function_or_class):
242
- return self._wrap_class(function_or_class)
243
- else:
244
- return self._wrap_function(function_or_class)
245
-
246
- def _wrap_class(self, cls):
247
- for attr_name, attr in cls.__dict__.items():
248
- # Coroutines are functions, but snooping them is not supported
249
- # at the moment
250
- if pycompat.iscoroutinefunction(attr):
251
- continue
252
-
253
- if inspect.isfunction(attr):
254
- setattr(cls, attr_name, self._wrap_function(attr))
255
- return cls
256
-
257
- def _wrap_function(self, function):
258
- self.target_codes.add(function.__code__)
259
-
260
- @functools.wraps(function)
261
- def simple_wrapper(*args, **kwargs):
262
- with self:
263
- return function(*args, **kwargs)
264
-
265
- @functools.wraps(function)
266
- def generator_wrapper(*args, **kwargs):
267
- gen = function(*args, **kwargs)
268
- method, incoming = gen.send, None
269
- while True:
270
- with self:
271
- try:
272
- outgoing = method(incoming)
273
- except StopIteration:
274
- return
275
- try:
276
- method, incoming = gen.send, (yield outgoing)
277
- except BaseException as e:
278
- method, incoming = gen.throw, e
279
-
280
- if pycompat.iscoroutinefunction(function):
281
- raise NotImplementedError
282
- if pycompat.isasyncgenfunction(function):
283
- raise NotImplementedError
284
- elif inspect.isgeneratorfunction(function):
285
- return generator_wrapper
286
- else:
287
- return simple_wrapper
288
-
289
- def write(self, s):
290
- s = u'{self.prefix}{s}\n'.format(**locals())
291
- self._write(s)
292
-
293
- def __enter__(self):
294
- if DISABLED:
295
- return
296
- calling_frame = inspect.currentframe().f_back
297
- if not self._is_internal_frame(calling_frame):
298
- calling_frame.f_trace = self.trace
299
- self.target_frames.add(calling_frame)
300
-
301
- stack = self.thread_local.__dict__.setdefault(
302
- 'original_trace_functions', []
303
- )
304
- stack.append(sys.gettrace())
305
- sys.settrace(self.trace)
306
-
307
- def __exit__(self, exc_type, exc_value, exc_traceback):
308
- if DISABLED:
309
- return
310
- stack = self.thread_local.original_trace_functions
311
- sys.settrace(stack.pop())
312
- calling_frame = inspect.currentframe().f_back
313
- self.target_frames.discard(calling_frame)
314
- self.frame_to_local_reprs.pop(calling_frame, None)
315
- utils.nb_print(f'解释运行的总代码行数是 {self.total_run_line} 行')
316
-
317
- def _is_internal_frame(self, frame):
318
- return frame.f_code.co_filename == Tracer.__enter__.__code__.co_filename
319
-
320
- def set_thread_info_padding(self, thread_info):
321
- current_thread_len = len(thread_info)
322
- self.thread_info_padding = max(self.thread_info_padding,
323
- current_thread_len)
324
- return thread_info.ljust(self.thread_info_padding)
325
-
326
- def trace(self, frame, event, arg):
327
-
328
- ### Checking whether we should trace this line: #######################
329
- # #
330
- # We should trace this line either if it's in the decorated function,
331
- # or the user asked to go a few levels deeper and we're within that
332
- # number of levels deeper.
333
-
334
- if not (frame.f_code in self.target_codes or frame in self.target_frames):
335
- if self.depth == 1:
336
- # We did the most common and quickest check above, because the
337
- # trace function runs so incredibly often, therefore it's
338
- # crucial to hyper-optimize it for the common case.
339
- return None
340
- elif self._is_internal_frame(frame):
341
- return None
342
- else:
343
- _frame_candidate = frame
344
- for i in range(1, self.depth):
345
- _frame_candidate = _frame_candidate.f_back
346
- if _frame_candidate is None:
347
- return None
348
- elif _frame_candidate.f_code in self.target_codes or _frame_candidate in self.target_frames:
349
- break
350
- else:
351
- return None
352
-
353
- thread_global.__dict__.setdefault('depth', -1)
354
- if event == 'call':
355
- thread_global.depth += 1
356
- indent = ' ' * 4 * thread_global.depth
357
-
358
- # #
359
- ### Finished checking whether we should trace this line. ##############
360
-
361
- now = datetime_module.datetime.now().time()
362
- now_string = pycompat.time_isoformat(now, timespec='microseconds')
363
- line_no = frame.f_lineno
364
- source_path, source = get_path_and_source_from_frame(frame)
365
- if self.last_source_path != source_path:
366
- self.write(u'\033[0;35m{indent}跳转到文件。。。 {source_path}\033[0m'.
367
- format(**locals()))
368
- self.last_source_path = source_path
369
- source_line = source[line_no - 1]
370
- thread_info = ""
371
- if self.thread_info:
372
- current_thread = threading.current_thread()
373
- thread_info = "{ident}-{name} ".format(
374
- ident=current_thread.ident, name=current_thread.getName())
375
- thread_info = self.set_thread_info_padding(thread_info)
376
-
377
- ### Reporting newish and modified variables: ##########################
378
- # #
379
- old_local_reprs = self.frame_to_local_reprs.get(frame, {})
380
- self.frame_to_local_reprs[frame] = local_reprs = \
381
- get_local_reprs(frame,
382
- watch=self.watch, custom_repr=self.custom_repr,
383
- max_length=self.max_variable_length)
384
-
385
- newish_string = ('开始变量:。。。 ' if event == 'call' else
386
- '新变量:。。。 ')
387
-
388
- for name, value_repr in local_reprs.items():
389
- if name not in old_local_reprs:
390
- self.write('\033[0;34m{indent}{newish_string}{name} = {value_repr}\033[0m'.format(
391
- **locals()))
392
- elif old_local_reprs[name] != value_repr:
393
- self.write('\033[0;33m{indent}修改变量:。。。 {name} = {value_repr}\033[0m'.format(
394
- **locals()))
395
-
396
- # #
397
- ### Finished newish and modified variables. ###########################
398
-
399
-
400
- ### Dealing with misplaced function definition: #######################
401
- # #
402
- if event == 'call' and source_line.lstrip().startswith('@'):
403
- # If a function decorator is found, skip lines until an actual
404
- # function definition is found.
405
- for candidate_line_no in itertools.count(line_no):
406
- try:
407
- candidate_source_line = source[candidate_line_no - 1]
408
- except IndexError:
409
- # End of source file reached without finding a function
410
- # definition. Fall back to original source line.
411
- break
412
-
413
- if candidate_source_line.lstrip().startswith('def'):
414
- # Found the def line!
415
- line_no = candidate_line_no
416
- source_line = candidate_source_line
417
- break
418
- # #
419
- ### Finished dealing with misplaced function definition. ##############
420
-
421
- # If a call ends due to an exception, we still get a 'return' event
422
- # with arg = None. This seems to be the only way to tell the difference
423
- # https://stackoverflow.com/a/12800909/2482744
424
- code_byte = frame.f_code.co_code[frame.f_lasti]
425
- if not isinstance(code_byte, int):
426
- code_byte = ord(code_byte)
427
- ended_by_exception = (
428
- event == 'return'
429
- and arg is None
430
- and (opcode.opname[code_byte]
431
- not in ('RETURN_VALUE', 'YIELD_VALUE'))
432
- )
433
-
434
- if ended_by_exception:
435
- self.write('\033[0;31m{indent}调用结束被异常\033[0m'.
436
- format(**locals()))
437
- else:
438
- """
439
- \033[0;94m{"".join(args)}\033[0m
440
- """
441
- self.total_run_line +=1
442
- # self.write('{indent}{now_string} {event:9} '
443
- # '{frame.f_lineno:4} {source_line}'.format(**locals()))
444
-
445
- # REMIND 这里改了,改成能显示运行轨迹可点击。
446
- # utils.nb_print(locals())
447
- # utils.nb_print(source_line)
448
- file_name_and_line = f'{frame.f_code.co_filename}:{frame.f_lineno}'
449
-
450
- file_name_and_line2 = f'"{file_name_and_line}"'
451
- event_map = {
452
- 'call':'调用',
453
- 'line':'代码行',
454
- 'return': '返回',
455
- 'exception':'异常',
456
- }
457
- event_cn = event_map.get(event,event)
458
-
459
- self.write(u'\033[0;32m{indent}{now_string} {thread_info} {event_cn:9} {file_name_and_line2:100} {source_line}\033[0m'.format(**locals()))
460
-
461
- if event == 'return':
462
- del self.frame_to_local_reprs[frame]
463
- thread_global.depth -= 1
464
-
465
- if not ended_by_exception:
466
- return_value_repr = utils.get_shortish_repr(arg,
467
- custom_repr=self.custom_repr,
468
- max_length=self.max_variable_length)
469
- self.write('\033[0;36m{indent}返回值:。。。 {return_value_repr}\033[0m'.
470
- format(**locals()))
471
-
472
- if event == 'exception':
473
- exception = '\n'.join(traceback.format_exception_only(*arg[:2])).strip()
474
- if self.max_variable_length:
475
- exception = utils.truncate(exception, self.max_variable_length)
476
- self.write('\033[0;31m{indent}{exception}\033[0m'.
477
- format(**locals()))
478
-
479
- return self.trace
@@ -1,101 +0,0 @@
1
- # Copyright 2019 Ram Rachum and collaborators.
2
- # This program is distributed under the MIT license.
3
-
4
- import abc
5
- import time
6
- import sys
7
- from .pycompat import ABC, string_types, collections_abc
8
-
9
- def _check_methods(C, *methods):
10
- mro = C.__mro__
11
- for method in methods:
12
- for B in mro:
13
- if method in B.__dict__:
14
- if B.__dict__[method] is None:
15
- return NotImplemented
16
- break
17
- else:
18
- return NotImplemented
19
- return True
20
-
21
-
22
- class WritableStream(ABC):
23
- @abc.abstractmethod
24
- def write(self, s):
25
- pass
26
-
27
- @classmethod
28
- def __subclasshook__(cls, C):
29
- if cls is WritableStream:
30
- return _check_methods(C, 'write')
31
- return NotImplemented
32
-
33
-
34
-
35
- file_reading_errors = (
36
- IOError,
37
- OSError,
38
- ValueError # IronPython weirdness.
39
- )
40
-
41
-
42
-
43
- def shitcode(s):
44
- return ''.join(
45
- (c if (0 < ord(c) < 256) else '?') for c in s
46
- )
47
-
48
-
49
- def get_repr_function(item, custom_repr):
50
- for condition, action in custom_repr:
51
- if isinstance(condition, type):
52
- condition = lambda x, y=condition: isinstance(x, y)
53
- if condition(item):
54
- return action
55
- return repr
56
-
57
-
58
- def get_shortish_repr(item, custom_repr=(), max_length=None):
59
- repr_function = get_repr_function(item, custom_repr)
60
- try:
61
- r = repr_function(item)
62
- except BaseException :
63
- r = 'REPR FAILED'
64
- r = r.replace('\r', '').replace('\n', '')
65
- if max_length:
66
- r = truncate(r, max_length)
67
- return r
68
-
69
-
70
- def truncate(string, max_length):
71
- if (max_length is None) or (len(string) <= max_length):
72
- return string
73
- else:
74
- left = (max_length - 3) // 2
75
- right = max_length - 3 - left
76
- return u'{}...{}'.format(string[:left], string[-right:])
77
-
78
-
79
- def ensure_tuple(x):
80
- if isinstance(x, collections_abc.Iterable) and \
81
- not isinstance(x, string_types):
82
- return tuple(x)
83
- else:
84
- return (x,)
85
-
86
-
87
- def nb_print( *args, sep=' ', end='\n', file=None):
88
- """
89
- 超流弊的print补丁
90
- :param x:
91
- :return:
92
- """
93
- # 获取被调用函数在被调用时所处代码行数
94
- line = sys._getframe().f_back.f_lineno
95
- # 获取被调用函数所在模块文件名
96
- file_name = sys._getframe(1).f_code.co_filename
97
- # sys.stdout.write(f'"{__file__}:{sys._getframe().f_lineno}" {x}\n')
98
- args = (str(arg) for arg in args) # REMIND 防止是数字不能被join
99
- sys.stdout.write(f'"{file_name}:{line}" {time.strftime("%H:%M:%S")} \033[0;94m{"".join(args)}\033[0m\n')
100
- sys.stdout.flush()# 36 93 96 94
101
-