kkpyutil 1.40.0__py3-none-any.whl → 1.40.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {kkpyutil-1.40.0.dist-info → kkpyutil-1.40.1.dist-info}/METADATA +1 -1
- kkpyutil-1.40.1.dist-info/RECORD +7 -0
- kkpyutil.py +316 -315
- kkpyutil-1.40.0.dist-info/RECORD +0 -7
- {kkpyutil-1.40.0.dist-info → kkpyutil-1.40.1.dist-info}/LICENSE +0 -0
- {kkpyutil-1.40.0.dist-info → kkpyutil-1.40.1.dist-info}/WHEEL +0 -0
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
kkpyutil.py,sha256=zsn7VZE8iDbFxZJlI2Y3600v4MhbXU7HNqTKdag2aks,111547
|
|
2
|
+
kkpyutil_helper/windows/kkttssave.ps1,sha256=xa3-WzqNh2rGYlOx_I4ewOuCE94gkTO5cEwYH0M67_0,446
|
|
3
|
+
kkpyutil_helper/windows/kkttsspeak.ps1,sha256=7WUUHMmjTQroUWA2Mvdt4JtSt475nZUHQx-qP-7rS6o,305
|
|
4
|
+
kkpyutil-1.40.1.dist-info/LICENSE,sha256=uISevGnCxB5QOU0ftbofN75_yUtd6E2h_MWE1zqagC8,1065
|
|
5
|
+
kkpyutil-1.40.1.dist-info/METADATA,sha256=PiZkRJf1LJo1Bae4EHQelLRyqVBDQfb6Yik0LYNkuXs,1136
|
|
6
|
+
kkpyutil-1.40.1.dist-info/WHEEL,sha256=kLuE8m1WYU0Ig0_YEGrXyTtiJvKPpLpDEiChiNyei5Y,88
|
|
7
|
+
kkpyutil-1.40.1.dist-info/RECORD,,
|
kkpyutil.py
CHANGED
|
@@ -58,9 +58,8 @@ import uuid
|
|
|
58
58
|
import warnings
|
|
59
59
|
from types import SimpleNamespace
|
|
60
60
|
|
|
61
|
-
#
|
|
62
|
-
|
|
63
|
-
#
|
|
61
|
+
# region globals
|
|
62
|
+
|
|
64
63
|
_script_dir = osp.abspath(osp.dirname(__file__))
|
|
65
64
|
TXT_CODEC = 'utf-8' # Importable.
|
|
66
65
|
LOCALE_CODEC = locale.getpreferredencoding()
|
|
@@ -70,6 +69,10 @@ PLATFORM = platform.system()
|
|
|
70
69
|
if PLATFORM == 'Windows':
|
|
71
70
|
import winreg
|
|
72
71
|
|
|
72
|
+
# endregion
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
# region classes
|
|
73
76
|
|
|
74
77
|
class SingletonDecorator:
|
|
75
78
|
"""
|
|
@@ -127,6 +130,288 @@ class BandPassLogFilter(object):
|
|
|
127
130
|
return self.__levelbounds[0] <= log.levelno <= self.__levelbounds[1]
|
|
128
131
|
|
|
129
132
|
|
|
133
|
+
class OfflineJSON:
|
|
134
|
+
def __init__(self, file_path):
|
|
135
|
+
self.path = file_path
|
|
136
|
+
|
|
137
|
+
def exists(self):
|
|
138
|
+
return osp.isfile(self.path)
|
|
139
|
+
|
|
140
|
+
def load(self):
|
|
141
|
+
return load_json(self.path) if self.exists() else None
|
|
142
|
+
|
|
143
|
+
def save(self, data: dict):
|
|
144
|
+
save_json(self.path, data)
|
|
145
|
+
|
|
146
|
+
def merge(self, props: dict):
|
|
147
|
+
data = self.load()
|
|
148
|
+
if not data:
|
|
149
|
+
return self.save(props)
|
|
150
|
+
data.update(props)
|
|
151
|
+
self.save(data)
|
|
152
|
+
return data
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def get_platform_tmp_dir():
|
|
156
|
+
plat_dir_map = {
|
|
157
|
+
'Windows': osp.join(str(os.getenv('LOCALAPPDATA')), 'Temp'),
|
|
158
|
+
'Darwin': osp.expanduser('~/Library/Caches'),
|
|
159
|
+
'Linux': '/tmp'
|
|
160
|
+
}
|
|
161
|
+
return plat_dir_map.get(PLATFORM)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
class RerunLock:
|
|
165
|
+
"""
|
|
166
|
+
- Lock process from reentering when seeing lock file on disk
|
|
167
|
+
- use semaphore-like behaviour with an instance limit
|
|
168
|
+
- Because lockfile is created by pyutil, we also save the occupier pid and .py path (name) in it
|
|
169
|
+
- if name is a path, e.g., __file__, then lockfile will be named after its basename
|
|
170
|
+
"""
|
|
171
|
+
|
|
172
|
+
def __init__(self, name, folder=None, logger=None, max_instances=1):
|
|
173
|
+
folder = folder or osp.join(get_platform_tmp_dir(), '_util')
|
|
174
|
+
filename = f'lock_{extract_path_stem(name)}.{os.getpid()}.lock.json'
|
|
175
|
+
self.name = name
|
|
176
|
+
self.lockFile = osp.join(folder, filename)
|
|
177
|
+
self.nMaxInstances = max_instances
|
|
178
|
+
self.logger = logger or glogger
|
|
179
|
+
# CAUTION:
|
|
180
|
+
# - windows grpc server crashes with signals:
|
|
181
|
+
# - ValueError: signal only works in main thread of the main interpreter
|
|
182
|
+
# - signals are disabled for windows
|
|
183
|
+
if threading.current_thread() is threading.main_thread():
|
|
184
|
+
common_sigs = [
|
|
185
|
+
signal.SIGABRT,
|
|
186
|
+
signal.SIGFPE,
|
|
187
|
+
signal.SIGILL,
|
|
188
|
+
signal.SIGINT,
|
|
189
|
+
signal.SIGSEGV,
|
|
190
|
+
signal.SIGTERM,
|
|
191
|
+
]
|
|
192
|
+
plat_sigs = [
|
|
193
|
+
signal.SIGBREAK,
|
|
194
|
+
# CAUTION
|
|
195
|
+
# - CTRL_C_EVENT, CTRL_BREAK_EVENT not working on Windows
|
|
196
|
+
# signal.CTRL_C_EVENT,
|
|
197
|
+
# signal.CTRL_BREAK_EVENT,
|
|
198
|
+
] if PLATFORM == 'Windows' else [
|
|
199
|
+
# CAUTION:
|
|
200
|
+
# - SIGCHLD as an alias is safe to ignore
|
|
201
|
+
# - SIGKILL must be handled by os.kill()
|
|
202
|
+
signal.SIGALRM,
|
|
203
|
+
signal.SIGBUS,
|
|
204
|
+
# signal.SIGCHLD,
|
|
205
|
+
# - SIGCONT: CTRL+Z is allowed for bg process
|
|
206
|
+
# signal.SIGCONT,
|
|
207
|
+
signal.SIGHUP,
|
|
208
|
+
# signal.SIGKILL,
|
|
209
|
+
signal.SIGPIPE,
|
|
210
|
+
]
|
|
211
|
+
for sig in common_sigs + plat_sigs:
|
|
212
|
+
signal.signal(sig, self.handle_signal)
|
|
213
|
+
# cleanup zombie locks due to runtime exceptions
|
|
214
|
+
locks = [osp.basename(lock) for lock in glob.glob(osp.join(osp.dirname(self.lockFile), f'lock_{extract_path_stem(self.name)}.*.lock.json'))]
|
|
215
|
+
zombie_locks = [lock for lock in locks if not is_pid_running(int(lock.split(".")[1]))]
|
|
216
|
+
for lock in zombie_locks:
|
|
217
|
+
safe_remove(osp.join(osp.dirname(self.lockFile), lock))
|
|
218
|
+
|
|
219
|
+
def lock(self):
|
|
220
|
+
locks = [osp.basename(lock) for lock in glob.glob(osp.join(osp.dirname(self.lockFile), f'lock_{extract_path_stem(self.name)}.*.lock.json'))]
|
|
221
|
+
is_locked = len(locks) >= self.nMaxInstances
|
|
222
|
+
if is_locked:
|
|
223
|
+
locker_pids = [int(lock.split(".")[1]) for lock in locks]
|
|
224
|
+
self.logger.warning(f'{self.name} is locked by processes: {locker_pids}. Will block new instances until unlocked.')
|
|
225
|
+
return False
|
|
226
|
+
save_json(self.lockFile, {
|
|
227
|
+
'pid': os.getpid(),
|
|
228
|
+
'name': self.name,
|
|
229
|
+
})
|
|
230
|
+
# CAUTION: race condition: saving needs a sec, it's up to application to await lockfile
|
|
231
|
+
return True
|
|
232
|
+
|
|
233
|
+
def unlock(self):
|
|
234
|
+
try:
|
|
235
|
+
os.remove(self.lockFile)
|
|
236
|
+
except FileNotFoundError:
|
|
237
|
+
self.logger.warning(f'{self.name} already unlocked. Safely ignored.')
|
|
238
|
+
return False
|
|
239
|
+
except Exception:
|
|
240
|
+
failure = traceback.format_exc()
|
|
241
|
+
self.logger.error(f""""\
|
|
242
|
+
Failed to unlock {self.name}:
|
|
243
|
+
Details:
|
|
244
|
+
{failure}
|
|
245
|
+
|
|
246
|
+
Advice:
|
|
247
|
+
- Delete the lock by hand: {self.lockFile}""")
|
|
248
|
+
return False
|
|
249
|
+
return True
|
|
250
|
+
|
|
251
|
+
def unlock_all(self):
|
|
252
|
+
locks = glob.glob(osp.join(osp.dirname(self.lockFile), f'lock_{osp.basename(self.name)}.*.lock.json'))
|
|
253
|
+
for lock in locks:
|
|
254
|
+
os.remove(lock)
|
|
255
|
+
return True
|
|
256
|
+
|
|
257
|
+
def is_locked(self):
|
|
258
|
+
return osp.isfile(self.lockFile)
|
|
259
|
+
|
|
260
|
+
def handle_signal(self, sig, frame):
|
|
261
|
+
msg = f'Terminated due to signal: {signal.Signals(sig).name}; Will unlock'
|
|
262
|
+
self.logger.warning(msg)
|
|
263
|
+
self.unlock()
|
|
264
|
+
raise RuntimeError(msg)
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
class Tracer:
|
|
268
|
+
"""
|
|
269
|
+
- custom module-ignore rules
|
|
270
|
+
- trace calls and returns
|
|
271
|
+
- exclude first, then include
|
|
272
|
+
- usage: use in source code
|
|
273
|
+
- tracer = util.Tracer(exclude_funcname_pattern='stop')
|
|
274
|
+
- tracer.start()
|
|
275
|
+
- # add traceable code here
|
|
276
|
+
- tracer.stop()
|
|
277
|
+
"""
|
|
278
|
+
|
|
279
|
+
def __init__(self,
|
|
280
|
+
excluded_modules: set[str] = None,
|
|
281
|
+
exclude_filename_pattern: str = None,
|
|
282
|
+
include_filename_pattern: str = None,
|
|
283
|
+
exclude_funcname_pattern: str = None,
|
|
284
|
+
include_funcname_pattern: str = None,
|
|
285
|
+
trace_func=None,
|
|
286
|
+
exclude_builtins=True):
|
|
287
|
+
self.exclMods = {'builtins'} if excluded_modules is None else excluded_modules
|
|
288
|
+
self.exclFilePatt = re.compile(exclude_filename_pattern) if exclude_filename_pattern else None
|
|
289
|
+
self.inclFilePatt = re.compile(include_filename_pattern) if include_filename_pattern else None
|
|
290
|
+
self.exclFuncPatt = re.compile(exclude_funcname_pattern) if exclude_funcname_pattern else None
|
|
291
|
+
self.inclFuncPatt = re.compile(include_funcname_pattern) if include_funcname_pattern else None
|
|
292
|
+
self.traceFunc = trace_func
|
|
293
|
+
if exclude_builtins:
|
|
294
|
+
self.ignore_stdlibs()
|
|
295
|
+
|
|
296
|
+
def start(self):
|
|
297
|
+
sys.settrace(self.traceFunc or self._trace_calls_and_returns)
|
|
298
|
+
|
|
299
|
+
@staticmethod
|
|
300
|
+
def stop():
|
|
301
|
+
sys.settrace(None)
|
|
302
|
+
|
|
303
|
+
def ignore_stdlibs(self):
|
|
304
|
+
def _get_stdlib_module_names():
|
|
305
|
+
import distutils.sysconfig
|
|
306
|
+
stdlib_dir = distutils.sysconfig.get_python_lib(standard_lib=True)
|
|
307
|
+
return {f.replace(".py", "") for f in os.listdir(stdlib_dir)}
|
|
308
|
+
|
|
309
|
+
py_ver = sys.version_info
|
|
310
|
+
std_libs = set(sys.stdlib_module_names) if py_ver.major >= 3 and py_ver.minor >= 10 else _get_stdlib_module_names()
|
|
311
|
+
self.exclMods.update(std_libs)
|
|
312
|
+
|
|
313
|
+
def _trace_calls_and_returns(self, frame, event, arg):
|
|
314
|
+
"""
|
|
315
|
+
track hook for function calls. Usage:
|
|
316
|
+
sys.settrace(trace_calls_and_returns)
|
|
317
|
+
"""
|
|
318
|
+
if event not in ('call', 'return'):
|
|
319
|
+
return
|
|
320
|
+
module_name = frame.f_globals.get('__name__')
|
|
321
|
+
if module_name is not None and module_name in self.exclMods:
|
|
322
|
+
return
|
|
323
|
+
filename = frame.f_code.co_filename
|
|
324
|
+
if self.exclFilePatt and self.exclFuncPatt.search(filename):
|
|
325
|
+
return
|
|
326
|
+
if self.inclFilePatt and not self.inclFilePatt.search(filename):
|
|
327
|
+
return
|
|
328
|
+
func_name = frame.f_code.co_name
|
|
329
|
+
if self.exclFuncPatt and self.exclFuncPatt.search(func_name):
|
|
330
|
+
return
|
|
331
|
+
if self.inclFuncPatt and not self.inclFuncPatt.search(func_name):
|
|
332
|
+
return
|
|
333
|
+
line_number = frame.f_lineno
|
|
334
|
+
line = linecache.getline(filename, line_number).strip()
|
|
335
|
+
if event == 'call':
|
|
336
|
+
args = ', '.join(f'{arg}={repr(frame.f_locals[arg])}' for arg in frame.f_code.co_varnames[:frame.f_code.co_argcount])
|
|
337
|
+
print(f'Call: {module_name}.{func_name}({args}) - {line}')
|
|
338
|
+
return self._trace_calls_and_returns
|
|
339
|
+
print(f'Call: {module_name}.{func_name} => {arg} - {line}')
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
class Cache:
|
|
343
|
+
"""
|
|
344
|
+
cross-session caching: using temp-file to retrieve data based on hash changes
|
|
345
|
+
- constraints:
|
|
346
|
+
- data retrieval/parsing is expensive
|
|
347
|
+
- one cache per data-source
|
|
348
|
+
- cache is a mediator b/w app and data-source as a retriever only, cuz user's saving intent is always towards source, no need to cache a saving action
|
|
349
|
+
- for cross-session caching, save hash into cache, then when instantiate cache object, always load hash from cache to compare with incoming hash
|
|
350
|
+
- app must provide retriever function: retriever(src) -> json_data
|
|
351
|
+
- because it'd cost the same to retrieve data from a json-file source as from cache, so no json default is provided
|
|
352
|
+
- e.g., loading a complex tree-structure from a file:
|
|
353
|
+
- tree_cache = Cache('/path/to/file.tree', lambda: src: load_data(src), '/tmp/my_app')
|
|
354
|
+
- # ... later
|
|
355
|
+
- cached_tree_data = tree_cache.retrieve()
|
|
356
|
+
"""
|
|
357
|
+
|
|
358
|
+
def __init__(self, data_source, data_retriever, cache_dir=get_platform_tmp_dir(), cache_type='cache', algo='checksum', source_seed='6ba7b810-9dad-11d1-80b4-00c04fd430c8'):
|
|
359
|
+
assert algo in ['checksum', 'mtime']
|
|
360
|
+
self.srcURL = data_source
|
|
361
|
+
self.retriever = data_retriever
|
|
362
|
+
# use a fixed namespace for each data-source to ensure inter-session consistency
|
|
363
|
+
namespace = uuid.UUID(str(source_seed))
|
|
364
|
+
uid = str(uuid.uuid5(namespace, self.srcURL))
|
|
365
|
+
self.cacheFile = osp.join(cache_dir, f'{uid}.{cache_type}.json')
|
|
366
|
+
self.hashAlgo = algo
|
|
367
|
+
# first comparison needs
|
|
368
|
+
self.prevSrcHash = load_json(self.cacheFile).get('hash') if osp.isfile(self.cacheFile) else None
|
|
369
|
+
|
|
370
|
+
def retrieve(self):
|
|
371
|
+
if self._compare_hash():
|
|
372
|
+
return self.update()
|
|
373
|
+
return load_json(self.cacheFile)['data']
|
|
374
|
+
|
|
375
|
+
def update(self):
|
|
376
|
+
"""
|
|
377
|
+
- update cache directly
|
|
378
|
+
- useful when app needs to force update cache
|
|
379
|
+
"""
|
|
380
|
+
data = self.retriever(self.srcURL)
|
|
381
|
+
container = {
|
|
382
|
+
'data': data,
|
|
383
|
+
'hash': self.prevSrcHash,
|
|
384
|
+
}
|
|
385
|
+
save_json(self.cacheFile, container)
|
|
386
|
+
return data
|
|
387
|
+
|
|
388
|
+
def _compare_hash(self):
|
|
389
|
+
in_src_hash = self._compute_hash()
|
|
390
|
+
if changed := in_src_hash != self.prevSrcHash or self.prevSrcHash is None:
|
|
391
|
+
self.prevSrcHash = in_src_hash
|
|
392
|
+
return changed
|
|
393
|
+
|
|
394
|
+
def _compute_hash(self):
|
|
395
|
+
hash_algo_map = {
|
|
396
|
+
'checksum': self._compute_hash_as_checksum,
|
|
397
|
+
'mtime': self._compute_hash_as_modified_time,
|
|
398
|
+
}
|
|
399
|
+
return hash_algo_map[self.hashAlgo]()
|
|
400
|
+
|
|
401
|
+
def _compute_hash_as_checksum(self):
|
|
402
|
+
return get_md5_checksum(self.srcURL)
|
|
403
|
+
|
|
404
|
+
def _compute_hash_as_modified_time(self):
|
|
405
|
+
try:
|
|
406
|
+
return osp.getmtime(self.srcURL)
|
|
407
|
+
except FileNotFoundError:
|
|
408
|
+
return None
|
|
409
|
+
|
|
410
|
+
# endregion
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
# region functions
|
|
414
|
+
|
|
130
415
|
def get_platform_home_dir():
|
|
131
416
|
home_envvar = 'USERPROFILE' if PLATFORM == 'Windows' else 'HOME'
|
|
132
417
|
return os.getenv(home_envvar)
|
|
@@ -141,15 +426,6 @@ def get_platform_appdata_dir(winroam=True):
|
|
|
141
426
|
return plat_dir_map.get(PLATFORM)
|
|
142
427
|
|
|
143
428
|
|
|
144
|
-
def get_platform_tmp_dir():
|
|
145
|
-
plat_dir_map = {
|
|
146
|
-
'Windows': osp.join(str(os.getenv('LOCALAPPDATA')), 'Temp'),
|
|
147
|
-
'Darwin': osp.expanduser('~/Library/Caches'),
|
|
148
|
-
'Linux': '/tmp'
|
|
149
|
-
}
|
|
150
|
-
return plat_dir_map.get(PLATFORM)
|
|
151
|
-
|
|
152
|
-
|
|
153
429
|
def get_posix_shell_cfgfile():
|
|
154
430
|
return os.path.expanduser('~/.bash_profile' if os.getenv('SHELL') == '/bin/bash' else '~/.zshrc')
|
|
155
431
|
|
|
@@ -232,6 +508,10 @@ def build_default_logger(logdir, name=None, verbose=False):
|
|
|
232
508
|
return logging.getLogger(name or 'default')
|
|
233
509
|
|
|
234
510
|
|
|
511
|
+
def find_log_path(logger):
|
|
512
|
+
return next((handler.baseFilename for handler in logger.handlers if isinstance(handler, logging.FileHandler)), None)
|
|
513
|
+
|
|
514
|
+
|
|
235
515
|
glogger = build_default_logger(logdir=osp.join(get_platform_tmp_dir(), '_util'), name='util', verbose=True)
|
|
236
516
|
glogger.setLevel(logging.DEBUG)
|
|
237
517
|
|
|
@@ -332,107 +612,32 @@ def throw(err_cls, detail, advice):
|
|
|
332
612
|
|
|
333
613
|
|
|
334
614
|
def is_python3():
|
|
335
|
-
return sys.version_info[0] > 2
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
def load_json(path, as_namespace=False, encoding=TXT_CODEC):
|
|
339
|
-
"""
|
|
340
|
-
- Load Json configuration file.
|
|
341
|
-
- supports UTF-8 only, due to no way to support mixed encodings
|
|
342
|
-
- most usecases involve either utf-8 or mixed encodings
|
|
343
|
-
- windows users must fix their region and localization setup via control panel
|
|
344
|
-
"""
|
|
345
|
-
with open(path, 'r', encoding=encoding, errors='backslashreplace', newline=None) as f:
|
|
346
|
-
text = f.read()
|
|
347
|
-
return json.loads(text) if not as_namespace else json.loads(text, object_hook=lambda d: SimpleNamespace(**d))
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
def save_json(path, config, encoding=TXT_CODEC):
|
|
351
|
-
"""
|
|
352
|
-
Use io.open(), aka open() with py3 to produce a file object that encodes
|
|
353
|
-
Unicode as you write, then use json.dump() to write to that file.
|
|
354
|
-
Validate keys to avoid JSON and program out-of-sync.
|
|
355
|
-
"""
|
|
356
|
-
dict_config = vars(config) if isinstance(config, types.SimpleNamespace) else config
|
|
357
|
-
par_dir = osp.split(path)[0]
|
|
358
|
-
os.makedirs(par_dir, exist_ok=True)
|
|
359
|
-
with open(path, 'w', encoding=encoding) as f:
|
|
360
|
-
return json.dump(dict_config, f, ensure_ascii=False, indent=4)
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
class Tracer:
|
|
364
|
-
"""
|
|
365
|
-
- custom module-ignore rules
|
|
366
|
-
- trace calls and returns
|
|
367
|
-
- exclude first, then include
|
|
368
|
-
- usage: use in source code
|
|
369
|
-
- tracer = util.Tracer(exclude_funcname_pattern='stop')
|
|
370
|
-
- tracer.start()
|
|
371
|
-
- # add traceable code here
|
|
372
|
-
- tracer.stop()
|
|
373
|
-
"""
|
|
374
|
-
|
|
375
|
-
def __init__(self,
|
|
376
|
-
excluded_modules: set[str] = None,
|
|
377
|
-
exclude_filename_pattern: str = None,
|
|
378
|
-
include_filename_pattern: str = None,
|
|
379
|
-
exclude_funcname_pattern: str = None,
|
|
380
|
-
include_funcname_pattern: str = None,
|
|
381
|
-
trace_func=None,
|
|
382
|
-
exclude_builtins=True):
|
|
383
|
-
self.exclMods = {'builtins'} if excluded_modules is None else excluded_modules
|
|
384
|
-
self.exclFilePatt = re.compile(exclude_filename_pattern) if exclude_filename_pattern else None
|
|
385
|
-
self.inclFilePatt = re.compile(include_filename_pattern) if include_filename_pattern else None
|
|
386
|
-
self.exclFuncPatt = re.compile(exclude_funcname_pattern) if exclude_funcname_pattern else None
|
|
387
|
-
self.inclFuncPatt = re.compile(include_funcname_pattern) if include_funcname_pattern else None
|
|
388
|
-
self.traceFunc = trace_func
|
|
389
|
-
if exclude_builtins:
|
|
390
|
-
self.ignore_stdlibs()
|
|
391
|
-
|
|
392
|
-
def start(self):
|
|
393
|
-
sys.settrace(self.traceFunc or self._trace_calls_and_returns)
|
|
394
|
-
|
|
395
|
-
@staticmethod
|
|
396
|
-
def stop():
|
|
397
|
-
sys.settrace(None)
|
|
615
|
+
return sys.version_info[0] > 2
|
|
398
616
|
|
|
399
|
-
def ignore_stdlibs(self):
|
|
400
|
-
def _get_stdlib_module_names():
|
|
401
|
-
import distutils.sysconfig
|
|
402
|
-
stdlib_dir = distutils.sysconfig.get_python_lib(standard_lib=True)
|
|
403
|
-
return {f.replace(".py", "") for f in os.listdir(stdlib_dir)}
|
|
404
617
|
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
618
|
+
def load_json(path, as_namespace=False, encoding=TXT_CODEC):
|
|
619
|
+
"""
|
|
620
|
+
- Load Json configuration file.
|
|
621
|
+
- supports UTF-8 only, due to no way to support mixed encodings
|
|
622
|
+
- most usecases involve either utf-8 or mixed encodings
|
|
623
|
+
- windows users must fix their region and localization setup via control panel
|
|
624
|
+
"""
|
|
625
|
+
with open(path, 'r', encoding=encoding, errors='backslashreplace', newline=None) as f:
|
|
626
|
+
text = f.read()
|
|
627
|
+
return json.loads(text) if not as_namespace else json.loads(text, object_hook=lambda d: SimpleNamespace(**d))
|
|
408
628
|
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
return
|
|
422
|
-
if self.inclFilePatt and not self.inclFilePatt.search(filename):
|
|
423
|
-
return
|
|
424
|
-
func_name = frame.f_code.co_name
|
|
425
|
-
if self.exclFuncPatt and self.exclFuncPatt.search(func_name):
|
|
426
|
-
return
|
|
427
|
-
if self.inclFuncPatt and not self.inclFuncPatt.search(func_name):
|
|
428
|
-
return
|
|
429
|
-
line_number = frame.f_lineno
|
|
430
|
-
line = linecache.getline(filename, line_number).strip()
|
|
431
|
-
if event == 'call':
|
|
432
|
-
args = ', '.join(f'{arg}={repr(frame.f_locals[arg])}' for arg in frame.f_code.co_varnames[:frame.f_code.co_argcount])
|
|
433
|
-
print(f'Call: {module_name}.{func_name}({args}) - {line}')
|
|
434
|
-
return self._trace_calls_and_returns
|
|
435
|
-
print(f'Call: {module_name}.{func_name} => {arg} - {line}')
|
|
629
|
+
|
|
630
|
+
def save_json(path, config, encoding=TXT_CODEC):
|
|
631
|
+
"""
|
|
632
|
+
Use io.open(), aka open() with py3 to produce a file object that encodes
|
|
633
|
+
Unicode as you write, then use json.dump() to write to that file.
|
|
634
|
+
Validate keys to avoid JSON and program out-of-sync.
|
|
635
|
+
"""
|
|
636
|
+
dict_config = vars(config) if isinstance(config, types.SimpleNamespace) else config
|
|
637
|
+
par_dir = osp.split(path)[0]
|
|
638
|
+
os.makedirs(par_dir, exist_ok=True)
|
|
639
|
+
with open(path, 'w', encoding=encoding) as f:
|
|
640
|
+
return json.dump(dict_config, f, ensure_ascii=False, indent=4)
|
|
436
641
|
|
|
437
642
|
|
|
438
643
|
def get_md5_checksum(file):
|
|
@@ -776,109 +981,6 @@ def match_files_except_lines(file1, file2, excluded=None):
|
|
|
776
981
|
return content1 == content2
|
|
777
982
|
|
|
778
983
|
|
|
779
|
-
class RerunLock:
|
|
780
|
-
"""
|
|
781
|
-
- Lock process from reentering when seeing lock file on disk
|
|
782
|
-
- use semaphore-like behaviour with an instance limit
|
|
783
|
-
- Because lockfile is created by pyutil, we also save the occupier pid and .py path (name) in it
|
|
784
|
-
- if name is a path, e.g., __file__, then lockfile will be named after its basename
|
|
785
|
-
"""
|
|
786
|
-
|
|
787
|
-
def __init__(self, name, folder=None, logger=None, max_instances=1):
|
|
788
|
-
folder = folder or osp.join(get_platform_tmp_dir(), '_util')
|
|
789
|
-
filename = f'lock_{extract_path_stem(name)}.{os.getpid()}.lock.json'
|
|
790
|
-
self.name = name
|
|
791
|
-
self.lockFile = osp.join(folder, filename)
|
|
792
|
-
self.nMaxInstances = max_instances
|
|
793
|
-
self.logger = logger or glogger
|
|
794
|
-
# CAUTION:
|
|
795
|
-
# - windows grpc server crashes with signals:
|
|
796
|
-
# - ValueError: signal only works in main thread of the main interpreter
|
|
797
|
-
# - signals are disabled for windows
|
|
798
|
-
if threading.current_thread() is threading.main_thread():
|
|
799
|
-
common_sigs = [
|
|
800
|
-
signal.SIGABRT,
|
|
801
|
-
signal.SIGFPE,
|
|
802
|
-
signal.SIGILL,
|
|
803
|
-
signal.SIGINT,
|
|
804
|
-
signal.SIGSEGV,
|
|
805
|
-
signal.SIGTERM,
|
|
806
|
-
]
|
|
807
|
-
plat_sigs = [
|
|
808
|
-
signal.SIGBREAK,
|
|
809
|
-
# CAUTION
|
|
810
|
-
# - CTRL_C_EVENT, CTRL_BREAK_EVENT not working on Windows
|
|
811
|
-
# signal.CTRL_C_EVENT,
|
|
812
|
-
# signal.CTRL_BREAK_EVENT,
|
|
813
|
-
] if PLATFORM == 'Windows' else [
|
|
814
|
-
# CAUTION:
|
|
815
|
-
# - SIGCHLD as an alias is safe to ignore
|
|
816
|
-
# - SIGKILL must be handled by os.kill()
|
|
817
|
-
signal.SIGALRM,
|
|
818
|
-
signal.SIGBUS,
|
|
819
|
-
# signal.SIGCHLD,
|
|
820
|
-
# - SIGCONT: CTRL+Z is allowed for bg process
|
|
821
|
-
# signal.SIGCONT,
|
|
822
|
-
signal.SIGHUP,
|
|
823
|
-
# signal.SIGKILL,
|
|
824
|
-
signal.SIGPIPE,
|
|
825
|
-
]
|
|
826
|
-
for sig in common_sigs + plat_sigs:
|
|
827
|
-
signal.signal(sig, self.handle_signal)
|
|
828
|
-
# cleanup zombie locks due to runtime exceptions
|
|
829
|
-
locks = [osp.basename(lock) for lock in glob.glob(osp.join(osp.dirname(self.lockFile), f'lock_{extract_path_stem(self.name)}.*.lock.json'))]
|
|
830
|
-
zombie_locks = [lock for lock in locks if not is_pid_running(int(lock.split(".")[1]))]
|
|
831
|
-
for lock in zombie_locks:
|
|
832
|
-
safe_remove(osp.join(osp.dirname(self.lockFile), lock))
|
|
833
|
-
|
|
834
|
-
def lock(self):
|
|
835
|
-
locks = [osp.basename(lock) for lock in glob.glob(osp.join(osp.dirname(self.lockFile), f'lock_{extract_path_stem(self.name)}.*.lock.json'))]
|
|
836
|
-
is_locked = len(locks) >= self.nMaxInstances
|
|
837
|
-
if is_locked:
|
|
838
|
-
locker_pids = [int(lock.split(".")[1]) for lock in locks]
|
|
839
|
-
self.logger.warning(f'{self.name} is locked by processes: {locker_pids}. Will block new instances until unlocked.')
|
|
840
|
-
return False
|
|
841
|
-
save_json(self.lockFile, {
|
|
842
|
-
'pid': os.getpid(),
|
|
843
|
-
'name': self.name,
|
|
844
|
-
})
|
|
845
|
-
# CAUTION: race condition: saving needs a sec, it's up to application to await lockfile
|
|
846
|
-
return True
|
|
847
|
-
|
|
848
|
-
def unlock(self):
|
|
849
|
-
try:
|
|
850
|
-
os.remove(self.lockFile)
|
|
851
|
-
except FileNotFoundError:
|
|
852
|
-
self.logger.warning(f'{self.name} already unlocked. Safely ignored.')
|
|
853
|
-
return False
|
|
854
|
-
except Exception:
|
|
855
|
-
failure = traceback.format_exc()
|
|
856
|
-
self.logger.error(f""""\
|
|
857
|
-
Failed to unlock {self.name}:
|
|
858
|
-
Details:
|
|
859
|
-
{failure}
|
|
860
|
-
|
|
861
|
-
Advice:
|
|
862
|
-
- Delete the lock by hand: {self.lockFile}""")
|
|
863
|
-
return False
|
|
864
|
-
return True
|
|
865
|
-
|
|
866
|
-
def unlock_all(self):
|
|
867
|
-
locks = glob.glob(osp.join(osp.dirname(self.lockFile), f'lock_{osp.basename(self.name)}.*.lock.json'))
|
|
868
|
-
for lock in locks:
|
|
869
|
-
os.remove(lock)
|
|
870
|
-
return True
|
|
871
|
-
|
|
872
|
-
def is_locked(self):
|
|
873
|
-
return osp.isfile(self.lockFile)
|
|
874
|
-
|
|
875
|
-
def handle_signal(self, sig, frame):
|
|
876
|
-
msg = f'Terminated due to signal: {signal.Signals(sig).name}; Will unlock'
|
|
877
|
-
self.logger.warning(msg)
|
|
878
|
-
self.unlock()
|
|
879
|
-
raise RuntimeError(msg)
|
|
880
|
-
|
|
881
|
-
|
|
882
984
|
def rerun_lock(name, folder=None, logger=glogger, max_instances=1):
|
|
883
985
|
"""Decorator for reentrance locking on functions"""
|
|
884
986
|
|
|
@@ -2534,75 +2636,6 @@ def inspect_obj(obj):
|
|
|
2534
2636
|
return {'type': type_name, 'attrs': attrs, 'repr': repr(obj), 'details': details}
|
|
2535
2637
|
|
|
2536
2638
|
|
|
2537
|
-
class Cache:
|
|
2538
|
-
"""
|
|
2539
|
-
cross-session caching: using temp-file to retrieve data based on hash changes
|
|
2540
|
-
- constraints:
|
|
2541
|
-
- data retrieval/parsing is expensive
|
|
2542
|
-
- one cache per data-source
|
|
2543
|
-
- cache is a mediator b/w app and data-source as a retriever only, cuz user's saving intent is always towards source, no need to cache a saving action
|
|
2544
|
-
- for cross-session caching, save hash into cache, then when instantiate cache object, always load hash from cache to compare with incoming hash
|
|
2545
|
-
- app must provide retriever function: retriever(src) -> json_data
|
|
2546
|
-
- because it'd cost the same to retrieve data from a json-file source as from cache, so no json default is provided
|
|
2547
|
-
- e.g., loading a complex tree-structure from a file:
|
|
2548
|
-
- tree_cache = Cache('/path/to/file.tree', lambda: src: load_data(src), '/tmp/my_app')
|
|
2549
|
-
- # ... later
|
|
2550
|
-
- cached_tree_data = tree_cache.retrieve()
|
|
2551
|
-
"""
|
|
2552
|
-
|
|
2553
|
-
def __init__(self, data_source, data_retriever, cache_dir=get_platform_tmp_dir(), cache_type='cache', algo='checksum', source_seed='6ba7b810-9dad-11d1-80b4-00c04fd430c8'):
|
|
2554
|
-
assert algo in ['checksum', 'mtime']
|
|
2555
|
-
self.srcURL = data_source
|
|
2556
|
-
self.retriever = data_retriever
|
|
2557
|
-
# use a fixed namespace for each data-source to ensure inter-session consistency
|
|
2558
|
-
namespace = uuid.UUID(str(source_seed))
|
|
2559
|
-
uid = str(uuid.uuid5(namespace, self.srcURL))
|
|
2560
|
-
self.cacheFile = osp.join(cache_dir, f'{uid}.{cache_type}.json')
|
|
2561
|
-
self.hashAlgo = algo
|
|
2562
|
-
# first comparison needs
|
|
2563
|
-
self.prevSrcHash = load_json(self.cacheFile).get('hash') if osp.isfile(self.cacheFile) else None
|
|
2564
|
-
|
|
2565
|
-
def retrieve(self):
|
|
2566
|
-
if self._compare_hash():
|
|
2567
|
-
return self.update()
|
|
2568
|
-
return load_json(self.cacheFile)['data']
|
|
2569
|
-
|
|
2570
|
-
def update(self):
|
|
2571
|
-
"""
|
|
2572
|
-
- update cache directly
|
|
2573
|
-
- useful when app needs to force update cache
|
|
2574
|
-
"""
|
|
2575
|
-
data = self.retriever(self.srcURL)
|
|
2576
|
-
container = {
|
|
2577
|
-
'data': data,
|
|
2578
|
-
'hash': self.prevSrcHash,
|
|
2579
|
-
}
|
|
2580
|
-
save_json(self.cacheFile, container)
|
|
2581
|
-
return data
|
|
2582
|
-
|
|
2583
|
-
def _compare_hash(self):
|
|
2584
|
-
in_src_hash = self._compute_hash()
|
|
2585
|
-
if changed := in_src_hash != self.prevSrcHash or self.prevSrcHash is None:
|
|
2586
|
-
self.prevSrcHash = in_src_hash
|
|
2587
|
-
return changed
|
|
2588
|
-
|
|
2589
|
-
def _compute_hash(self):
|
|
2590
|
-
hash_algo_map = {
|
|
2591
|
-
'checksum': self._compute_hash_as_checksum,
|
|
2592
|
-
'mtime': self._compute_hash_as_modified_time,
|
|
2593
|
-
}
|
|
2594
|
-
return hash_algo_map[self.hashAlgo]()
|
|
2595
|
-
|
|
2596
|
-
def _compute_hash_as_checksum(self):
|
|
2597
|
-
return get_md5_checksum(self.srcURL)
|
|
2598
|
-
|
|
2599
|
-
def _compute_hash_as_modified_time(self):
|
|
2600
|
-
try:
|
|
2601
|
-
return osp.getmtime(self.srcURL)
|
|
2602
|
-
except FileNotFoundError:
|
|
2603
|
-
return None
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
2639
|
def mem_caching(maxsize=None):
|
|
2607
2640
|
"""
|
|
2608
2641
|
- per-process lru caching for multiple data sources
|
|
@@ -2844,18 +2877,6 @@ def indent(code_or_lines, spaces_per_indent=4):
|
|
|
2844
2877
|
return '\n'.join(indented) if isinstance(code_or_lines, str) else indented
|
|
2845
2878
|
|
|
2846
2879
|
|
|
2847
|
-
def find_log_path(logger):
|
|
2848
|
-
"""
|
|
2849
|
-
- logger must be a python logger
|
|
2850
|
-
"""
|
|
2851
|
-
for handler in logger.handlers:
|
|
2852
|
-
if isinstance(handler, logging.FileHandler):
|
|
2853
|
-
return handler.baseFilename
|
|
2854
|
-
# use next() to get the first handler
|
|
2855
|
-
# if not found, raise StopIteration
|
|
2856
|
-
return next(filter(lambda h: isinstance(h, logging.FileHandler), logger.handlers), None)
|
|
2857
|
-
|
|
2858
|
-
|
|
2859
2880
|
def collect_file_tree(root):
|
|
2860
2881
|
return [file for file in glob.glob(osp.join(root, '**'), recursive=True) if osp.isfile(file)]
|
|
2861
2882
|
|
|
@@ -2951,27 +2972,7 @@ def json_from_text(json_str):
|
|
|
2951
2972
|
except json.JSONDecodeError as e:
|
|
2952
2973
|
return None, e
|
|
2953
2974
|
|
|
2954
|
-
|
|
2955
|
-
class OfflineJSON:
|
|
2956
|
-
def __init__(self, file_path):
|
|
2957
|
-
self.path = file_path
|
|
2958
|
-
|
|
2959
|
-
def exists(self):
|
|
2960
|
-
return osp.isfile(self.path)
|
|
2961
|
-
|
|
2962
|
-
def load(self):
|
|
2963
|
-
return load_json(self.path) if self.exists() else None
|
|
2964
|
-
|
|
2965
|
-
def save(self, data: dict):
|
|
2966
|
-
save_json(self.path, data)
|
|
2967
|
-
|
|
2968
|
-
def merge(self, props: dict):
|
|
2969
|
-
data = self.load()
|
|
2970
|
-
if not data:
|
|
2971
|
-
return self.save(props)
|
|
2972
|
-
data.update(props)
|
|
2973
|
-
self.save(data)
|
|
2974
|
-
return data
|
|
2975
|
+
# endregion
|
|
2975
2976
|
|
|
2976
2977
|
|
|
2977
2978
|
def _test():
|
kkpyutil-1.40.0.dist-info/RECORD
DELETED
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
kkpyutil.py,sha256=Lm_9sYcqDI9d_AR5qdvrn0fcMi6n1cWrGwOOC7d28ZE,111702
|
|
2
|
-
kkpyutil_helper/windows/kkttssave.ps1,sha256=xa3-WzqNh2rGYlOx_I4ewOuCE94gkTO5cEwYH0M67_0,446
|
|
3
|
-
kkpyutil_helper/windows/kkttsspeak.ps1,sha256=7WUUHMmjTQroUWA2Mvdt4JtSt475nZUHQx-qP-7rS6o,305
|
|
4
|
-
kkpyutil-1.40.0.dist-info/LICENSE,sha256=uISevGnCxB5QOU0ftbofN75_yUtd6E2h_MWE1zqagC8,1065
|
|
5
|
-
kkpyutil-1.40.0.dist-info/METADATA,sha256=OQeESTvqKUHtgPSSf5KwiKEZdBlVpsANpi5Hk9s882M,1136
|
|
6
|
-
kkpyutil-1.40.0.dist-info/WHEEL,sha256=kLuE8m1WYU0Ig0_YEGrXyTtiJvKPpLpDEiChiNyei5Y,88
|
|
7
|
-
kkpyutil-1.40.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|