abstract-utilities 0.2.2.593__py3-none-any.whl → 0.2.2.679__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of abstract-utilities might be problematic. Click here for more details.
- abstract_utilities/__init__.py +18 -6
- abstract_utilities/class_utils/abstract_classes.py +104 -34
- abstract_utilities/class_utils/caller_utils.py +39 -0
- abstract_utilities/class_utils/global_utils.py +35 -21
- abstract_utilities/class_utils/imports/imports.py +1 -1
- abstract_utilities/directory_utils/src/directory_utils.py +2 -0
- abstract_utilities/file_utils/imports/classes.py +59 -55
- abstract_utilities/file_utils/imports/module_imports.py +1 -1
- abstract_utilities/file_utils/src/file_filters/__init__.py +0 -3
- abstract_utilities/file_utils/src/file_filters/ensure_utils.py +382 -10
- abstract_utilities/file_utils/src/file_filters/filter_params.py +64 -0
- abstract_utilities/file_utils/src/file_filters/predicate_utils.py +21 -91
- abstract_utilities/file_utils/src/find_collect.py +10 -0
- abstract_utilities/file_utils/src/initFunctionsGen.py +36 -23
- abstract_utilities/file_utils/src/initFunctionsGens.py +280 -0
- abstract_utilities/import_utils/imports/__init__.py +1 -1
- abstract_utilities/import_utils/imports/init_imports.py +3 -0
- abstract_utilities/import_utils/imports/module_imports.py +2 -1
- abstract_utilities/import_utils/imports/utils.py +1 -1
- abstract_utilities/import_utils/src/__init__.py +1 -0
- abstract_utilities/import_utils/src/extract_utils.py +2 -2
- abstract_utilities/import_utils/src/import_functions.py +30 -10
- abstract_utilities/import_utils/src/import_utils.py +41 -0
- abstract_utilities/import_utils/src/layze_import_utils/__init__.py +2 -0
- abstract_utilities/import_utils/src/layze_import_utils/lazy_utils.py +41 -0
- abstract_utilities/import_utils/src/layze_import_utils/nullProxy.py +37 -0
- abstract_utilities/import_utils/src/nullProxy.py +30 -0
- abstract_utilities/import_utils/src/sysroot_utils.py +1 -1
- abstract_utilities/imports.py +5 -2
- abstract_utilities/json_utils/imports/imports.py +1 -1
- abstract_utilities/json_utils/json_utils.py +37 -3
- abstract_utilities/list_utils/list_utils.py +3 -0
- abstract_utilities/log_utils/log_file.py +137 -34
- abstract_utilities/parse_utils/parse_utils.py +23 -0
- abstract_utilities/path_utils/imports/module_imports.py +1 -1
- abstract_utilities/path_utils/path_utils.py +7 -12
- abstract_utilities/read_write_utils/imports/imports.py +1 -1
- abstract_utilities/read_write_utils/read_write_utils.py +137 -36
- abstract_utilities/type_utils/__init__.py +5 -1
- abstract_utilities/type_utils/get_type.py +120 -0
- abstract_utilities/type_utils/imports/__init__.py +1 -0
- abstract_utilities/type_utils/imports/constants.py +134 -0
- abstract_utilities/type_utils/imports/module_imports.py +25 -1
- abstract_utilities/type_utils/is_type.py +455 -0
- abstract_utilities/type_utils/make_type.py +126 -0
- abstract_utilities/type_utils/mime_types.py +68 -0
- abstract_utilities/type_utils/type_utils.py +0 -877
- {abstract_utilities-0.2.2.593.dist-info → abstract_utilities-0.2.2.679.dist-info}/METADATA +1 -1
- {abstract_utilities-0.2.2.593.dist-info → abstract_utilities-0.2.2.679.dist-info}/RECORD +51 -40
- {abstract_utilities-0.2.2.593.dist-info → abstract_utilities-0.2.2.679.dist-info}/WHEEL +0 -0
- {abstract_utilities-0.2.2.593.dist-info → abstract_utilities-0.2.2.679.dist-info}/top_level.txt +0 -0
|
@@ -1,4 +1,82 @@
|
|
|
1
|
-
from
|
|
1
|
+
from ...imports import *
|
|
2
|
+
import re
|
|
3
|
+
def combine_params(*values,typ=None):
|
|
4
|
+
nu_values = None
|
|
5
|
+
for value in values:
|
|
6
|
+
if value is not None:
|
|
7
|
+
typ = typ or type(value)
|
|
8
|
+
if nu_values is None:
|
|
9
|
+
nu_values = typ()
|
|
10
|
+
|
|
11
|
+
if typ is set:
|
|
12
|
+
nu_values = nu_values | typ(value)
|
|
13
|
+
if typ is list:
|
|
14
|
+
nu_values += typ(value)
|
|
15
|
+
return nu_values
|
|
16
|
+
def get_safe_kwargs(canonical_map, **kwargs):
|
|
17
|
+
# Lowercase all keys for safety
|
|
18
|
+
canonical_map = canonical_map or CANONICAL_MAP
|
|
19
|
+
norm_kwargs = {k.lower(): v for k, v in kwargs.items() if v is not None}
|
|
20
|
+
|
|
21
|
+
# Inverse lookup: alias → canonical key
|
|
22
|
+
alias_lookup = {
|
|
23
|
+
alias: canon
|
|
24
|
+
for canon, aliases in canonical_map.items()
|
|
25
|
+
if aliases
|
|
26
|
+
for alias in aliases
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
# Preserve correctly named keys
|
|
30
|
+
safe_kwargs = {k: v for k, v in norm_kwargs.items() if k in canonical_map}
|
|
31
|
+
|
|
32
|
+
for k, v in norm_kwargs.items():
|
|
33
|
+
if k in alias_lookup:
|
|
34
|
+
canonical_key = alias_lookup[k]
|
|
35
|
+
prev = safe_kwargs.get(canonical_key)
|
|
36
|
+
if prev is None:
|
|
37
|
+
safe_kwargs[canonical_key] = v
|
|
38
|
+
else:
|
|
39
|
+
# merge intelligently if both exist
|
|
40
|
+
if isinstance(prev, (set, list)) and isinstance(v, (set, list)):
|
|
41
|
+
safe_kwargs[canonical_key] = list(set(prev) | set(v))
|
|
42
|
+
else:
|
|
43
|
+
safe_kwargs[canonical_key] = v # overwrite for non-iterables
|
|
44
|
+
|
|
45
|
+
# fill defaults if missing
|
|
46
|
+
for canon in canonical_map:
|
|
47
|
+
safe_kwargs.setdefault(canon, None)
|
|
48
|
+
|
|
49
|
+
return safe_kwargs
|
|
50
|
+
|
|
51
|
+
def create_canonical_map(*args,canonical_map=None):
|
|
52
|
+
keys = [arg for arg in args if arg]
|
|
53
|
+
if not keys:
|
|
54
|
+
return CANONICAL_MAP
|
|
55
|
+
canonical_map = canonical_map or CANONICAL_MAP
|
|
56
|
+
|
|
57
|
+
return {key:canonical_map.get(key) for key in keys}
|
|
58
|
+
def get_safe_canonical_kwargs(*args,canonical_map=None,**kwargs):
|
|
59
|
+
canonical_map = canonical_map or create_canonical_map(*args)
|
|
60
|
+
|
|
61
|
+
return get_safe_kwargs(canonical_map=canonical_map,**kwargs)
|
|
62
|
+
def get_dir_filter_kwargs(**kwargs):
|
|
63
|
+
canonical_map = create_canonical_map("directories")
|
|
64
|
+
return get_safe_kwargs(canonical_map=canonical_map,**kwargs)
|
|
65
|
+
def get_file_filter_kwargs(**kwargs):
|
|
66
|
+
"""
|
|
67
|
+
Normalize arbitrary keyword arguments for file scanning configuration.
|
|
68
|
+
|
|
69
|
+
Examples:
|
|
70
|
+
- 'excluded_ext' or 'unallowed_exts' → 'exclude_exts'
|
|
71
|
+
- 'include_dirs' or 'allow_dir' → 'allowed_dirs'
|
|
72
|
+
- 'excludePattern' or 'excluded_patterns' → 'exclude_patterns'
|
|
73
|
+
- 'allowed_type' or 'include_types' → 'allowed_types'
|
|
74
|
+
"""
|
|
75
|
+
# Canonical keys and aliases
|
|
76
|
+
canonical_keys =["allowed_exts","exclude_exts","allowed_types","exclude_types","allowed_dirs","exclude_dirs","allowed_patterns","exclude_patterns"]
|
|
77
|
+
|
|
78
|
+
return get_safe_canonical_kwargs(*canonical_keys,**kwargs)
|
|
79
|
+
|
|
2
80
|
def normalize_listlike(value, typ=list, sep=','):
|
|
3
81
|
"""Normalize comma-separated or iterable values into the desired type."""
|
|
4
82
|
if value in [True, None, False]:
|
|
@@ -10,12 +88,12 @@ def normalize_listlike(value, typ=list, sep=','):
|
|
|
10
88
|
def ensure_exts(exts):
|
|
11
89
|
if exts in [True, None, False]:
|
|
12
90
|
return exts
|
|
13
|
-
|
|
91
|
+
cleaned = set()
|
|
14
92
|
for ext in normalize_listlike(exts, list):
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
return
|
|
93
|
+
ext = ext.strip().lower()
|
|
94
|
+
ext = ext.lstrip(".") # remove ALL leading dots
|
|
95
|
+
cleaned.add("." + ext) # add exactly one
|
|
96
|
+
return cleaned
|
|
19
97
|
|
|
20
98
|
def ensure_patterns(patterns):
|
|
21
99
|
"""Normalize pattern list and ensure they are valid globs."""
|
|
@@ -41,14 +119,17 @@ def ensure_directories(*args,**kwargs):
|
|
|
41
119
|
|
|
42
120
|
if run_pruned_func(is_dir,arg_str,**kwargs):
|
|
43
121
|
directories.append(arg_str)
|
|
122
|
+
|
|
44
123
|
elif run_pruned_func(is_file,arg_str,**kwargs):
|
|
45
124
|
dirname = os.path.dirname(arg_str)
|
|
46
125
|
directories.append(dirname)
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
126
|
+
if not directories:
|
|
127
|
+
safe_directories = get_dir_filter_kwargs(**kwargs)
|
|
128
|
+
safe_dirs = safe_directories.get('directories')
|
|
129
|
+
safe_dirs = if_none_change(safe_dirs or None,get_initial_caller_dir())
|
|
130
|
+
directories+= make_list(safe_dirs)
|
|
51
131
|
return list(set([r for r in directories if r]))
|
|
132
|
+
|
|
52
133
|
def get_proper_type_str(string):
|
|
53
134
|
if not string:
|
|
54
135
|
return None
|
|
@@ -116,3 +197,294 @@ def check_path_type(
|
|
|
116
197
|
return output if output in ("file", "directory", "missing") else "unknown"
|
|
117
198
|
except Exception:
|
|
118
199
|
return "unknown"
|
|
200
|
+
def get_allowed_predicate(allowed=None,cfg=None,**kwargs):
|
|
201
|
+
if allowed != False:
|
|
202
|
+
if allowed == True:
|
|
203
|
+
allowed = None
|
|
204
|
+
allowed = allowed or make_allowed_predicate(cfg=cfg,**kwargs)
|
|
205
|
+
else:
|
|
206
|
+
def allowed(*args):
|
|
207
|
+
return True
|
|
208
|
+
allowed = allowed
|
|
209
|
+
return allowed
|
|
210
|
+
def get_globs(items,recursive: bool = True,allowed=None,cfg=None,**kwargs):
|
|
211
|
+
glob_paths = []
|
|
212
|
+
allowed = get_allowed_predicate(allowed=allowed,cfg=cfg,**kwargs)
|
|
213
|
+
items = [item for item in make_list(items) if item]
|
|
214
|
+
for item in items:
|
|
215
|
+
pattern = os.path.join(item, "**/*") # include all files recursively\n
|
|
216
|
+
nuItems = glob.glob(pattern, recursive=recursive)
|
|
217
|
+
if allowed:
|
|
218
|
+
nuItems = [nuItem for nuItem in nuItems if nuItem and allowed(nuItem)]
|
|
219
|
+
glob_paths += nuItems
|
|
220
|
+
return glob_paths
|
|
221
|
+
def get_allowed_files(items,allowed=True,cfg=None,**kwargs):
|
|
222
|
+
allowed = get_allowed_predicate(allowed=allowed,cfg=cfg,**kwargs)
|
|
223
|
+
return [item for item in items if item and os.path.isfile(item) and allowed(item)]
|
|
224
|
+
def get_allowed_dirs(items,allowed=False,cfg=None,**kwargs):
|
|
225
|
+
allowed = get_allowed_predicate(allowed=allowed,cfg=cfg,**kwargs)
|
|
226
|
+
return [item for item in items if item and os.path.isdir(item) and allowed(item)]
|
|
227
|
+
|
|
228
|
+
def get_filtered_files(items,allowed=None,files = [],cfg=None,**kwargs):
|
|
229
|
+
allowed = get_allowed_predicate(allowed=allowed,cfg=cfg,**kwargs)
|
|
230
|
+
glob_paths = get_globs(items,allowed=allowed,cfg=cfg,**kwargs)
|
|
231
|
+
return [glob_path for glob_path in glob_paths if glob_path and os.path.isfile(glob_path) and glob_path not in files and allowed(glob_path)]
|
|
232
|
+
def get_filtered_dirs(items,allowed=None,dirs = [],cfg=None,**kwargs):
|
|
233
|
+
allowed = get_allowed_predicate(allowed=allowed,cfg=cfg,**kwargs)
|
|
234
|
+
glob_paths = get_globs(items,allowed=allowed,cfg=cfg,**kwargs)
|
|
235
|
+
return [glob_path for glob_path in glob_paths if glob_path and os.path.isdir(glob_path) and glob_path not in dirs and allowed(glob_path)]
|
|
236
|
+
|
|
237
|
+
def get_all_allowed_files(items,allowed=None,cfg=None,**kwargs):
|
|
238
|
+
dirs = get_all_allowed_dirs(items,allowed=allowed,cfg=cfg,**kwargs)
|
|
239
|
+
files = get_allowed_files(items,allowed=allowed,cfg=cfg,**kwargs)
|
|
240
|
+
nu_files = []
|
|
241
|
+
for directory in dirs:
|
|
242
|
+
files += get_filtered_files(directory,allowed=allowed,files=files,cfg=cfg,**kwargs)
|
|
243
|
+
return files
|
|
244
|
+
def get_all_allowed_dirs(items,allowed=None,cfg=None,**kwargs):
|
|
245
|
+
allowed = get_allowed_predicate(allowed=allowed,cfg=cfg,**kwargs)
|
|
246
|
+
dirs = get_allowed_dirs(items,allowed=allowed,cfg=cfg,**kwargs)
|
|
247
|
+
nu_dirs=[]
|
|
248
|
+
for directory in dirs:
|
|
249
|
+
nu_dirs += get_filtered_dirs(directory,allowed=allowed,dirs=nu_dirs,cfg=cfg,**kwargs)
|
|
250
|
+
return nu_dirs
|
|
251
|
+
|
|
252
|
+
def make_allowed_predicate(cfg: ScanConfig=None,**kwargs) -> Callable[[str], bool]:
|
|
253
|
+
"""
|
|
254
|
+
Build a predicate that returns True if a given path is considered allowed
|
|
255
|
+
under the given ScanConfig. Applies allowed_* and exclude_* logic symmetrically.
|
|
256
|
+
"""
|
|
257
|
+
cfg=cfg or define_defaults(**kwargs)
|
|
258
|
+
def allowed(path: str=None,p=None) -> bool:
|
|
259
|
+
p = p or Path(path)
|
|
260
|
+
name = p.name.lower()
|
|
261
|
+
path_str = str(p).lower()
|
|
262
|
+
|
|
263
|
+
# --------------------
|
|
264
|
+
# A) directory filters
|
|
265
|
+
# --------------------
|
|
266
|
+
if cfg.exclude_dirs:
|
|
267
|
+
for dpat in cfg.exclude_dirs:
|
|
268
|
+
dpat_l = dpat.lower()
|
|
269
|
+
if dpat_l in path_str or fnmatch.fnmatch(name, dpat_l):
|
|
270
|
+
if p.is_dir() or dpat_l in path_str:
|
|
271
|
+
return False
|
|
272
|
+
|
|
273
|
+
if cfg.allowed_dirs and "*" not in cfg.allowed_dirs:
|
|
274
|
+
# must be in at least one allowed dir
|
|
275
|
+
if not any(
|
|
276
|
+
fnmatch.fnmatch(path_str, f"*{dpat.lower()}*") for dpat in cfg.allowed_dirs
|
|
277
|
+
):
|
|
278
|
+
return False
|
|
279
|
+
|
|
280
|
+
# --------------------
|
|
281
|
+
# B) pattern filters
|
|
282
|
+
# --------------------
|
|
283
|
+
if cfg.allowed_patterns and "*" not in cfg.allowed_patterns:
|
|
284
|
+
if not any(fnmatch.fnmatch(name, pat.lower()) for pat in cfg.allowed_patterns):
|
|
285
|
+
return False
|
|
286
|
+
|
|
287
|
+
if cfg.exclude_patterns:
|
|
288
|
+
for pat in cfg.exclude_patterns:
|
|
289
|
+
if fnmatch.fnmatch(name, pat.lower()):
|
|
290
|
+
return False
|
|
291
|
+
|
|
292
|
+
# --------------------
|
|
293
|
+
# C) extension filters
|
|
294
|
+
# --------------------
|
|
295
|
+
if p.is_file():
|
|
296
|
+
ext = p.suffix.lower()
|
|
297
|
+
if cfg.allowed_exts and ext not in cfg.allowed_exts:
|
|
298
|
+
return False
|
|
299
|
+
if cfg.exclude_exts and ext in cfg.exclude_exts:
|
|
300
|
+
return False
|
|
301
|
+
|
|
302
|
+
# --------------------
|
|
303
|
+
# D) type filters (optional)
|
|
304
|
+
# --------------------
|
|
305
|
+
if cfg.allowed_types and "*" not in cfg.allowed_types:
|
|
306
|
+
if not any(t in path_str for t in cfg.allowed_types):
|
|
307
|
+
return False
|
|
308
|
+
if cfg.exclude_types:
|
|
309
|
+
if any(t in path_str for t in cfg.exclude_types):
|
|
310
|
+
return False
|
|
311
|
+
|
|
312
|
+
return True
|
|
313
|
+
|
|
314
|
+
return allowed
|
|
315
|
+
def _get_default_modular(value, default, add=False, typ=set):
|
|
316
|
+
"""Merge user and default values intelligently."""
|
|
317
|
+
if value == None:
|
|
318
|
+
value = add
|
|
319
|
+
if value in [True]:
|
|
320
|
+
return default
|
|
321
|
+
if value is False:
|
|
322
|
+
return value
|
|
323
|
+
if add:
|
|
324
|
+
return combine_params(value,default,typ=None)
|
|
325
|
+
|
|
326
|
+
return typ(value)
|
|
327
|
+
|
|
328
|
+
# -------------------------
|
|
329
|
+
# Default derivation logic
|
|
330
|
+
# -------------------------
|
|
331
|
+
def _get_default_modular(value, default, add=None, typ=set):
|
|
332
|
+
"""Merge user and default values intelligently."""
|
|
333
|
+
add = add or False
|
|
334
|
+
if value == None:
|
|
335
|
+
value = add
|
|
336
|
+
if value in [True]:
|
|
337
|
+
return default
|
|
338
|
+
if value is False:
|
|
339
|
+
return value
|
|
340
|
+
if add:
|
|
341
|
+
return combine_params(value,default,typ=None)
|
|
342
|
+
return typ(value)
|
|
343
|
+
def make_allowed_predicate(cfg: ScanConfig = None, **kwargs) -> Callable[[str], bool]:
|
|
344
|
+
"""
|
|
345
|
+
Build and return a function `allowed(path)` that evaluates the given ScanConfig.
|
|
346
|
+
Unlike substring-based matching, this version avoids accidental matches inside
|
|
347
|
+
unrelated names (e.g., 'abstract' matching 'archive').
|
|
348
|
+
"""
|
|
349
|
+
|
|
350
|
+
cfg = cfg or define_defaults(**kwargs)
|
|
351
|
+
|
|
352
|
+
def allowed(path: str) -> bool:
|
|
353
|
+
p = Path(path)
|
|
354
|
+
name = p.name.lower()
|
|
355
|
+
path_str = str(p).lower()
|
|
356
|
+
|
|
357
|
+
# --------------------
|
|
358
|
+
# A) directory filters
|
|
359
|
+
# --------------------
|
|
360
|
+
# Excluded dirs: reject if any directory in the path matches exactly
|
|
361
|
+
if cfg.exclude_dirs:
|
|
362
|
+
parts = path_str.split("/")
|
|
363
|
+
if any(d.lower() in parts for d in cfg.exclude_dirs):
|
|
364
|
+
print(f"[exclude_dirs] → {path}")
|
|
365
|
+
return False
|
|
366
|
+
|
|
367
|
+
# Allowed dirs: require at least one match (unless "*")
|
|
368
|
+
if cfg.allowed_dirs and cfg.allowed_dirs != ["*"]:
|
|
369
|
+
parts = path_str.split("/")
|
|
370
|
+
if not any(d.lower() in parts for d in cfg.allowed_dirs):
|
|
371
|
+
print(f"[allowed_dirs] → {path}")
|
|
372
|
+
return False
|
|
373
|
+
|
|
374
|
+
# --------------------
|
|
375
|
+
# B) pattern filters
|
|
376
|
+
# --------------------
|
|
377
|
+
if cfg.allowed_patterns and cfg.allowed_patterns != ["*"]:
|
|
378
|
+
if not any(fnmatch.fnmatch(name, pat.lower()) for pat in cfg.allowed_patterns):
|
|
379
|
+
print(f"[allowed_patterns] → {path}")
|
|
380
|
+
return False
|
|
381
|
+
|
|
382
|
+
if cfg.exclude_patterns:
|
|
383
|
+
if any(fnmatch.fnmatch(name, pat.lower()) for pat in cfg.exclude_patterns):
|
|
384
|
+
print(f"[exclude_patterns] → {path}")
|
|
385
|
+
return False
|
|
386
|
+
|
|
387
|
+
# --------------------
|
|
388
|
+
# C) extension filters
|
|
389
|
+
# --------------------
|
|
390
|
+
if p.is_file():
|
|
391
|
+
ext = p.suffix.lower()
|
|
392
|
+
|
|
393
|
+
if cfg.allowed_exts and ext not in cfg.allowed_exts:
|
|
394
|
+
print(f"[allowed_exts] → {path}")
|
|
395
|
+
return False
|
|
396
|
+
|
|
397
|
+
if cfg.exclude_exts and ext in cfg.exclude_exts:
|
|
398
|
+
print(f"[exclude_exts] → {path}")
|
|
399
|
+
return False
|
|
400
|
+
|
|
401
|
+
# --------------------
|
|
402
|
+
# D) type filters (SAFE SEMANTIC MATCHING)
|
|
403
|
+
# --------------------
|
|
404
|
+
if cfg.allowed_types and "*" not in cfg.allowed_types:
|
|
405
|
+
if not any(t.lower() in path_str.split("/") for t in cfg.allowed_types):
|
|
406
|
+
print(f"[allowed_types] → {path}")
|
|
407
|
+
return False
|
|
408
|
+
|
|
409
|
+
if cfg.exclude_types:
|
|
410
|
+
if any(t.lower() in path_str.split("/") for t in cfg.exclude_types):
|
|
411
|
+
print(f"[exclude_types] → {path}")
|
|
412
|
+
return False
|
|
413
|
+
|
|
414
|
+
return True
|
|
415
|
+
|
|
416
|
+
# Preserve real name for debugging and repr
|
|
417
|
+
allowed.__name__ = "allowed"
|
|
418
|
+
return allowed
|
|
419
|
+
|
|
420
|
+
def filter_allowed_items(items, cfg=None, **kwargs):
|
|
421
|
+
"""
|
|
422
|
+
Apply ScanConfig allow/exclude rules to a flat list of file or directory paths.
|
|
423
|
+
No recursion. No globs. No shell calls.
|
|
424
|
+
Just pure deterministic filtering.
|
|
425
|
+
"""
|
|
426
|
+
allowed_items = []
|
|
427
|
+
allowed = make_allowed_predicate(cfg=cfg, **kwargs)
|
|
428
|
+
for item in items:
|
|
429
|
+
if allowed(item):
|
|
430
|
+
allowed_items.append(item)
|
|
431
|
+
|
|
432
|
+
return allowed_items
|
|
433
|
+
|
|
434
|
+
def derive_all_defaults(**kwargs):
|
|
435
|
+
kwargs = get_safe_canonical_kwargs(**kwargs)
|
|
436
|
+
add = kwargs.get("add",False)
|
|
437
|
+
nu_defaults = {}
|
|
438
|
+
for key,values in DEFAULT_CANONICAL_MAP.items():
|
|
439
|
+
default = values.get("default")
|
|
440
|
+
typ = values.get("type")
|
|
441
|
+
key_value = kwargs.get(key)
|
|
442
|
+
if key in DEFAULT_ALLOWED_EXCLUDE_MAP:
|
|
443
|
+
|
|
444
|
+
if key.endswith('exts'):
|
|
445
|
+
input_value = ensure_exts(key_value)
|
|
446
|
+
if key.endswith('patterns'):
|
|
447
|
+
input_value = ensure_patterns(key_value)
|
|
448
|
+
else:
|
|
449
|
+
input_value = normalize_listlike(key_value, typ)
|
|
450
|
+
nu_defaults[key] = _get_default_modular(input_value, default, add, typ)
|
|
451
|
+
else:
|
|
452
|
+
value = default if key_value is None else key_value
|
|
453
|
+
if typ == list:
|
|
454
|
+
value = make_list(value)
|
|
455
|
+
elif typ == bool:
|
|
456
|
+
value = bool(value)
|
|
457
|
+
nu_defaults[key] = value
|
|
458
|
+
|
|
459
|
+
return nu_defaults
|
|
460
|
+
# -------------------------
|
|
461
|
+
# Default derivation logic
|
|
462
|
+
# -------------------------
|
|
463
|
+
def derive_file_defaults(**kwargs):
|
|
464
|
+
kwargs = derive_all_defaults(**kwargs)
|
|
465
|
+
add = kwargs.get("add",True)
|
|
466
|
+
nu_defaults = {}
|
|
467
|
+
for key,values in DEFAULT_ALLOWED_EXCLUDE_MAP.items():
|
|
468
|
+
default = values.get("default")
|
|
469
|
+
typ = values.get("type")
|
|
470
|
+
key_value = kwargs.get(key)
|
|
471
|
+
if key.endswith('exts'):
|
|
472
|
+
input_value = ensure_exts(key_value)
|
|
473
|
+
if key.endswith('patterns'):
|
|
474
|
+
input_value = ensure_patterns(key_value)
|
|
475
|
+
else:
|
|
476
|
+
input_value = normalize_listlike(key_value, typ)
|
|
477
|
+
nu_defaults[key] = _get_default_modular(input_value, default, add, typ)
|
|
478
|
+
return nu_defaults
|
|
479
|
+
|
|
480
|
+
def define_defaults(**kwargs):
|
|
481
|
+
defaults = derive_file_defaults(**kwargs)
|
|
482
|
+
return ScanConfig(**defaults)
|
|
483
|
+
|
|
484
|
+
def get_file_filters(*args,**kwargs):
|
|
485
|
+
directories = ensure_directories(*args,**kwargs)
|
|
486
|
+
recursive = kwargs.get('recursive',True)
|
|
487
|
+
include_files = kwargs.get('include_files',True)
|
|
488
|
+
cfg = define_defaults(**kwargs)
|
|
489
|
+
allowed = kwargs.get("allowed") or make_allowed_predicate(cfg)
|
|
490
|
+
return directories,cfg,allowed,include_files,recursive
|
|
@@ -27,6 +27,70 @@ def _get_default_modular(value, default, add=None, typ=set):
|
|
|
27
27
|
if add:
|
|
28
28
|
return combine_params(value,default,typ=None)
|
|
29
29
|
return typ(value)
|
|
30
|
+
def make_allowed_predicate(cfg: ScanConfig=None,**kwargs) -> Callable[[str], bool]:
|
|
31
|
+
"""
|
|
32
|
+
Build a predicate that returns True if a given path is considered allowed
|
|
33
|
+
under the given ScanConfig. Applies allowed_* and exclude_* logic symmetrically.
|
|
34
|
+
"""
|
|
35
|
+
cfg=cfg or define_defaults(**kwargs)
|
|
36
|
+
def allowed(path: str=None,p=None) -> bool:
|
|
37
|
+
p = p or Path(path)
|
|
38
|
+
name = p.name.lower()
|
|
39
|
+
path_str = str(p).lower()
|
|
40
|
+
|
|
41
|
+
# --------------------
|
|
42
|
+
# A) directory filters
|
|
43
|
+
# --------------------
|
|
44
|
+
if cfg.exclude_dirs:
|
|
45
|
+
for dpat in cfg.exclude_dirs:
|
|
46
|
+
dpat_l = dpat.lower()
|
|
47
|
+
if dpat_l in path_str or fnmatch.fnmatch(name, dpat_l):
|
|
48
|
+
if p.is_dir() or dpat_l in path_str:
|
|
49
|
+
return False
|
|
50
|
+
|
|
51
|
+
if cfg.allowed_dirs and cfg.allowed_dirs != ["*"]:
|
|
52
|
+
# must be in at least one allowed dir
|
|
53
|
+
if not any(
|
|
54
|
+
fnmatch.fnmatch(path_str, f"*{dpat.lower()}*") for dpat in cfg.allowed_dirs
|
|
55
|
+
):
|
|
56
|
+
return False
|
|
57
|
+
|
|
58
|
+
# --------------------
|
|
59
|
+
# B) pattern filters
|
|
60
|
+
# --------------------
|
|
61
|
+
if cfg.allowed_patterns and cfg.allowed_patterns != ["*"]:
|
|
62
|
+
if not any(fnmatch.fnmatch(name, pat.lower()) for pat in cfg.allowed_patterns):
|
|
63
|
+
return False
|
|
64
|
+
|
|
65
|
+
if cfg.exclude_patterns:
|
|
66
|
+
for pat in cfg.exclude_patterns:
|
|
67
|
+
if fnmatch.fnmatch(name, pat.lower()):
|
|
68
|
+
return False
|
|
69
|
+
|
|
70
|
+
# --------------------
|
|
71
|
+
# C) extension filters
|
|
72
|
+
# --------------------
|
|
73
|
+
if p.is_file():
|
|
74
|
+
ext = p.suffix.lower()
|
|
75
|
+
if cfg.allowed_exts and ext not in cfg.allowed_exts:
|
|
76
|
+
return False
|
|
77
|
+
if cfg.exclude_exts and ext in cfg.exclude_exts:
|
|
78
|
+
return False
|
|
79
|
+
|
|
80
|
+
# --------------------
|
|
81
|
+
# D) type filters (optional)
|
|
82
|
+
# --------------------
|
|
83
|
+
if cfg.allowed_types and cfg.allowed_types != {"*"}:
|
|
84
|
+
if not any(t in path_str for t in cfg.allowed_types):
|
|
85
|
+
return False
|
|
86
|
+
if cfg.exclude_types and cfg.exclude_types != {"*"}:
|
|
87
|
+
if any(t in path_str for t in cfg.exclude_types):
|
|
88
|
+
return False
|
|
89
|
+
|
|
90
|
+
return True
|
|
91
|
+
|
|
92
|
+
return allowed
|
|
93
|
+
|
|
30
94
|
def derive_all_defaults(**kwargs):
|
|
31
95
|
kwargs = get_safe_canonical_kwargs(**kwargs)
|
|
32
96
|
add = kwargs.get("add",False)
|
|
@@ -1,16 +1,8 @@
|
|
|
1
1
|
from .ensure_utils import *
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
if allowed == True:
|
|
5
|
-
allowed = None
|
|
6
|
-
allowed = allowed or make_allowed_predicate()
|
|
7
|
-
else:
|
|
8
|
-
def allowed(*args):
|
|
9
|
-
return True
|
|
10
|
-
allowed = allowed
|
|
11
|
-
return allowed
|
|
12
|
-
def get_globs(items,recursive: bool = True,allowed=None):
|
|
2
|
+
|
|
3
|
+
def get_globs(items,recursive: bool = True,allowed=None,cfg=None,**kwargs):
|
|
13
4
|
glob_paths = []
|
|
5
|
+
allowed = get_allowed_predicate(allowed=allowed,cfg=cfg,**kwargs)
|
|
14
6
|
items = [item for item in make_list(items) if item]
|
|
15
7
|
for item in items:
|
|
16
8
|
pattern = os.path.join(item, "**/*") # include all files recursively\n
|
|
@@ -19,96 +11,34 @@ def get_globs(items,recursive: bool = True,allowed=None):
|
|
|
19
11
|
nuItems = [nuItem for nuItem in nuItems if nuItem and allowed(nuItem)]
|
|
20
12
|
glob_paths += nuItems
|
|
21
13
|
return glob_paths
|
|
22
|
-
def get_allowed_files(items,allowed=True):
|
|
23
|
-
allowed = get_allowed_predicate(allowed=allowed)
|
|
14
|
+
def get_allowed_files(items,allowed=True,cfg=None,**kwargs):
|
|
15
|
+
allowed = get_allowed_predicate(allowed=allowed,cfg=cfg,**kwargs)
|
|
24
16
|
return [item for item in items if item and os.path.isfile(item) and allowed(item)]
|
|
25
|
-
def get_allowed_dirs(items,allowed=False):
|
|
26
|
-
allowed = get_allowed_predicate(allowed=allowed)
|
|
17
|
+
def get_allowed_dirs(items,allowed=False,cfg=None,**kwargs):
|
|
18
|
+
allowed = get_allowed_predicate(allowed=allowed,cfg=cfg,**kwargs)
|
|
27
19
|
return [item for item in items if item and os.path.isdir(item) and allowed(item)]
|
|
28
20
|
|
|
29
|
-
def get_filtered_files(items,allowed=None,files = []):
|
|
30
|
-
allowed = get_allowed_predicate(allowed=allowed)
|
|
31
|
-
glob_paths = get_globs(items)
|
|
21
|
+
def get_filtered_files(items,allowed=None,files = [],cfg=None,**kwargs):
|
|
22
|
+
allowed = get_allowed_predicate(allowed=allowed,cfg=cfg,**kwargs)
|
|
23
|
+
glob_paths = get_globs(items,allowed=allowed,cfg=cfg,**kwargs)
|
|
32
24
|
return [glob_path for glob_path in glob_paths if glob_path and os.path.isfile(glob_path) and glob_path not in files and allowed(glob_path)]
|
|
33
|
-
def get_filtered_dirs(items,allowed=None,dirs = []):
|
|
34
|
-
allowed = get_allowed_predicate(allowed=allowed)
|
|
35
|
-
glob_paths = get_globs(items)
|
|
25
|
+
def get_filtered_dirs(items,allowed=None,dirs = [],cfg=None,**kwargs):
|
|
26
|
+
allowed = get_allowed_predicate(allowed=allowed,cfg=cfg,**kwargs)
|
|
27
|
+
glob_paths = get_globs(items,allowed=allowed,cfg=cfg,**kwargs)
|
|
36
28
|
return [glob_path for glob_path in glob_paths if glob_path and os.path.isdir(glob_path) and glob_path not in dirs and allowed(glob_path)]
|
|
37
29
|
|
|
38
|
-
def get_all_allowed_files(items,allowed=None):
|
|
39
|
-
dirs = get_all_allowed_dirs(items)
|
|
40
|
-
files = get_allowed_files(items)
|
|
30
|
+
def get_all_allowed_files(items,allowed=None,cfg=None,**kwargs):
|
|
31
|
+
dirs = get_all_allowed_dirs(items,allowed=allowed,cfg=cfg,**kwargs)
|
|
32
|
+
files = get_allowed_files(items,allowed=allowed,cfg=cfg,**kwargs)
|
|
41
33
|
nu_files = []
|
|
42
34
|
for directory in dirs:
|
|
43
|
-
files += get_filtered_files(directory,allowed=allowed,files=files)
|
|
35
|
+
files += get_filtered_files(directory,allowed=allowed,files=files,cfg=cfg,**kwargs)
|
|
44
36
|
return files
|
|
45
|
-
def get_all_allowed_dirs(items,allowed=None):
|
|
46
|
-
allowed = get_allowed_predicate(allowed=allowed)
|
|
47
|
-
dirs = get_allowed_dirs(items)
|
|
37
|
+
def get_all_allowed_dirs(items,allowed=None,cfg=None,**kwargs):
|
|
38
|
+
allowed = get_allowed_predicate(allowed=allowed,cfg=cfg,**kwargs)
|
|
39
|
+
dirs = get_allowed_dirs(items,allowed=allowed,cfg=cfg,**kwargs)
|
|
48
40
|
nu_dirs=[]
|
|
49
41
|
for directory in dirs:
|
|
50
|
-
nu_dirs += get_filtered_dirs(directory,allowed=allowed,dirs=nu_dirs)
|
|
42
|
+
nu_dirs += get_filtered_dirs(directory,allowed=allowed,dirs=nu_dirs,cfg=cfg,**kwargs)
|
|
51
43
|
return nu_dirs
|
|
52
44
|
|
|
53
|
-
def make_allowed_predicate(cfg: ScanConfig) -> Callable[[str], bool]:
|
|
54
|
-
"""
|
|
55
|
-
Build a predicate that returns True if a given path is considered allowed
|
|
56
|
-
under the given ScanConfig. Applies allowed_* and exclude_* logic symmetrically.
|
|
57
|
-
"""
|
|
58
|
-
def allowed(path: str=None,p=None) -> bool:
|
|
59
|
-
p = p or Path(path)
|
|
60
|
-
name = p.name.lower()
|
|
61
|
-
path_str = str(p).lower()
|
|
62
|
-
|
|
63
|
-
# --------------------
|
|
64
|
-
# A) directory filters
|
|
65
|
-
# --------------------
|
|
66
|
-
if cfg.exclude_dirs:
|
|
67
|
-
for dpat in cfg.exclude_dirs:
|
|
68
|
-
dpat_l = dpat.lower()
|
|
69
|
-
if dpat_l in path_str or fnmatch.fnmatch(name, dpat_l):
|
|
70
|
-
if p.is_dir() or dpat_l in path_str:
|
|
71
|
-
return False
|
|
72
|
-
|
|
73
|
-
if cfg.allowed_dirs and cfg.allowed_dirs != ["*"]:
|
|
74
|
-
# must be in at least one allowed dir
|
|
75
|
-
if not any(
|
|
76
|
-
fnmatch.fnmatch(path_str, f"*{dpat.lower()}*") for dpat in cfg.allowed_dirs
|
|
77
|
-
):
|
|
78
|
-
return False
|
|
79
|
-
|
|
80
|
-
# --------------------
|
|
81
|
-
# B) pattern filters
|
|
82
|
-
# --------------------
|
|
83
|
-
if cfg.allowed_patterns and cfg.allowed_patterns != ["*"]:
|
|
84
|
-
if not any(fnmatch.fnmatch(name, pat.lower()) for pat in cfg.allowed_patterns):
|
|
85
|
-
return False
|
|
86
|
-
|
|
87
|
-
if cfg.exclude_patterns:
|
|
88
|
-
for pat in cfg.exclude_patterns:
|
|
89
|
-
if fnmatch.fnmatch(name, pat.lower()):
|
|
90
|
-
return False
|
|
91
|
-
|
|
92
|
-
# --------------------
|
|
93
|
-
# C) extension filters
|
|
94
|
-
# --------------------
|
|
95
|
-
if p.is_file():
|
|
96
|
-
ext = p.suffix.lower()
|
|
97
|
-
if cfg.allowed_exts and ext not in cfg.allowed_exts:
|
|
98
|
-
return False
|
|
99
|
-
if cfg.exclude_exts and ext in cfg.exclude_exts:
|
|
100
|
-
return False
|
|
101
|
-
|
|
102
|
-
# --------------------
|
|
103
|
-
# D) type filters (optional)
|
|
104
|
-
# --------------------
|
|
105
|
-
if cfg.allowed_types and cfg.allowed_types != {"*"}:
|
|
106
|
-
if not any(t in path_str for t in cfg.allowed_types):
|
|
107
|
-
return False
|
|
108
|
-
if cfg.exclude_types and cfg.exclude_types != {"*"}:
|
|
109
|
-
if any(t in path_str for t in cfg.exclude_types):
|
|
110
|
-
return False
|
|
111
|
-
|
|
112
|
-
return True
|
|
113
|
-
|
|
114
|
-
return allowed
|
|
@@ -188,3 +188,13 @@ def collect_filepaths(
|
|
|
188
188
|
) -> List[str]:
|
|
189
189
|
kwargs['file_type']='f'
|
|
190
190
|
return collect_globs(*args,**kwargs)
|
|
191
|
+
|
|
192
|
+
def get_filename(path):
|
|
193
|
+
basename = os.path.basename(path)
|
|
194
|
+
filename,ext = os.path.splitext(basename)
|
|
195
|
+
return filename
|
|
196
|
+
def find_files(filename,directory=None,add=None):
|
|
197
|
+
add = if_not_bool_default(add,default=True)
|
|
198
|
+
directory = directory or os.getcwd()
|
|
199
|
+
dirs,files = get_files_and_dirs(directory,add=add)
|
|
200
|
+
return [file for file in files if get_filename(file) == filename]
|