abstract-utilities 0.2.2.413__py3-none-any.whl → 0.2.2.415__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of abstract-utilities might be problematic. Click here for more details.
- abstract_utilities/__init__.py +12 -9
- abstract_utilities/cmd_utils/imports/__init__.py +1 -0
- abstract_utilities/cmd_utils/imports/imports.py +10 -0
- abstract_utilities/cmd_utils/pexpect_utils.py +310 -0
- abstract_utilities/cmd_utils/user_utils.py +1 -1
- abstract_utilities/env_utils/__init__.py +3 -0
- abstract_utilities/env_utils/abstractEnv.py +129 -0
- abstract_utilities/env_utils/envy_it.py +33 -0
- abstract_utilities/env_utils/imports/__init__.py +2 -0
- abstract_utilities/env_utils/imports/imports.py +8 -0
- abstract_utilities/env_utils/imports/utils.py +122 -0
- abstract_utilities/{path_utils → file_utils}/__init__.py +0 -3
- abstract_utilities/file_utils/file_utils/__init__.py +4 -0
- abstract_utilities/file_utils/file_utils/file_filters.py +104 -0
- abstract_utilities/file_utils/file_utils/file_utils.py +194 -0
- abstract_utilities/file_utils/file_utils/imports.py +1 -0
- abstract_utilities/{path_utils → file_utils}/imports/__init__.py +1 -0
- abstract_utilities/file_utils/imports/classes.py +125 -0
- abstract_utilities/file_utils/imports/imports.py +11 -0
- abstract_utilities/{path_utils → file_utils}/imports/module_imports.py +3 -1
- abstract_utilities/{path_utils → file_utils}/req.py +2 -5
- abstract_utilities/{path_utils/path_utils.py → path_utils.py} +5 -1
- abstract_utilities/robust_readers/__init__.py +1 -0
- abstract_utilities/robust_readers/imports.py +1 -1
- abstract_utilities/ssh_utils/__init__.py +3 -0
- abstract_utilities/ssh_utils/classes.py +127 -0
- abstract_utilities/ssh_utils/imports.py +11 -0
- abstract_utilities/ssh_utils/pexpect_utils.py +315 -0
- abstract_utilities/ssh_utils/utils.py +122 -0
- {abstract_utilities-0.2.2.413.dist-info → abstract_utilities-0.2.2.415.dist-info}/METADATA +1 -1
- {abstract_utilities-0.2.2.413.dist-info → abstract_utilities-0.2.2.415.dist-info}/RECORD +36 -18
- abstract_utilities/path_utils/file_filters.py +0 -213
- abstract_utilities/path_utils/imports/imports.py +0 -6
- /abstract_utilities/{path_utils → file_utils/file_utils}/filter_params.py +0 -0
- /abstract_utilities/{path_utils/file_utils.py → file_utils/file_utils/map_utils.py} +0 -0
- /abstract_utilities/{path_utils → file_utils}/imports/constants.py +0 -0
- {abstract_utilities-0.2.2.413.dist-info → abstract_utilities-0.2.2.415.dist-info}/WHEEL +0 -0
- {abstract_utilities-0.2.2.413.dist-info → abstract_utilities-0.2.2.415.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
from .imports import *
|
|
2
|
+
from .filter_params import *
|
|
3
|
+
from .file_utils import *
|
|
4
|
+
##from abstract_utilities import make_list,get_media_exts, is_media_type
|
|
5
|
+
|
|
6
|
+
def collect_filepaths(
|
|
7
|
+
directory: List[str],
|
|
8
|
+
cfg: ScanConfig=None,
|
|
9
|
+
allowed_exts: Optional[Set[str]] = False,
|
|
10
|
+
unallowed_exts: Optional[Set[str]] = False,
|
|
11
|
+
exclude_types: Optional[Set[str]] = False,
|
|
12
|
+
exclude_dirs: Optional[List[str]] = False,
|
|
13
|
+
exclude_patterns: Optional[List[str]] = False,
|
|
14
|
+
add=False,
|
|
15
|
+
allowed: Optional[Callable[[str], bool]] = None,
|
|
16
|
+
**kwargs
|
|
17
|
+
) -> List[str]:
|
|
18
|
+
cfg = cfg or define_defaults(
|
|
19
|
+
allowed_exts=allowed_exts,
|
|
20
|
+
unallowed_exts=unallowed_exts,
|
|
21
|
+
exclude_types=exclude_types,
|
|
22
|
+
exclude_dirs=exclude_dirs,
|
|
23
|
+
exclude_patterns=exclude_patterns,
|
|
24
|
+
add = add
|
|
25
|
+
)
|
|
26
|
+
allowed = allowed or make_allowed_predicate(cfg)
|
|
27
|
+
directories = make_list(directory)
|
|
28
|
+
roots = [r for r in directories if r]
|
|
29
|
+
|
|
30
|
+
# your existing helpers (get_dirs, get_globs, etc.) stay the same
|
|
31
|
+
original_dirs = get_allowed_dirs(roots, allowed=allowed)
|
|
32
|
+
original_globs = get_globs(original_dirs)
|
|
33
|
+
files = get_allowed_files(original_globs, allowed=allowed)
|
|
34
|
+
|
|
35
|
+
for d in get_filtered_dirs(original_dirs, allowed=allowed):
|
|
36
|
+
files += get_filtered_files(d, allowed=allowed, files=files)
|
|
37
|
+
|
|
38
|
+
# de-dupe while preserving order
|
|
39
|
+
seen, out = set(), []
|
|
40
|
+
for f in files:
|
|
41
|
+
if f not in seen:
|
|
42
|
+
seen.add(f)
|
|
43
|
+
out.append(f)
|
|
44
|
+
return out
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _fast_walk(
|
|
48
|
+
root: Path,
|
|
49
|
+
exts: Iterable[str],
|
|
50
|
+
skip_dirs: Iterable[str] = (),
|
|
51
|
+
skip_patterns: Iterable[str] = (),
|
|
52
|
+
) -> List[Path]:
|
|
53
|
+
exts = tuple(exts)
|
|
54
|
+
skip_dirs = set(sd.lower() for sd in skip_dirs or ())
|
|
55
|
+
skip_patterns = tuple(sp.lower() for sp in (skip_patterns or ()))
|
|
56
|
+
|
|
57
|
+
out = []
|
|
58
|
+
for p in root.rglob("*"):
|
|
59
|
+
# skip directories by name hit
|
|
60
|
+
if p.is_dir():
|
|
61
|
+
name = p.name.lower()
|
|
62
|
+
if name in skip_dirs:
|
|
63
|
+
# rglob doesn't let us prune mid-iteration cleanly; we just won't collect under it
|
|
64
|
+
continue
|
|
65
|
+
# nothing to collect for dirs
|
|
66
|
+
continue
|
|
67
|
+
|
|
68
|
+
# file filters
|
|
69
|
+
name = p.name.lower()
|
|
70
|
+
if any(fnmatch.fnmatch(name, pat) for pat in skip_patterns):
|
|
71
|
+
continue
|
|
72
|
+
if p.suffix.lower() in exts:
|
|
73
|
+
out.append(p)
|
|
74
|
+
|
|
75
|
+
# de-dup and normalize
|
|
76
|
+
return sorted({pp.resolve() for pp in out})
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def enumerate_source_files(
|
|
80
|
+
src_root: Path,
|
|
81
|
+
cfg: Optional["ScanConfig"] = None,
|
|
82
|
+
*,
|
|
83
|
+
exts: Optional[Iterable[str]] = None,
|
|
84
|
+
fast_skip_dirs: Optional[Iterable[str]] = None,
|
|
85
|
+
fast_skip_patterns: Optional[Iterable[str]] = None,
|
|
86
|
+
) -> List[Path]:
|
|
87
|
+
"""
|
|
88
|
+
Unified enumerator:
|
|
89
|
+
- If `cfg` is provided: use collect_filepaths(...) with full rules.
|
|
90
|
+
- Else: fast walk using rglob over `exts` (defaults to EXTS) with optional light excludes.
|
|
91
|
+
"""
|
|
92
|
+
src_root = Path(src_root)
|
|
93
|
+
|
|
94
|
+
if cfg is not None:
|
|
95
|
+
files = collect_filepaths([str(src_root)], cfg=cfg)
|
|
96
|
+
return sorted({Path(f).resolve() for f in files})
|
|
97
|
+
|
|
98
|
+
# Fast mode
|
|
99
|
+
return _fast_walk(
|
|
100
|
+
src_root,
|
|
101
|
+
exts or EXTS,
|
|
102
|
+
skip_dirs=fast_skip_dirs or (),
|
|
103
|
+
skip_patterns=fast_skip_patterns or (),
|
|
104
|
+
)
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
from .imports import *
|
|
2
|
+
# -------- Public API drop-ins that mirror your originals --------
|
|
3
|
+
from .filter_params import *
|
|
4
|
+
from .file_filters import *
|
|
5
|
+
def make_allowed_predicate(cfg: ScanConfig, *, isdir=None, isfile=None) -> Callable[[str], bool]:
|
|
6
|
+
def _isdir(p: str) -> bool: return isdir(p) if isdir else Path(p).is_dir()
|
|
7
|
+
def _isfile(p: str) -> bool: return isfile(p) if isfile else Path(p).is_file()
|
|
8
|
+
def allowed(path: str) -> bool:
|
|
9
|
+
name = os.path.basename(path).lower()
|
|
10
|
+
path_str = path.lower()
|
|
11
|
+
|
|
12
|
+
if cfg.exclude_dirs:
|
|
13
|
+
for dpat in cfg.exclude_dirs:
|
|
14
|
+
d = dpat.lower()
|
|
15
|
+
if d in path_str or fnmatch.fnmatch(name, d):
|
|
16
|
+
if _isdir(path) or d in path_str:
|
|
17
|
+
return False
|
|
18
|
+
|
|
19
|
+
if cfg.exclude_patterns:
|
|
20
|
+
for pat in cfg.exclude_patterns:
|
|
21
|
+
if fnmatch.fnmatch(name, pat.lower()):
|
|
22
|
+
return False
|
|
23
|
+
|
|
24
|
+
if _isfile(path):
|
|
25
|
+
ext = os.path.splitext(name)[1].lower()
|
|
26
|
+
if (cfg.allowed_exts and ext not in cfg.allowed_exts) or (cfg.unallowed_exts and ext in cfg.unallowed_exts):
|
|
27
|
+
return False
|
|
28
|
+
return True
|
|
29
|
+
return allowed
|
|
30
|
+
|
|
31
|
+
def get_globs(paths, recursive: bool = True, allowed=None, **kwargs) -> List[str]:
|
|
32
|
+
roots = [p for p in make_list(paths) if p]
|
|
33
|
+
res: List[str] = []
|
|
34
|
+
if recursive:
|
|
35
|
+
kwargs.setdefault("mindepth", 1)
|
|
36
|
+
else:
|
|
37
|
+
kwargs.setdefault("mindepth", 1)
|
|
38
|
+
kwargs.setdefault("maxdepth", 1)
|
|
39
|
+
for fs, root in normalize_items(roots, **kwargs):
|
|
40
|
+
gl = fs.glob_recursive(root, **kwargs)
|
|
41
|
+
if allowed:
|
|
42
|
+
gl = [p for p in gl if allowed(p)]
|
|
43
|
+
res.extend(gl)
|
|
44
|
+
return res
|
|
45
|
+
|
|
46
|
+
def get_allowed_files(paths, allowed=True, **kwargs) -> List[str]:
|
|
47
|
+
if allowed is not False:
|
|
48
|
+
if allowed is True:
|
|
49
|
+
allowed = None
|
|
50
|
+
allowed = allowed or make_allowed_predicate(ScanConfig())
|
|
51
|
+
elif allowed is False:
|
|
52
|
+
allowed = (lambda *_: True)
|
|
53
|
+
|
|
54
|
+
roots = [p for p in make_list(paths) if p]
|
|
55
|
+
out: List[str] = []
|
|
56
|
+
kwargs = {**kwargs, "include_files": True, "include_dirs": False}
|
|
57
|
+
for fs, root in normalize_items(roots):
|
|
58
|
+
if fs.isdir(root):
|
|
59
|
+
for p in fs.glob_recursive(root, **kwargs):
|
|
60
|
+
if allowed and not allowed(p): continue
|
|
61
|
+
if fs.isfile(p): out.append(p)
|
|
62
|
+
else:
|
|
63
|
+
if allowed and not allowed(root): continue
|
|
64
|
+
if fs.isfile(root): out.append(root)
|
|
65
|
+
return out
|
|
66
|
+
|
|
67
|
+
def get_allowed_dirs(paths, allowed=False, **kwargs) -> List[str]:
|
|
68
|
+
if allowed is not False:
|
|
69
|
+
if allowed is True:
|
|
70
|
+
allowed = None
|
|
71
|
+
allowed = allowed or make_allowed_predicate(ScanConfig())
|
|
72
|
+
else:
|
|
73
|
+
allowed = (lambda *_: True)
|
|
74
|
+
|
|
75
|
+
roots = [p for p in make_list(paths) if p]
|
|
76
|
+
out: List[str] = []
|
|
77
|
+
kwargs = {**kwargs, "include_files": False, "include_dirs": True}
|
|
78
|
+
for fs, root in normalize_items(roots):
|
|
79
|
+
if fs.isdir(root):
|
|
80
|
+
if (not allowed) or allowed(root):
|
|
81
|
+
out.append(root)
|
|
82
|
+
for p in fs.glob_recursive(root, **kwargs):
|
|
83
|
+
if (not allowed) or allowed(p):
|
|
84
|
+
if fs.isdir(p): out.append(p)
|
|
85
|
+
return out
|
|
86
|
+
|
|
87
|
+
def get_filtered_files(paths, allowed=None, files: List[str] | None = None, **kwargs) -> List[str]:
|
|
88
|
+
if allowed is not False:
|
|
89
|
+
if allowed is True:
|
|
90
|
+
allowed = None
|
|
91
|
+
allowed = allowed or make_allowed_predicate(ScanConfig())
|
|
92
|
+
else:
|
|
93
|
+
allowed = (lambda *_: True)
|
|
94
|
+
|
|
95
|
+
files = files or []
|
|
96
|
+
roots = [p for p in make_list(paths) if p]
|
|
97
|
+
out: List[str] = []
|
|
98
|
+
kwargs = {**kwargs, "include_files": True, "include_dirs": False}
|
|
99
|
+
for fs, root in normalize_items(roots):
|
|
100
|
+
for p in fs.glob_recursive(root, **kwargs):
|
|
101
|
+
if p in files: continue
|
|
102
|
+
if allowed and not allowed(p): continue
|
|
103
|
+
if fs.isfile(p): out.append(p)
|
|
104
|
+
return out
|
|
105
|
+
|
|
106
|
+
def get_filtered_dirs(paths, allowed=None, dirs: List[str] | None = None, **kwargs) -> List[str]:
|
|
107
|
+
if allowed is not False:
|
|
108
|
+
if allowed is True:
|
|
109
|
+
allowed = None
|
|
110
|
+
allowed = allowed or make_allowed_predicate(ScanConfig())
|
|
111
|
+
else:
|
|
112
|
+
allowed = (lambda *_: True)
|
|
113
|
+
|
|
114
|
+
dirs = dirs or []
|
|
115
|
+
roots = [p for p in make_list(paths) if p]
|
|
116
|
+
out: List[str] = []
|
|
117
|
+
kwargs = {**kwargs, "include_files": False, "include_dirs": True}
|
|
118
|
+
for fs, root in normalize_items(roots):
|
|
119
|
+
for p in fs.glob_recursive(root, **kwargs):
|
|
120
|
+
if p in dirs: continue
|
|
121
|
+
if allowed and not allowed(p): continue
|
|
122
|
+
if fs.isdir(p): out.append(p)
|
|
123
|
+
return out
|
|
124
|
+
|
|
125
|
+
def get_all_allowed_files(paths, allowed=None, **kwargs) -> List[str]:
|
|
126
|
+
dirs = get_all_allowed_dirs(paths, allowed=allowed, **kwargs)
|
|
127
|
+
files = get_allowed_files(paths, allowed=allowed, **kwargs)
|
|
128
|
+
seen = set(files)
|
|
129
|
+
for fs, directory in normalize_items(dirs):
|
|
130
|
+
for p in fs.glob_recursive(directory, **kwargs):
|
|
131
|
+
if p in seen: continue
|
|
132
|
+
if allowed and not allowed(p): continue
|
|
133
|
+
if fs.isfile(p):
|
|
134
|
+
files.append(p); seen.add(p)
|
|
135
|
+
return files
|
|
136
|
+
|
|
137
|
+
def get_all_allowed_dirs(paths, allowed=None, **kwargs) -> List[str]:
|
|
138
|
+
if allowed is not False:
|
|
139
|
+
if allowed is True:
|
|
140
|
+
allowed = None
|
|
141
|
+
allowed = allowed or make_allowed_predicate(ScanConfig())
|
|
142
|
+
else:
|
|
143
|
+
allowed = (lambda *_: True)
|
|
144
|
+
|
|
145
|
+
roots = [p for p in make_list(paths) if p]
|
|
146
|
+
out: List[str] = []
|
|
147
|
+
seen = set()
|
|
148
|
+
kwargs = {**kwargs, "include_dirs": True}
|
|
149
|
+
for fs, root in normalize_items(roots):
|
|
150
|
+
if fs.isdir(root) and ((not allowed) or allowed(root)):
|
|
151
|
+
out.append(root); seen.add(root)
|
|
152
|
+
for p in fs.glob_recursive(root, **kwargs):
|
|
153
|
+
if p in seen: continue
|
|
154
|
+
if allowed and not allowed(p): continue
|
|
155
|
+
if fs.isdir(p):
|
|
156
|
+
out.append(p); seen.add(p)
|
|
157
|
+
return out
|
|
158
|
+
def get_files_and_dirs(
|
|
159
|
+
directory: str | list[str]=None,
|
|
160
|
+
cfg: Optional["ScanConfig"] = None,
|
|
161
|
+
allowed_exts: Optional[Set[str]] = False,
|
|
162
|
+
unallowed_exts: Optional[Set[str]] = False,
|
|
163
|
+
exclude_types: Optional[Set[str]] = False,
|
|
164
|
+
exclude_dirs: Optional[List[str]] = False,
|
|
165
|
+
exclude_patterns: Optional[List[str]] = False,
|
|
166
|
+
add: bool = False,
|
|
167
|
+
recursive: bool = True,
|
|
168
|
+
include_files: bool = True,
|
|
169
|
+
paths: str | list[str]=None,
|
|
170
|
+
**kwargs
|
|
171
|
+
):
|
|
172
|
+
cfg = cfg or define_defaults(
|
|
173
|
+
allowed_exts=allowed_exts,
|
|
174
|
+
unallowed_exts=unallowed_exts,
|
|
175
|
+
exclude_types=exclude_types,
|
|
176
|
+
exclude_dirs=exclude_dirs,
|
|
177
|
+
exclude_patterns=exclude_patterns,
|
|
178
|
+
add=add,
|
|
179
|
+
)
|
|
180
|
+
# make predicate backend-agnostic here; the per-backend calls below pass strings only
|
|
181
|
+
allowed = make_allowed_predicate(cfg)
|
|
182
|
+
|
|
183
|
+
if recursive:
|
|
184
|
+
kwargs.setdefault("mindepth", 1)
|
|
185
|
+
else:
|
|
186
|
+
kwargs.setdefault("mindepth", 1)
|
|
187
|
+
kwargs.setdefault("maxdepth", 1)
|
|
188
|
+
|
|
189
|
+
roots = make_list(directory or paths)
|
|
190
|
+
items = get_globs(roots, recursive=recursive, allowed=allowed, **kwargs)
|
|
191
|
+
|
|
192
|
+
dirs = get_allowed_dirs(items, allowed=allowed, **kwargs)
|
|
193
|
+
files = get_allowed_files(items, allowed=allowed, **kwargs) if include_files else []
|
|
194
|
+
return dirs, files
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from ..imports import *
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
from .imports import *
|
|
2
|
+
from .module_imports import *
|
|
3
|
+
from .constants import *
|
|
4
|
+
class PathBackend(Protocol):
|
|
5
|
+
def join(self, *parts: str) -> str: ...
|
|
6
|
+
def isfile(self, path: str) -> bool: ...
|
|
7
|
+
def isdir(self, path: str) -> bool: ...
|
|
8
|
+
def glob_recursive(self, base: str, **opts) -> List[str]: ...
|
|
9
|
+
def listdir(self, base: str) -> List[str]: ...
|
|
10
|
+
|
|
11
|
+
class LocalFS:
|
|
12
|
+
def join(self, *parts: str) -> str:
|
|
13
|
+
return os.path.join(*parts)
|
|
14
|
+
def isfile(self, path: str) -> bool:
|
|
15
|
+
return os.path.isfile(path)
|
|
16
|
+
def isdir(self, path: str) -> bool:
|
|
17
|
+
return os.path.isdir(path)
|
|
18
|
+
def glob_recursive(self, base: str, **opts) -> List[str]:
|
|
19
|
+
"""
|
|
20
|
+
opts:
|
|
21
|
+
- maxdepth: int | None
|
|
22
|
+
- mindepth: int (default 1)
|
|
23
|
+
- follow_symlinks: bool
|
|
24
|
+
- include_dirs: bool
|
|
25
|
+
- include_files: bool
|
|
26
|
+
- exclude_hidden: bool
|
|
27
|
+
"""
|
|
28
|
+
maxdepth = opts.get("maxdepth")
|
|
29
|
+
mindepth = opts.get("mindepth", 1)
|
|
30
|
+
follow = opts.get("follow_symlinks", False)
|
|
31
|
+
want_d = opts.get("include_dirs", True)
|
|
32
|
+
want_f = opts.get("include_files", True)
|
|
33
|
+
hide = opts.get("exclude_hidden", False)
|
|
34
|
+
|
|
35
|
+
results: List[str] = []
|
|
36
|
+
base_depth = os.path.normpath(base).count(os.sep)
|
|
37
|
+
|
|
38
|
+
for root, dirs, files in os.walk(base, followlinks=follow):
|
|
39
|
+
depth = os.path.normpath(root).count(os.sep) - base_depth
|
|
40
|
+
if maxdepth is not None and depth > maxdepth:
|
|
41
|
+
dirs[:] = []
|
|
42
|
+
continue
|
|
43
|
+
if want_d and depth >= mindepth:
|
|
44
|
+
for d in dirs:
|
|
45
|
+
if hide and d.startswith("."): continue
|
|
46
|
+
results.append(os.path.join(root, d))
|
|
47
|
+
if want_f and depth >= mindepth:
|
|
48
|
+
for f in files:
|
|
49
|
+
if hide and f.startswith("."): continue
|
|
50
|
+
results.append(os.path.join(root, f))
|
|
51
|
+
return results
|
|
52
|
+
|
|
53
|
+
def listdir(self, base: str) -> List[str]:
|
|
54
|
+
try:
|
|
55
|
+
return [os.path.join(base, name) for name in os.listdir(base)]
|
|
56
|
+
except Exception:
|
|
57
|
+
return []
|
|
58
|
+
|
|
59
|
+
class SSHFS:
|
|
60
|
+
"""Remote POSIX backend via your run_remote_cmd."""
|
|
61
|
+
def __init__(self, user_at_host: str):
|
|
62
|
+
self.user_at_host = user_at_host
|
|
63
|
+
|
|
64
|
+
def join(self, *parts: str) -> str:
|
|
65
|
+
return posixpath.join(*parts)
|
|
66
|
+
|
|
67
|
+
def isfile(self, path: str) -> bool:
|
|
68
|
+
cmd = f"test -f {shlex.quote(path)} && echo __OK__ || true"
|
|
69
|
+
out = run_remote_cmd(self.user_at_host, cmd)
|
|
70
|
+
return "__OK__" in (out or "")
|
|
71
|
+
|
|
72
|
+
def isdir(self, path: str) -> bool:
|
|
73
|
+
cmd = f"test -d {shlex.quote(path)} && echo __OK__ || true"
|
|
74
|
+
out = run_remote_cmd(self.user_at_host, cmd)
|
|
75
|
+
return "__OK__" in (out or "")
|
|
76
|
+
|
|
77
|
+
def glob_recursive(self, base: str, **opts) -> List[str]:
|
|
78
|
+
maxdepth = opts.get("maxdepth")
|
|
79
|
+
mindepth = opts.get("mindepth", 1)
|
|
80
|
+
follow = opts.get("follow_symlinks", False)
|
|
81
|
+
want_d = opts.get("include_dirs", True)
|
|
82
|
+
want_f = opts.get("include_files", True)
|
|
83
|
+
hide = opts.get("exclude_hidden", False)
|
|
84
|
+
|
|
85
|
+
parts = []
|
|
86
|
+
if follow:
|
|
87
|
+
parts.append("-L")
|
|
88
|
+
parts += ["find", shlex.quote(base)]
|
|
89
|
+
if mindepth is not None:
|
|
90
|
+
parts += ["-mindepth", str(mindepth)]
|
|
91
|
+
if maxdepth is not None:
|
|
92
|
+
parts += ["-maxdepth", str(maxdepth)]
|
|
93
|
+
|
|
94
|
+
type_filters = []
|
|
95
|
+
if want_d and not want_f:
|
|
96
|
+
type_filters = ["-type", "d"]
|
|
97
|
+
elif want_f and not want_d:
|
|
98
|
+
type_filters = ["-type", "f"]
|
|
99
|
+
|
|
100
|
+
hidden_filter = []
|
|
101
|
+
if hide:
|
|
102
|
+
hidden_filter = ["!", "-regex", r".*/\..*"]
|
|
103
|
+
|
|
104
|
+
cmd = " ".join(parts + type_filters + hidden_filter + ["-printf", r"'%p\n'"]) + " 2>/dev/null"
|
|
105
|
+
out = run_remote_cmd(self.user_at_host, cmd)
|
|
106
|
+
return [line.strip().strip("'") for line in (out or "").splitlines() if line.strip()]
|
|
107
|
+
|
|
108
|
+
def listdir(self, base: str) -> List[str]:
|
|
109
|
+
cmd = f"find {shlex.quote(base)} -maxdepth 1 -mindepth 1 -printf '%p\\n' 2>/dev/null"
|
|
110
|
+
out = run_remote_cmd(self.user_at_host, cmd)
|
|
111
|
+
return [line.strip() for line in (out or "").splitlines() if line.strip()]
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def normalize_items(paths: Iterable[str],user_at_host=None,**kwargs) -> List[tuple[PathBackend, str]]:
|
|
115
|
+
pairs: List[tuple[PathBackend, str]] = []
|
|
116
|
+
host = user_at_host or kwargs.get('host')
|
|
117
|
+
|
|
118
|
+
for item in paths:
|
|
119
|
+
if not item: continue
|
|
120
|
+
m = REMOTE_RE.match(item)
|
|
121
|
+
if m:
|
|
122
|
+
pairs.append((SSHFS(m.group("host") or user_at_host), m.group("path")))
|
|
123
|
+
else:
|
|
124
|
+
pairs.append((LocalFS(), item))
|
|
125
|
+
return pairs
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from typing import *
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
import fnmatch, fnmatch, os, glob, platform, textwrap, pkgutil, re, textwrap, sys, types, importlib, importlib.util, inspect
|
|
5
|
+
from types import ModuleType
|
|
6
|
+
# Accept strings like:
|
|
7
|
+
# "/local/abs/dir"
|
|
8
|
+
# "relative/local/dir"
|
|
9
|
+
# "user@host:/abs/dir"
|
|
10
|
+
REMOTE_RE = re.compile(r"^(?P<host>[^:\s]+@[^:\s]+):(?P<path>/.*)$")
|
|
11
|
+
AllowedPredicate = Optional[Callable[[str], bool]]
|
|
@@ -2,4 +2,6 @@ from ...read_write_utils import read_from_file,write_to_file
|
|
|
2
2
|
from ...string_clean import eatAll
|
|
3
3
|
from ...list_utils import make_list
|
|
4
4
|
from ...type_utils import get_media_exts, is_media_type,MIME_TYPES
|
|
5
|
-
import
|
|
5
|
+
from ...ssh_utils import run_local_cmd, run_ssh_cmd,run_any_cmd,run_cmd
|
|
6
|
+
from ...env_utils import *
|
|
7
|
+
from ...path_utils import *
|
|
@@ -1,10 +1,8 @@
|
|
|
1
1
|
# attach_functions.py — single helper you can import anywhere
|
|
2
2
|
# attach_dynamic.py
|
|
3
3
|
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
from .file_filters import define_defaults,get_files_and_dirs
|
|
4
|
+
from .file_utils import define_defaults,get_files_and_dirs
|
|
6
5
|
from .imports import *
|
|
7
|
-
|
|
8
6
|
ABSPATH = os.path.abspath(__file__)
|
|
9
7
|
ABSROOT = os.path.dirname(ABSPATH)
|
|
10
8
|
def caller_path():
|
|
@@ -217,8 +215,7 @@ def attach_functions(
|
|
|
217
215
|
return attached
|
|
218
216
|
|
|
219
217
|
|
|
220
|
-
|
|
221
|
-
ABSROOT = os.path.dirname(ABSPATH)
|
|
218
|
+
|
|
222
219
|
def clean_imports():
|
|
223
220
|
alls = str(list(set("""os,re,subprocess,sys,re,traceback,pydot, enum, inspect, sys, traceback, threading,json,traceback,logging,requests""".replace('\n','').replace(' ','').replace('\t','').split(','))))[1:-1].replace('"','').replace("'",'')
|
|
224
221
|
input(alls)
|
|
@@ -22,7 +22,11 @@ Author: putkoff
|
|
|
22
22
|
Date: 05/31/2023
|
|
23
23
|
Version: 0.1.2
|
|
24
24
|
"""
|
|
25
|
-
|
|
25
|
+
import os
|
|
26
|
+
from .read_write_utils import read_from_file,write_to_file
|
|
27
|
+
from .string_clean import eatAll
|
|
28
|
+
from .list_utils import make_list
|
|
29
|
+
from .type_utils import get_media_exts, is_media_type,MIME_TYPES
|
|
26
30
|
def get_caller_path(i=1):
|
|
27
31
|
frame = inspect.stack()[i]
|
|
28
32
|
return os.path.abspath(frame.filename)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from types import ModuleType
|
|
2
2
|
from typing import Iterable
|
|
3
|
-
from ..
|
|
3
|
+
from ..file_utils import get_caller_dir,get_caller_path,define_defaults,get_files_and_dirs
|
|
4
4
|
from ..read_write_utils import *
|
|
5
5
|
import textwrap, pkgutil, os, re, textwrap, sys, types, importlib, importlib.util
|
|
6
6
|
from typing import *
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
from .imports import *
|
|
2
|
+
from .utils import run_local_cmd, run_ssh_cmd,run_any_cmd,run_cmd
|
|
3
|
+
|
|
4
|
+
class PathBackend(Protocol):
|
|
5
|
+
def join(self, *parts: str) -> str: ...
|
|
6
|
+
def isfile(self, path: str) -> bool: ...
|
|
7
|
+
def isdir(self, path: str) -> bool: ...
|
|
8
|
+
def glob_recursive(self, base: str, **opts) -> List[str]: ...
|
|
9
|
+
def listdir(self, base: str) -> List[str]: ...
|
|
10
|
+
|
|
11
|
+
class LocalFS:
|
|
12
|
+
def join(self, *parts: str) -> str:
|
|
13
|
+
return os.path.join(*parts)
|
|
14
|
+
def isfile(self, path: str) -> bool:
|
|
15
|
+
return os.path.isfile(path)
|
|
16
|
+
def isdir(self, path: str) -> bool:
|
|
17
|
+
return os.path.isdir(path)
|
|
18
|
+
def glob_recursive(self, base: str, **opts) -> List[str]:
|
|
19
|
+
"""
|
|
20
|
+
opts:
|
|
21
|
+
- maxdepth: int | None
|
|
22
|
+
- mindepth: int (default 1)
|
|
23
|
+
- follow_symlinks: bool
|
|
24
|
+
- include_dirs: bool
|
|
25
|
+
- include_files: bool
|
|
26
|
+
- exclude_hidden: bool
|
|
27
|
+
"""
|
|
28
|
+
maxdepth = opts.get("maxdepth")
|
|
29
|
+
mindepth = opts.get("mindepth", 1)
|
|
30
|
+
follow = opts.get("follow_symlinks", False)
|
|
31
|
+
want_d = opts.get("include_dirs", True)
|
|
32
|
+
want_f = opts.get("include_files", True)
|
|
33
|
+
hide = opts.get("exclude_hidden", False)
|
|
34
|
+
|
|
35
|
+
results: List[str] = []
|
|
36
|
+
base_depth = os.path.normpath(base).count(os.sep)
|
|
37
|
+
|
|
38
|
+
for root, dirs, files in os.walk(base, followlinks=follow):
|
|
39
|
+
depth = os.path.normpath(root).count(os.sep) - base_depth
|
|
40
|
+
if maxdepth is not None and depth > maxdepth:
|
|
41
|
+
dirs[:] = []
|
|
42
|
+
continue
|
|
43
|
+
if want_d and depth >= mindepth:
|
|
44
|
+
for d in dirs:
|
|
45
|
+
if hide and d.startswith("."): continue
|
|
46
|
+
results.append(os.path.join(root, d))
|
|
47
|
+
if want_f and depth >= mindepth:
|
|
48
|
+
for f in files:
|
|
49
|
+
if hide and f.startswith("."): continue
|
|
50
|
+
results.append(os.path.join(root, f))
|
|
51
|
+
return results
|
|
52
|
+
|
|
53
|
+
def listdir(self, base: str) -> List[str]:
|
|
54
|
+
try:
|
|
55
|
+
return [os.path.join(base, name) for name in os.listdir(base)]
|
|
56
|
+
except Exception:
|
|
57
|
+
return []
|
|
58
|
+
|
|
59
|
+
class SSHFS:
|
|
60
|
+
"""Remote POSIX backend via your run_remote_cmd."""
|
|
61
|
+
def __init__(self, user_at_host: str):
|
|
62
|
+
self.user_at_host = user_at_host
|
|
63
|
+
|
|
64
|
+
def join(self, *parts: str) -> str:
|
|
65
|
+
return posixpath.join(*parts)
|
|
66
|
+
|
|
67
|
+
def isfile(self, path: str) -> bool:
|
|
68
|
+
cmd = f"test -f {shlex.quote(path)} && echo __OK__ || true"
|
|
69
|
+
out = run_remote_cmd(self.user_at_host, cmd)
|
|
70
|
+
return "__OK__" in (out or "")
|
|
71
|
+
|
|
72
|
+
def isdir(self, path: str) -> bool:
|
|
73
|
+
cmd = f"test -d {shlex.quote(path)} && echo __OK__ || true"
|
|
74
|
+
out = run_remote_cmd(self.user_at_host, cmd)
|
|
75
|
+
return "__OK__" in (out or "")
|
|
76
|
+
|
|
77
|
+
def glob_recursive(self, base: str, **opts) -> List[str]:
|
|
78
|
+
maxdepth = opts.get("maxdepth")
|
|
79
|
+
mindepth = opts.get("mindepth", 1)
|
|
80
|
+
follow = opts.get("follow_symlinks", False)
|
|
81
|
+
want_d = opts.get("include_dirs", True)
|
|
82
|
+
want_f = opts.get("include_files", True)
|
|
83
|
+
hide = opts.get("exclude_hidden", False)
|
|
84
|
+
|
|
85
|
+
parts = []
|
|
86
|
+
if follow:
|
|
87
|
+
parts.append("-L")
|
|
88
|
+
parts += ["find", shlex.quote(base)]
|
|
89
|
+
if mindepth is not None:
|
|
90
|
+
parts += ["-mindepth", str(mindepth)]
|
|
91
|
+
if maxdepth is not None:
|
|
92
|
+
parts += ["-maxdepth", str(maxdepth)]
|
|
93
|
+
|
|
94
|
+
type_filters = []
|
|
95
|
+
if want_d and not want_f:
|
|
96
|
+
type_filters = ["-type", "d"]
|
|
97
|
+
elif want_f and not want_d:
|
|
98
|
+
type_filters = ["-type", "f"]
|
|
99
|
+
|
|
100
|
+
hidden_filter = []
|
|
101
|
+
if hide:
|
|
102
|
+
hidden_filter = ["!", "-regex", r".*/\..*"]
|
|
103
|
+
|
|
104
|
+
cmd = " ".join(parts + type_filters + hidden_filter + ["-printf", r"'%p\n'"]) + " 2>/dev/null"
|
|
105
|
+
out = run_remote_cmd(self.user_at_host, cmd)
|
|
106
|
+
return [line.strip().strip("'") for line in (out or "").splitlines() if line.strip()]
|
|
107
|
+
|
|
108
|
+
def listdir(self, base: str) -> List[str]:
|
|
109
|
+
cmd = f"find {shlex.quote(base)} -maxdepth 1 -mindepth 1 -printf '%p\\n' 2>/dev/null"
|
|
110
|
+
out = run_remote_cmd(self.user_at_host, cmd)
|
|
111
|
+
return [line.strip() for line in (out or "").splitlines() if line.strip()]
|
|
112
|
+
|
|
113
|
+
# ---- auto-detect "user@host:/abs/path" ----
|
|
114
|
+
REMOTE_RE = re.compile(r"^(?P<host>[^:\s]+@[^:\s]+):(?P<path>/.*)$")
|
|
115
|
+
|
|
116
|
+
def normalize_items(paths: Iterable[str],user_at_host=None,**kwargs) -> List[tuple[PathBackend, str]]:
|
|
117
|
+
pairs: List[tuple[PathBackend, str]] = []
|
|
118
|
+
host = user_at_host or kwargs.get('host')
|
|
119
|
+
|
|
120
|
+
for item in paths:
|
|
121
|
+
if not item: continue
|
|
122
|
+
m = REMOTE_RE.match(item)
|
|
123
|
+
if m:
|
|
124
|
+
pairs.append((SSHFS(m.group("host") or user_at_host), m.group("path")))
|
|
125
|
+
else:
|
|
126
|
+
pairs.append((LocalFS(), item))
|
|
127
|
+
return pairs
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# remote_fs.py
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
from typing import *
|
|
4
|
+
import subprocess, shlex, os, fnmatch, glob, posixpath, re
|
|
5
|
+
# exec_api.py
|
|
6
|
+
# ---- import your existing pieces ----
|
|
7
|
+
from ..type_utils import make_list # whatever you already have
|
|
8
|
+
from ..time_utils import get_sleep
|
|
9
|
+
from ..ssh_utils import *
|
|
10
|
+
from ..env_utils import *
|
|
11
|
+
from ..string_clean import eatOuter
|