abstract-utilities 0.2.2.513__py3-none-any.whl → 0.2.2.627__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- abstract_utilities/__init__.py +11 -3
- abstract_utilities/class_utils/caller_utils.py +19 -0
- abstract_utilities/class_utils/global_utils.py +35 -20
- abstract_utilities/class_utils/imports/imports.py +1 -1
- abstract_utilities/directory_utils/__init__.py +2 -4
- abstract_utilities/directory_utils/imports/__init__.py +2 -0
- abstract_utilities/directory_utils/imports/imports.py +1 -0
- abstract_utilities/directory_utils/imports/module_imports.py +2 -0
- abstract_utilities/directory_utils/src/__init__.py +4 -0
- abstract_utilities/directory_utils/src/directory_utils.py +110 -0
- abstract_utilities/directory_utils/src/name_utils.py +43 -0
- abstract_utilities/directory_utils/src/size_utils.py +57 -0
- abstract_utilities/directory_utils/src/utils.py +116 -0
- abstract_utilities/file_utils/imports/constants.py +81 -7
- abstract_utilities/file_utils/imports/imports.py +0 -4
- abstract_utilities/file_utils/imports/module_imports.py +1 -1
- abstract_utilities/file_utils/src/__init__.py +2 -4
- abstract_utilities/file_utils/src/file_filters/__init__.py +4 -0
- abstract_utilities/file_utils/src/file_filters/ensure_utils.py +118 -0
- abstract_utilities/file_utils/src/file_filters/filter_params.py +86 -0
- abstract_utilities/file_utils/src/file_filters/filter_utils.py +78 -0
- abstract_utilities/file_utils/src/file_filters/predicate_utils.py +116 -0
- abstract_utilities/file_utils/src/file_filters.py +114 -47
- abstract_utilities/file_utils/src/file_reader.py +0 -64
- abstract_utilities/file_utils/src/file_utils.py +7 -130
- abstract_utilities/file_utils/src/filter_params.py +128 -86
- abstract_utilities/file_utils/src/find_collect.py +85 -165
- abstract_utilities/file_utils/src/find_content.py +210 -0
- abstract_utilities/file_utils/src/initFunctionsGen.py +35 -28
- abstract_utilities/file_utils/src/initFunctionsGens.py +280 -0
- abstract_utilities/file_utils/src/reader_utils/__init__.py +4 -0
- abstract_utilities/file_utils/src/reader_utils/directory_reader.py +53 -0
- abstract_utilities/file_utils/src/reader_utils/file_reader.py +543 -0
- abstract_utilities/file_utils/src/reader_utils/file_readers.py +376 -0
- abstract_utilities/file_utils/src/reader_utils/imports.py +18 -0
- abstract_utilities/file_utils/src/reader_utils/pdf_utils.py +300 -0
- abstract_utilities/file_utils (2)/__init__.py +2 -0
- abstract_utilities/file_utils (2)/imports/__init__.py +2 -0
- abstract_utilities/file_utils (2)/imports/constants.py +118 -0
- abstract_utilities/file_utils (2)/imports/imports/__init__.py +3 -0
- abstract_utilities/file_utils (2)/imports/imports/constants.py +119 -0
- abstract_utilities/file_utils (2)/imports/imports/imports.py +46 -0
- abstract_utilities/file_utils (2)/imports/imports/module_imports.py +8 -0
- abstract_utilities/file_utils (2)/imports/utils/__init__.py +3 -0
- abstract_utilities/file_utils (2)/imports/utils/classes.py +379 -0
- abstract_utilities/file_utils (2)/imports/utils/clean_imps.py +155 -0
- abstract_utilities/file_utils (2)/imports/utils/filter_utils.py +341 -0
- abstract_utilities/file_utils (2)/src/__init__.py +8 -0
- abstract_utilities/file_utils (2)/src/file_filters.py +155 -0
- abstract_utilities/file_utils (2)/src/file_reader.py +604 -0
- abstract_utilities/file_utils (2)/src/find_collect.py +258 -0
- abstract_utilities/file_utils (2)/src/initFunctionsGen.py +286 -0
- abstract_utilities/file_utils (2)/src/map_utils.py +28 -0
- abstract_utilities/file_utils (2)/src/pdf_utils.py +300 -0
- abstract_utilities/import_utils/circular_import_finder.py +222 -0
- abstract_utilities/import_utils/circular_import_finder2.py +118 -0
- abstract_utilities/import_utils/imports/module_imports.py +3 -1
- abstract_utilities/import_utils/src/clean_imports.py +156 -25
- abstract_utilities/import_utils/src/dot_utils.py +11 -0
- abstract_utilities/import_utils/src/extract_utils.py +4 -0
- abstract_utilities/import_utils/src/import_functions.py +66 -2
- abstract_utilities/import_utils/src/pkg_utils.py +58 -4
- abstract_utilities/import_utils/src/sysroot_utils.py +56 -1
- abstract_utilities/log_utils/log_file.py +73 -24
- abstract_utilities/parse_utils/parse_utils.py +23 -0
- abstract_utilities/path_utils/path_utils.py +25 -23
- abstract_utilities/read_write_utils/imports/imports.py +1 -1
- abstract_utilities/read_write_utils/read_write_utils.py +99 -31
- abstract_utilities/safe_utils/safe_utils.py +30 -0
- {abstract_utilities-0.2.2.513.dist-info → abstract_utilities-0.2.2.627.dist-info}/METADATA +1 -1
- {abstract_utilities-0.2.2.513.dist-info → abstract_utilities-0.2.2.627.dist-info}/RECORD +73 -32
- {abstract_utilities-0.2.2.513.dist-info → abstract_utilities-0.2.2.627.dist-info}/WHEEL +0 -0
- {abstract_utilities-0.2.2.513.dist-info → abstract_utilities-0.2.2.627.dist-info}/top_level.txt +0 -0
|
@@ -1,12 +1,64 @@
|
|
|
1
1
|
from ..imports import *
|
|
2
2
|
from .pkg_utils import *
|
|
3
|
-
|
|
3
|
+
from ...safe_utils import *
|
|
4
|
+
def is_local_import(line):
|
|
5
|
+
print(line)
|
|
6
|
+
imports_from_import_pkg = clean_imports(line)
|
|
7
|
+
input(imports_from_import_pkg)
|
|
8
|
+
def try_is_file(file_path):
|
|
9
|
+
try:
|
|
10
|
+
return os.path.isfile(file_path)
|
|
11
|
+
except:
|
|
12
|
+
return False
|
|
13
|
+
def try_is_dir(file_path):
|
|
14
|
+
try:
|
|
15
|
+
return os.path.isdir(file_path)
|
|
16
|
+
except:
|
|
17
|
+
return False
|
|
18
|
+
def try_join(*args):
|
|
19
|
+
try:
|
|
20
|
+
return safe_join(*args)
|
|
21
|
+
except:
|
|
22
|
+
return False
|
|
23
|
+
def get_pkg_or_init(pkg_path):
|
|
24
|
+
if pkg_path:
|
|
25
|
+
if try_is_file(pkg_path):
|
|
26
|
+
return pkg_path
|
|
27
|
+
pkg_py_path = f"{pkg_path}.py"
|
|
28
|
+
if try_is_file(pkg_py_path):
|
|
29
|
+
return pkg_py_path
|
|
30
|
+
pkg_init_path = try_join(pkg_path,'__init__.py')
|
|
31
|
+
if try_is_dir(pkg_path):
|
|
32
|
+
if os.path.isfile(pkg_init_path):
|
|
33
|
+
return pkg_init_path
|
|
34
|
+
def get_text_and_file_and_js(text=None,file_path=None,import_pkg_js=None):
|
|
35
|
+
inputs = {"text":text,"file_path":file_path,"import_pkg_js":import_pkg_js}
|
|
36
|
+
for key,value in inputs.items():
|
|
37
|
+
if value:
|
|
38
|
+
if isinstance(value,str):
|
|
39
|
+
_file_path = get_pkg_or_init(file_path)
|
|
40
|
+
if _file_path:
|
|
41
|
+
file_path=_file_path
|
|
42
|
+
if key == "text" or text == None:
|
|
43
|
+
text=read_from_file(file_path)
|
|
44
|
+
if isinstance(value,dict):
|
|
45
|
+
if key in ["text","file_path"]:
|
|
46
|
+
if key == "text":
|
|
47
|
+
text = None
|
|
48
|
+
if key == "file_path":
|
|
49
|
+
file_path = None
|
|
50
|
+
import_pkg_js=value
|
|
51
|
+
import_pkg_js = ensure_import_pkg_js(import_pkg_js,file_path=file_path)
|
|
52
|
+
return text,file_path,import_pkg_js
|
|
4
53
|
def get_text_or_read(text=None,file_path=None):
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
if not text and file_path and os.path.isfile(file_path):
|
|
54
|
+
file_path = get_pkg_or_init(file_path)
|
|
55
|
+
if not text and file_path:
|
|
8
56
|
text=read_from_file(file_path)
|
|
9
|
-
|
|
57
|
+
if text and not file_path:
|
|
58
|
+
file_path=get_pkg_or_init(text)
|
|
59
|
+
if file_path:
|
|
60
|
+
text = None
|
|
61
|
+
return text,file_path
|
|
10
62
|
def is_line_import(line):
|
|
11
63
|
if line and (line.startswith(FROM_TAG) or line.startswith(IMPORT_TAG)):
|
|
12
64
|
return True
|
|
@@ -25,19 +77,18 @@ def is_from_line_group(line):
|
|
|
25
77
|
return False
|
|
26
78
|
|
|
27
79
|
def get_all_imports(text=None,file_path=None,import_pkg_js=None):
|
|
28
|
-
|
|
29
|
-
file_path = text
|
|
30
|
-
text = read_from_file(text)
|
|
31
|
-
text = get_text_or_read(text=text,file_path=file_path)
|
|
80
|
+
text,file_path = get_text_or_read(text=text,file_path=file_path)
|
|
32
81
|
lines = text.split('\n')
|
|
33
82
|
cleaned_import_list=[]
|
|
34
83
|
nu_lines = []
|
|
35
84
|
is_from_group = False
|
|
36
85
|
import_pkg_js = ensure_import_pkg_js(import_pkg_js,file_path=file_path)
|
|
37
86
|
for line in lines:
|
|
87
|
+
|
|
38
88
|
if line.startswith(IMPORT_TAG) and ' from ' not in line:
|
|
89
|
+
|
|
39
90
|
cleaned_import_list = get_cleaned_import_list(line)
|
|
40
|
-
import_pkg_js = add_imports_to_import_pkg_js("import",cleaned_import_list,import_pkg_js=import_pkg_js)
|
|
91
|
+
import_pkg_js = add_imports_to_import_pkg_js("import",cleaned_import_list,import_pkg_js=import_pkg_js,file_path=file_path)
|
|
41
92
|
else:
|
|
42
93
|
if is_from_group:
|
|
43
94
|
import_pkg=is_from_group
|
|
@@ -46,18 +97,16 @@ def get_all_imports(text=None,file_path=None,import_pkg_js=None):
|
|
|
46
97
|
is_from_group=False
|
|
47
98
|
line=line[:-1]
|
|
48
99
|
imports_from_import_pkg = clean_imports(line)
|
|
49
|
-
import_pkg_js = add_imports_to_import_pkg_js(import_pkg,imports_from_import_pkg,import_pkg_js=import_pkg_js)
|
|
100
|
+
import_pkg_js = add_imports_to_import_pkg_js(import_pkg,imports_from_import_pkg,import_pkg_js=import_pkg_js,file_path=file_path)
|
|
50
101
|
|
|
51
102
|
else:
|
|
52
|
-
import_pkg_js=update_import_pkg_js(line,import_pkg_js=import_pkg_js)
|
|
103
|
+
import_pkg_js=update_import_pkg_js(line,import_pkg_js=import_pkg_js,file_path=file_path)
|
|
53
104
|
if is_from_line_group(line) and is_from_group == False:
|
|
54
105
|
is_from_group=get_import_pkg(line)
|
|
55
106
|
return import_pkg_js
|
|
56
107
|
|
|
57
|
-
def
|
|
58
|
-
|
|
59
|
-
file_path = text
|
|
60
|
-
text = read_from_file(text)
|
|
108
|
+
def get_clean_imports(text=None,file_path=None,import_pkg_js=None,fill_nulines=False):
|
|
109
|
+
text,file_path,_ = get_text_and_file_and_js(text=text,file_path=file_path,import_pkg_js=import_pkg_js)
|
|
61
110
|
if not import_pkg_js:
|
|
62
111
|
import_pkg_js = get_all_imports(text=text,file_path=file_path)
|
|
63
112
|
import_pkg_js = ensure_import_pkg_js(import_pkg_js,file_path=file_path)
|
|
@@ -87,11 +136,11 @@ def clean_imports(text=None,file_path=None,import_pkg_js=None,fill_nulines=False
|
|
|
87
136
|
nu_lines[line] += imports
|
|
88
137
|
return import_pkg_js
|
|
89
138
|
def clean_all_imports(text=None,file_path=None,import_pkg_js=None,fill_nulines=False):
|
|
90
|
-
|
|
91
|
-
import_pkg_js["context"]["nulines"]=
|
|
139
|
+
import_pkg_js = get_clean_imports(text=text,file_path=file_path,import_pkg_js=import_pkg_js,fill_nulines=import_pkg_js)
|
|
140
|
+
import_pkg_js["context"]["nulines"]=import_pkg_js["context"]["nulines"]
|
|
92
141
|
return import_pkg_js
|
|
93
142
|
def get_clean_import_string(import_pkg_js,fill_nulines=False,get_locals=False):
|
|
94
|
-
import_pkg_js =
|
|
143
|
+
import_pkg_js = get_clean_imports(import_pkg_js=import_pkg_js,fill_nulines=fill_nulines)
|
|
95
144
|
import_ls = []
|
|
96
145
|
for key,values in import_pkg_js.items():
|
|
97
146
|
if key not in ['context','nulines']:
|
|
@@ -130,18 +179,100 @@ def get_dot_fro_lines(lines,file_path,all_imps):
|
|
|
130
179
|
all_imps.append(line)
|
|
131
180
|
return all_imps
|
|
132
181
|
def get_all_real_imps(text=None,file_path=None,all_imps=None):
|
|
133
|
-
|
|
134
|
-
file_path = text
|
|
135
|
-
text = read_from_file(text)
|
|
182
|
+
|
|
136
183
|
all_imps = all_imps or []
|
|
137
|
-
|
|
138
|
-
lines =
|
|
184
|
+
text,file_path = get_text_or_read(text=text,file_path=file_path)
|
|
185
|
+
lines = text.split('\n')
|
|
139
186
|
all_imps = get_dot_fro_lines(lines,file_path,all_imps)
|
|
140
187
|
return '\n'.join(all_imps)
|
|
141
188
|
def save_cleaned_imports(text=None,file_path=None,write=False,import_pkg_js=None):
|
|
142
|
-
import_pkg_js=
|
|
189
|
+
text,file_path,import_pkg_js = get_text_and_file_and_js(text=text,file_path=file_path,import_pkg_js=import_pkg_js)
|
|
143
190
|
import_pkg_js = clean_all_imports(text=text,file_path=file_path,import_pkg_js=import_pkg_js)
|
|
144
191
|
contents = '\n'.join(import_pkg_js["context"]["nulines"])
|
|
145
192
|
if file_path and write:
|
|
146
193
|
write_to_file(contents=contents,file_path=file_path)
|
|
147
194
|
return contents
|
|
195
|
+
def convert_to_sysroot_relative(import_pkg, file_path, sysroot):
|
|
196
|
+
"""
|
|
197
|
+
Convert an absolute package import into a dotted relative import based on
|
|
198
|
+
the file's depth inside sysroot.
|
|
199
|
+
"""
|
|
200
|
+
|
|
201
|
+
if not sysroot:
|
|
202
|
+
return import_pkg # no conversion
|
|
203
|
+
|
|
204
|
+
file_path = os.path.abspath(file_path)
|
|
205
|
+
sysroot = os.path.abspath(sysroot)
|
|
206
|
+
|
|
207
|
+
# Ignore imports outside sysroot
|
|
208
|
+
file_dir = os.path.dirname(file_path)
|
|
209
|
+
if not file_dir.startswith(sysroot):
|
|
210
|
+
return import_pkg
|
|
211
|
+
|
|
212
|
+
# Compute how many directories deep the file is
|
|
213
|
+
rel = os.path.relpath(file_dir, sysroot)
|
|
214
|
+
depth = 0 if rel == "." else len(rel.split(os.sep))
|
|
215
|
+
|
|
216
|
+
# Depth N means N dots (i.e. N relative levels)
|
|
217
|
+
dots = "." * depth
|
|
218
|
+
|
|
219
|
+
return f"{dots}{import_pkg}"
|
|
220
|
+
|
|
221
|
+
import os
|
|
222
|
+
|
|
223
|
+
def rewrite_import_with_sysroot(line, file_path, sysroot):
|
|
224
|
+
"""
|
|
225
|
+
Rewrite imports like:
|
|
226
|
+
from imports.constants import *
|
|
227
|
+
into:
|
|
228
|
+
from <relative_path>.imports.constants import *
|
|
229
|
+
Where <relative_path> is computed relative to sysroot.
|
|
230
|
+
"""
|
|
231
|
+
|
|
232
|
+
line = line.rstrip()
|
|
233
|
+
if not line.startswith("from "):
|
|
234
|
+
return line
|
|
235
|
+
|
|
236
|
+
# Split import structure
|
|
237
|
+
try:
|
|
238
|
+
after_from = line[len("from "):]
|
|
239
|
+
pkg, after_import = after_from.split(" import ", 1)
|
|
240
|
+
except ValueError:
|
|
241
|
+
return line # Not a normal from X import Y
|
|
242
|
+
|
|
243
|
+
# Absolute paths
|
|
244
|
+
file_dir = os.path.dirname(os.path.abspath(file_path))
|
|
245
|
+
sysroot = os.path.abspath(sysroot)
|
|
246
|
+
|
|
247
|
+
# Compute relative path
|
|
248
|
+
relpath = os.path.relpath(file_dir, sysroot)
|
|
249
|
+
|
|
250
|
+
# Turn filesystem path into dotted python path
|
|
251
|
+
if relpath == ".":
|
|
252
|
+
dotted = ""
|
|
253
|
+
else:
|
|
254
|
+
dotted = ".".join(part for part in relpath.split(os.sep) if part)
|
|
255
|
+
|
|
256
|
+
# Import path you want to append the old import to
|
|
257
|
+
new_pkg = f"{dotted}.{pkg}".lstrip('.')
|
|
258
|
+
|
|
259
|
+
# Build final rewritten import
|
|
260
|
+
return f"from {new_pkg} import {after_import}"
|
|
261
|
+
|
|
262
|
+
def trace_all_imports(file_path, sysroot=None):
|
|
263
|
+
import_pkg_js = {}
|
|
264
|
+
files = collect_filepaths(file_path, allowed_exts='.py', add=True)
|
|
265
|
+
|
|
266
|
+
for file in files:
|
|
267
|
+
text = get_all_real_imps(file_path=file)
|
|
268
|
+
lines = text.split("\n")
|
|
269
|
+
|
|
270
|
+
if sysroot:
|
|
271
|
+
new_lines = []
|
|
272
|
+
for line in lines:
|
|
273
|
+
new_lines.append(rewrite_import_with_sysroot(line, file, sysroot))
|
|
274
|
+
text = "\n".join(new_lines)
|
|
275
|
+
|
|
276
|
+
import_pkg_js = get_all_imports(text=text, file_path=file, import_pkg_js=import_pkg_js)
|
|
277
|
+
|
|
278
|
+
return get_clean_import_string(import_pkg_js)
|
|
@@ -1,4 +1,15 @@
|
|
|
1
1
|
from ..imports import *
|
|
2
|
+
def find_top_package_dir(p: Path) -> Path | None:
|
|
3
|
+
p = p.resolve()
|
|
4
|
+
if p.is_file():
|
|
5
|
+
p = p.parent
|
|
6
|
+
top = None
|
|
7
|
+
while (p / "__init__.py").exists():
|
|
8
|
+
top = p
|
|
9
|
+
if p.parent == p:
|
|
10
|
+
break
|
|
11
|
+
p = p.parent
|
|
12
|
+
return top
|
|
2
13
|
def get_Path(path):
|
|
3
14
|
if isinstance(path,str):
|
|
4
15
|
path = Path(str(path))
|
|
@@ -40,3 +40,7 @@ def extract_class(path: str):
|
|
|
40
40
|
if m:
|
|
41
41
|
funcs.append(m.group(1))
|
|
42
42
|
return funcs
|
|
43
|
+
def get_all_py_file_paths(directory,*args,**kwargs):
|
|
44
|
+
globs = collect_globs(directory,*args,allowed_exts='.py',**kwargs)
|
|
45
|
+
globs = [glo for glo in globs.get('files') if glo]
|
|
46
|
+
return globs
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
# --- auto-package bootstrap (run-safe) ---------------------------------
|
|
2
2
|
from ..imports import *
|
|
3
3
|
from .dot_utils import get_dot_range
|
|
4
|
-
from .sysroot_utils import get_sysroot
|
|
4
|
+
from .sysroot_utils import get_sysroot,get_import_with_sysroot,get_py_files,get_all_py_sysroots
|
|
5
|
+
from .extract_utils import get_all_py_file_paths
|
|
5
6
|
def clean_imports(imports,commaClean=True):
|
|
6
7
|
chars=["*"]
|
|
7
8
|
if not commaClean:
|
|
@@ -9,6 +10,13 @@ def clean_imports(imports,commaClean=True):
|
|
|
9
10
|
if isinstance(imports,str):
|
|
10
11
|
imports = imports.split(',')
|
|
11
12
|
return [eatElse(imp,chars=chars) for imp in imports if imp]
|
|
13
|
+
def get_dot_range(import_pkg):
|
|
14
|
+
count = 0
|
|
15
|
+
for char in import_pkg:
|
|
16
|
+
if char != '.':
|
|
17
|
+
break
|
|
18
|
+
count+=1
|
|
19
|
+
return count
|
|
12
20
|
def get_cleaned_import_list(line,commaClean=True):
|
|
13
21
|
cleaned_import_list=[]
|
|
14
22
|
if IMPORT_TAG in line:
|
|
@@ -17,7 +25,7 @@ def get_cleaned_import_list(line,commaClean=True):
|
|
|
17
25
|
return cleaned_import_list
|
|
18
26
|
def get_module_from_import(imp,path=None):
|
|
19
27
|
path = path or os.getcwd()
|
|
20
|
-
i = get_dot_range(
|
|
28
|
+
i = get_dot_range(imp)
|
|
21
29
|
imp = eatAll(imp,'.')
|
|
22
30
|
sysroot = get_sysroot(path,i)
|
|
23
31
|
return os.path.join(sysroot, imp)
|
|
@@ -42,5 +50,61 @@ def safe_import(name: str, *, package: str | None = None, member: str | None = N
|
|
|
42
50
|
mod = importlib.import_module(name, package=package)
|
|
43
51
|
return getattr(mod, member) if member else mod
|
|
44
52
|
|
|
53
|
+
def dynamic_import(module_path: str, namespace: dict, all_imports = None):
|
|
54
|
+
"""
|
|
55
|
+
Emulates:
|
|
56
|
+
from module_path import *
|
|
57
|
+
but includes private (_xxx) names too.
|
|
58
|
+
"""
|
|
59
|
+
all_imports = if_none_change(all_imports,True)
|
|
60
|
+
if module_path:
|
|
61
|
+
module = importlib.import_module(module_path)
|
|
62
|
+
# Import literally everything except dunders, unless you want them too.
|
|
63
|
+
names = [n for n in dir(module) if n and ((not all_imports and not n.startswith("_")) or all_imports)]
|
|
64
|
+
for name in names:
|
|
65
|
+
namespace[name] = getattr(module, name)
|
|
66
|
+
return module
|
|
67
|
+
def get_monorepo_root(directory=None,files=None):
|
|
68
|
+
directory = directory or get_initial_caller_dir()
|
|
69
|
+
|
|
70
|
+
py_files = get_all_py_file_paths(directory,add=True)
|
|
71
|
+
sysroots = get_all_py_sysroots(directory=directory,files=py_files)
|
|
72
|
+
monorepo_root = get_common_root(sysroots)
|
|
73
|
+
return monorepo_root
|
|
74
|
+
def switch_to_monorepo_root(directory=None,files=None):
|
|
75
|
+
monorepo_root = get_monorepo_root(directory=directory,files=files)
|
|
76
|
+
if str(monorepo_root) not in sys.path:
|
|
77
|
+
sys.path.insert(0, str(monorepo_root))
|
|
78
|
+
return str(monorepo_root)
|
|
79
|
+
def get_all_imports(directory=None,sysroot=None,globs=None):
|
|
80
|
+
globs = globs or get_true_globals() or globals()
|
|
81
|
+
directory = directory or get_initial_caller_dir()
|
|
82
|
+
files = collect_globs(directory=directory,allowed_exts='.py').get('files')
|
|
83
|
+
sysroot = sysroot or switch_to_monorepo_root(directory=directory,files=files)
|
|
84
|
+
for glo in files:
|
|
85
|
+
imp = get_import_with_sysroot(glo, sysroot)
|
|
86
|
+
dynamic_import(imp, globs)
|
|
87
|
+
def get_all_imports_for_class(self, directory=None, sysroot=None, include_private=True):
|
|
88
|
+
"""
|
|
89
|
+
Load all modules under `directory` and assign their exports as attributes
|
|
90
|
+
on the class instance (self).
|
|
91
|
+
"""
|
|
92
|
+
directory = directory or get_initial_caller_dir()
|
|
93
|
+
files = collect_globs(directory=directory, allowed_exts='.py').get("files")
|
|
94
|
+
|
|
95
|
+
# Compute sysroot (monorepo root)
|
|
96
|
+
sysroot = sysroot or switch_to_monorepo_root(directory=directory, files=files)
|
|
97
|
+
|
|
98
|
+
for glo in files:
|
|
99
|
+
mod_path = get_import_with_sysroot(glo, sysroot)
|
|
100
|
+
module = importlib.import_module(mod_path)
|
|
101
|
+
|
|
102
|
+
for name in dir(module):
|
|
103
|
+
if name.startswith("__"):
|
|
104
|
+
continue
|
|
105
|
+
if not include_private and name.startswith("_"):
|
|
106
|
+
continue
|
|
45
107
|
|
|
108
|
+
setattr(self, name, getattr(module, name))
|
|
46
109
|
|
|
110
|
+
return self
|
|
@@ -1,6 +1,36 @@
|
|
|
1
1
|
# safe_import_utils.py
|
|
2
2
|
from ..imports import *
|
|
3
3
|
from .import_functions import *
|
|
4
|
+
from ...safe_utils import *
|
|
5
|
+
from .sysroot_utils import get_import_with_sysroot
|
|
6
|
+
def try_is_file(file_path):
|
|
7
|
+
try:
|
|
8
|
+
return os.path.isfile(file_path)
|
|
9
|
+
except:
|
|
10
|
+
return False
|
|
11
|
+
def try_is_dir(file_path):
|
|
12
|
+
try:
|
|
13
|
+
return os.path.isdir(file_path)
|
|
14
|
+
except:
|
|
15
|
+
return False
|
|
16
|
+
def try_join(*args):
|
|
17
|
+
try:
|
|
18
|
+
return safe_join(*args)
|
|
19
|
+
except:
|
|
20
|
+
return False
|
|
21
|
+
def get_pkg_or_init(pkg_path):
|
|
22
|
+
if pkg_path:
|
|
23
|
+
if try_is_file(pkg_path):
|
|
24
|
+
return pkg_path
|
|
25
|
+
pkg_py_path = f"{pkg_path}.py"
|
|
26
|
+
if try_is_file(pkg_py_path):
|
|
27
|
+
return pkg_py_path
|
|
28
|
+
pkg_init_path = try_join(pkg_path,'__init__.py')
|
|
29
|
+
if try_is_dir(pkg_path):
|
|
30
|
+
if os.path.isfile(pkg_init_path):
|
|
31
|
+
return pkg_init_path
|
|
32
|
+
|
|
33
|
+
|
|
4
34
|
|
|
5
35
|
def ensure_import_pkg_js(import_pkg_js,file_path=None):
|
|
6
36
|
import_pkg_js = import_pkg_js or {"context":{}}
|
|
@@ -79,18 +109,42 @@ def ensure_caller_package(caller_file: str, caller_globals: dict | None = None)
|
|
|
79
109
|
if caller_globals and caller_globals.get("__name__") == "__main__" and not caller_globals.get("__package__"):
|
|
80
110
|
caller_globals["__package__"] = pkg
|
|
81
111
|
return pkg
|
|
112
|
+
def get_init_dots(import_pkg):
|
|
113
|
+
count = 0
|
|
114
|
+
for char in import_pkg:
|
|
115
|
+
if char != '.':
|
|
116
|
+
break
|
|
117
|
+
count+=1
|
|
118
|
+
return count
|
|
119
|
+
def make_relative_pkg(import_pkg,file_path):
|
|
120
|
+
full_pkg = eatAll(import_pkg,'.')
|
|
121
|
+
if file_path:
|
|
122
|
+
dirname = file_path
|
|
123
|
+
dot_count = get_init_dots(import_pkg)
|
|
124
|
+
for i in range(dot_count):
|
|
125
|
+
dirname = os.path.dirname(dirname)
|
|
126
|
+
dirbase = os.path.basename(dirname)
|
|
127
|
+
full_pkg=f"{dirbase}.{full_pkg}"
|
|
128
|
+
return full_pkg
|
|
129
|
+
def is_local_import(import_pkg,file_path=None):
|
|
130
|
+
relative_pkg = get_module_from_import(import_pkg,file_path)
|
|
131
|
+
if get_pkg_or_init(relative_pkg):
|
|
132
|
+
return True
|
|
133
|
+
return False
|
|
82
134
|
def get_import_pkg(line):
|
|
83
135
|
if is_line_group_import(line):
|
|
84
136
|
return clean_line(line.split(FROM_TAG)[1].split(IMPORT_TAG)[0])
|
|
85
137
|
def get_imports_from_import_pkg(line):
|
|
86
138
|
if is_line_group_import(line):
|
|
87
139
|
return get_cleaned_import_list(line,commaClean=True)
|
|
88
|
-
def add_imports_to_import_pkg_js(import_pkg,imports,import_pkg_js=None):
|
|
140
|
+
def add_imports_to_import_pkg_js(import_pkg,imports,import_pkg_js=None,file_path=None):
|
|
89
141
|
import_pkg_js = ensure_import_pkg_js(import_pkg_js)
|
|
90
142
|
imports = clean_imports(imports)
|
|
143
|
+
pkg_path = get_module_from_import(import_pkg,file_path)
|
|
144
|
+
local_import = get_pkg_or_init(pkg_path)
|
|
91
145
|
if import_pkg not in import_pkg_js:
|
|
92
146
|
i = len(import_pkg_js["context"]["nulines"])
|
|
93
|
-
import_pkg_js[import_pkg]={"imports":imports,"line":i}
|
|
147
|
+
import_pkg_js[import_pkg]={"imports":imports,"line":i,"file_path":file_path,"local_import":local_import,"is_local":is_local_import(import_pkg,file_path=file_path)}
|
|
94
148
|
import_line = f"from {import_pkg} import "
|
|
95
149
|
if import_pkg == "import":
|
|
96
150
|
import_line = IMPORT_TAG
|
|
@@ -98,12 +152,12 @@ def add_imports_to_import_pkg_js(import_pkg,imports,import_pkg_js=None):
|
|
|
98
152
|
else:
|
|
99
153
|
import_pkg_js[import_pkg]["imports"]+=imports
|
|
100
154
|
return import_pkg_js
|
|
101
|
-
def update_import_pkg_js(line,import_pkg_js=None):
|
|
155
|
+
def update_import_pkg_js(line,import_pkg_js=None,file_path=None):
|
|
102
156
|
import_pkg_js = ensure_import_pkg_js(import_pkg_js)
|
|
103
157
|
if is_line_group_import(line):
|
|
104
158
|
import_pkg = get_import_pkg(line)
|
|
105
159
|
imports = get_imports_from_import_pkg(line)
|
|
106
|
-
import_pkg_js = add_imports_to_import_pkg_js(import_pkg,imports,import_pkg_js=import_pkg_js)
|
|
160
|
+
import_pkg_js = add_imports_to_import_pkg_js(import_pkg,imports,import_pkg_js=import_pkg_js,file_path=file_path)
|
|
107
161
|
else:
|
|
108
162
|
if len(import_pkg_js["context"]["nulines"]) >0 and line == '' and is_line_import(import_pkg_js["context"]["nulines"][-1]):
|
|
109
163
|
pass
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
from ..imports import *
|
|
2
2
|
from .dot_utils import *
|
|
3
|
-
|
|
3
|
+
from .extract_utils import get_all_py_file_paths
|
|
4
|
+
def get_py_files(directory=None):
|
|
5
|
+
directory = directory or get_initial_caller_dir()
|
|
6
|
+
return get_all_py_file_paths(directory,add=True)
|
|
4
7
|
def ensure_on_path(p: Path):
|
|
5
8
|
s = str(p)
|
|
6
9
|
if s not in sys.path:
|
|
@@ -19,7 +22,59 @@ def get_dot_range_sysroot(filepath):
|
|
|
19
22
|
sysroot = get_sysroot(sysroot,dot_range)
|
|
20
23
|
|
|
21
24
|
return sysroot
|
|
25
|
+
def get_import_with_sysroot(file_path, sysroot):
|
|
26
|
+
"""
|
|
27
|
+
Rewrite imports like:
|
|
28
|
+
from imports.constants import *
|
|
29
|
+
into:
|
|
30
|
+
from <relative_path>.imports.constants import *
|
|
31
|
+
Where <relative_path> is computed relative to sysroot.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
# Absolute paths
|
|
36
|
+
file_dir = os.path.dirname(os.path.abspath(file_path))
|
|
37
|
+
sysroot = os.path.abspath(sysroot)
|
|
38
|
+
|
|
39
|
+
# Compute relative path
|
|
40
|
+
relpath = os.path.relpath(file_dir, sysroot)
|
|
41
|
+
|
|
42
|
+
bare_rel = eatAll(relpath,'.')
|
|
43
|
+
|
|
44
|
+
# Turn filesystem path into dotted python path
|
|
45
|
+
if relpath == ".":
|
|
46
|
+
dotted = ""
|
|
47
|
+
else:
|
|
48
|
+
dotted = ".".join(part for part in relpath.split(os.sep) if part)
|
|
49
|
+
if bare_rel.startswith('/') and dotted.startswith('.'):
|
|
50
|
+
dotted = dotted[1:]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
# Build final rewritten import
|
|
54
|
+
return dotted
|
|
55
|
+
def get_all_sysroots(files):
|
|
56
|
+
sysroots=[]
|
|
57
|
+
for glo in files:
|
|
58
|
+
imp = compute_dotted_and_sysroot(glo)
|
|
59
|
+
sysroots.append(imp[-1])
|
|
60
|
+
return sysroots
|
|
61
|
+
def get_shortest_sysroot(files):
|
|
62
|
+
sysroots = get_all_sysroots(files)
|
|
63
|
+
return get_shortest_path(*sysroots)
|
|
64
|
+
def get_all_py_sysroots(directory=None,files=None):
|
|
65
|
+
py_files = files or get_py_files(directory=directory)
|
|
66
|
+
return [compute_dotted_and_sysroot(glo)[1] for glo in py_files]
|
|
22
67
|
|
|
68
|
+
def get__imports(directory=None, sysroot=None):
|
|
69
|
+
directory = directory or get_caller_dir(1)
|
|
70
|
+
globs = collect_globs(directory, allowed_exts='.py')
|
|
71
|
+
globs = [glo for glo in globs.get('files') if glo]
|
|
72
|
+
sysroots = [compute_dotted_and_sysroot(glo)[1] for glo in globs]
|
|
73
|
+
# ⭐ Get unified monorepo root
|
|
74
|
+
monorepo_root = get_common_root(sysroots)
|
|
75
|
+
if str(monorepo_root) not in sys.path:
|
|
76
|
+
sys.path.insert(0, str(monorepo_root))
|
|
77
|
+
get_all_imports(directory=directory, sysroot=monorepo_root)
|
|
23
78
|
def is_import_or_init(sysroot,likely=None):
|
|
24
79
|
file_data = get_file_parts(sysroot)
|
|
25
80
|
nuroot = sysroot
|
|
@@ -1,41 +1,90 @@
|
|
|
1
1
|
from .imports import *
|
|
2
|
-
|
|
2
|
+
import os, sys, inspect, logging
|
|
3
|
+
from logging.handlers import RotatingFileHandler
|
|
4
|
+
|
|
5
|
+
PACKAGE_NAME = "abstract_utilities" # ← update if needed
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _resolve_log_root():
|
|
9
|
+
"""
|
|
10
|
+
Returns a safe writable logging directory depending on environment:
|
|
11
|
+
- If running in a virtualenv → <venv>/.logs/<package>
|
|
12
|
+
- Else if user writable → ~/.cache/<package>/logs
|
|
13
|
+
- Else → /var/log/<package>
|
|
14
|
+
"""
|
|
15
|
+
# 1) Virtualenv or Conda environment
|
|
16
|
+
venv = os.getenv("VIRTUAL_ENV") or os.getenv("CONDA_PREFIX")
|
|
17
|
+
if venv:
|
|
18
|
+
root = os.path.join(venv, ".logs", PACKAGE_NAME)
|
|
19
|
+
os.makedirs(root, exist_ok=True)
|
|
20
|
+
return root
|
|
21
|
+
|
|
22
|
+
# 2) User home cache folder
|
|
23
|
+
home = os.path.expanduser("~")
|
|
24
|
+
user_cache_root = os.path.join(home, ".cache", PACKAGE_NAME, "logs")
|
|
25
|
+
try:
|
|
26
|
+
os.makedirs(user_cache_root, exist_ok=True)
|
|
27
|
+
return user_cache_root
|
|
28
|
+
except PermissionError:
|
|
29
|
+
pass
|
|
30
|
+
|
|
31
|
+
# 3) Last resort: system log dir (requires correct service user permissions)
|
|
32
|
+
system_root = f"/var/log/{PACKAGE_NAME}"
|
|
33
|
+
try:
|
|
34
|
+
os.makedirs(system_root, exist_ok=True)
|
|
35
|
+
return system_root
|
|
36
|
+
except PermissionError:
|
|
37
|
+
# Fail-safe fallback to /tmp
|
|
38
|
+
fallback = f"/tmp/{PACKAGE_NAME}/logs"
|
|
39
|
+
os.makedirs(fallback, exist_ok=True)
|
|
40
|
+
return fallback
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
LOG_ROOT = _resolve_log_root()
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def get_logFile(bpName=None, maxBytes=100_000, backupCount=3):
|
|
3
47
|
"""
|
|
4
|
-
|
|
5
|
-
|
|
48
|
+
A logger that always writes to a safe OS-appropriate path.
|
|
49
|
+
Works even when installed through pip.
|
|
6
50
|
"""
|
|
7
51
|
if bpName is None:
|
|
8
|
-
# Find the first frame outside logging_utils.py
|
|
9
52
|
frame_idx = _find_caller_frame_index()
|
|
10
53
|
frame_info = inspect.stack()[frame_idx]
|
|
11
|
-
caller_path = frame_info.filename
|
|
54
|
+
caller_path = frame_info.filename
|
|
12
55
|
bpName = os.path.splitext(os.path.basename(caller_path))[0]
|
|
13
56
|
del frame_info
|
|
14
57
|
|
|
15
|
-
|
|
16
|
-
log_path = os.path.join(log_dir, f"{bpName}.log")
|
|
17
|
-
|
|
18
|
-
logger = logging.getLogger(bpName)
|
|
58
|
+
logger = logging.getLogger(f"{PACKAGE_NAME}.{bpName}")
|
|
19
59
|
logger.setLevel(logging.INFO)
|
|
20
60
|
|
|
21
61
|
if not logger.handlers:
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
print(f"{e}")
|
|
62
|
+
log_file = os.path.join(LOG_ROOT, f"{bpName}.log")
|
|
63
|
+
handler = RotatingFileHandler(log_file, maxBytes=maxBytes, backupCount=backupCount)
|
|
64
|
+
|
|
65
|
+
fmt = "%(asctime)s - %(levelname)s - %(pathname)s:%(lineno)d - %(message)s"
|
|
66
|
+
formatter = logging.Formatter(fmt)
|
|
67
|
+
handler.setFormatter(formatter)
|
|
68
|
+
|
|
69
|
+
logger.addHandler(handler)
|
|
70
|
+
|
|
71
|
+
# Console handler (optional; can disable for gunicorn)
|
|
72
|
+
console = logging.StreamHandler(sys.stdout)
|
|
73
|
+
console.setFormatter(formatter)
|
|
74
|
+
logger.addHandler(console)
|
|
75
|
+
|
|
37
76
|
return logger
|
|
38
77
|
|
|
78
|
+
|
|
79
|
+
def _find_caller_frame_index():
|
|
80
|
+
"""Find the correct caller module outside this logger."""
|
|
81
|
+
for idx, frame_info in enumerate(inspect.stack()):
|
|
82
|
+
if idx == 0:
|
|
83
|
+
continue
|
|
84
|
+
module = inspect.getmodule(frame_info.frame)
|
|
85
|
+
if module and module.__name__ not in (__name__, "logging"):
|
|
86
|
+
return idx
|
|
87
|
+
return 1
|
|
39
88
|
def _find_caller_frame_index():
|
|
40
89
|
"""
|
|
41
90
|
Scan up the call stack until we find a frame whose module is NOT logging_utils.
|
|
@@ -64,6 +64,29 @@ def detect_language_from_text(text: str):
|
|
|
64
64
|
|
|
65
65
|
likely = [lang for lang, score in scores.items() if score == max_score]
|
|
66
66
|
return likely[0] if len(likely) == 1 else 'uncertain'
|
|
67
|
+
def get_tripple_string(string):
|
|
68
|
+
nustring = ''
|
|
69
|
+
for i in range(3):
|
|
70
|
+
nustring +=string
|
|
71
|
+
return nustring
|
|
72
|
+
def get_within_quotes(text,quotes=None):
|
|
73
|
+
quotes_strings = quotes or ["'",'"']
|
|
74
|
+
in_quotes = []
|
|
75
|
+
for quotes_string in quotes_strings:
|
|
76
|
+
if not isinstance(quotes_string,list):
|
|
77
|
+
tripple= get_tripple_string(quotes_string)
|
|
78
|
+
texts = [text]
|
|
79
|
+
if tripple in text:
|
|
80
|
+
texts= text.split(tripple)
|
|
81
|
+
for text_part in texts:
|
|
82
|
+
quote_count = len(text_part) - len(text_part.replace(quotes_string,''))
|
|
83
|
+
quote_spl = text_part.split(quotes_string)
|
|
84
|
+
in_quotes+=[quote_spl[i] for i in range(quote_count) if ((i == 1 or i%2 != float(0)) and len(quote_spl) > i)]
|
|
85
|
+
else:
|
|
86
|
+
texts= text.split(quotes_string[0])
|
|
87
|
+
for text in texts:
|
|
88
|
+
in_quotes.append(text.split(quotes_string[1])[0])
|
|
89
|
+
return in_quotes
|
|
67
90
|
|
|
68
91
|
def search_code(code_languages, parts):
|
|
69
92
|
return [data for datas in parts for data in make_list(datas)
|