webscout 3.3__py3-none-any.whl → 3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of webscout might be problematic. Click here for more details.
- webscout/AIutel.py +1 -0
- webscout/DWEBS.py +772 -176
- webscout/Local/_version.py +1 -1
- webscout/Provider/Deepinfra.py +479 -0
- webscout/Provider/__init__.py +5 -0
- webscout/__init__.py +4 -2
- webscout/cli.py +17 -15
- webscout/exceptions.py +1 -1
- webscout/version.py +1 -1
- webscout/webai.py +15 -0
- webscout/webscout_search.py +48 -39
- webscout/webscout_search_async.py +11 -10
- webscout/websx_search.py +370 -0
- {webscout-3.3.dist-info → webscout-3.5.dist-info}/METADATA +149 -217
- {webscout-3.3.dist-info → webscout-3.5.dist-info}/RECORD +19 -29
- {webscout-3.3.dist-info → webscout-3.5.dist-info}/top_level.txt +0 -1
- DeepWEBS/__init__.py +0 -0
- DeepWEBS/documents/__init__.py +0 -0
- DeepWEBS/documents/query_results_extractor.py +0 -99
- DeepWEBS/documents/webpage_content_extractor.py +0 -145
- DeepWEBS/networks/__init__.py +0 -0
- DeepWEBS/networks/filepath_converter.py +0 -109
- DeepWEBS/networks/google_searcher.py +0 -52
- DeepWEBS/networks/network_configs.py +0 -30
- DeepWEBS/networks/webpage_fetcher.py +0 -95
- DeepWEBS/utilsdw/__init__.py +0 -0
- DeepWEBS/utilsdw/enver.py +0 -78
- DeepWEBS/utilsdw/logger.py +0 -269
- {webscout-3.3.dist-info → webscout-3.5.dist-info}/LICENSE.md +0 -0
- {webscout-3.3.dist-info → webscout-3.5.dist-info}/WHEEL +0 -0
- {webscout-3.3.dist-info → webscout-3.5.dist-info}/entry_points.txt +0 -0
DeepWEBS/utilsdw/logger.py
DELETED
|
@@ -1,269 +0,0 @@
|
|
|
1
|
-
import datetime
|
|
2
|
-
import functools
|
|
3
|
-
import inspect
|
|
4
|
-
import logging
|
|
5
|
-
import os
|
|
6
|
-
import shutil
|
|
7
|
-
import subprocess
|
|
8
|
-
from termcolor import colored
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
def add_fillers(text, filler="=", fill_side="both"):
|
|
12
|
-
terminal_width = shutil.get_terminal_size().columns
|
|
13
|
-
text = text.strip()
|
|
14
|
-
text_width = len(text)
|
|
15
|
-
if text_width >= terminal_width:
|
|
16
|
-
return text
|
|
17
|
-
|
|
18
|
-
if fill_side[0].lower() == "b":
|
|
19
|
-
leading_fill_str = filler * ((terminal_width - text_width) // 2 - 1) + " "
|
|
20
|
-
trailing_fill_str = " " + filler * (
|
|
21
|
-
terminal_width - text_width - len(leading_fill_str) - 1
|
|
22
|
-
)
|
|
23
|
-
elif fill_side[0].lower() == "l":
|
|
24
|
-
leading_fill_str = filler * (terminal_width - text_width - 1) + " "
|
|
25
|
-
trailing_fill_str = ""
|
|
26
|
-
elif fill_side[0].lower() == "r":
|
|
27
|
-
leading_fill_str = ""
|
|
28
|
-
trailing_fill_str = " " + filler * (terminal_width - text_width - 1)
|
|
29
|
-
else:
|
|
30
|
-
raise ValueError("Invalid fill_side")
|
|
31
|
-
|
|
32
|
-
filled_str = f"{leading_fill_str}{text}{trailing_fill_str}"
|
|
33
|
-
return filled_str
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
class OSLogger(logging.Logger):
|
|
37
|
-
LOG_METHODS = {
|
|
38
|
-
"err": ("error", "red"),
|
|
39
|
-
"warn": ("warning", "light_red"),
|
|
40
|
-
"note": ("info", "light_magenta"),
|
|
41
|
-
"mesg": ("info", "light_cyan"),
|
|
42
|
-
"file": ("info", "light_blue"),
|
|
43
|
-
"line": ("info", "white"),
|
|
44
|
-
"success": ("info", "light_green"),
|
|
45
|
-
"fail": ("info", "light_red"),
|
|
46
|
-
"back": ("debug", "light_cyan"),
|
|
47
|
-
}
|
|
48
|
-
INDENT_METHODS = [
|
|
49
|
-
"indent",
|
|
50
|
-
"set_indent",
|
|
51
|
-
"reset_indent",
|
|
52
|
-
"store_indent",
|
|
53
|
-
"restore_indent",
|
|
54
|
-
"log_indent",
|
|
55
|
-
]
|
|
56
|
-
LEVEL_METHODS = [
|
|
57
|
-
"set_level",
|
|
58
|
-
"store_level",
|
|
59
|
-
"restore_level",
|
|
60
|
-
"quiet",
|
|
61
|
-
"enter_quiet",
|
|
62
|
-
"exit_quiet",
|
|
63
|
-
]
|
|
64
|
-
LEVEL_NAMES = {
|
|
65
|
-
"critical": logging.CRITICAL,
|
|
66
|
-
"error": logging.ERROR,
|
|
67
|
-
"warning": logging.WARNING,
|
|
68
|
-
"info": logging.INFO,
|
|
69
|
-
"debug": logging.DEBUG,
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
def __init__(self, name=None, prefix=False):
|
|
73
|
-
if not name:
|
|
74
|
-
frame = inspect.stack()[1]
|
|
75
|
-
module = inspect.getmodule(frame[0])
|
|
76
|
-
name = module.__name__
|
|
77
|
-
|
|
78
|
-
super().__init__(name)
|
|
79
|
-
self.setLevel(logging.INFO)
|
|
80
|
-
|
|
81
|
-
if prefix:
|
|
82
|
-
formatter_prefix = "[%(asctime)s] - [%(name)s] - [%(levelname)s]\n"
|
|
83
|
-
else:
|
|
84
|
-
formatter_prefix = ""
|
|
85
|
-
|
|
86
|
-
self.formatter = logging.Formatter(formatter_prefix + "%(message)s")
|
|
87
|
-
|
|
88
|
-
stream_handler = logging.StreamHandler()
|
|
89
|
-
stream_handler.setLevel(logging.INFO)
|
|
90
|
-
stream_handler.setFormatter(self.formatter)
|
|
91
|
-
self.addHandler(stream_handler)
|
|
92
|
-
|
|
93
|
-
self.log_indent = 0
|
|
94
|
-
self.log_indents = []
|
|
95
|
-
|
|
96
|
-
self.log_level = "info"
|
|
97
|
-
self.log_levels = []
|
|
98
|
-
|
|
99
|
-
def indent(self, indent=2):
|
|
100
|
-
self.log_indent += indent
|
|
101
|
-
|
|
102
|
-
def set_indent(self, indent=2):
|
|
103
|
-
self.log_indent = indent
|
|
104
|
-
|
|
105
|
-
def reset_indent(self):
|
|
106
|
-
self.log_indent = 0
|
|
107
|
-
|
|
108
|
-
def store_indent(self):
|
|
109
|
-
self.log_indents.append(self.log_indent)
|
|
110
|
-
|
|
111
|
-
def restore_indent(self):
|
|
112
|
-
self.log_indent = self.log_indents.pop(-1)
|
|
113
|
-
|
|
114
|
-
def set_level(self, level):
|
|
115
|
-
self.log_level = level
|
|
116
|
-
self.setLevel(self.LEVEL_NAMES[level])
|
|
117
|
-
|
|
118
|
-
def store_level(self):
|
|
119
|
-
self.log_levels.append(self.log_level)
|
|
120
|
-
|
|
121
|
-
def restore_level(self):
|
|
122
|
-
self.log_level = self.log_levels.pop(-1)
|
|
123
|
-
self.set_level(self.log_level)
|
|
124
|
-
|
|
125
|
-
def quiet(self):
|
|
126
|
-
self.set_level("critical")
|
|
127
|
-
|
|
128
|
-
def enter_quiet(self, quiet=False):
|
|
129
|
-
if quiet:
|
|
130
|
-
self.store_level()
|
|
131
|
-
self.quiet()
|
|
132
|
-
|
|
133
|
-
def exit_quiet(self, quiet=False):
|
|
134
|
-
if quiet:
|
|
135
|
-
self.restore_level()
|
|
136
|
-
|
|
137
|
-
def log(
|
|
138
|
-
self,
|
|
139
|
-
level,
|
|
140
|
-
color,
|
|
141
|
-
msg,
|
|
142
|
-
indent=0,
|
|
143
|
-
fill=False,
|
|
144
|
-
fill_side="both",
|
|
145
|
-
end="\n",
|
|
146
|
-
*args,
|
|
147
|
-
**kwargs,
|
|
148
|
-
):
|
|
149
|
-
if type(msg) == str:
|
|
150
|
-
msg_str = msg
|
|
151
|
-
else:
|
|
152
|
-
msg_str = repr(msg)
|
|
153
|
-
quotes = ["'", '"']
|
|
154
|
-
if msg_str[0] in quotes and msg_str[-1] in quotes:
|
|
155
|
-
msg_str = msg_str[1:-1]
|
|
156
|
-
|
|
157
|
-
indent_str = " " * (self.log_indent + indent)
|
|
158
|
-
indented_msg = "\n".join([indent_str + line for line in msg_str.split("\n")])
|
|
159
|
-
|
|
160
|
-
if fill:
|
|
161
|
-
indented_msg = add_fillers(indented_msg, fill_side=fill_side)
|
|
162
|
-
|
|
163
|
-
handler = self.handlers[0]
|
|
164
|
-
handler.terminator = end
|
|
165
|
-
|
|
166
|
-
getattr(self, level)(colored(indented_msg, color), *args, **kwargs)
|
|
167
|
-
|
|
168
|
-
def route_log(self, method, msg, *args, **kwargs):
|
|
169
|
-
level, method = method
|
|
170
|
-
functools.partial(self.log, level, method, msg)(*args, **kwargs)
|
|
171
|
-
|
|
172
|
-
def err(self, msg: str = "", *args, **kwargs):
|
|
173
|
-
self.route_log(("error", "red"), msg, *args, **kwargs)
|
|
174
|
-
|
|
175
|
-
def warn(self, msg: str = "", *args, **kwargs):
|
|
176
|
-
self.route_log(("warning", "light_red"), msg, *args, **kwargs)
|
|
177
|
-
|
|
178
|
-
def note(self, msg: str = "", *args, **kwargs):
|
|
179
|
-
self.route_log(("info", "light_magenta"), msg, *args, **kwargs)
|
|
180
|
-
|
|
181
|
-
def mesg(self, msg: str = "", *args, **kwargs):
|
|
182
|
-
self.route_log(("info", "light_cyan"), msg, *args, **kwargs)
|
|
183
|
-
|
|
184
|
-
def file(self, msg: str = "", *args, **kwargs):
|
|
185
|
-
self.route_log(("info", "light_blue"), msg, *args, **kwargs)
|
|
186
|
-
|
|
187
|
-
def line(self, msg: str = "", *args, **kwargs):
|
|
188
|
-
self.route_log(("info", "white"), msg, *args, **kwargs)
|
|
189
|
-
|
|
190
|
-
def success(self, msg: str = "", *args, **kwargs):
|
|
191
|
-
self.route_log(("info", "light_green"), msg, *args, **kwargs)
|
|
192
|
-
|
|
193
|
-
def fail(self, msg: str = "", *args, **kwargs):
|
|
194
|
-
self.route_log(("info", "light_red"), msg, *args, **kwargs)
|
|
195
|
-
|
|
196
|
-
def back(self, msg: str = "", *args, **kwargs):
|
|
197
|
-
self.route_log(("debug", "light_cyan"), msg, *args, **kwargs)
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
logger = OSLogger()
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
def shell_cmd(cmd, getoutput=False, showcmd=True, env=None):
|
|
204
|
-
if showcmd:
|
|
205
|
-
logger.info(colored(f"\n$ [{os.getcwd()}]", "light_blue"))
|
|
206
|
-
logger.info(colored(f" $ {cmd}\n", "light_cyan"))
|
|
207
|
-
if getoutput:
|
|
208
|
-
output = subprocess.getoutput(cmd, env=env)
|
|
209
|
-
return output
|
|
210
|
-
else:
|
|
211
|
-
subprocess.run(cmd, shell=True, env=env)
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
class Runtimer:
|
|
215
|
-
def __enter__(self):
|
|
216
|
-
self.t1, _ = self.start_time()
|
|
217
|
-
return self
|
|
218
|
-
|
|
219
|
-
def __exit__(self, exc_type, exc_value, traceback):
|
|
220
|
-
self.t2, _ = self.end_time()
|
|
221
|
-
self.elapsed_time(self.t2 - self.t1)
|
|
222
|
-
|
|
223
|
-
def start_time(self):
|
|
224
|
-
t1 = datetime.datetime.now()
|
|
225
|
-
self.logger_time("start", t1)
|
|
226
|
-
return t1, self.time2str(t1)
|
|
227
|
-
|
|
228
|
-
def end_time(self):
|
|
229
|
-
t2 = datetime.datetime.now()
|
|
230
|
-
self.logger_time("end", t2)
|
|
231
|
-
return t2, self.time2str(t2)
|
|
232
|
-
|
|
233
|
-
def elapsed_time(self, dt=None):
|
|
234
|
-
if dt is None:
|
|
235
|
-
dt = self.t2 - self.t1
|
|
236
|
-
self.logger_time("elapsed", dt)
|
|
237
|
-
return dt, self.time2str(dt)
|
|
238
|
-
|
|
239
|
-
def logger_time(self, time_type, t):
|
|
240
|
-
time_types = {
|
|
241
|
-
"start": "Start",
|
|
242
|
-
"end": "End",
|
|
243
|
-
"elapsed": "Elapsed",
|
|
244
|
-
}
|
|
245
|
-
time_str = add_fillers(
|
|
246
|
-
colored(
|
|
247
|
-
f"{time_types[time_type]} time: [ {self.time2str(t)} ]",
|
|
248
|
-
"light_magenta",
|
|
249
|
-
),
|
|
250
|
-
fill_side="both",
|
|
251
|
-
)
|
|
252
|
-
logger.line(time_str)
|
|
253
|
-
|
|
254
|
-
# Convert time to string
|
|
255
|
-
def time2str(self, t):
|
|
256
|
-
datetime_str_format = "%Y-%m-%d %H:%M:%S"
|
|
257
|
-
if isinstance(t, datetime.datetime):
|
|
258
|
-
return t.strftime(datetime_str_format)
|
|
259
|
-
elif isinstance(t, datetime.timedelta):
|
|
260
|
-
hours = t.seconds // 3600
|
|
261
|
-
hour_str = f"{hours} hr" if hours > 0 else ""
|
|
262
|
-
minutes = (t.seconds // 60) % 60
|
|
263
|
-
minute_str = f"{minutes:>2} min" if minutes > 0 else ""
|
|
264
|
-
seconds = t.seconds % 60
|
|
265
|
-
second_str = f"{seconds:>2} s"
|
|
266
|
-
time_str = " ".join([hour_str, minute_str, second_str]).strip()
|
|
267
|
-
return time_str
|
|
268
|
-
else:
|
|
269
|
-
return str(t)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|