im-internals 0.4.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- im_internals/__init__.py +34 -0
- im_internals/api.py +471 -0
- im_internals/cfg_commands.py +185 -0
- im_internals/email.py +224 -0
- im_internals/file.py +216 -0
- im_internals/folder.py +83 -0
- im_internals/ftp.py +630 -0
- im_internals/logging.py +221 -0
- im_internals/sanitize.py +177 -0
- im_internals/sftp.py +691 -0
- im_internals/sql.py +251 -0
- im_internals/transfer.py +536 -0
- im_internals-0.4.2.dist-info/METADATA +156 -0
- im_internals-0.4.2.dist-info/RECORD +18 -0
- im_internals-0.4.2.dist-info/WHEEL +5 -0
- im_internals-0.4.2.dist-info/entry_points.txt +2 -0
- im_internals-0.4.2.dist-info/licenses/LICENSE +16 -0
- im_internals-0.4.2.dist-info/top_level.txt +1 -0
im_internals/__init__.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""
|
|
2
|
+
im_internals package
|
|
3
|
+
Bundled helpers for database access, (S)FTP/file transfer, REST APIs,
|
|
4
|
+
e-mail sending/processing, and assorted utilities, for internal use only.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
from importlib import import_module as _mod
|
|
9
|
+
from importlib.metadata import version, PackageNotFoundError
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"api",
|
|
13
|
+
"cfg_commands",
|
|
14
|
+
"email",
|
|
15
|
+
"file",
|
|
16
|
+
"folder",
|
|
17
|
+
"ftp",
|
|
18
|
+
"logging",
|
|
19
|
+
"sanitize",
|
|
20
|
+
"sftp",
|
|
21
|
+
"sql",
|
|
22
|
+
"transfer",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
# Lazy-import the sub-modules so startup remains fast
|
|
26
|
+
for _name in __all__:
|
|
27
|
+
globals()[_name] = _mod(f".{_name}", __name__)
|
|
28
|
+
|
|
29
|
+
try:
|
|
30
|
+
__version__ = version("im_internals")
|
|
31
|
+
except PackageNotFoundError:
|
|
32
|
+
__version__ = "unknown"
|
|
33
|
+
|
|
34
|
+
|
im_internals/api.py
ADDED
|
@@ -0,0 +1,471 @@
|
|
|
1
|
+
"""
|
|
2
|
+
API client
|
|
3
|
+
==========
|
|
4
|
+
HTTP API client with token auth, per-call timeouts, and retry-on-timeout, built on `requests`.
|
|
5
|
+
|
|
6
|
+
Two classes:
|
|
7
|
+
|
|
8
|
+
- `Api` — token-lifecycle management (fetch/verify/refresh) plus `send_json`/`send_files` for the
|
|
9
|
+
bespoke document-submission APIs used by several client pipelines in this repo (e.g. AXA's
|
|
10
|
+
Mountaineer API). `get_files()` is an unimplemented placeholder (always raises
|
|
11
|
+
`NotImplementedError`) — no client currently needs a download path.
|
|
12
|
+
- `ApiClientCRUD(Api)` — adds `get`/`post`/`put`/`delete` against a `base_url`-prefixed REST API.
|
|
13
|
+
This is the closest existing analogue for a future replacement of the legacy
|
|
14
|
+
`Frequently_used_Prod.send_files_prod.JSON` class (see this repo's root CLAUDE.md — deferred
|
|
15
|
+
migration blocker for `McDonald_SFTP_prod.py`/`AXA_temp_manual_prod.py`), though nothing currently
|
|
16
|
+
uses it for that purpose.
|
|
17
|
+
|
|
18
|
+
Usage::
|
|
19
|
+
|
|
20
|
+
api = Api(web_url="https://example.com/api", login_web_url="https://example.com/login",
|
|
21
|
+
username="user", pwd="pass")
|
|
22
|
+
api.send_json([{"key": "value"}])
|
|
23
|
+
|
|
24
|
+
Every network-calling method is wrapped in `@retry_on_timeout` (retries `TimeoutException` up to 3
|
|
25
|
+
times with exponential backoff) and `@timeout_decorator(seconds)` (runs the call on a background
|
|
26
|
+
daemon thread and raises `TimeoutException` if it doesn't finish in time).
|
|
27
|
+
|
|
28
|
+
`timeout_decorator` used to run the call inside a `with ThreadPoolExecutor(max_workers=1) as exec:`
|
|
29
|
+
block (fixed 2026-09-23). That looked equivalent but wasn't: a Python thread can't be killed once
|
|
30
|
+
started, so a call that never actually returns left that worker thread running forever - and exiting
|
|
31
|
+
the `with` block calls `exec.shutdown(wait=True)`, which blocks until every submitted thread
|
|
32
|
+
finishes, timeout or not. `concurrent.futures.thread` also registers a process-wide `atexit` hook
|
|
33
|
+
that joins *every* worker thread any `ThreadPoolExecutor` in the process ever created, so this could
|
|
34
|
+
hang the whole interpreter at shutdown even for code that never touched this decorator's return
|
|
35
|
+
value. The replacement spawns a plain `daemon=True` thread instead: `Thread.join(timeout)` still
|
|
36
|
+
gives up after `timeout` seconds (execution stays just as sequential from the caller's side - the
|
|
37
|
+
call already ran on a background thread before, this doesn't add new concurrency), but a daemon
|
|
38
|
+
thread is never joined at interpreter exit, so an abandoned one can't block the process from exiting.
|
|
39
|
+
"""
|
|
40
|
+
import logging
|
|
41
|
+
import requests
|
|
42
|
+
import functools
|
|
43
|
+
import threading
|
|
44
|
+
from typing import List, Dict, Any
|
|
45
|
+
from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential, before_sleep_log
|
|
46
|
+
|
|
47
|
+
from . import logging as pl
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class TimeoutException(Exception):
|
|
51
|
+
"""
|
|
52
|
+
Custom exception to indicate that a function has exceeded its time limit.
|
|
53
|
+
"""
|
|
54
|
+
pass
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
# Standard logger kept solely for tenacity's before_sleep_log (requires stdlib Logger)
|
|
58
|
+
_tenacity_logger = logging.getLogger(__name__)
|
|
59
|
+
|
|
60
|
+
# -----------------------------
|
|
61
|
+
# Retry decorator for timeouts
|
|
62
|
+
# -----------------------------
|
|
63
|
+
retry_on_timeout = retry(
|
|
64
|
+
retry=retry_if_exception_type(TimeoutException),
|
|
65
|
+
reraise=True,
|
|
66
|
+
stop=stop_after_attempt(3),
|
|
67
|
+
wait=wait_exponential(multiplier=1, min=3, max=10),
|
|
68
|
+
before_sleep=before_sleep_log(_tenacity_logger, logging.WARNING)
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def timeout_decorator(timeout: int):
|
|
73
|
+
"""
|
|
74
|
+
Decorator to enforce a maximum execution time on a function.
|
|
75
|
+
|
|
76
|
+
:param timeout: Maximum seconds the function is allowed to run.
|
|
77
|
+
:type timeout: int
|
|
78
|
+
:return: Decorated function that raises TimeoutException if time limit exceeded.
|
|
79
|
+
:rtype: Callable
|
|
80
|
+
|
|
81
|
+
:raises TimeoutException: When function execution surpasses the timeout.
|
|
82
|
+
"""
|
|
83
|
+
assert timeout > 0, "Timeout must be positive"
|
|
84
|
+
|
|
85
|
+
def decorator(func):
|
|
86
|
+
@functools.wraps(func)
|
|
87
|
+
def wrapper(*args, **kwargs):
|
|
88
|
+
result_box = {}
|
|
89
|
+
|
|
90
|
+
def target():
|
|
91
|
+
try:
|
|
92
|
+
result_box["value"] = func(*args, **kwargs)
|
|
93
|
+
except BaseException as e:
|
|
94
|
+
result_box["error"] = e
|
|
95
|
+
|
|
96
|
+
worker = threading.Thread(target=target, daemon=True)
|
|
97
|
+
worker.start()
|
|
98
|
+
worker.join(timeout)
|
|
99
|
+
|
|
100
|
+
if worker.is_alive():
|
|
101
|
+
# Call is still running (likely a stalled network call). We can't kill it, so we
|
|
102
|
+
# abandon it as a daemon thread - it won't block interpreter shutdown - and give
|
|
103
|
+
# up here instead of blocking the caller (and every caller of this decorator, and
|
|
104
|
+
# potentially interpreter exit) forever.
|
|
105
|
+
raise TimeoutException(f"Timed out after {timeout}s")
|
|
106
|
+
if "error" in result_box:
|
|
107
|
+
raise result_box["error"]
|
|
108
|
+
return result_box.get("value")
|
|
109
|
+
return wrapper
|
|
110
|
+
|
|
111
|
+
return decorator
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def ensure_token(func):
|
|
115
|
+
"""
|
|
116
|
+
Decorator to ensure a valid API token exists before method execution.
|
|
117
|
+
|
|
118
|
+
If no token is set, it calls get_api_token(). If verification fails, it refreshes the token.
|
|
119
|
+
|
|
120
|
+
:param func: Method requiring a valid token.
|
|
121
|
+
:type func: Callable
|
|
122
|
+
:return: Wrapped method with token validation.
|
|
123
|
+
:rtype: Callable
|
|
124
|
+
"""
|
|
125
|
+
def wrapper(self, *args, **kwargs):
|
|
126
|
+
if not getattr(self, 'api_token', None):
|
|
127
|
+
pl.progress('No token found, fetching new token.')
|
|
128
|
+
self.api_token = self.get_api_token()
|
|
129
|
+
else:
|
|
130
|
+
self.verify_and_refresh_api_token()
|
|
131
|
+
return func(self, *args, **kwargs)
|
|
132
|
+
return wrapper
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class Api:
|
|
136
|
+
"""
|
|
137
|
+
Core API client handling authentication, timeouts, and retries.
|
|
138
|
+
"""
|
|
139
|
+
|
|
140
|
+
def __init__(self, web_url: str, login_web_url: str, token: str = None, username: str = "", pwd: str = ""):
|
|
141
|
+
"""
|
|
142
|
+
Initializes the API class with the necessary URLs and login credentials.
|
|
143
|
+
|
|
144
|
+
:param web_url: The base URL for data-related API operations.
|
|
145
|
+
:type web_url: str
|
|
146
|
+
:param login_web_url: The URL for obtaining a new API token.
|
|
147
|
+
:type login_web_url: str
|
|
148
|
+
:param token: Optional, an existing API token for immediate use.
|
|
149
|
+
:type token: str, optional
|
|
150
|
+
:param username: The username for the API login.
|
|
151
|
+
:type username: str
|
|
152
|
+
:param pwd: The password for the API login.
|
|
153
|
+
:type pwd: str
|
|
154
|
+
"""
|
|
155
|
+
assert isinstance(web_url, str) and ("http://" in web_url or "https://" in web_url), \
|
|
156
|
+
"Web URL should be a string with the structure of URL like https://www.example.com"
|
|
157
|
+
assert isinstance(login_web_url, str) and ("http://" in login_web_url or "https://" in login_web_url), \
|
|
158
|
+
"Login URL should be a string with the structure of URL like https://www.login-page.com"
|
|
159
|
+
assert isinstance(username, str), "Username must always be only string!"
|
|
160
|
+
assert isinstance(pwd, str), "As we now do not support crypted password, we need a string!"
|
|
161
|
+
|
|
162
|
+
self.username = username
|
|
163
|
+
self.pwd = pwd
|
|
164
|
+
self.api_token = token
|
|
165
|
+
self.data_url = web_url
|
|
166
|
+
self.login_url = login_web_url
|
|
167
|
+
self.session = requests.Session()
|
|
168
|
+
self.api_token = token if token else self.get_api_token()
|
|
169
|
+
|
|
170
|
+
@property
|
|
171
|
+
def token(self) -> str:
|
|
172
|
+
"""
|
|
173
|
+
Return a fresh API token, refreshing if expired.
|
|
174
|
+
|
|
175
|
+
:return: Valid API token string.
|
|
176
|
+
:rtype: str
|
|
177
|
+
"""
|
|
178
|
+
return self.verify_and_refresh_api_token()
|
|
179
|
+
|
|
180
|
+
@retry_on_timeout
|
|
181
|
+
@timeout_decorator(5 * 60)
|
|
182
|
+
def verify_and_refresh_api_token(self) -> str:
|
|
183
|
+
"""
|
|
184
|
+
Verify current token validity and refresh if expired.
|
|
185
|
+
|
|
186
|
+
:return: Current or newly fetched API token.
|
|
187
|
+
:rtype: str
|
|
188
|
+
:raises TimeoutException: When checking token exceeds time limit.
|
|
189
|
+
:raises requests.RequestException: On HTTP errors during verification.
|
|
190
|
+
"""
|
|
191
|
+
pl.progress('Verifying token validity.')
|
|
192
|
+
try:
|
|
193
|
+
if self.user_is_logged_out():
|
|
194
|
+
pl.progress('Token expired, fetching new one.')
|
|
195
|
+
self.api_token = self.get_api_token()
|
|
196
|
+
except TimeoutException:
|
|
197
|
+
pl.error('Token verification timed out, will retry fetch.')
|
|
198
|
+
self.api_token = self.get_api_token()
|
|
199
|
+
except requests.RequestException as e:
|
|
200
|
+
pl.error(f'Failed to verify token: {e}, fetching new one.')
|
|
201
|
+
self.api_token = self.get_api_token()
|
|
202
|
+
return self.api_token
|
|
203
|
+
|
|
204
|
+
@retry_on_timeout
|
|
205
|
+
@timeout_decorator(5 * 60)
|
|
206
|
+
def user_is_logged_out(self) -> bool:
|
|
207
|
+
"""
|
|
208
|
+
Determine if the current API token is expired based on a test request.
|
|
209
|
+
|
|
210
|
+
:return: True if expired (status 403), False otherwise.
|
|
211
|
+
:rtype: bool
|
|
212
|
+
"""
|
|
213
|
+
try:
|
|
214
|
+
resp = self.session.get(self.data_url, headers=self._get_headers(), timeout=30)
|
|
215
|
+
return resp.status_code == 403
|
|
216
|
+
except requests.RequestException:
|
|
217
|
+
return True
|
|
218
|
+
|
|
219
|
+
@retry_on_timeout
|
|
220
|
+
@timeout_decorator(5 * 60)
|
|
221
|
+
def get_api_token(self) -> str:
|
|
222
|
+
"""
|
|
223
|
+
Obtain a new API token using stored credentials.
|
|
224
|
+
|
|
225
|
+
:return: New API token string.
|
|
226
|
+
:rtype: str
|
|
227
|
+
:raises TimeoutException: When token request exceeds time limit.
|
|
228
|
+
:raises requests.RequestException: On HTTP errors during token fetch.
|
|
229
|
+
"""
|
|
230
|
+
pl.progress('Requesting new API token.')
|
|
231
|
+
try:
|
|
232
|
+
resp = self.session.post(self.login_url, data={'login': self.username, 'pwd': self.pwd}, timeout=30)
|
|
233
|
+
resp.raise_for_status()
|
|
234
|
+
token = resp.json().get('token', '')
|
|
235
|
+
self.api_token = token
|
|
236
|
+
return token
|
|
237
|
+
except TimeoutException:
|
|
238
|
+
pl.error('get_api_token function exceeded the time limit of 5 minutes.')
|
|
239
|
+
raise
|
|
240
|
+
except requests.RequestException as e:
|
|
241
|
+
pl.error(f'Failed to retrieve new API token. Error: {e}')
|
|
242
|
+
return ''
|
|
243
|
+
|
|
244
|
+
def _get_headers(self) -> dict:
|
|
245
|
+
"""
|
|
246
|
+
Construct authorization headers for API requests.
|
|
247
|
+
|
|
248
|
+
:return: Headers containing the Bearer token.
|
|
249
|
+
:rtype: dict
|
|
250
|
+
"""
|
|
251
|
+
return {'Authorization': f'Bearer {self.api_token}'}
|
|
252
|
+
|
|
253
|
+
@ensure_token
|
|
254
|
+
@retry_on_timeout
|
|
255
|
+
@timeout_decorator(20 * 60)
|
|
256
|
+
def send_json(self, files_list: List[str | Dict[str, Any]] | Dict[str, Any]) -> requests.Response:
|
|
257
|
+
"""
|
|
258
|
+
Send a list of items as JSON payload to the API endpoint.
|
|
259
|
+
|
|
260
|
+
:param files_list: List of items to serialize and send.
|
|
261
|
+
:type files_list: list
|
|
262
|
+
:return: Response from the API.
|
|
263
|
+
:rtype: requests.Response
|
|
264
|
+
:raises TimeoutException: If request exceeds allowed time.
|
|
265
|
+
:raises requests.RequestException: On HTTP errors during send.
|
|
266
|
+
"""
|
|
267
|
+
assert isinstance(files_list, list), "List of files must contain a JSON list-like structure!!"
|
|
268
|
+
|
|
269
|
+
pl.progress('Sending JSON payload.')
|
|
270
|
+
try:
|
|
271
|
+
resp = self.session.post(self.data_url, json=files_list, headers=self._get_headers(), timeout=60)
|
|
272
|
+
resp.raise_for_status()
|
|
273
|
+
pl.log_kv("Response", text=resp.text)
|
|
274
|
+
return resp
|
|
275
|
+
except TimeoutException:
|
|
276
|
+
pl.error('send_json function exceeded the time limit of 20 minutes.')
|
|
277
|
+
raise
|
|
278
|
+
except requests.RequestException as e:
|
|
279
|
+
pl.error(f'Failed to send JSON data to API. Error: {e}')
|
|
280
|
+
raise
|
|
281
|
+
|
|
282
|
+
@ensure_token
|
|
283
|
+
@retry_on_timeout
|
|
284
|
+
@timeout_decorator(20 * 60)
|
|
285
|
+
def send_files(self, files_dict: Dict[str, str | bytes], form_data: Dict[str, Any]):
|
|
286
|
+
"""
|
|
287
|
+
Send files via multipart/form-data to the API endpoint.
|
|
288
|
+
|
|
289
|
+
:param files_dict: Mapping of filename to file-like object or bytes.
|
|
290
|
+
:type files_dict: dict
|
|
291
|
+
:param form_data: Additional form fields.
|
|
292
|
+
:type form_data: dict
|
|
293
|
+
:return: Response from the API.
|
|
294
|
+
:rtype: requests.Response
|
|
295
|
+
:raises TimeoutException: If upload exceeds allowed time.
|
|
296
|
+
:raises requests.RequestException: On HTTP errors during upload.
|
|
297
|
+
"""
|
|
298
|
+
assert isinstance(files_dict, dict), ("Files_dict must contain a dictionary in structure: "
|
|
299
|
+
"{'filename': b'file_bytes', ...}")
|
|
300
|
+
assert isinstance(form_data, dict), "Metadata must be in a dictionary strcture: {'metadata1': value, ...}"
|
|
301
|
+
|
|
302
|
+
pl.progress('Sending multipart files.')
|
|
303
|
+
try:
|
|
304
|
+
resp = self.session.post(self.data_url, files=files_dict, data=form_data, headers=self._get_headers(),
|
|
305
|
+
timeout=60)
|
|
306
|
+
resp.raise_for_status()
|
|
307
|
+
return resp
|
|
308
|
+
except TimeoutException:
|
|
309
|
+
pl.error('send_files function exceeded the time limit of 20 minutes.')
|
|
310
|
+
raise
|
|
311
|
+
except requests.RequestException as e:
|
|
312
|
+
pl.error(f'Failed to send files to the API. Error: {e}')
|
|
313
|
+
raise
|
|
314
|
+
|
|
315
|
+
@ensure_token
|
|
316
|
+
@retry_on_timeout
|
|
317
|
+
@timeout_decorator(20 * 60)
|
|
318
|
+
def get_files(self):
|
|
319
|
+
"""
|
|
320
|
+
Placeholder for downloading files from the API.
|
|
321
|
+
|
|
322
|
+
:raises NotImplementedError: Always, until implemented.
|
|
323
|
+
"""
|
|
324
|
+
pl.progress("get_files is not implemented yet.")
|
|
325
|
+
raise NotImplementedError
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
# -----------------------------
|
|
329
|
+
# CRUD extension
|
|
330
|
+
# -----------------------------
|
|
331
|
+
class ApiClientCRUD(Api):
|
|
332
|
+
"""
|
|
333
|
+
Extension of Api providing standard CRUD HTTP methods.
|
|
334
|
+
"""
|
|
335
|
+
|
|
336
|
+
def __init__(self, base_url: str, *args, api_key: str = None, **kwargs):
|
|
337
|
+
"""
|
|
338
|
+
Initialize CRUD client with base URL and credentials.
|
|
339
|
+
|
|
340
|
+
:param base_url: Root URL for CRUD endpoints.
|
|
341
|
+
:param web_url: Base URL for data operations (inherited).
|
|
342
|
+
:param login_web_url: URL for authentication requests (inherited).
|
|
343
|
+
:param token: Optional initial API token.
|
|
344
|
+
:param username: Username for login.
|
|
345
|
+
:param pwd: Password for login.
|
|
346
|
+
"""
|
|
347
|
+
assert isinstance(base_url, str) and ("http://" in base_url or "https://" in base_url), \
|
|
348
|
+
"Base URL that is used as a prefix for all endpoints must be a string and contain 'http://' or 'https://'"
|
|
349
|
+
|
|
350
|
+
super().__init__(*args, **kwargs)
|
|
351
|
+
self.base_url = base_url
|
|
352
|
+
if api_key:
|
|
353
|
+
self.api_token = api_key
|
|
354
|
+
|
|
355
|
+
@ensure_token
|
|
356
|
+
@retry_on_timeout
|
|
357
|
+
@timeout_decorator(10)
|
|
358
|
+
def get(self, endpoint: str, params: dict = None) -> dict:
|
|
359
|
+
"""
|
|
360
|
+
Perform an HTTP GET request.
|
|
361
|
+
|
|
362
|
+
:param endpoint: API endpoint path to append to base_url.
|
|
363
|
+
:type endpoint: str
|
|
364
|
+
:param params: Query parameters for the request.
|
|
365
|
+
:type params: dict
|
|
366
|
+
:return: JSON-decoded response body.
|
|
367
|
+
:rtype: dict
|
|
368
|
+
:raises TimeoutException: If request exceeds time limit.
|
|
369
|
+
:raises requests.RequestException: On HTTP errors.
|
|
370
|
+
"""
|
|
371
|
+
assert isinstance(endpoint, str), "Endpoint must be an existing web-path"
|
|
372
|
+
|
|
373
|
+
pl.progress(f'GET {endpoint}')
|
|
374
|
+
try:
|
|
375
|
+
resp = self.session.get(f"{self.base_url}/{endpoint}", headers=self._get_headers(), params=params,
|
|
376
|
+
timeout=10)
|
|
377
|
+
resp.raise_for_status()
|
|
378
|
+
return resp.json()
|
|
379
|
+
except TimeoutException:
|
|
380
|
+
pl.error(f'GET {endpoint} timed out.')
|
|
381
|
+
raise
|
|
382
|
+
except requests.RequestException as e:
|
|
383
|
+
pl.error(f'Failed GET {endpoint}: {e}')
|
|
384
|
+
raise
|
|
385
|
+
|
|
386
|
+
@ensure_token
|
|
387
|
+
@retry_on_timeout
|
|
388
|
+
@timeout_decorator(10)
|
|
389
|
+
def post(self, endpoint: str, body: Dict[str, Any]) -> dict:
|
|
390
|
+
"""
|
|
391
|
+
Perform an HTTP POST request with a JSON body.
|
|
392
|
+
|
|
393
|
+
:param endpoint: API endpoint path to append to base_url.
|
|
394
|
+
:type endpoint: str
|
|
395
|
+
:param body: JSON-serializable dictionary to send.
|
|
396
|
+
:type body: dict
|
|
397
|
+
:return: JSON-decoded response body.
|
|
398
|
+
:rtype: dict
|
|
399
|
+
:raises TimeoutException: If request exceeds time limit.
|
|
400
|
+
:raises requests.RequestException: On HTTP errors.
|
|
401
|
+
"""
|
|
402
|
+
assert isinstance(endpoint, str), "Endpoint must be an existing web-path"
|
|
403
|
+
assert isinstance(body, dict), "Body must a dictionary/JSON like structure. Example: {'metadata': value, ...}"
|
|
404
|
+
|
|
405
|
+
pl.progress(f'POST {endpoint}')
|
|
406
|
+
try:
|
|
407
|
+
resp = self.session.post(f"{self.base_url}/{endpoint}", headers=self._get_headers(), json=body, timeout=10)
|
|
408
|
+
resp.raise_for_status()
|
|
409
|
+
return resp.json()
|
|
410
|
+
except TimeoutException:
|
|
411
|
+
pl.error(f'POST {endpoint} timed out.')
|
|
412
|
+
raise
|
|
413
|
+
except requests.RequestException as e:
|
|
414
|
+
pl.error(f'Failed POST {endpoint}: {e}')
|
|
415
|
+
raise
|
|
416
|
+
|
|
417
|
+
@ensure_token
|
|
418
|
+
@retry_on_timeout
|
|
419
|
+
@timeout_decorator(10)
|
|
420
|
+
def put(self, endpoint: str, body: dict) -> dict:
|
|
421
|
+
"""
|
|
422
|
+
Perform an HTTP PUT request with a JSON body.
|
|
423
|
+
|
|
424
|
+
:param endpoint: API endpoint path to append to base_url.
|
|
425
|
+
:type endpoint: str
|
|
426
|
+
:param body: JSON-serializable dictionary to send.
|
|
427
|
+
:type body: dict
|
|
428
|
+
:return: JSON-decoded response body.
|
|
429
|
+
:rtype: dict
|
|
430
|
+
:raises TimeoutException: If request exceeds time limit.
|
|
431
|
+
:raises requests.RequestException: On HTTP errors.
|
|
432
|
+
"""
|
|
433
|
+
assert isinstance(endpoint, str), "Endpoint must be an existing web-path"
|
|
434
|
+
assert isinstance(body, dict), "Body must a dictionary/JSON like structure. Example: {'metadata': value, ...}"
|
|
435
|
+
|
|
436
|
+
pl.progress(f'PUT {endpoint}')
|
|
437
|
+
try:
|
|
438
|
+
resp = self.session.put(f"{self.base_url}/{endpoint}", headers=self._get_headers(), json=body, timeout=10)
|
|
439
|
+
resp.raise_for_status()
|
|
440
|
+
return resp.json()
|
|
441
|
+
except TimeoutException:
|
|
442
|
+
pl.error(f'PUT {endpoint} timed out.')
|
|
443
|
+
raise
|
|
444
|
+
except requests.RequestException as e:
|
|
445
|
+
pl.error(f'Failed PUT {endpoint}: {e}')
|
|
446
|
+
raise
|
|
447
|
+
|
|
448
|
+
@ensure_token
|
|
449
|
+
@retry_on_timeout
|
|
450
|
+
@timeout_decorator(10)
|
|
451
|
+
def delete(self, endpoint: str) -> None:
|
|
452
|
+
"""
|
|
453
|
+
Perform an HTTP DELETE request.
|
|
454
|
+
|
|
455
|
+
:param endpoint: API endpoint path to append to base_url.
|
|
456
|
+
:type endpoint: str
|
|
457
|
+
:raises TimeoutException: If request exceeds time limit.
|
|
458
|
+
:raises requests.RequestException: On HTTP errors.
|
|
459
|
+
"""
|
|
460
|
+
assert isinstance(endpoint, str), "Endpoint must be an existing web-path"
|
|
461
|
+
|
|
462
|
+
pl.progress(f'DELETE {endpoint}')
|
|
463
|
+
try:
|
|
464
|
+
resp = self.session.delete(f"{self.base_url}/{endpoint}", headers=self._get_headers(), timeout=10)
|
|
465
|
+
resp.raise_for_status()
|
|
466
|
+
except TimeoutException:
|
|
467
|
+
pl.error(f'DELETE {endpoint} timed out.')
|
|
468
|
+
raise
|
|
469
|
+
except requests.RequestException as e:
|
|
470
|
+
pl.error(f'Failed DELETE {endpoint}: {e}')
|
|
471
|
+
raise
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
"""
|
|
2
|
+
cfg_commands
|
|
3
|
+
============
|
|
4
|
+
Read the shared company `.cfg` file (INI-format, one section per client, with a
|
|
5
|
+
`[DEFAULT]` section whose keys every other section inherits via `configparser`'s
|
|
6
|
+
built-in fallback). Provides both a Python API (:func:`read_cfg_file`,
|
|
7
|
+
:func:`list_sections`) and a CLI entry point (this module's own :func:`main`, run via
|
|
8
|
+
`python -m im_internals.cfg_commands ...`).
|
|
9
|
+
|
|
10
|
+
Every production script in this repo is launched by a `.bat` file that passes
|
|
11
|
+
`-p`/`-c` (path/client) on the command line — see :func:`main`'s own docstring for
|
|
12
|
+
the exact "Legacy single-script calls" pattern that convention follows.
|
|
13
|
+
"""
|
|
14
|
+
import argparse
|
|
15
|
+
import configparser
|
|
16
|
+
import os
|
|
17
|
+
import sys
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def find_cfg_file(path: str) -> str:
|
|
21
|
+
"""
|
|
22
|
+
Locate a .cfg file on disk.
|
|
23
|
+
|
|
24
|
+
:param path: Path to a directory or .cfg file.
|
|
25
|
+
:return: Absolute path to the found .cfg file.
|
|
26
|
+
:raises FileNotFoundError: If no .cfg file exists in the directory,
|
|
27
|
+
the file does not exist, or the path is not a .cfg file.
|
|
28
|
+
|
|
29
|
+
Example:
|
|
30
|
+
>>> find_cfg_file('/etc/myapp')
|
|
31
|
+
'/etc/myapp/config.cfg'
|
|
32
|
+
"""
|
|
33
|
+
path = os.path.abspath(path)
|
|
34
|
+
if os.path.isdir(path):
|
|
35
|
+
for entry in os.listdir(path):
|
|
36
|
+
if entry.lower().endswith('.cfg'):
|
|
37
|
+
return os.path.join(path, entry)
|
|
38
|
+
raise FileNotFoundError(f"No .cfg file found in directory: {path}")
|
|
39
|
+
if path.lower().endswith('.cfg'):
|
|
40
|
+
if os.path.exists(path):
|
|
41
|
+
return path
|
|
42
|
+
raise FileNotFoundError(f"Config file not found: {path}")
|
|
43
|
+
raise FileNotFoundError(f"Path is not a .cfg file: {path}")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def read_cfg_file(config_path: str, client_name: str) -> dict:
|
|
47
|
+
"""
|
|
48
|
+
Read and parse a specific client section from a .cfg file.
|
|
49
|
+
|
|
50
|
+
:param config_path: Directory or file path for the .cfg file.
|
|
51
|
+
:param client_name: Section name within the .cfg to read.
|
|
52
|
+
:return: Dictionary of key/value pairs from the specified section.
|
|
53
|
+
:raises FileNotFoundError: If the .cfg file or directory cannot be located.
|
|
54
|
+
:raises ValueError: If the .cfg file contains no sections.
|
|
55
|
+
:raises KeyError: If the requested client_name section is missing.
|
|
56
|
+
|
|
57
|
+
Example:
|
|
58
|
+
>>> read_cfg_file('/configs', 'AXA')
|
|
59
|
+
{'user': 'axa_user', 'password': 'secret'}
|
|
60
|
+
"""
|
|
61
|
+
cfg_file = find_cfg_file(config_path)
|
|
62
|
+
parser = configparser.ConfigParser()
|
|
63
|
+
parser.read(cfg_file)
|
|
64
|
+
if not parser.sections():
|
|
65
|
+
raise ValueError(f"No sections found in config file: {cfg_file}")
|
|
66
|
+
if client_name not in parser.sections():
|
|
67
|
+
raise KeyError(f"Client '{client_name}' not found. Available: {parser.sections()}")
|
|
68
|
+
return dict(parser[client_name])
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def list_sections(config_path: str) -> list:
|
|
72
|
+
"""
|
|
73
|
+
List all section names in a .cfg file.
|
|
74
|
+
|
|
75
|
+
:param config_path: Directory or file path for the .cfg file.
|
|
76
|
+
:return: List of configuration section names.
|
|
77
|
+
:raises FileNotFoundError: If the .cfg file or directory cannot be located.
|
|
78
|
+
|
|
79
|
+
Example:
|
|
80
|
+
>>> list_sections('settings.cfg')
|
|
81
|
+
['DEFAULT', 'AXA', 'XYZ']
|
|
82
|
+
"""
|
|
83
|
+
cfg_file = find_cfg_file(config_path)
|
|
84
|
+
parser = configparser.ConfigParser()
|
|
85
|
+
parser.read(cfg_file)
|
|
86
|
+
return parser.sections()
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def parse_cli_args(args=None) -> dict:
|
|
90
|
+
"""
|
|
91
|
+
Parse command-line arguments for both subcommands and legacy flags.
|
|
92
|
+
|
|
93
|
+
Supports:
|
|
94
|
+
- list: --path/-p
|
|
95
|
+
- get: --path/-p, --client/-c
|
|
96
|
+
- legacy: top-level -p/--path and -c/--client without subcommand.
|
|
97
|
+
|
|
98
|
+
:param args: List of arguments (defaults to sys.argv[1:]).
|
|
99
|
+
:return: Dictionary with keys 'command', 'path', and 'client'.
|
|
100
|
+
:raises SystemExit: On argument parsing errors.
|
|
101
|
+
|
|
102
|
+
Example:
|
|
103
|
+
>>> parse_cli_args(['list', '-p', '/configs'])
|
|
104
|
+
{'command': 'list', 'path': '/configs', 'client': None}
|
|
105
|
+
|
|
106
|
+
>>> parse_cli_args(['-c', 'AXA', '-p', 'cfgs'])
|
|
107
|
+
{'command': 'get', 'path': 'cfgs', 'client': 'AXA'}
|
|
108
|
+
"""
|
|
109
|
+
parser = argparse.ArgumentParser(prog='cfg_commands', description='Manage .cfg configuration files')
|
|
110
|
+
subparsers = parser.add_subparsers(dest='command', help="Specify the method! For LEGACY, None/empty works as well")
|
|
111
|
+
|
|
112
|
+
# list subcommand
|
|
113
|
+
list_p = subparsers.add_parser('list', help='List all available configuration sections')
|
|
114
|
+
list_p.add_argument('-p', '--path', default='.', help='Path to .cfg file or directory')
|
|
115
|
+
|
|
116
|
+
# get subcommand
|
|
117
|
+
get_p = subparsers.add_parser('get', help='Get configuration values for a client')
|
|
118
|
+
get_p.add_argument('-p', '--path', default='.', help='Path to .cfg file or directory')
|
|
119
|
+
get_p.add_argument('-c', '--client', required=True, help='Client section name')
|
|
120
|
+
|
|
121
|
+
# legacy compatibility: allow direct -p/-c without subcommand
|
|
122
|
+
parser.add_argument('-p', '--path', help=argparse.SUPPRESS)
|
|
123
|
+
parser.add_argument('-c', '--client', help=argparse.SUPPRESS)
|
|
124
|
+
|
|
125
|
+
parsed = parser.parse_args(args or sys.argv[1:])
|
|
126
|
+
|
|
127
|
+
# Detect legacy invocation
|
|
128
|
+
if parsed.command is None and getattr(parsed, 'client', None):
|
|
129
|
+
parsed.command = 'get'
|
|
130
|
+
return vars(parsed)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def main():
|
|
134
|
+
"""
|
|
135
|
+
This module provides programmatic functions and an argparse-based CLI for managing .cfg files.
|
|
136
|
+
|
|
137
|
+
Usage in Python scripts:
|
|
138
|
+
from cfg_commands import read_cfg_file, list_sections
|
|
139
|
+
|
|
140
|
+
# Read a specific client section
|
|
141
|
+
cfg = read_cfg_file('/path/to/configs', 'ClientName')
|
|
142
|
+
|
|
143
|
+
# List all available sections in the config
|
|
144
|
+
sections = list_sections('/path/to/configs')
|
|
145
|
+
|
|
146
|
+
CLI integration (argparse):
|
|
147
|
+
# List all clients
|
|
148
|
+
python cfg_commands.py list -p /path/to/configs
|
|
149
|
+
|
|
150
|
+
# Get settings for a specific client
|
|
151
|
+
python cfg_commands.py get -c ClientName -p /path/to/configs
|
|
152
|
+
|
|
153
|
+
Legacy single-script calls:
|
|
154
|
+
# Scripts consuming only -p/-c flags can do:
|
|
155
|
+
python your_script.py -p /path/to/configs -c ClientName
|
|
156
|
+
|
|
157
|
+
# Then inside your_script.py:
|
|
158
|
+
from cfg_commands import parse_cli_args
|
|
159
|
+
params = parse_cli_args()
|
|
160
|
+
path = params['path']
|
|
161
|
+
client = params['client']
|
|
162
|
+
|
|
163
|
+
:return: None
|
|
164
|
+
:raises SystemExit: Exits with code 1 on execution errors.
|
|
165
|
+
"""
|
|
166
|
+
params = parse_cli_args()
|
|
167
|
+
cmd = params.pop('command')
|
|
168
|
+
|
|
169
|
+
try:
|
|
170
|
+
if cmd == 'list':
|
|
171
|
+
sections = list_sections(params['path'])
|
|
172
|
+
print('Available configurations:')
|
|
173
|
+
for sec in sections:
|
|
174
|
+
print(f'- {sec}')
|
|
175
|
+
elif cmd == 'get':
|
|
176
|
+
data = read_cfg_file(params['path'], params['client'])
|
|
177
|
+
for key, val in data.items():
|
|
178
|
+
print(f"{key}={val}")
|
|
179
|
+
except Exception as e:
|
|
180
|
+
print(f"Error: {e}", file=sys.stderr)
|
|
181
|
+
sys.exit(1)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
if __name__ == '__main__':
|
|
185
|
+
main()
|