async-url-caller-base 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- async_url_caller_base-0.1.0/LICENSE.txt +23 -0
- async_url_caller_base-0.1.0/MANIFEST.in +2 -0
- async_url_caller_base-0.1.0/PKG-INFO +29 -0
- async_url_caller_base-0.1.0/README.md +6 -0
- async_url_caller_base-0.1.0/pyproject.toml +53 -0
- async_url_caller_base-0.1.0/setup.cfg +43 -0
- async_url_caller_base-0.1.0/src/async_url_caller_base/__init__.py +1 -0
- async_url_caller_base-0.1.0/src/async_url_caller_base/async_url_caller_base.py +430 -0
- async_url_caller_base-0.1.0/src/async_url_caller_base/constants.py +10 -0
- async_url_caller_base-0.1.0/src/async_url_caller_base/constants_version.py +9 -0
- async_url_caller_base-0.1.0/src/async_url_caller_base.egg-info/PKG-INFO +29 -0
- async_url_caller_base-0.1.0/src/async_url_caller_base.egg-info/SOURCES.txt +14 -0
- async_url_caller_base-0.1.0/src/async_url_caller_base.egg-info/dependency_links.txt +1 -0
- async_url_caller_base-0.1.0/src/async_url_caller_base.egg-info/requires.txt +1 -0
- async_url_caller_base-0.1.0/src/async_url_caller_base.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) since 2026: async-url-caller-base Python Module
|
|
4
|
+
description: python async base class for calling URLs
|
|
5
|
+
author: J. Arrizza email: cppgent0@gmail.com
|
|
6
|
+
|
|
7
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
8
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
9
|
+
in the Software without restriction, including without limitation the rights
|
|
10
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
11
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
12
|
+
furnished to do so, subject to the following conditions:
|
|
13
|
+
|
|
14
|
+
The above copyright notice and this permission notice shall be included in all
|
|
15
|
+
copies or substantial portions of the Software.
|
|
16
|
+
|
|
17
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
18
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
19
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
20
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
21
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
22
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
23
|
+
SOFTWARE.
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: async-url-caller-base
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: python async base class for calling URLs
|
|
5
|
+
Author-email: "J. Arrizza" <cppgent0@gmail.com>
|
|
6
|
+
Maintainer-email: "J. Arrizza" <cppgent0@gmail.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Project-URL: Website, https://arrizza.com/async-url-caller-base
|
|
9
|
+
Project-URL: Source, https://bitbucket.org/arrizza-public/async-url-caller-base/src/master
|
|
10
|
+
Project-URL: Download, https://bitbucket.org/arrizza-public/async-url-caller-base/get/master.zip
|
|
11
|
+
Keywords: async,aiohttp
|
|
12
|
+
Classifier: Development Status :: 1 - Planning
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Topic :: Utilities
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Requires-Python: >=3.9
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE.txt
|
|
21
|
+
Requires-Dist: falcon-logger
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
* website: [Website](https://arrizza.com/TODO.html)
|
|
25
|
+
* installation: [Common Setup](https://arrizza.com/setup-common.html)
|
|
26
|
+
|
|
27
|
+
## Summary
|
|
28
|
+
|
|
29
|
+
***Under Construction***
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
description = 'python async base class for calling URLs'
|
|
3
|
+
keywords = ['async', 'aiohttp']
|
|
4
|
+
|
|
5
|
+
classifiers = [
|
|
6
|
+
# "Private :: Do Not Upload", # use to prevent upload
|
|
7
|
+
# Choose either '3 - Alpha', '4 - Beta' or '5 - Production/Stable'
|
|
8
|
+
'Development Status :: 1 - Planning',
|
|
9
|
+
'Environment :: Console',
|
|
10
|
+
'Topic :: Utilities',
|
|
11
|
+
'Intended Audience :: Developers',
|
|
12
|
+
'Programming Language :: Python :: 3.10',
|
|
13
|
+
'Programming Language :: Python :: 3.12',
|
|
14
|
+
]
|
|
15
|
+
# additional packages to install
|
|
16
|
+
dependencies = [
|
|
17
|
+
'falcon-logger',
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
# common values
|
|
21
|
+
name = 'async-url-caller-base'
|
|
22
|
+
version = '0.1.0'
|
|
23
|
+
requires-python = '>=3.9'
|
|
24
|
+
authors = [{ name = 'J. Arrizza', email = 'cppgent0@gmail.com' }]
|
|
25
|
+
maintainers = [
|
|
26
|
+
{ name = 'J. Arrizza', email = 'cppgent0@gmail.com' },
|
|
27
|
+
]
|
|
28
|
+
readme = { file = 'README.md', content-type = 'text/markdown' }
|
|
29
|
+
license = 'MIT'
|
|
30
|
+
|
|
31
|
+
[project.optional-dependencies]
|
|
32
|
+
# None
|
|
33
|
+
[project.scripts]
|
|
34
|
+
# None e.g. spam-cli = "spam:main_cli"
|
|
35
|
+
|
|
36
|
+
[project.urls]
|
|
37
|
+
'Website' = 'https://arrizza.com/async-url-caller-base'
|
|
38
|
+
'Source' = 'https://bitbucket.org/arrizza-public/async-url-caller-base/src/master'
|
|
39
|
+
'Download' = 'https://bitbucket.org/arrizza-public/async-url-caller-base/get/master.zip'
|
|
40
|
+
# 'Bug Reports' = 'TODO https://github.com/pypa/sampleproject/issues'
|
|
41
|
+
# 'Funding' = 'TODO https://donate.pypi.org'
|
|
42
|
+
# 'Say Thanks!' = 'TODO http://saythanks.io/to/example'
|
|
43
|
+
|
|
44
|
+
[tool.setuptools.packages.find]
|
|
45
|
+
where = ['src']
|
|
46
|
+
exclude = ['.gitignore']
|
|
47
|
+
|
|
48
|
+
[tool.setuptools]
|
|
49
|
+
include-package-data = true
|
|
50
|
+
|
|
51
|
+
[build-system]
|
|
52
|
+
requires = ['setuptools >=61.0']
|
|
53
|
+
build-backend = 'setuptools.build_meta'
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
[metadata]
|
|
2
|
+
description_file = README.md
|
|
3
|
+
|
|
4
|
+
[pylint.MESSAGES CONTROL]
|
|
5
|
+
max-line-length = 140
|
|
6
|
+
disable = C0114,C0115,C0116,C0415,R0801,R0903,R0912,R0913,R0917,W0511
|
|
7
|
+
|
|
8
|
+
[pycodestyle]
|
|
9
|
+
max-line-length = 140
|
|
10
|
+
ignore = E266,E402,E731,E203
|
|
11
|
+
statistics = True
|
|
12
|
+
exclude = venv/*, out/*, pf_*, setup.py, debug/*, release/*
|
|
13
|
+
|
|
14
|
+
[coverage:run]
|
|
15
|
+
omit =
|
|
16
|
+
./conftest.py
|
|
17
|
+
./out/*
|
|
18
|
+
./ut/*
|
|
19
|
+
./venv/*
|
|
20
|
+
./ver/*
|
|
21
|
+
build_info.py
|
|
22
|
+
branch = true
|
|
23
|
+
|
|
24
|
+
[coverage:report]
|
|
25
|
+
exclude_lines =
|
|
26
|
+
pragma: no cover
|
|
27
|
+
|
|
28
|
+
def __repr__
|
|
29
|
+
if self\.debug
|
|
30
|
+
|
|
31
|
+
raise NotImplementedError
|
|
32
|
+
|
|
33
|
+
if 0:
|
|
34
|
+
if __main__ == .__main__:
|
|
35
|
+
ignore_errors = True
|
|
36
|
+
|
|
37
|
+
[coverage:html]
|
|
38
|
+
directory = ./out/coverage
|
|
39
|
+
|
|
40
|
+
[egg_info]
|
|
41
|
+
tag_build =
|
|
42
|
+
tag_date = 0
|
|
43
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# TODO add imports as needed
|
|
@@ -0,0 +1,430 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import os
|
|
3
|
+
import re
|
|
4
|
+
import ssl
|
|
5
|
+
import time
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
|
|
8
|
+
import aiohttp
|
|
9
|
+
from falcon_logger import FalconLogger
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# --------------------
|
|
13
|
+
## do async calls for a URL list
|
|
14
|
+
class AsyncUrlCallerBase:
|
|
15
|
+
# --------------------
|
|
16
|
+
## constructor
|
|
17
|
+
# @param logger the logger to use. if None, then FalconLogger is used
|
|
18
|
+
# the logger must have these functions:
|
|
19
|
+
# line - print a line
|
|
20
|
+
# dbg - print a debug line e.g. prefix "DBG"
|
|
21
|
+
# exception - print exception information
|
|
22
|
+
def __init__(self, logger=None):
|
|
23
|
+
if logger is None:
|
|
24
|
+
## reference to the logger to use
|
|
25
|
+
self._log = FalconLogger(mode='immediate')
|
|
26
|
+
self._log.set_format('prefix')
|
|
27
|
+
else:
|
|
28
|
+
self._log = logger
|
|
29
|
+
## prefix tax in log lines
|
|
30
|
+
self._tag = self.__class__.__name__
|
|
31
|
+
|
|
32
|
+
## ip address to use for raw calls
|
|
33
|
+
self._ip_address = None
|
|
34
|
+
## ip port to use for raw calls
|
|
35
|
+
self._port = None
|
|
36
|
+
|
|
37
|
+
## holds ssl information
|
|
38
|
+
self._ssl_context = None
|
|
39
|
+
## allow redirect status codes
|
|
40
|
+
self._allow_redirects = True
|
|
41
|
+
|
|
42
|
+
## max time out to wait for a response
|
|
43
|
+
self._max_timeout = 9.0
|
|
44
|
+
## the timeout per wait
|
|
45
|
+
self._timeout = 4.0
|
|
46
|
+
## how long to pause the async thread when a timeout occurs
|
|
47
|
+
self._delay = 0.300
|
|
48
|
+
|
|
49
|
+
## multiplier for cpu count for total number of threads
|
|
50
|
+
self._thread_multiplier = 3
|
|
51
|
+
|
|
52
|
+
# --------------------
|
|
53
|
+
## the thread multiplier.
|
|
54
|
+
# This is used to calculated the number of threads:
|
|
55
|
+
# * number of cpus * this multiplier
|
|
56
|
+
#
|
|
57
|
+
# @param new_val the new multiplier value
|
|
58
|
+
def _set_thread_multiplier(self, new_val):
|
|
59
|
+
self._thread_multiplier = new_val
|
|
60
|
+
|
|
61
|
+
## the thread multiplier
|
|
62
|
+
thread_multiplier = property(fset=_set_thread_multiplier)
|
|
63
|
+
|
|
64
|
+
# --------------------
|
|
65
|
+
## these are parameters needed for raw calls.
|
|
66
|
+
# @param ip_address the IP address to use for these calls
|
|
67
|
+
# @param port the IP port to use, typically 443
|
|
68
|
+
def set_raw_params(self, ip_address, port):
|
|
69
|
+
self._ip_address = ip_address
|
|
70
|
+
self._port = port
|
|
71
|
+
|
|
72
|
+
# --------------------
|
|
73
|
+
## set timeout parameters
|
|
74
|
+
# @param max_timeout the maximum total accumulated timeout
|
|
75
|
+
# @param timeout the individual timeout for each call
|
|
76
|
+
# @param delay the delay between calls
|
|
77
|
+
def set_timeouts(self, max_timeout, timeout, delay):
|
|
78
|
+
self._max_timeout = max_timeout
|
|
79
|
+
self._timeout = timeout
|
|
80
|
+
self._delay = delay
|
|
81
|
+
|
|
82
|
+
# -------------------
|
|
83
|
+
## default - exclude a url from running in a normal get()
|
|
84
|
+
# @param url the url to skip
|
|
85
|
+
# @return if True, the url is skipped, otherwise it is run
|
|
86
|
+
def is_excluded_get(self, url):
|
|
87
|
+
# don't exclude
|
|
88
|
+
return False
|
|
89
|
+
|
|
90
|
+
# -------------------
|
|
91
|
+
## default - exclude a url from running in a raw request
|
|
92
|
+
# @param url the url to skip
|
|
93
|
+
# @return if True, the url is skipped, otherwise it is run
|
|
94
|
+
def is_excluded_raw(self, url):
|
|
95
|
+
# don't exclude
|
|
96
|
+
return False
|
|
97
|
+
|
|
98
|
+
# --------------------
|
|
99
|
+
## run tests on all provided urls.
|
|
100
|
+
# @param urls a dict of urls to use;
|
|
101
|
+
# key : the url with no scheme
|
|
102
|
+
# info: a triplet or longer:
|
|
103
|
+
# * 0: "raw" vs "get"
|
|
104
|
+
# * 1: full url (including scheme)
|
|
105
|
+
# * 2: additional info for this call required by cb_xx function
|
|
106
|
+
# @param cb_get the callback function for HTTP GETs URLs
|
|
107
|
+
# @param cb_raw the callback function for URLs sent to ip_address/port
|
|
108
|
+
# @param need_content if True, the content is retrieved; default is False
|
|
109
|
+
# @param allow_redirects if True, reports 301 and 302's,
|
|
110
|
+
# otherwise the final result is returned; default is True
|
|
111
|
+
def run_all_tests(self, urls, cb_get=None, cb_raw=None, need_content=False, allow_redirects=True):
|
|
112
|
+
self._allow_redirects = allow_redirects
|
|
113
|
+
self._ssl_context = ssl.create_default_context()
|
|
114
|
+
self._ssl_context.check_hostname = False
|
|
115
|
+
self._ssl_context.verify_mode = ssl.CERT_NONE
|
|
116
|
+
|
|
117
|
+
if not cb_get:
|
|
118
|
+
cb_get = self.cb_get_default
|
|
119
|
+
|
|
120
|
+
if not cb_raw:
|
|
121
|
+
cb_raw = self.cb_raw_default
|
|
122
|
+
|
|
123
|
+
time_start = time.time()
|
|
124
|
+
asyncio.run(self._run_tests_async(urls, cb_get, cb_raw, need_content))
|
|
125
|
+
elapsed = time.time() - time_start
|
|
126
|
+
return elapsed
|
|
127
|
+
|
|
128
|
+
# === callbacks
|
|
129
|
+
|
|
130
|
+
# --------------------
|
|
131
|
+
## a default callback function for GET URLs
|
|
132
|
+
# @param url url w/ no scheme e.g. https://ip_addr
|
|
133
|
+
# @param info triplet:
|
|
134
|
+
# * 0: "raw" vs "get": for this call should be "get"
|
|
135
|
+
# * 1: full url
|
|
136
|
+
# * 2: additional callers info for this call e.g. list of IPs
|
|
137
|
+
# @param response reference to response from call
|
|
138
|
+
# @param content content returned by call
|
|
139
|
+
def cb_get_default(self, url, info, response, content=None):
|
|
140
|
+
status_code = response.status
|
|
141
|
+
|
|
142
|
+
# url => url with no scheme
|
|
143
|
+
# info[0] => raw/get call
|
|
144
|
+
# info[1] => url (with scheme)
|
|
145
|
+
self._log.dbg(f'{info[0]}: {status_code} {url} {content}')
|
|
146
|
+
|
|
147
|
+
# --------------------
|
|
148
|
+
## a default callback function for raw URLs
|
|
149
|
+
# @param url url w/ no scheme e.g. https://ip_addr
|
|
150
|
+
# @param info triplet:
|
|
151
|
+
# * 0: "raw" vs "get": for this call should be "raw"
|
|
152
|
+
# * 1: full url
|
|
153
|
+
# * 2: additional callers info for this call e.g. list of IPs
|
|
154
|
+
# @param response reference to response from call
|
|
155
|
+
# @param content content returned by call
|
|
156
|
+
def cb_raw_default(self, url, info, response, content):
|
|
157
|
+
status_code = response.status
|
|
158
|
+
|
|
159
|
+
# url => url with no scheme
|
|
160
|
+
# info[0] => raw/get call
|
|
161
|
+
# info[1] => url (with scheme)
|
|
162
|
+
self._log.dbg(f'{info[0]}: {status_code} {url} {content}')
|
|
163
|
+
|
|
164
|
+
# # --- handle exceptions in calls
|
|
165
|
+
|
|
166
|
+
# --------------------
|
|
167
|
+
## overridable function to handle reporting TimeoutError exceptions
|
|
168
|
+
# @param excp reference to the exception
|
|
169
|
+
# @param call_type 'raw' or 'get'
|
|
170
|
+
# @param url the url that was called
|
|
171
|
+
# @param info the associated info from the dict for this url
|
|
172
|
+
# @param loop which delay loop
|
|
173
|
+
# @param elapsed total time elasped since the call was invoked
|
|
174
|
+
def handle_excp_timeout_error(self, excp, call_type, url, info, loop, elapsed):
|
|
175
|
+
self._log.err(f'({call_type}) Timeout excp occurred, loop:{loop: >3} elapsed:{round(elapsed, 1)} {type(excp).__name__}')
|
|
176
|
+
self._log.err(f' {loop: >3}] {url} {info[1]}')
|
|
177
|
+
|
|
178
|
+
# --------------------
|
|
179
|
+
## overridable function to handle reporting ClientError exceptions
|
|
180
|
+
# @param excp reference to the exception
|
|
181
|
+
# @param call_type 'raw' or 'get'
|
|
182
|
+
# @param url the url that was called
|
|
183
|
+
# @param info the associated info from the dict for this url
|
|
184
|
+
def handle_excp_client_error(self, excp, call_type, url, info):
|
|
185
|
+
self._log.err(f'({call_type}) Client excp occurred: {type(excp).__name__}')
|
|
186
|
+
self._log.err(f' {url} {info[1]}')
|
|
187
|
+
|
|
188
|
+
# --------------------
|
|
189
|
+
## overridable function to handle reporting ResetError or IncompleteReadError exceptions
|
|
190
|
+
# @param excp reference to the exception
|
|
191
|
+
# @param call_type 'raw' or 'get'
|
|
192
|
+
# @param url the url that was called
|
|
193
|
+
# @param info the associated info from the dict for this url
|
|
194
|
+
def handle_excp_reset_error(self, excp, call_type, url, info):
|
|
195
|
+
self._log.err(f'({call_type}) Reset excp occurred: {type(excp).__name__}')
|
|
196
|
+
self._log.err(f' {url} {info[1]}')
|
|
197
|
+
|
|
198
|
+
# --------------------
|
|
199
|
+
## overridable function to handle reporting general exceptions
|
|
200
|
+
# @param excp reference to the exception
|
|
201
|
+
# @param call_type 'raw' or 'get'
|
|
202
|
+
# @param url the url that was called
|
|
203
|
+
# @param info the associated info from the dict for this url
|
|
204
|
+
# @param response the response from the call
|
|
205
|
+
def handle_excp_general(self, excp, call_type, url, info, response):
|
|
206
|
+
self._log.err(f'({call_type}) General excp: unexpected error: {type(excp).__name__}')
|
|
207
|
+
self._log.err(f' url : {url}')
|
|
208
|
+
self._log.err(f' full : {info[1]}')
|
|
209
|
+
redir_url = response.headers.get('Location')
|
|
210
|
+
if redir_url is not None:
|
|
211
|
+
self._log.err(f' redir: {redir_url}')
|
|
212
|
+
self._log.exception(excp)
|
|
213
|
+
|
|
214
|
+
# --------------------
|
|
215
|
+
## overridable function to handle responses with unknown status codes
|
|
216
|
+
# @param call_type 'raw' or 'get'
|
|
217
|
+
# @param url the url that was called
|
|
218
|
+
# @param info the associated info from the dict for this url
|
|
219
|
+
# @param content the returned content
|
|
220
|
+
# @param status_code the response status code
|
|
221
|
+
def handle_unknown_status_code(self, call_type, url, info, content, status_code):
|
|
222
|
+
self._log.err(f'({call_type}): unknown status code in response: assuming {status_code}')
|
|
223
|
+
self._log.err(f' url : {url} {info[1]}')
|
|
224
|
+
self._log.err(f' content: "{content}"')
|
|
225
|
+
|
|
226
|
+
# --- private from here on
|
|
227
|
+
|
|
228
|
+
## used to simulate a Response object when there isn't one e.g. nginx 444s
|
|
229
|
+
@dataclass
|
|
230
|
+
class MockResponse:
|
|
231
|
+
status: int
|
|
232
|
+
history: dict
|
|
233
|
+
url: str
|
|
234
|
+
headers: str
|
|
235
|
+
reason: str
|
|
236
|
+
|
|
237
|
+
# --------------------
|
|
238
|
+
## run the async tests
|
|
239
|
+
# @param urls dict of urls:
|
|
240
|
+
# key : the url with no scheme
|
|
241
|
+
# info: a triplet or longer:
|
|
242
|
+
# * 0: "raw" vs "get"
|
|
243
|
+
# * 1: full url (including scheme)
|
|
244
|
+
# * 2: additional info for this call required by cb_xx function
|
|
245
|
+
# @param cb_get callback for "get" urls
|
|
246
|
+
# @param cb_raw callback for "raw" urls
|
|
247
|
+
# @param need_content if true, get the content and pass it to the cb_xx function
|
|
248
|
+
async def _run_tests_async(self, urls, cb_get, cb_raw, need_content):
|
|
249
|
+
num_threads = os.cpu_count() * self._thread_multiplier
|
|
250
|
+
self._log.line(f'{self._tag}: num cpus: {os.cpu_count()}, threads: {num_threads}')
|
|
251
|
+
semaphore = asyncio.Semaphore(num_threads)
|
|
252
|
+
|
|
253
|
+
headers = {
|
|
254
|
+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
|
|
255
|
+
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'
|
|
256
|
+
}
|
|
257
|
+
async with aiohttp.ClientSession(headers=headers, connector=aiohttp.TCPConnector(ssl=self._ssl_context)) as session:
|
|
258
|
+
# --- async call to do a normal get
|
|
259
|
+
async def bounded_fetch(url, info, cb_get, need_content):
|
|
260
|
+
# Acquire the semaphore to respect the concurrency limit
|
|
261
|
+
async with semaphore:
|
|
262
|
+
await self._fetch_get(session, url, info, cb_get, need_content)
|
|
263
|
+
|
|
264
|
+
# --- async call to do a raw call e.g. CONNECT, PRI, etc.
|
|
265
|
+
async def raw_fetch(url, info, cb_raw):
|
|
266
|
+
async with semaphore:
|
|
267
|
+
await self._fetch_raw(url, info, cb_raw)
|
|
268
|
+
|
|
269
|
+
# load tasks with all calls to do
|
|
270
|
+
tasks = []
|
|
271
|
+
for url, info in sorted(urls.items()):
|
|
272
|
+
if info[0] == 'raw':
|
|
273
|
+
tasks.append(raw_fetch(url, info, cb_raw))
|
|
274
|
+
else:
|
|
275
|
+
tasks.append(bounded_fetch(url, info, cb_get, need_content))
|
|
276
|
+
|
|
277
|
+
await asyncio.gather(*tasks)
|
|
278
|
+
|
|
279
|
+
# --------------------
|
|
280
|
+
## do the GET fetch for on the URL
|
|
281
|
+
# @param session reference to session object
|
|
282
|
+
# @param key raw url, no leading root e.g. https://your_server
|
|
283
|
+
# @param info triplet:
|
|
284
|
+
# * 0: "raw" vs "get": for this call should be "get"
|
|
285
|
+
# * 1: full url
|
|
286
|
+
# * 2: additional callers info for this call e.g. list of IPs
|
|
287
|
+
# @param cb_get callback function when
|
|
288
|
+
# @param need_content if true, then read and return the content of the call's response, else return None for content
|
|
289
|
+
async def _fetch_get(self, session, key, info, cb_get, need_content):
|
|
290
|
+
url = info[1]
|
|
291
|
+
|
|
292
|
+
if self.is_excluded_get(url):
|
|
293
|
+
return
|
|
294
|
+
|
|
295
|
+
done = False
|
|
296
|
+
loop = 0
|
|
297
|
+
start_time = time.time()
|
|
298
|
+
while not done:
|
|
299
|
+
loop += 1
|
|
300
|
+
|
|
301
|
+
try:
|
|
302
|
+
async with session.get(
|
|
303
|
+
url, timeout=self._timeout, ssl=self._ssl_context, allow_redirects=self._allow_redirects
|
|
304
|
+
) as response:
|
|
305
|
+
if need_content:
|
|
306
|
+
content = await response.read()
|
|
307
|
+
else:
|
|
308
|
+
content = None
|
|
309
|
+
if response.status == 503:
|
|
310
|
+
elapsed = time.time() - start_time
|
|
311
|
+
if (elapsed) > self._max_timeout:
|
|
312
|
+
done = True
|
|
313
|
+
else:
|
|
314
|
+
time.sleep(self._delay)
|
|
315
|
+
self._delay = self._delay * 1.5
|
|
316
|
+
else:
|
|
317
|
+
cb_get(key, info, response, content)
|
|
318
|
+
done = True
|
|
319
|
+
except asyncio.TimeoutError as excp:
|
|
320
|
+
elapsed = time.time() - start_time
|
|
321
|
+
self.handle_excp_timeout_error(excp, 'get', key, info, loop, elapsed)
|
|
322
|
+
if elapsed > self._max_timeout:
|
|
323
|
+
done = True
|
|
324
|
+
else:
|
|
325
|
+
time.sleep(self._delay)
|
|
326
|
+
self._delay = self._delay * 1.5
|
|
327
|
+
except aiohttp.ClientError as excp:
|
|
328
|
+
if str(excp) in ['Server disconnected', 'Session is closed']:
|
|
329
|
+
response = self.MockResponse(status=444, history={}, url=url, headers=None, reason='clienterror excp')
|
|
330
|
+
content = b''
|
|
331
|
+
cb_get(key, info, response, content)
|
|
332
|
+
done = True
|
|
333
|
+
else:
|
|
334
|
+
self.handle_excp_client_error(excp, 'get', key, info)
|
|
335
|
+
# give up
|
|
336
|
+
done = True
|
|
337
|
+
except (ConnectionResetError, asyncio.IncompleteReadError) as excp:
|
|
338
|
+
self.handle_excp_reset_error(excp, 'get', key, info)
|
|
339
|
+
done = True
|
|
340
|
+
except Exception as excp:
|
|
341
|
+
self.handle_excp_general(excp, 'get', key, info, response)
|
|
342
|
+
done = True
|
|
343
|
+
|
|
344
|
+
# -------------------
|
|
345
|
+
## do a fetch on a raw url
|
|
346
|
+
# @param key raw url, no leading root e.g. https://ip_addr
|
|
347
|
+
# @param info triplet:
|
|
348
|
+
# * 0: "raw" vs "get": for this call should be "get"
|
|
349
|
+
# * 1: full url
|
|
350
|
+
# * 2: additional callers info for this call e.g. list of IPs
|
|
351
|
+
# @param cb_raw callback function when
|
|
352
|
+
async def _fetch_raw(self, key, info, cb_raw):
|
|
353
|
+
url = info[1]
|
|
354
|
+
|
|
355
|
+
if self.is_excluded_raw(url):
|
|
356
|
+
return
|
|
357
|
+
|
|
358
|
+
done = False
|
|
359
|
+
loop = 0
|
|
360
|
+
start_time = time.time()
|
|
361
|
+
while not done:
|
|
362
|
+
loop += 1
|
|
363
|
+
|
|
364
|
+
try:
|
|
365
|
+
request_lines = [
|
|
366
|
+
f'{url} HTTP/1.1',
|
|
367
|
+
f'Host: {self._ip_address}',
|
|
368
|
+
'Content-Length: 0',
|
|
369
|
+
'',
|
|
370
|
+
]
|
|
371
|
+
full_request = '\r\n'.join(request_lines)
|
|
372
|
+
encoded_request = full_request.encode('ascii')
|
|
373
|
+
|
|
374
|
+
reader, writer = await asyncio.open_connection(self._ip_address, self._port, ssl=self._ssl_context)
|
|
375
|
+
writer.write(encoded_request)
|
|
376
|
+
await writer.drain() # Ensure buffer is flushed
|
|
377
|
+
|
|
378
|
+
response = None
|
|
379
|
+
try:
|
|
380
|
+
response = await asyncio.wait_for(reader.read(10), timeout=self._timeout)
|
|
381
|
+
except asyncio.TimeoutError as excp:
|
|
382
|
+
elapsed = time.time() - start_time
|
|
383
|
+
self.handle_excp_timeout_error(excp, 'raw', key, info, loop, elapsed)
|
|
384
|
+
if elapsed > self._max_timeout:
|
|
385
|
+
done = True
|
|
386
|
+
else:
|
|
387
|
+
time.sleep(self._delay)
|
|
388
|
+
self._delay = self._delay * 1.5
|
|
389
|
+
except aiohttp.ClientError as excp:
|
|
390
|
+
if str(excp) in ['Server disconnected', 'Session is closed']:
|
|
391
|
+
response = self.MockResponse(status=444, history={}, url=url, headers=None, reason='clienterror excp')
|
|
392
|
+
content = b''
|
|
393
|
+
cb_raw(key, info, response, content)
|
|
394
|
+
done = True
|
|
395
|
+
else:
|
|
396
|
+
self.handle_excp_client_error(excp, 'raw', key, info)
|
|
397
|
+
done = True
|
|
398
|
+
except (ConnectionResetError, asyncio.IncompleteReadError) as excp:
|
|
399
|
+
self.handle_excp_reset_error(excp, 'raw', key, info)
|
|
400
|
+
done = True
|
|
401
|
+
except Exception as excp:
|
|
402
|
+
self.handle_excp_general(excp, 'raw', key, info, response)
|
|
403
|
+
done = True
|
|
404
|
+
|
|
405
|
+
content = ''
|
|
406
|
+
if response is not None and response != b'':
|
|
407
|
+
remaining_data = await reader.read(4096)
|
|
408
|
+
response = response + remaining_data
|
|
409
|
+
content = response.decode('utf-8')
|
|
410
|
+
|
|
411
|
+
if response is None or response == b'':
|
|
412
|
+
# this is an assumption based on what occurs with MKCOL calls
|
|
413
|
+
response = self.MockResponse(status=444, history={}, url=url, headers=None, reason='response is empty')
|
|
414
|
+
else:
|
|
415
|
+
m = re.search(r'HTTP/.* (\d+) ', content)
|
|
416
|
+
if m:
|
|
417
|
+
status_code = m.group(1)
|
|
418
|
+
else:
|
|
419
|
+
status_code = 600
|
|
420
|
+
self.handle_unknown_status_code('raw', key, info, content, status_code)
|
|
421
|
+
response = self.MockResponse(status=status_code, history={}, url=url, headers=None, reason='response is empty')
|
|
422
|
+
|
|
423
|
+
# close after the wait_for otherwise socket may close
|
|
424
|
+
writer.close()
|
|
425
|
+
await writer.wait_closed()
|
|
426
|
+
cb_raw(key, info, response, content)
|
|
427
|
+
done = True
|
|
428
|
+
except Exception as excp:
|
|
429
|
+
self.handle_excp_general(excp, 'raw', key, info, response)
|
|
430
|
+
done = True
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: async-url-caller-base
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: python async base class for calling URLs
|
|
5
|
+
Author-email: "J. Arrizza" <cppgent0@gmail.com>
|
|
6
|
+
Maintainer-email: "J. Arrizza" <cppgent0@gmail.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Project-URL: Website, https://arrizza.com/async-url-caller-base
|
|
9
|
+
Project-URL: Source, https://bitbucket.org/arrizza-public/async-url-caller-base/src/master
|
|
10
|
+
Project-URL: Download, https://bitbucket.org/arrizza-public/async-url-caller-base/get/master.zip
|
|
11
|
+
Keywords: async,aiohttp
|
|
12
|
+
Classifier: Development Status :: 1 - Planning
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Topic :: Utilities
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Requires-Python: >=3.9
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE.txt
|
|
21
|
+
Requires-Dist: falcon-logger
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
* website: [Website](https://arrizza.com/TODO.html)
|
|
25
|
+
* installation: [Common Setup](https://arrizza.com/setup-common.html)
|
|
26
|
+
|
|
27
|
+
## Summary
|
|
28
|
+
|
|
29
|
+
***Under Construction***
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
LICENSE.txt
|
|
2
|
+
MANIFEST.in
|
|
3
|
+
README.md
|
|
4
|
+
pyproject.toml
|
|
5
|
+
setup.cfg
|
|
6
|
+
src/async_url_caller_base/__init__.py
|
|
7
|
+
src/async_url_caller_base/async_url_caller_base.py
|
|
8
|
+
src/async_url_caller_base/constants.py
|
|
9
|
+
src/async_url_caller_base/constants_version.py
|
|
10
|
+
src/async_url_caller_base.egg-info/PKG-INFO
|
|
11
|
+
src/async_url_caller_base.egg-info/SOURCES.txt
|
|
12
|
+
src/async_url_caller_base.egg-info/dependency_links.txt
|
|
13
|
+
src/async_url_caller_base.egg-info/requires.txt
|
|
14
|
+
src/async_url_caller_base.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
falcon-logger
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
async_url_caller_base
|