requence 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- requence-0.0.1/MANIFEST.in +4 -0
- requence-0.0.1/PKG-INFO +13 -0
- requence-0.0.1/README.md +3 -0
- requence-0.0.1/pyproject.toml +6 -0
- requence-0.0.1/requence/__init__.py +0 -0
- requence-0.0.1/requence/consumer.py +272 -0
- requence-0.0.1/requence.egg-info/PKG-INFO +13 -0
- requence-0.0.1/requence.egg-info/SOURCES.txt +11 -0
- requence-0.0.1/requence.egg-info/dependency_links.txt +1 -0
- requence-0.0.1/requence.egg-info/requires.txt +2 -0
- requence-0.0.1/requence.egg-info/top_level.txt +1 -0
- requence-0.0.1/setup.cfg +4 -0
- requence-0.0.1/setup.py +15 -0
requence-0.0.1/PKG-INFO
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: requence
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Consumer implementation
|
|
5
|
+
Author: Torsten Blindert
|
|
6
|
+
Requires-Python: >=3.10
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: pika>=1.3.2
|
|
9
|
+
Requires-Dist: semver>=3.0.2
|
|
10
|
+
|
|
11
|
+
# requence-consumer
|
|
12
|
+
|
|
13
|
+
This is the python implementation of the @requence/consumer package
|
requence-0.0.1/README.md
ADDED
|
File without changes
|
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
from distutils.util import execute
|
|
2
|
+
from pika.adapters.blocking_connection import BlockingChannel
|
|
3
|
+
from pika.spec import Basic, Channel, BasicProperties
|
|
4
|
+
from typing import Any, Tuple, TypedDict, Awaitable, Callable, Dict, List, Literal, Optional, Protocol, Union, cast
|
|
5
|
+
import os
|
|
6
|
+
import pika
|
|
7
|
+
import semver
|
|
8
|
+
import json
|
|
9
|
+
import uuid
|
|
10
|
+
import re
|
|
11
|
+
import asyncio
|
|
12
|
+
import datetime
|
|
13
|
+
|
|
14
|
+
version_pattern = r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
|
|
15
|
+
|
|
16
|
+
def parse_iso_format(iso: str):
|
|
17
|
+
iso_without_z = iso.rstrip("Z")
|
|
18
|
+
return datetime.datetime.fromisoformat(iso_without_z).replace(tzinfo=datetime.timezone.utc)
|
|
19
|
+
|
|
20
|
+
def is_valid_version(version):
|
|
21
|
+
return re.match(version_pattern, version)
|
|
22
|
+
|
|
23
|
+
def is_valid_uuid(uuid_to_test):
|
|
24
|
+
try:
|
|
25
|
+
uuid_obj = uuid.UUID(uuid_to_test, version=4)
|
|
26
|
+
return str(uuid_obj) == uuid_to_test
|
|
27
|
+
except ValueError:
|
|
28
|
+
return False
|
|
29
|
+
|
|
30
|
+
class ConsumerConfig(TypedDict, total=False):
|
|
31
|
+
url: Optional[str]
|
|
32
|
+
version: Optional[str]
|
|
33
|
+
prefetch: Optional[int]
|
|
34
|
+
|
|
35
|
+
class RetryException(Exception):
|
|
36
|
+
def __init__(self, delay: Optional[int]):
|
|
37
|
+
self.delay = delay
|
|
38
|
+
|
|
39
|
+
class AbortException(Exception):
|
|
40
|
+
pass
|
|
41
|
+
|
|
42
|
+
class Context(TypedDict, total=True):
|
|
43
|
+
input: Any
|
|
44
|
+
meta: Any
|
|
45
|
+
data: Dict[str, Any]
|
|
46
|
+
dates: Dict[str, str]
|
|
47
|
+
errors: Dict[str, Any]
|
|
48
|
+
|
|
49
|
+
class Service(TypedDict):
|
|
50
|
+
id: str
|
|
51
|
+
alias: Optional[str]
|
|
52
|
+
name: str
|
|
53
|
+
version: str
|
|
54
|
+
configuration: Any
|
|
55
|
+
|
|
56
|
+
class ServiceMeta(Service):
|
|
57
|
+
execution_data: Union[None, datetime.datetime]
|
|
58
|
+
|
|
59
|
+
class Meta(TypedDict):
|
|
60
|
+
services: List[Service]
|
|
61
|
+
|
|
62
|
+
class Payload(TypedDict, total=True):
|
|
63
|
+
context: Context
|
|
64
|
+
meta: Meta
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class ContextHelper:
|
|
68
|
+
def __init__(self, id: str, context: Context, serviceList: List[Service]):
|
|
69
|
+
self.__context = context
|
|
70
|
+
self.__serviceList = serviceList
|
|
71
|
+
self.__id = id
|
|
72
|
+
|
|
73
|
+
def retry(self, delay: Optional[int] = None):
|
|
74
|
+
raise RetryException(delay)
|
|
75
|
+
|
|
76
|
+
def abort(self, reason: Optional[str] = ''):
|
|
77
|
+
raise AbortException(reason)
|
|
78
|
+
|
|
79
|
+
def __get_service_ids(self, service_identifier: str):
|
|
80
|
+
if (is_valid_uuid(service_identifier)):
|
|
81
|
+
return [service_identifier]
|
|
82
|
+
|
|
83
|
+
return [service['id'] for service in self.__serviceList if service.get('alias') == service_identifier or service.get('name') == service_identifier]
|
|
84
|
+
|
|
85
|
+
def __get_from_context(self, field: Literal["data", "errors"], service_identifier: str) -> list[Tuple[Any, datetime.datetime]]:
|
|
86
|
+
service_ids = self.__get_service_ids(service_identifier)
|
|
87
|
+
|
|
88
|
+
service_ids = self.__get_service_ids(service_identifier)
|
|
89
|
+
results = []
|
|
90
|
+
for service_id in service_ids:
|
|
91
|
+
if service_id in self.__context[field]:
|
|
92
|
+
results.append(
|
|
93
|
+
(
|
|
94
|
+
self.__context[field][service_id],
|
|
95
|
+
parse_iso_format(self.__context["dates"][service_id])
|
|
96
|
+
)
|
|
97
|
+
)
|
|
98
|
+
return results
|
|
99
|
+
|
|
100
|
+
def get_input(self):
|
|
101
|
+
return self.__context["input"]
|
|
102
|
+
|
|
103
|
+
def get_meta(self):
|
|
104
|
+
return self.__context["meta"]
|
|
105
|
+
|
|
106
|
+
def get_configuration(self):
|
|
107
|
+
meta = self.get_service_meta(self.__id)
|
|
108
|
+
if (not meta):
|
|
109
|
+
return None
|
|
110
|
+
return meta["configuration"]
|
|
111
|
+
|
|
112
|
+
def get_service_meta(self, service_identifier: str):
|
|
113
|
+
service_id = self.__get_service_ids(service_identifier)[0]
|
|
114
|
+
if (not service_id):
|
|
115
|
+
return None
|
|
116
|
+
|
|
117
|
+
service = next((service for service in self.__serviceList if service['id'] == service_id), None)
|
|
118
|
+
if (not service):
|
|
119
|
+
return None
|
|
120
|
+
|
|
121
|
+
execution_date = parse_iso_format(self.__context['dates'][service_id]) if service_id in self.__context['dates'] else None
|
|
122
|
+
|
|
123
|
+
return cast(ServiceMeta, {
|
|
124
|
+
**service,
|
|
125
|
+
"executionDate": execution_date,
|
|
126
|
+
})
|
|
127
|
+
|
|
128
|
+
def get_results(self):
|
|
129
|
+
return [
|
|
130
|
+
{
|
|
131
|
+
**service,
|
|
132
|
+
"executionDate": parse_iso_format(self.__context['dates'][service['id']]) if service['id'] in self.__context['dates'] else None,
|
|
133
|
+
"data": self.__context['data'].get(service['id']),
|
|
134
|
+
"error": self.__context['errors'].get(service['id']),
|
|
135
|
+
}
|
|
136
|
+
for service in self.__serviceList
|
|
137
|
+
]
|
|
138
|
+
|
|
139
|
+
def get_last_service_data(self, service_identifier: str):
|
|
140
|
+
data_with_dates = self.__get_from_context('data', service_identifier)
|
|
141
|
+
data_with_dates.sort(key=lambda x: x[1], reverse=True)
|
|
142
|
+
return data_with_dates[0][0] if data_with_dates else None
|
|
143
|
+
|
|
144
|
+
def get_service_data(self, service_identifier: str):
|
|
145
|
+
return self.__get_from_context('data', service_identifier)[0][0] or None
|
|
146
|
+
|
|
147
|
+
def get_last_service_error(self, service_identifier: str):
|
|
148
|
+
data_with_dates = self.__get_from_context('errors', service_identifier)
|
|
149
|
+
data_with_dates.sort(key=lambda x: x[1], reverse=True)
|
|
150
|
+
return data_with_dates[0][0] if data_with_dates else None
|
|
151
|
+
|
|
152
|
+
def get_service_error(self, service_identifier: str):
|
|
153
|
+
return self.__get_from_context('errors', service_identifier)[0][0] or None
|
|
154
|
+
|
|
155
|
+
class OnMessageCallback(Protocol):
|
|
156
|
+
def __call__(self, context: ContextHelper) -> Any | None | Awaitable[Any | None]: ...
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
class Consumer:
|
|
160
|
+
|
|
161
|
+
def __init__(self, config: ConsumerConfig, on_message_callback: OnMessageCallback):
|
|
162
|
+
|
|
163
|
+
url = config.get("url") or os.getenv("REQUENCE_URL")
|
|
164
|
+
if not isinstance(url, str) or not url.startswith("amqp://"):
|
|
165
|
+
raise ValueError("URL must be an amqp connection string.")
|
|
166
|
+
|
|
167
|
+
version = config.get("version") or os.getenv("VERSION")
|
|
168
|
+
if not isinstance(version, str) or not is_valid_version(version):
|
|
169
|
+
raise ValueError("Version must be a valid version")
|
|
170
|
+
|
|
171
|
+
prefetch = config.get("prefetch") or 1
|
|
172
|
+
if not isinstance(prefetch, int):
|
|
173
|
+
raise ValueError("Prefetch must be an integer")
|
|
174
|
+
|
|
175
|
+
parsed_url = pika.URLParameters(url)
|
|
176
|
+
if not parsed_url.credentials:
|
|
177
|
+
raise ValueError("Url connection string needs a username")
|
|
178
|
+
|
|
179
|
+
username = parsed_url.credentials.username
|
|
180
|
+
|
|
181
|
+
parameters = pika.URLParameters(url)
|
|
182
|
+
connection = pika.BlockingConnection(parameters)
|
|
183
|
+
|
|
184
|
+
queue = username
|
|
185
|
+
exchange = username
|
|
186
|
+
|
|
187
|
+
channel = connection.channel()
|
|
188
|
+
channel.exchange_declare(exchange=exchange, passive=True)
|
|
189
|
+
channel.exchange_declare(exchange=exchange+"-retry", passive=True)
|
|
190
|
+
channel.queue_declare(queue=queue, passive=True)
|
|
191
|
+
channel.basic_qos(prefetch_count=prefetch)
|
|
192
|
+
|
|
193
|
+
self.__version = version
|
|
194
|
+
self.__channel = channel
|
|
195
|
+
self.__exchange = exchange
|
|
196
|
+
self.__on_message_callback = on_message_callback
|
|
197
|
+
channel.basic_consume(queue=queue, on_message_callback=self.__on_message)
|
|
198
|
+
channel.start_consuming()
|
|
199
|
+
|
|
200
|
+
def close(self):
|
|
201
|
+
self.__channel.stop_consuming()
|
|
202
|
+
self.__channel.close()
|
|
203
|
+
|
|
204
|
+
async def __handle_response(self, body: bytes, delivery_tag: int, properties: pika.BasicProperties):
|
|
205
|
+
try:
|
|
206
|
+
correlation_id = properties.correlation_id
|
|
207
|
+
if not is_valid_uuid(correlation_id):
|
|
208
|
+
raise Exception("Invalid or missing correlation ID.")
|
|
209
|
+
|
|
210
|
+
payloadData = json.loads(body.decode('utf-8'))
|
|
211
|
+
if not isinstance(payloadData, dict):
|
|
212
|
+
raise ValueError("Context must be a dictionary.")
|
|
213
|
+
|
|
214
|
+
payload = cast(Payload, payloadData)
|
|
215
|
+
extended_context = self.__on_message_callback(ContextHelper(correlation_id, payload.get("context"), payload.get("meta").get("services")))
|
|
216
|
+
|
|
217
|
+
if (isinstance(extended_context, Awaitable)):
|
|
218
|
+
extended_context = await extended_context
|
|
219
|
+
|
|
220
|
+
self.__channel.basic_ack(delivery_tag)
|
|
221
|
+
self.__channel.basic_publish(
|
|
222
|
+
exchange=self.__exchange,
|
|
223
|
+
routing_key="",
|
|
224
|
+
body=json.dumps(extended_context).encode('utf-8'),
|
|
225
|
+
properties=pika.BasicProperties(correlation_id=correlation_id)
|
|
226
|
+
)
|
|
227
|
+
except RetryException as e:
|
|
228
|
+
properties.expiration = str(e.delay or 1000)
|
|
229
|
+
self.__channel.basic_publish(
|
|
230
|
+
exchange=self.__exchange + "-retry",
|
|
231
|
+
routing_key="retry",
|
|
232
|
+
body=body,
|
|
233
|
+
properties=properties
|
|
234
|
+
)
|
|
235
|
+
self.__channel.basic_ack(delivery_tag)
|
|
236
|
+
except AbortException as e:
|
|
237
|
+
message = str(e)
|
|
238
|
+
if (len(message) == 0):
|
|
239
|
+
raise e
|
|
240
|
+
|
|
241
|
+
properties.expiration = str(1)
|
|
242
|
+
properties.priority = 0
|
|
243
|
+
properties.content_type = "text/plain"
|
|
244
|
+
properties.headers["error"] = True
|
|
245
|
+
|
|
246
|
+
self.__channel.basic_publish(
|
|
247
|
+
exchange=self.__exchange + "-retry",
|
|
248
|
+
routing_key="requeue",
|
|
249
|
+
body=message,
|
|
250
|
+
properties=properties
|
|
251
|
+
)
|
|
252
|
+
self.__channel.basic_ack(delivery_tag)
|
|
253
|
+
except Exception as e:
|
|
254
|
+
print("Encountered an error inside consumer handler:")
|
|
255
|
+
print(str(e))
|
|
256
|
+
self.__channel.basic_nack(delivery_tag, multiple=False, requeue=False)
|
|
257
|
+
|
|
258
|
+
def __on_message(self, channel: BlockingChannel, method: Basic.Deliver, properties: BasicProperties, body: bytes):
|
|
259
|
+
try:
|
|
260
|
+
expected_version = properties.headers.get('version')
|
|
261
|
+
if not is_valid_version(expected_version):
|
|
262
|
+
raise Exception("Invalid version received in message properties.")
|
|
263
|
+
|
|
264
|
+
if not semver.match(self.__version, expected_version):
|
|
265
|
+
channel.basic_nack(delivery_tag=method.delivery_tag, multiple=False, requeue=True)
|
|
266
|
+
return
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
asyncio.run(self.__handle_response(body, method.delivery_tag, properties))
|
|
270
|
+
|
|
271
|
+
except:
|
|
272
|
+
channel.basic_nack(delivery_tag=method.delivery_tag, multiple=False, requeue=False)
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: requence
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Consumer implementation
|
|
5
|
+
Author: Torsten Blindert
|
|
6
|
+
Requires-Python: >=3.10
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: pika>=1.3.2
|
|
9
|
+
Requires-Dist: semver>=3.0.2
|
|
10
|
+
|
|
11
|
+
# requence-consumer
|
|
12
|
+
|
|
13
|
+
This is the python implementation of the @requence/consumer package
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
MANIFEST.in
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
setup.py
|
|
5
|
+
requence/__init__.py
|
|
6
|
+
requence/consumer.py
|
|
7
|
+
requence.egg-info/PKG-INFO
|
|
8
|
+
requence.egg-info/SOURCES.txt
|
|
9
|
+
requence.egg-info/dependency_links.txt
|
|
10
|
+
requence.egg-info/requires.txt
|
|
11
|
+
requence.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
requence
|
requence-0.0.1/setup.cfg
ADDED
requence-0.0.1/setup.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import pathlib
|
|
2
|
+
import setuptools
|
|
3
|
+
|
|
4
|
+
setuptools.setup(
|
|
5
|
+
name="requence",
|
|
6
|
+
version="0.0.1",
|
|
7
|
+
description="Consumer implementation",
|
|
8
|
+
long_description=pathlib.Path("README.md").read_text(),
|
|
9
|
+
long_description_content_type="text/markdown",
|
|
10
|
+
author="Torsten Blindert",
|
|
11
|
+
python_requires=">=3.10",
|
|
12
|
+
install_requires=["pika>=1.3.2", "semver>=3.0.2"],
|
|
13
|
+
packages=setuptools.find_packages(),
|
|
14
|
+
include_package_data=True,
|
|
15
|
+
)
|