requence 0.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,4 @@
1
+ graft operator-consumer
2
+ include pyproject.toml
3
+ include setup.py
4
+ global-exclude *.py[cod]
@@ -0,0 +1,13 @@
1
+ Metadata-Version: 2.1
2
+ Name: requence
3
+ Version: 0.0.1
4
+ Summary: Consumer implementation
5
+ Author: Torsten Blindert
6
+ Requires-Python: >=3.10
7
+ Description-Content-Type: text/markdown
8
+ Requires-Dist: pika>=1.3.2
9
+ Requires-Dist: semver>=3.0.2
10
+
11
+ # requence-consumer
12
+
13
+ This is the python implementation of the @requence/consumer package
@@ -0,0 +1,3 @@
1
+ # requence-consumer
2
+
3
+ This is the python implementation of the @requence/consumer package
@@ -0,0 +1,6 @@
1
+ [build-system]
2
+ requires = [
3
+ "setuptools>=42",
4
+ "wheel",
5
+ ]
6
+ build-backend = "setuptools.build_meta"
File without changes
@@ -0,0 +1,272 @@
1
+ from distutils.util import execute
2
+ from pika.adapters.blocking_connection import BlockingChannel
3
+ from pika.spec import Basic, Channel, BasicProperties
4
+ from typing import Any, Tuple, TypedDict, Awaitable, Callable, Dict, List, Literal, Optional, Protocol, Union, cast
5
+ import os
6
+ import pika
7
+ import semver
8
+ import json
9
+ import uuid
10
+ import re
11
+ import asyncio
12
+ import datetime
13
+
14
+ version_pattern = r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
15
+
16
+ def parse_iso_format(iso: str):
17
+ iso_without_z = iso.rstrip("Z")
18
+ return datetime.datetime.fromisoformat(iso_without_z).replace(tzinfo=datetime.timezone.utc)
19
+
20
+ def is_valid_version(version):
21
+ return re.match(version_pattern, version)
22
+
23
+ def is_valid_uuid(uuid_to_test):
24
+ try:
25
+ uuid_obj = uuid.UUID(uuid_to_test, version=4)
26
+ return str(uuid_obj) == uuid_to_test
27
+ except ValueError:
28
+ return False
29
+
30
+ class ConsumerConfig(TypedDict, total=False):
31
+ url: Optional[str]
32
+ version: Optional[str]
33
+ prefetch: Optional[int]
34
+
35
+ class RetryException(Exception):
36
+ def __init__(self, delay: Optional[int]):
37
+ self.delay = delay
38
+
39
+ class AbortException(Exception):
40
+ pass
41
+
42
+ class Context(TypedDict, total=True):
43
+ input: Any
44
+ meta: Any
45
+ data: Dict[str, Any]
46
+ dates: Dict[str, str]
47
+ errors: Dict[str, Any]
48
+
49
+ class Service(TypedDict):
50
+ id: str
51
+ alias: Optional[str]
52
+ name: str
53
+ version: str
54
+ configuration: Any
55
+
56
+ class ServiceMeta(Service):
57
+ execution_data: Union[None, datetime.datetime]
58
+
59
+ class Meta(TypedDict):
60
+ services: List[Service]
61
+
62
+ class Payload(TypedDict, total=True):
63
+ context: Context
64
+ meta: Meta
65
+
66
+
67
+ class ContextHelper:
68
+ def __init__(self, id: str, context: Context, serviceList: List[Service]):
69
+ self.__context = context
70
+ self.__serviceList = serviceList
71
+ self.__id = id
72
+
73
+ def retry(self, delay: Optional[int] = None):
74
+ raise RetryException(delay)
75
+
76
+ def abort(self, reason: Optional[str] = ''):
77
+ raise AbortException(reason)
78
+
79
+ def __get_service_ids(self, service_identifier: str):
80
+ if (is_valid_uuid(service_identifier)):
81
+ return [service_identifier]
82
+
83
+ return [service['id'] for service in self.__serviceList if service.get('alias') == service_identifier or service.get('name') == service_identifier]
84
+
85
+ def __get_from_context(self, field: Literal["data", "errors"], service_identifier: str) -> list[Tuple[Any, datetime.datetime]]:
86
+ service_ids = self.__get_service_ids(service_identifier)
87
+
88
+ service_ids = self.__get_service_ids(service_identifier)
89
+ results = []
90
+ for service_id in service_ids:
91
+ if service_id in self.__context[field]:
92
+ results.append(
93
+ (
94
+ self.__context[field][service_id],
95
+ parse_iso_format(self.__context["dates"][service_id])
96
+ )
97
+ )
98
+ return results
99
+
100
+ def get_input(self):
101
+ return self.__context["input"]
102
+
103
+ def get_meta(self):
104
+ return self.__context["meta"]
105
+
106
+ def get_configuration(self):
107
+ meta = self.get_service_meta(self.__id)
108
+ if (not meta):
109
+ return None
110
+ return meta["configuration"]
111
+
112
+ def get_service_meta(self, service_identifier: str):
113
+ service_id = self.__get_service_ids(service_identifier)[0]
114
+ if (not service_id):
115
+ return None
116
+
117
+ service = next((service for service in self.__serviceList if service['id'] == service_id), None)
118
+ if (not service):
119
+ return None
120
+
121
+ execution_date = parse_iso_format(self.__context['dates'][service_id]) if service_id in self.__context['dates'] else None
122
+
123
+ return cast(ServiceMeta, {
124
+ **service,
125
+ "executionDate": execution_date,
126
+ })
127
+
128
+ def get_results(self):
129
+ return [
130
+ {
131
+ **service,
132
+ "executionDate": parse_iso_format(self.__context['dates'][service['id']]) if service['id'] in self.__context['dates'] else None,
133
+ "data": self.__context['data'].get(service['id']),
134
+ "error": self.__context['errors'].get(service['id']),
135
+ }
136
+ for service in self.__serviceList
137
+ ]
138
+
139
+ def get_last_service_data(self, service_identifier: str):
140
+ data_with_dates = self.__get_from_context('data', service_identifier)
141
+ data_with_dates.sort(key=lambda x: x[1], reverse=True)
142
+ return data_with_dates[0][0] if data_with_dates else None
143
+
144
+ def get_service_data(self, service_identifier: str):
145
+ return self.__get_from_context('data', service_identifier)[0][0] or None
146
+
147
+ def get_last_service_error(self, service_identifier: str):
148
+ data_with_dates = self.__get_from_context('errors', service_identifier)
149
+ data_with_dates.sort(key=lambda x: x[1], reverse=True)
150
+ return data_with_dates[0][0] if data_with_dates else None
151
+
152
+ def get_service_error(self, service_identifier: str):
153
+ return self.__get_from_context('errors', service_identifier)[0][0] or None
154
+
155
+ class OnMessageCallback(Protocol):
156
+ def __call__(self, context: ContextHelper) -> Any | None | Awaitable[Any | None]: ...
157
+
158
+
159
+ class Consumer:
160
+
161
+ def __init__(self, config: ConsumerConfig, on_message_callback: OnMessageCallback):
162
+
163
+ url = config.get("url") or os.getenv("REQUENCE_URL")
164
+ if not isinstance(url, str) or not url.startswith("amqp://"):
165
+ raise ValueError("URL must be an amqp connection string.")
166
+
167
+ version = config.get("version") or os.getenv("VERSION")
168
+ if not isinstance(version, str) or not is_valid_version(version):
169
+ raise ValueError("Version must be a valid version")
170
+
171
+ prefetch = config.get("prefetch") or 1
172
+ if not isinstance(prefetch, int):
173
+ raise ValueError("Prefetch must be an integer")
174
+
175
+ parsed_url = pika.URLParameters(url)
176
+ if not parsed_url.credentials:
177
+ raise ValueError("Url connection string needs a username")
178
+
179
+ username = parsed_url.credentials.username
180
+
181
+ parameters = pika.URLParameters(url)
182
+ connection = pika.BlockingConnection(parameters)
183
+
184
+ queue = username
185
+ exchange = username
186
+
187
+ channel = connection.channel()
188
+ channel.exchange_declare(exchange=exchange, passive=True)
189
+ channel.exchange_declare(exchange=exchange+"-retry", passive=True)
190
+ channel.queue_declare(queue=queue, passive=True)
191
+ channel.basic_qos(prefetch_count=prefetch)
192
+
193
+ self.__version = version
194
+ self.__channel = channel
195
+ self.__exchange = exchange
196
+ self.__on_message_callback = on_message_callback
197
+ channel.basic_consume(queue=queue, on_message_callback=self.__on_message)
198
+ channel.start_consuming()
199
+
200
+ def close(self):
201
+ self.__channel.stop_consuming()
202
+ self.__channel.close()
203
+
204
+ async def __handle_response(self, body: bytes, delivery_tag: int, properties: pika.BasicProperties):
205
+ try:
206
+ correlation_id = properties.correlation_id
207
+ if not is_valid_uuid(correlation_id):
208
+ raise Exception("Invalid or missing correlation ID.")
209
+
210
+ payloadData = json.loads(body.decode('utf-8'))
211
+ if not isinstance(payloadData, dict):
212
+ raise ValueError("Context must be a dictionary.")
213
+
214
+ payload = cast(Payload, payloadData)
215
+ extended_context = self.__on_message_callback(ContextHelper(correlation_id, payload.get("context"), payload.get("meta").get("services")))
216
+
217
+ if (isinstance(extended_context, Awaitable)):
218
+ extended_context = await extended_context
219
+
220
+ self.__channel.basic_ack(delivery_tag)
221
+ self.__channel.basic_publish(
222
+ exchange=self.__exchange,
223
+ routing_key="",
224
+ body=json.dumps(extended_context).encode('utf-8'),
225
+ properties=pika.BasicProperties(correlation_id=correlation_id)
226
+ )
227
+ except RetryException as e:
228
+ properties.expiration = str(e.delay or 1000)
229
+ self.__channel.basic_publish(
230
+ exchange=self.__exchange + "-retry",
231
+ routing_key="retry",
232
+ body=body,
233
+ properties=properties
234
+ )
235
+ self.__channel.basic_ack(delivery_tag)
236
+ except AbortException as e:
237
+ message = str(e)
238
+ if (len(message) == 0):
239
+ raise e
240
+
241
+ properties.expiration = str(1)
242
+ properties.priority = 0
243
+ properties.content_type = "text/plain"
244
+ properties.headers["error"] = True
245
+
246
+ self.__channel.basic_publish(
247
+ exchange=self.__exchange + "-retry",
248
+ routing_key="requeue",
249
+ body=message,
250
+ properties=properties
251
+ )
252
+ self.__channel.basic_ack(delivery_tag)
253
+ except Exception as e:
254
+ print("Encountered an error inside consumer handler:")
255
+ print(str(e))
256
+ self.__channel.basic_nack(delivery_tag, multiple=False, requeue=False)
257
+
258
+ def __on_message(self, channel: BlockingChannel, method: Basic.Deliver, properties: BasicProperties, body: bytes):
259
+ try:
260
+ expected_version = properties.headers.get('version')
261
+ if not is_valid_version(expected_version):
262
+ raise Exception("Invalid version received in message properties.")
263
+
264
+ if not semver.match(self.__version, expected_version):
265
+ channel.basic_nack(delivery_tag=method.delivery_tag, multiple=False, requeue=True)
266
+ return
267
+
268
+
269
+ asyncio.run(self.__handle_response(body, method.delivery_tag, properties))
270
+
271
+ except:
272
+ channel.basic_nack(delivery_tag=method.delivery_tag, multiple=False, requeue=False)
@@ -0,0 +1,13 @@
1
+ Metadata-Version: 2.1
2
+ Name: requence
3
+ Version: 0.0.1
4
+ Summary: Consumer implementation
5
+ Author: Torsten Blindert
6
+ Requires-Python: >=3.10
7
+ Description-Content-Type: text/markdown
8
+ Requires-Dist: pika>=1.3.2
9
+ Requires-Dist: semver>=3.0.2
10
+
11
+ # requence-consumer
12
+
13
+ This is the python implementation of the @requence/consumer package
@@ -0,0 +1,11 @@
1
+ MANIFEST.in
2
+ README.md
3
+ pyproject.toml
4
+ setup.py
5
+ requence/__init__.py
6
+ requence/consumer.py
7
+ requence.egg-info/PKG-INFO
8
+ requence.egg-info/SOURCES.txt
9
+ requence.egg-info/dependency_links.txt
10
+ requence.egg-info/requires.txt
11
+ requence.egg-info/top_level.txt
@@ -0,0 +1,2 @@
1
+ pika>=1.3.2
2
+ semver>=3.0.2
@@ -0,0 +1 @@
1
+ requence
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,15 @@
1
+ import pathlib
2
+ import setuptools
3
+
4
+ setuptools.setup(
5
+ name="requence",
6
+ version="0.0.1",
7
+ description="Consumer implementation",
8
+ long_description=pathlib.Path("README.md").read_text(),
9
+ long_description_content_type="text/markdown",
10
+ author="Torsten Blindert",
11
+ python_requires=">=3.10",
12
+ install_requires=["pika>=1.3.2", "semver>=3.0.2"],
13
+ packages=setuptools.find_packages(),
14
+ include_package_data=True,
15
+ )