utf-queue-client 1.15.1__tar.gz → 1.16.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/PKG-INFO +1 -1
  2. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/setup.py +1 -1
  3. utf-queue-client-1.16.0/test/test_amqp_connections.py +46 -0
  4. utf-queue-client-1.16.0/test/test_http_producer.py +90 -0
  5. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_sqa_test_result_producer.py +6 -1
  6. utf-queue-client-1.16.0/test/test_utils.py +77 -0
  7. utf-queue-client-1.16.0/utf_queue_client/clients/http_producer.py +207 -0
  8. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/clients/sqa_test_result_producer.py +23 -5
  9. utf-queue-client-1.16.0/utf_queue_client/utils/__init__.py +89 -0
  10. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client.egg-info/PKG-INFO +1 -1
  11. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client.egg-info/SOURCES.txt +4 -0
  12. utf-queue-client-1.15.1/utf_queue_client/utils/__init__.py +0 -14
  13. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/README.md +0 -0
  14. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/setup.cfg +0 -0
  15. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_async_consumer.py +0 -0
  16. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_connection_retry.py +0 -0
  17. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_data_lake_data_producer.py +0 -0
  18. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_legacy_base_model.py +0 -0
  19. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_models.py +0 -0
  20. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_opentelemetry_data_producer.py +0 -0
  21. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_publish_release_info.py +0 -0
  22. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_publish_test_results.py +0 -0
  23. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_ubai_copy.py +0 -0
  24. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_ubai_search.py +0 -0
  25. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_ubai_update_retention.py +0 -0
  26. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_ubai_upload.py +0 -0
  27. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/__init__.py +0 -0
  28. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/clients/__init__.py +0 -0
  29. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/clients/async_consumer.py +0 -0
  30. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/clients/base_producer.py +0 -0
  31. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/clients/data_lake_data_producer.py +0 -0
  32. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/clients/opentelemetry_data_producer.py +0 -0
  33. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/clients/ubai_artifact_upload_request_producer.py +0 -0
  34. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/exceptions/__init__.py +0 -0
  35. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/models/__init__.py +0 -0
  36. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/models/base_model.py +0 -0
  37. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/models/model_factory.py +0 -0
  38. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/models/schemas/__init__.py +0 -0
  39. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/scripts/__init__.py +0 -0
  40. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/scripts/publish_release_info_cli.py +0 -0
  41. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/scripts/publish_test_results_cli.py +0 -0
  42. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/scripts/ubai_copy_cli.py +0 -0
  43. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/scripts/ubai_search_cli.py +0 -0
  44. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/scripts/ubai_update_retention_cli.py +0 -0
  45. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/scripts/ubai_upload_cli.py +0 -0
  46. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/utils/queue_metrics_exporter.py +0 -0
  47. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/utils/queue_span_exporter.py +0 -0
  48. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client.egg-info/dependency_links.txt +0 -0
  49. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client.egg-info/entry_points.txt +0 -0
  50. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client.egg-info/requires.txt +0 -0
  51. {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 1.2
2
2
  Name: utf-queue-client
3
- Version: 1.15.1
3
+ Version: 1.16.0
4
4
  Summary: UNKNOWN
5
5
  Home-page: UNKNOWN
6
6
  Author: Joe Savage
@@ -6,7 +6,7 @@
6
6
  from setuptools import setup, find_packages # noqa: H301
7
7
 
8
8
  NAME = "utf-queue-client"
9
- VERSION = "1.15.1"
9
+ VERSION = "1.16.0"
10
10
  # To install the library, run the following
11
11
  #
12
12
  # python setup.py install
@@ -0,0 +1,46 @@
1
+ import os
2
+ from utf_queue_client.clients.base_producer import BlockingProducer
3
+ import pytest
4
+ import logging
5
+ import pika
6
+ import time
7
+ from utf_queue_client.utils import UTF_QUEUE_REGION_MAP
8
+ from urllib.parse import quote
9
+
10
+
11
+ regions = list(UTF_QUEUE_REGION_MAP.values())
12
+ servers = [f"utf-queue-{region}" for region in regions]
13
+
14
+
15
+ @pytest.fixture(params=servers)
16
+ def queue_server(request):
17
+ yield request.param
18
+
19
+
20
+ @pytest.mark.parametrize(
21
+ "username, password",
22
+ [
23
+ (os.environ["UTF_QUEUE_USERNAME_LDAP"], os.environ["UTF_QUEUE_PASSWORD_LDAP"]),
24
+ (os.environ["UTF_QUEUE_USERNAME"], os.environ["UTF_QUEUE_PASSWORD"]),
25
+ ],
26
+ )
27
+ def test_producer(queue_server, username, password):
28
+ suffix = ".dev.silabs.net"
29
+ queue_server = f"{queue_server}{suffix}"
30
+ l = logging.getLogger("test_producer")
31
+ l.info("Creating producer...")
32
+ p = BlockingProducer(
33
+ url=f"amqps://{username}:{quote(password)}@{queue_server}:443/testing?stack_timeout=30",
34
+ )
35
+ l.info(f"{queue_server}: Created producer")
36
+ params = pika.URLParameters(p.url)
37
+ l.info(f"stack timeout: {params.stack_timeout}")
38
+ begin = time.time()
39
+ for _ in range(0, 10):
40
+ l.info(f"{queue_server}: declaring queue...")
41
+ start = time.time()
42
+ p.queue_declare("default", durable=True)
43
+ stop = time.time()
44
+ l.info(
45
+ f"{queue_server}: done declaring queue in {stop - start} sec. Total time: {stop - begin}"
46
+ )
@@ -0,0 +1,90 @@
1
+ import pytest
2
+ from utf_queue_client.clients.base_producer import (
3
+ ConnectionError as QueueServerConnectionError,
4
+ )
5
+ from utf_queue_client.clients.http_producer import (
6
+ HTTPProducer,
7
+ HTTPChannel,
8
+ CompositeHTTPProducer,
9
+ )
10
+ from pika.spec import BasicProperties, PERSISTENT_DELIVERY_MODE
11
+ from unittest.mock import patch
12
+ import utf_queue_client.clients.http_producer
13
+
14
+ FAKE_CREDS = "notarealusername:notarealpassword"
15
+
16
+
17
+ def test_basic_properties_to_json():
18
+ assert HTTPChannel.basic_properties_to_json(BasicProperties(app_id="foo")) == {
19
+ "app_id": "foo"
20
+ }
21
+ assert HTTPChannel.basic_properties_to_json(
22
+ BasicProperties(app_id="foo", delivery_mode=PERSISTENT_DELIVERY_MODE)
23
+ ) == {"app_id": "foo", "delivery_mode": 2}
24
+ assert HTTPChannel.basic_properties_to_json(None) == {}
25
+
26
+
27
+ def test_http_channel_connection_error():
28
+ channel = HTTPChannel(
29
+ "https://localshmost", ("user", "pass"), "/", retries=0, retry_delay=0
30
+ )
31
+ with pytest.raises(QueueServerConnectionError):
32
+ channel.queue_declare("queue")
33
+ with pytest.raises(QueueServerConnectionError):
34
+ channel.basic_publish("exchange", "routing_key", b"foo", None)
35
+
36
+
37
+ def test_composite_producer_errors():
38
+ with patch.object(
39
+ utf_queue_client.clients.http_producer, "DEFAULT_NUM_CONNECTION_RETRIES", 0
40
+ ):
41
+ composite_producer = CompositeHTTPProducer(
42
+ f"amqps://{FAKE_CREDS}@localshmost:443/%2F", "app_id"
43
+ )
44
+ composite_producer.producers.append(composite_producer.producers[0])
45
+ with pytest.raises(QueueServerConnectionError):
46
+ composite_producer.queue_declare("queue")
47
+ with pytest.raises(QueueServerConnectionError):
48
+ composite_producer.publish("exchange", "routing_key", {"foo": "bar"}, True)
49
+
50
+
51
+ @pytest.mark.parametrize(
52
+ ("amqp_url", "expected_http_urls"),
53
+ [
54
+ (
55
+ f"amqps://{FAKE_CREDS}@localshmost:443/%2F",
56
+ ["https://localshmost-admin:443"],
57
+ ),
58
+ (
59
+ f"amqps://{FAKE_CREDS}@utf-queue-central:443/%2F",
60
+ ["https://utf-queue-central-admin:443"],
61
+ ),
62
+ (
63
+ f"amqps://{FAKE_CREDS}@utf-queue-central.silabs.net:443/%2F",
64
+ [
65
+ "https://utf-queue-central-admin.silabs.net:443",
66
+ "https://utf-queue-aus-admin.silabs.net:443",
67
+ "https://utf-queue-bos-admin.silabs.net:443",
68
+ "https://utf-queue-bud-admin.silabs.net:443",
69
+ "https://utf-queue-hyd-admin.silabs.net:443",
70
+ "https://utf-queue-yul-admin.silabs.net:443",
71
+ ],
72
+ ),
73
+ (
74
+ f"amqps://{FAKE_CREDS}@utf-queue-aus.silabs.net:443/%2F",
75
+ [
76
+ "https://utf-queue-central-admin.silabs.net:443",
77
+ "https://utf-queue-aus-admin.silabs.net:443",
78
+ "https://utf-queue-bos-admin.silabs.net:443",
79
+ "https://utf-queue-bud-admin.silabs.net:443",
80
+ "https://utf-queue-hyd-admin.silabs.net:443",
81
+ "https://utf-queue-yul-admin.silabs.net:443",
82
+ ],
83
+ ),
84
+ ],
85
+ )
86
+ def test_composite_producer_creation(amqp_url, expected_http_urls):
87
+ composite_producer = CompositeHTTPProducer(amqp_url, "app_id")
88
+ assert len(composite_producer.producers) == len(expected_http_urls)
89
+ for index, value in enumerate(expected_http_urls):
90
+ assert composite_producer.producers[index].channel.url == value
@@ -15,6 +15,7 @@ def test_result_producer_empty_url():
15
15
 
16
16
 
17
17
  @pytest.mark.parametrize("queue_consumer", [60, 120], indirect=True)
18
+ @pytest.mark.parametrize("use_http_producer", [True, False])
18
19
  def test_result_producer_central_queue(
19
20
  request,
20
21
  sqa_app_build_result,
@@ -22,9 +23,13 @@ def test_result_producer_central_queue(
22
23
  sqa_test_session,
23
24
  amqp_url,
24
25
  queue_consumer,
26
+ use_http_producer,
25
27
  ):
26
28
  os.environ["UTF_PRODUCER_APP_ID"] = request.node.name
27
- producer = SqaTestResultProducerFactory.create_producer()
29
+ os.environ["UTF_QUEUE_USE_HTTP_PRODUCER"] = "true" if use_http_producer else "false"
30
+ producer = SqaTestResultProducerFactory.create_producer(
31
+ use_http_producer=use_http_producer
32
+ )
28
33
  producer.publish_app_build_result(sqa_app_build_result)
29
34
  queue_consumer.expect_messages(1)
30
35
  producer.publish_test_session_start(sqa_test_session)
@@ -0,0 +1,77 @@
1
+ import os
2
+ import pytest
3
+ from utf_queue_client.utils import (
4
+ get_queue_url_list,
5
+ UTF_QUEUE_REGION_MAP,
6
+ get_closest_queue_region,
7
+ )
8
+
9
+
10
+ NOMAD_DC_PARAMS = [
11
+ ("NOMAD_DC", value)
12
+ for value in [
13
+ None,
14
+ "aws-us-east-1",
15
+ "vmw-us-aus-1",
16
+ "vmw-us-bos-1",
17
+ "vmw-eu-bud-1",
18
+ "vmw-ap-hyd-1",
19
+ "vmw-ca-yul-1",
20
+ "vmw-ap-syd-1",
21
+ ]
22
+ ]
23
+ FAKE_CREDS = "notarealusername:notarealpassword"
24
+
25
+
26
+ @pytest.mark.parametrize(
27
+ "set_environment_variable",
28
+ NOMAD_DC_PARAMS,
29
+ indirect=True,
30
+ )
31
+ @pytest.mark.parametrize("domain", ["dev.silabs.net", "silabs.net"])
32
+ def test_get_queue_url_list(set_environment_variable, domain):
33
+ nomad_dc = os.getenv("NOMAD_DC")
34
+ url = f"amqps://{FAKE_CREDS}@utf-queue-central.{domain}:443/%2F?stack_timeout=30"
35
+ urls = get_queue_url_list(url)
36
+ assert len(urls) == 6
37
+
38
+ region_list = list(UTF_QUEUE_REGION_MAP.values())
39
+ if nomad_dc is None or nomad_dc not in UTF_QUEUE_REGION_MAP:
40
+ assert urls[0] == url
41
+ region_list.remove("central")
42
+ else:
43
+ region = UTF_QUEUE_REGION_MAP[nomad_dc]
44
+ assert urls[0] == url.replace("utf-queue-central", f"utf-queue-{region}")
45
+ region_list.remove(region)
46
+ assert urls[1:] == [
47
+ url.replace("utf-queue-central", f"utf-queue-{region}")
48
+ for region in region_list
49
+ ]
50
+
51
+
52
+ def test_get_queue_url_list_no_match():
53
+ url_list = [
54
+ f"amqps://{FAKE_CREDS}@custom-server.silabs.net:443/%2F?stack_timeout=30",
55
+ f"amqp://{FAKE_CREDS}@utf-queue-central.silabs.net:5672/%2F",
56
+ f"amqps://{FAKE_CREDS}@utf-queue-central.silabs.net:5672/%2F",
57
+ f"amqps://{FAKE_CREDS}@utf-queue-central.silabs.com:443/%2F",
58
+ ]
59
+ for url in url_list:
60
+ urls = get_queue_url_list(url)
61
+ assert urls == [url]
62
+
63
+
64
+ @pytest.mark.parametrize(
65
+ "set_environment_variable",
66
+ NOMAD_DC_PARAMS,
67
+ indirect=True,
68
+ )
69
+ def test_get_closest_queue_region(set_environment_variable):
70
+ nomad_dc = os.getenv("NOMAD_DC")
71
+ try:
72
+ if nomad_dc is None or nomad_dc not in UTF_QUEUE_REGION_MAP:
73
+ assert get_closest_queue_region() == "central"
74
+ else:
75
+ assert get_closest_queue_region() == UTF_QUEUE_REGION_MAP[nomad_dc]
76
+ finally:
77
+ os.environ.pop("NOMAD_DC", None)
@@ -0,0 +1,207 @@
1
+ import pika
2
+ from pika.spec import BasicProperties
3
+ from typing import Optional, Tuple
4
+
5
+ from utf_queue_client.clients.base_producer import (
6
+ BaseProducer,
7
+ PikaChannel,
8
+ ConnectionError,
9
+ )
10
+ import base64
11
+ from utf_queue_client.utils import get_queue_url_list
12
+ from requests import Session as RequestsSession, RequestException
13
+ from requests.adapters import HTTPAdapter
14
+ from urllib3.util import Retry
15
+ from . import Loggable
16
+ from dataclasses import dataclass
17
+ from urllib.parse import urlparse
18
+
19
+ __all__ = ["HTTPProducer", "HTTPChannel", "CompositeHTTPProducer"]
20
+
21
+ DEFAULT_NUM_CONNECTION_RETRIES = 6
22
+ DEFAULT_CONNECTION_RETRY_DELAY = 10
23
+
24
+
25
+ @dataclass
26
+ class UrlInfo:
27
+ amqp_url: str
28
+ http_url: str
29
+ virtual_host: str
30
+ username: str
31
+ password: str
32
+
33
+
34
+ class HTTPChannel(PikaChannel):
35
+ def __init__(
36
+ self,
37
+ url: str,
38
+ credentials: Tuple[str, str],
39
+ virtual_host: str,
40
+ retries=None,
41
+ retry_delay=None,
42
+ ):
43
+ if retries is None:
44
+ retries = DEFAULT_NUM_CONNECTION_RETRIES
45
+ if retry_delay is None:
46
+ retry_delay = DEFAULT_CONNECTION_RETRY_DELAY
47
+ self.client = RequestsSession()
48
+ self.client.verify = False
49
+ self.client.auth = credentials
50
+ self.client.mount(
51
+ "https://",
52
+ HTTPAdapter(
53
+ max_retries=Retry(
54
+ total=retries,
55
+ backoff_factor=retry_delay,
56
+ status_forcelist=[404, 429, 500, 502, 503, 504],
57
+ )
58
+ ),
59
+ )
60
+ self.url = url
61
+ self.virtual_host = virtual_host
62
+ if self.virtual_host == "/":
63
+ self.virtual_host = "%2F"
64
+
65
+ @classmethod
66
+ def basic_properties_to_json(cls, properties: Optional[BasicProperties]) -> dict:
67
+ props_json = {}
68
+ if properties:
69
+ for key in [
70
+ "app_id",
71
+ "content_type",
72
+ "content_encoding",
73
+ "correlation_id",
74
+ "cluster_id",
75
+ "delivery_mode",
76
+ "expiration",
77
+ "headers",
78
+ "message_id",
79
+ "priority",
80
+ "reply_to",
81
+ "timestamp",
82
+ "type",
83
+ "user_id",
84
+ ]:
85
+ val = getattr(properties, key)
86
+ if val is not None:
87
+ props_json[key] = val
88
+ return props_json
89
+
90
+ def basic_publish(
91
+ self,
92
+ exchange: str,
93
+ routing_key: str,
94
+ body: bytes,
95
+ properties: Optional[BasicProperties] = None,
96
+ mandatory: bool = False,
97
+ ):
98
+ if not exchange:
99
+ exchange = "amq.default"
100
+ publish_url = f"{self.url}/api/exchanges/{self.virtual_host}/{exchange}/publish"
101
+ try:
102
+ r = self.client.post(
103
+ publish_url,
104
+ json={
105
+ "properties": self.basic_properties_to_json(properties),
106
+ "routing_key": routing_key,
107
+ "payload": base64.b64encode(body).decode("utf-8"),
108
+ "payload_encoding": "base64",
109
+ },
110
+ )
111
+ r.raise_for_status()
112
+ except RequestException as e:
113
+ raise ConnectionError(e)
114
+
115
+ def queue_declare(
116
+ self,
117
+ queue,
118
+ passive=False,
119
+ durable=False,
120
+ exclusive=False,
121
+ auto_delete=False,
122
+ arguments=None,
123
+ callback=None,
124
+ ):
125
+ declare_url = f"{self.url}/api/queues/{self.virtual_host}/{queue}"
126
+ try:
127
+ r = self.client.put(
128
+ declare_url, json={"auto_delete": auto_delete, "durable": durable}
129
+ )
130
+ r.raise_for_status()
131
+ except RequestException as e:
132
+ raise ConnectionError(e)
133
+
134
+
135
+ class HTTPProducer(BaseProducer):
136
+ def __init__(
137
+ self,
138
+ url_info: UrlInfo,
139
+ producer_app_id: Optional[str] = None,
140
+ ):
141
+ super().__init__(url_info.amqp_url, producer_app_id)
142
+ self._channel = HTTPChannel(
143
+ url_info.http_url,
144
+ (url_info.username, url_info.password),
145
+ url_info.virtual_host,
146
+ )
147
+ self.hostname = urlparse(url_info.http_url).hostname
148
+
149
+ def _connect(self):
150
+ # no long-lived connection
151
+ pass
152
+
153
+ def is_connected(self):
154
+ return True # pragma: nocover
155
+
156
+ @property
157
+ def channel(self) -> HTTPChannel:
158
+ return self._channel
159
+
160
+
161
+ class CompositeHTTPProducer(Loggable):
162
+ def __init__(self, url: str, producer_app_id: str = None):
163
+ url_list = get_queue_url_list(url)
164
+ self.producers = [
165
+ HTTPProducer(self.convert_amqp_url_to_http_url(alt_url), producer_app_id)
166
+ for alt_url in url_list
167
+ ]
168
+
169
+ @classmethod
170
+ def convert_amqp_url_to_http_url(cls, amqp_url: str) -> UrlInfo:
171
+ url_params = pika.URLParameters(amqp_url)
172
+ host, delim, domain = url_params.host.partition(".")
173
+ hostname = f"{host}-admin{delim}{domain}:{url_params.port}"
174
+ scheme = "https" if url_params.ssl_options else "http"
175
+ return UrlInfo(
176
+ amqp_url=amqp_url,
177
+ http_url=f"{scheme}://{hostname}",
178
+ virtual_host=url_params.virtual_host,
179
+ username=url_params.credentials.username,
180
+ password=url_params.credentials.password,
181
+ )
182
+
183
+ def queue_declare(self, queue, **kwargs):
184
+ for index, producer in enumerate(self.producers):
185
+ if index:
186
+ self.logger.warning(
187
+ f"Failed to execute queue_declare for host {self.producers[index-1].hostname}, retrying with {producer.hostname}..."
188
+ )
189
+ try:
190
+ return producer.queue_declare(queue, **kwargs)
191
+ except ConnectionError:
192
+ if index == len(self.producers) - 1:
193
+ raise
194
+ continue
195
+
196
+ def publish(self, exchange: str, routing_key: str, payload: dict, persistent: bool):
197
+ for index, producer in enumerate(self.producers):
198
+ if index:
199
+ self.logger.warning(
200
+ f"Failed to execute publish for host {self.producers[index-1].hostname}, retrying with {producer.hostname}..."
201
+ )
202
+ try:
203
+ return producer.publish(exchange, routing_key, payload, persistent)
204
+ except ConnectionError:
205
+ if index == len(self.producers) - 1:
206
+ raise
207
+ continue
@@ -1,6 +1,8 @@
1
1
  import os
2
-
3
- from .base_producer import BlockingProducer, ConnectionError
2
+ import re
3
+ from .base_producer import BlockingProducer as AMQPBlockingProducer, ConnectionError
4
+ from .http_producer import CompositeHTTPProducer
5
+ from ..utils import UTF_QUEUE_SERVER_PATTERN
4
6
  from ..models import SqaAppBuildResult
5
7
  from ..models import SqaTestResult
6
8
  from ..models import SqaTestSession
@@ -10,6 +12,8 @@ from socket import gethostname
10
12
  from datetime import datetime
11
13
  from abc import ABC, abstractmethod
12
14
  from otel_extensions import instrumented
15
+ from typing import Optional
16
+
13
17
 
14
18
  __all__ = [
15
19
  "SqaTestResultProducer",
@@ -43,9 +47,16 @@ class BaseSqaTestResultProducer(ABC):
43
47
  class SqaTestResultProducer(BaseSqaTestResultProducer):
44
48
  RECORD_TYPE = "UTF_TEST_EVENT"
45
49
 
46
- def __init__(self, url=None, producer_app_id: str = None):
50
+ def __init__(
51
+ self,
52
+ url=None,
53
+ producer_app_id: str = None,
54
+ producer_class: Optional[type] = None,
55
+ ):
56
+ if producer_class is None:
57
+ producer_class = AMQPBlockingProducer
47
58
  self.queue_name = "default"
48
- self.__client = BlockingProducer(url, producer_app_id)
59
+ self.__client = producer_class(url, producer_app_id)
49
60
  self.__client.queue_declare(queue=self.queue_name, durable=True)
50
61
  self.producer_app_id = producer_app_id
51
62
 
@@ -136,13 +147,20 @@ class DummySqaTestResultProducer(BaseSqaTestResultProducer):
136
147
 
137
148
  class SqaTestResultProducerFactory(Loggable):
138
149
  @classmethod
139
- def create_producer(cls, raise_on_connection_error=False):
150
+ def create_producer(cls, raise_on_connection_error=False, use_http_producer=True):
140
151
  try:
141
152
  queue_server_url = os.environ.get("UTF_QUEUE_SERVER_URL")
142
153
  if queue_server_url is not None:
154
+ producer_class = None
155
+ use_http_producer = use_http_producer and os.environ.get("UTF_QUEUE_USE_HTTP_PRODUCER") == "true"
156
+ if use_http_producer and re.match(
157
+ UTF_QUEUE_SERVER_PATTERN, queue_server_url
158
+ ):
159
+ producer_class = CompositeHTTPProducer
143
160
  return SqaTestResultProducer(
144
161
  url=queue_server_url,
145
162
  producer_app_id=os.environ.get("UTF_PRODUCER_APP_ID"),
163
+ producer_class=producer_class,
146
164
  )
147
165
  else:
148
166
  return LocalSqaTestResultProducer()
@@ -0,0 +1,89 @@
1
+ import os
2
+ from urllib import parse
3
+ from typing import List
4
+ import re
5
+
6
+ QUEUE_APP_ID = "app_queue_client"
7
+ UTF_QUEUE_REGION_MAP = {
8
+ "aws-us-east-1": "central",
9
+ "vmw-us-aus-1": "aus",
10
+ "vmw-us-bos-1": "bos",
11
+ "vmw-eu-bud-1": "bud",
12
+ "vmw-ap-hyd-1": "hyd",
13
+ "vmw-ca-yul-1": "yul",
14
+ }
15
+ UTF_QUEUE_SERVER_PREFIX = "utf-queue-"
16
+ UTF_QUEUE_SERVER_PATTERN = (
17
+ "amqps://(.*):(.*)@"
18
+ + rf"({UTF_QUEUE_SERVER_PREFIX}({'|'.join(UTF_QUEUE_REGION_MAP.values())}))"
19
+ + r"\.(dev\.silabs\.net|silabs\.net):443"
20
+ )
21
+
22
+
23
+ def prepare_queue_central_url(username, password):
24
+ if username is None or password is None:
25
+ raise RuntimeError("username or password must be provided if using queue")
26
+ hostname = os.environ.get("UTF_QUEUE_HOSTNAME", "utf-queue-central.silabs.net")
27
+ scheme = os.environ.get("UTF_QUEUE_SCHEME", "amqps")
28
+ port = os.environ.get("UTF_QUEUE_PORT", "443")
29
+ virtual_host = os.environ.get("UTF_QUEUE_VIRTUAL_HOST", "%2f")
30
+ return f"{scheme}://{username}:{parse.quote(password)}@{hostname}:{port}/{virtual_host}"
31
+
32
+
33
+ def get_closest_queue_region():
34
+ """
35
+ Return the correct value from the UTF_QUEUE_REGION_MAP dictionary
36
+ based on the key contained in the NOMAD_DC environment variable (if it's defined).
37
+ If NOMAD_DC is not defined or the key is not in UTF_QUEUE_REGION_MAP, we'll just
38
+ return "central"
39
+ """
40
+ if "NOMAD_DC" in os.environ:
41
+ dc = os.environ["NOMAD_DC"]
42
+ if dc in UTF_QUEUE_REGION_MAP:
43
+ return UTF_QUEUE_REGION_MAP[dc]
44
+
45
+ return "central"
46
+
47
+
48
+ def get_queue_url_list(original_url: str) -> List[str]:
49
+ """
50
+ The purpose of this function is to return a list of alternative
51
+ utf queue urls given a single original url.
52
+ If 'original_url' matches the UTF_QUEUE_SERVER_PATTERN regex,
53
+ return a list of urls based on the values in the UTF_QUEUE_REGION_MAP dict,
54
+ where the order is determined by the NOMAD_DC environment variable.
55
+
56
+ Example:
57
+ 1) if original_url is "amqps://xxxx:yyyy@utf-queue-central.silabs.net:443/%2F"
58
+ and NOMAD_DC is "vmw-ap-hyd-1",
59
+ we'll return
60
+ - amqps://xxxx:yyyy@utf-queue-hyd.silabs.net:443/%2F
61
+ - amqps://xxxx:yyyy@utf-queue-central.silabs.net:443/%2F
62
+ - amqps://xxxx:yyyy@utf-queue-aus.silabs.net:443/%2F
63
+ - amqps://xxxx:yyyy@utf-queue-bos.silabs.net:443/%2F
64
+ etc.
65
+ 2) if original_url is "amqps://xxxx:yyyy@utf-queue-central.silabs.net:443/%2F"
66
+ and NOMAD_DC is not defined,
67
+ we'll return
68
+ - amqps://xxxx:yyyy@utf-queue-central.silabs.net:443/%2F
69
+ - amqps://xxxx:yyyy@utf-queue-aus.silabs.net:443/%2F
70
+ - amqps://xxxx:yyyy@utf-queue-bos.silabs.net:443/%2F
71
+ etc.
72
+ 3) if original_url doesn't match the UTF_QUEUE_SERVER_PATTERN pattern
73
+ we'll return a list containing original_url as the only item.
74
+ """
75
+ m = re.match(UTF_QUEUE_SERVER_PATTERN, original_url)
76
+ if not m:
77
+ return [original_url]
78
+
79
+ hostname_base = m.group(3)
80
+ domain = m.group(5)
81
+
82
+ url_list = []
83
+ region_list = [get_closest_queue_region()] + list(UTF_QUEUE_REGION_MAP.values())
84
+ for region in region_list:
85
+ url = original_url.replace(
86
+ f"{hostname_base}.{domain}", f"{UTF_QUEUE_SERVER_PREFIX}{region}.{domain}"
87
+ )
88
+ url_list.append(url)
89
+ return list(dict.fromkeys(url_list))
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 1.2
2
2
  Name: utf-queue-client
3
- Version: 1.15.1
3
+ Version: 1.16.0
4
4
  Summary: UNKNOWN
5
5
  Home-page: UNKNOWN
6
6
  Author: Joe Savage
@@ -1,9 +1,11 @@
1
1
  README.md
2
2
  setup.cfg
3
3
  setup.py
4
+ test/test_amqp_connections.py
4
5
  test/test_async_consumer.py
5
6
  test/test_connection_retry.py
6
7
  test/test_data_lake_data_producer.py
8
+ test/test_http_producer.py
7
9
  test/test_legacy_base_model.py
8
10
  test/test_models.py
9
11
  test/test_opentelemetry_data_producer.py
@@ -14,6 +16,7 @@ test/test_ubai_copy.py
14
16
  test/test_ubai_search.py
15
17
  test/test_ubai_update_retention.py
16
18
  test/test_ubai_upload.py
19
+ test/test_utils.py
17
20
  utf_queue_client/__init__.py
18
21
  utf_queue_client.egg-info/PKG-INFO
19
22
  utf_queue_client.egg-info/SOURCES.txt
@@ -25,6 +28,7 @@ utf_queue_client/clients/__init__.py
25
28
  utf_queue_client/clients/async_consumer.py
26
29
  utf_queue_client/clients/base_producer.py
27
30
  utf_queue_client/clients/data_lake_data_producer.py
31
+ utf_queue_client/clients/http_producer.py
28
32
  utf_queue_client/clients/opentelemetry_data_producer.py
29
33
  utf_queue_client/clients/sqa_test_result_producer.py
30
34
  utf_queue_client/clients/ubai_artifact_upload_request_producer.py
@@ -1,14 +0,0 @@
1
- import os
2
- from urllib import parse
3
-
4
- QUEUE_APP_ID = "app_queue_client"
5
-
6
-
7
- def prepare_queue_central_url(username, password):
8
- if username is None or password is None:
9
- raise RuntimeError("username or password must be provided if using queue")
10
- hostname = os.environ.get("UTF_QUEUE_HOSTNAME", "utf-queue-central.silabs.net")
11
- scheme = os.environ.get("UTF_QUEUE_SCHEME", "amqps")
12
- port = os.environ.get("UTF_QUEUE_PORT", "443")
13
- virtual_host = os.environ.get("UTF_QUEUE_VIRTUAL_HOST", "%2f")
14
- return f"{scheme}://{username}:{parse.quote(password)}@{hostname}:{port}/{virtual_host}"