utf-queue-client 1.15.1__tar.gz → 1.16.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/PKG-INFO +1 -1
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/setup.py +1 -1
- utf-queue-client-1.16.0/test/test_amqp_connections.py +46 -0
- utf-queue-client-1.16.0/test/test_http_producer.py +90 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_sqa_test_result_producer.py +6 -1
- utf-queue-client-1.16.0/test/test_utils.py +77 -0
- utf-queue-client-1.16.0/utf_queue_client/clients/http_producer.py +207 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/clients/sqa_test_result_producer.py +23 -5
- utf-queue-client-1.16.0/utf_queue_client/utils/__init__.py +89 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client.egg-info/PKG-INFO +1 -1
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client.egg-info/SOURCES.txt +4 -0
- utf-queue-client-1.15.1/utf_queue_client/utils/__init__.py +0 -14
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/README.md +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/setup.cfg +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_async_consumer.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_connection_retry.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_data_lake_data_producer.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_legacy_base_model.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_models.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_opentelemetry_data_producer.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_publish_release_info.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_publish_test_results.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_ubai_copy.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_ubai_search.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_ubai_update_retention.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_ubai_upload.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/__init__.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/clients/__init__.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/clients/async_consumer.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/clients/base_producer.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/clients/data_lake_data_producer.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/clients/opentelemetry_data_producer.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/clients/ubai_artifact_upload_request_producer.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/exceptions/__init__.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/models/__init__.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/models/base_model.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/models/model_factory.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/models/schemas/__init__.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/scripts/__init__.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/scripts/publish_release_info_cli.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/scripts/publish_test_results_cli.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/scripts/ubai_copy_cli.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/scripts/ubai_search_cli.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/scripts/ubai_update_retention_cli.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/scripts/ubai_upload_cli.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/utils/queue_metrics_exporter.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/utils/queue_span_exporter.py +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client.egg-info/dependency_links.txt +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client.egg-info/entry_points.txt +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client.egg-info/requires.txt +0 -0
- {utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from utf_queue_client.clients.base_producer import BlockingProducer
|
|
3
|
+
import pytest
|
|
4
|
+
import logging
|
|
5
|
+
import pika
|
|
6
|
+
import time
|
|
7
|
+
from utf_queue_client.utils import UTF_QUEUE_REGION_MAP
|
|
8
|
+
from urllib.parse import quote
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
regions = list(UTF_QUEUE_REGION_MAP.values())
|
|
12
|
+
servers = [f"utf-queue-{region}" for region in regions]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@pytest.fixture(params=servers)
|
|
16
|
+
def queue_server(request):
|
|
17
|
+
yield request.param
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@pytest.mark.parametrize(
|
|
21
|
+
"username, password",
|
|
22
|
+
[
|
|
23
|
+
(os.environ["UTF_QUEUE_USERNAME_LDAP"], os.environ["UTF_QUEUE_PASSWORD_LDAP"]),
|
|
24
|
+
(os.environ["UTF_QUEUE_USERNAME"], os.environ["UTF_QUEUE_PASSWORD"]),
|
|
25
|
+
],
|
|
26
|
+
)
|
|
27
|
+
def test_producer(queue_server, username, password):
|
|
28
|
+
suffix = ".dev.silabs.net"
|
|
29
|
+
queue_server = f"{queue_server}{suffix}"
|
|
30
|
+
l = logging.getLogger("test_producer")
|
|
31
|
+
l.info("Creating producer...")
|
|
32
|
+
p = BlockingProducer(
|
|
33
|
+
url=f"amqps://{username}:{quote(password)}@{queue_server}:443/testing?stack_timeout=30",
|
|
34
|
+
)
|
|
35
|
+
l.info(f"{queue_server}: Created producer")
|
|
36
|
+
params = pika.URLParameters(p.url)
|
|
37
|
+
l.info(f"stack timeout: {params.stack_timeout}")
|
|
38
|
+
begin = time.time()
|
|
39
|
+
for _ in range(0, 10):
|
|
40
|
+
l.info(f"{queue_server}: declaring queue...")
|
|
41
|
+
start = time.time()
|
|
42
|
+
p.queue_declare("default", durable=True)
|
|
43
|
+
stop = time.time()
|
|
44
|
+
l.info(
|
|
45
|
+
f"{queue_server}: done declaring queue in {stop - start} sec. Total time: {stop - begin}"
|
|
46
|
+
)
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
from utf_queue_client.clients.base_producer import (
|
|
3
|
+
ConnectionError as QueueServerConnectionError,
|
|
4
|
+
)
|
|
5
|
+
from utf_queue_client.clients.http_producer import (
|
|
6
|
+
HTTPProducer,
|
|
7
|
+
HTTPChannel,
|
|
8
|
+
CompositeHTTPProducer,
|
|
9
|
+
)
|
|
10
|
+
from pika.spec import BasicProperties, PERSISTENT_DELIVERY_MODE
|
|
11
|
+
from unittest.mock import patch
|
|
12
|
+
import utf_queue_client.clients.http_producer
|
|
13
|
+
|
|
14
|
+
FAKE_CREDS = "notarealusername:notarealpassword"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def test_basic_properties_to_json():
|
|
18
|
+
assert HTTPChannel.basic_properties_to_json(BasicProperties(app_id="foo")) == {
|
|
19
|
+
"app_id": "foo"
|
|
20
|
+
}
|
|
21
|
+
assert HTTPChannel.basic_properties_to_json(
|
|
22
|
+
BasicProperties(app_id="foo", delivery_mode=PERSISTENT_DELIVERY_MODE)
|
|
23
|
+
) == {"app_id": "foo", "delivery_mode": 2}
|
|
24
|
+
assert HTTPChannel.basic_properties_to_json(None) == {}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_http_channel_connection_error():
|
|
28
|
+
channel = HTTPChannel(
|
|
29
|
+
"https://localshmost", ("user", "pass"), "/", retries=0, retry_delay=0
|
|
30
|
+
)
|
|
31
|
+
with pytest.raises(QueueServerConnectionError):
|
|
32
|
+
channel.queue_declare("queue")
|
|
33
|
+
with pytest.raises(QueueServerConnectionError):
|
|
34
|
+
channel.basic_publish("exchange", "routing_key", b"foo", None)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def test_composite_producer_errors():
|
|
38
|
+
with patch.object(
|
|
39
|
+
utf_queue_client.clients.http_producer, "DEFAULT_NUM_CONNECTION_RETRIES", 0
|
|
40
|
+
):
|
|
41
|
+
composite_producer = CompositeHTTPProducer(
|
|
42
|
+
f"amqps://{FAKE_CREDS}@localshmost:443/%2F", "app_id"
|
|
43
|
+
)
|
|
44
|
+
composite_producer.producers.append(composite_producer.producers[0])
|
|
45
|
+
with pytest.raises(QueueServerConnectionError):
|
|
46
|
+
composite_producer.queue_declare("queue")
|
|
47
|
+
with pytest.raises(QueueServerConnectionError):
|
|
48
|
+
composite_producer.publish("exchange", "routing_key", {"foo": "bar"}, True)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@pytest.mark.parametrize(
|
|
52
|
+
("amqp_url", "expected_http_urls"),
|
|
53
|
+
[
|
|
54
|
+
(
|
|
55
|
+
f"amqps://{FAKE_CREDS}@localshmost:443/%2F",
|
|
56
|
+
["https://localshmost-admin:443"],
|
|
57
|
+
),
|
|
58
|
+
(
|
|
59
|
+
f"amqps://{FAKE_CREDS}@utf-queue-central:443/%2F",
|
|
60
|
+
["https://utf-queue-central-admin:443"],
|
|
61
|
+
),
|
|
62
|
+
(
|
|
63
|
+
f"amqps://{FAKE_CREDS}@utf-queue-central.silabs.net:443/%2F",
|
|
64
|
+
[
|
|
65
|
+
"https://utf-queue-central-admin.silabs.net:443",
|
|
66
|
+
"https://utf-queue-aus-admin.silabs.net:443",
|
|
67
|
+
"https://utf-queue-bos-admin.silabs.net:443",
|
|
68
|
+
"https://utf-queue-bud-admin.silabs.net:443",
|
|
69
|
+
"https://utf-queue-hyd-admin.silabs.net:443",
|
|
70
|
+
"https://utf-queue-yul-admin.silabs.net:443",
|
|
71
|
+
],
|
|
72
|
+
),
|
|
73
|
+
(
|
|
74
|
+
f"amqps://{FAKE_CREDS}@utf-queue-aus.silabs.net:443/%2F",
|
|
75
|
+
[
|
|
76
|
+
"https://utf-queue-central-admin.silabs.net:443",
|
|
77
|
+
"https://utf-queue-aus-admin.silabs.net:443",
|
|
78
|
+
"https://utf-queue-bos-admin.silabs.net:443",
|
|
79
|
+
"https://utf-queue-bud-admin.silabs.net:443",
|
|
80
|
+
"https://utf-queue-hyd-admin.silabs.net:443",
|
|
81
|
+
"https://utf-queue-yul-admin.silabs.net:443",
|
|
82
|
+
],
|
|
83
|
+
),
|
|
84
|
+
],
|
|
85
|
+
)
|
|
86
|
+
def test_composite_producer_creation(amqp_url, expected_http_urls):
|
|
87
|
+
composite_producer = CompositeHTTPProducer(amqp_url, "app_id")
|
|
88
|
+
assert len(composite_producer.producers) == len(expected_http_urls)
|
|
89
|
+
for index, value in enumerate(expected_http_urls):
|
|
90
|
+
assert composite_producer.producers[index].channel.url == value
|
|
@@ -15,6 +15,7 @@ def test_result_producer_empty_url():
|
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
@pytest.mark.parametrize("queue_consumer", [60, 120], indirect=True)
|
|
18
|
+
@pytest.mark.parametrize("use_http_producer", [True, False])
|
|
18
19
|
def test_result_producer_central_queue(
|
|
19
20
|
request,
|
|
20
21
|
sqa_app_build_result,
|
|
@@ -22,9 +23,13 @@ def test_result_producer_central_queue(
|
|
|
22
23
|
sqa_test_session,
|
|
23
24
|
amqp_url,
|
|
24
25
|
queue_consumer,
|
|
26
|
+
use_http_producer,
|
|
25
27
|
):
|
|
26
28
|
os.environ["UTF_PRODUCER_APP_ID"] = request.node.name
|
|
27
|
-
|
|
29
|
+
os.environ["UTF_QUEUE_USE_HTTP_PRODUCER"] = "true" if use_http_producer else "false"
|
|
30
|
+
producer = SqaTestResultProducerFactory.create_producer(
|
|
31
|
+
use_http_producer=use_http_producer
|
|
32
|
+
)
|
|
28
33
|
producer.publish_app_build_result(sqa_app_build_result)
|
|
29
34
|
queue_consumer.expect_messages(1)
|
|
30
35
|
producer.publish_test_session_start(sqa_test_session)
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import pytest
|
|
3
|
+
from utf_queue_client.utils import (
|
|
4
|
+
get_queue_url_list,
|
|
5
|
+
UTF_QUEUE_REGION_MAP,
|
|
6
|
+
get_closest_queue_region,
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
NOMAD_DC_PARAMS = [
|
|
11
|
+
("NOMAD_DC", value)
|
|
12
|
+
for value in [
|
|
13
|
+
None,
|
|
14
|
+
"aws-us-east-1",
|
|
15
|
+
"vmw-us-aus-1",
|
|
16
|
+
"vmw-us-bos-1",
|
|
17
|
+
"vmw-eu-bud-1",
|
|
18
|
+
"vmw-ap-hyd-1",
|
|
19
|
+
"vmw-ca-yul-1",
|
|
20
|
+
"vmw-ap-syd-1",
|
|
21
|
+
]
|
|
22
|
+
]
|
|
23
|
+
FAKE_CREDS = "notarealusername:notarealpassword"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@pytest.mark.parametrize(
|
|
27
|
+
"set_environment_variable",
|
|
28
|
+
NOMAD_DC_PARAMS,
|
|
29
|
+
indirect=True,
|
|
30
|
+
)
|
|
31
|
+
@pytest.mark.parametrize("domain", ["dev.silabs.net", "silabs.net"])
|
|
32
|
+
def test_get_queue_url_list(set_environment_variable, domain):
|
|
33
|
+
nomad_dc = os.getenv("NOMAD_DC")
|
|
34
|
+
url = f"amqps://{FAKE_CREDS}@utf-queue-central.{domain}:443/%2F?stack_timeout=30"
|
|
35
|
+
urls = get_queue_url_list(url)
|
|
36
|
+
assert len(urls) == 6
|
|
37
|
+
|
|
38
|
+
region_list = list(UTF_QUEUE_REGION_MAP.values())
|
|
39
|
+
if nomad_dc is None or nomad_dc not in UTF_QUEUE_REGION_MAP:
|
|
40
|
+
assert urls[0] == url
|
|
41
|
+
region_list.remove("central")
|
|
42
|
+
else:
|
|
43
|
+
region = UTF_QUEUE_REGION_MAP[nomad_dc]
|
|
44
|
+
assert urls[0] == url.replace("utf-queue-central", f"utf-queue-{region}")
|
|
45
|
+
region_list.remove(region)
|
|
46
|
+
assert urls[1:] == [
|
|
47
|
+
url.replace("utf-queue-central", f"utf-queue-{region}")
|
|
48
|
+
for region in region_list
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_get_queue_url_list_no_match():
|
|
53
|
+
url_list = [
|
|
54
|
+
f"amqps://{FAKE_CREDS}@custom-server.silabs.net:443/%2F?stack_timeout=30",
|
|
55
|
+
f"amqp://{FAKE_CREDS}@utf-queue-central.silabs.net:5672/%2F",
|
|
56
|
+
f"amqps://{FAKE_CREDS}@utf-queue-central.silabs.net:5672/%2F",
|
|
57
|
+
f"amqps://{FAKE_CREDS}@utf-queue-central.silabs.com:443/%2F",
|
|
58
|
+
]
|
|
59
|
+
for url in url_list:
|
|
60
|
+
urls = get_queue_url_list(url)
|
|
61
|
+
assert urls == [url]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@pytest.mark.parametrize(
|
|
65
|
+
"set_environment_variable",
|
|
66
|
+
NOMAD_DC_PARAMS,
|
|
67
|
+
indirect=True,
|
|
68
|
+
)
|
|
69
|
+
def test_get_closest_queue_region(set_environment_variable):
|
|
70
|
+
nomad_dc = os.getenv("NOMAD_DC")
|
|
71
|
+
try:
|
|
72
|
+
if nomad_dc is None or nomad_dc not in UTF_QUEUE_REGION_MAP:
|
|
73
|
+
assert get_closest_queue_region() == "central"
|
|
74
|
+
else:
|
|
75
|
+
assert get_closest_queue_region() == UTF_QUEUE_REGION_MAP[nomad_dc]
|
|
76
|
+
finally:
|
|
77
|
+
os.environ.pop("NOMAD_DC", None)
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
import pika
|
|
2
|
+
from pika.spec import BasicProperties
|
|
3
|
+
from typing import Optional, Tuple
|
|
4
|
+
|
|
5
|
+
from utf_queue_client.clients.base_producer import (
|
|
6
|
+
BaseProducer,
|
|
7
|
+
PikaChannel,
|
|
8
|
+
ConnectionError,
|
|
9
|
+
)
|
|
10
|
+
import base64
|
|
11
|
+
from utf_queue_client.utils import get_queue_url_list
|
|
12
|
+
from requests import Session as RequestsSession, RequestException
|
|
13
|
+
from requests.adapters import HTTPAdapter
|
|
14
|
+
from urllib3.util import Retry
|
|
15
|
+
from . import Loggable
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from urllib.parse import urlparse
|
|
18
|
+
|
|
19
|
+
__all__ = ["HTTPProducer", "HTTPChannel", "CompositeHTTPProducer"]
|
|
20
|
+
|
|
21
|
+
DEFAULT_NUM_CONNECTION_RETRIES = 6
|
|
22
|
+
DEFAULT_CONNECTION_RETRY_DELAY = 10
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class UrlInfo:
|
|
27
|
+
amqp_url: str
|
|
28
|
+
http_url: str
|
|
29
|
+
virtual_host: str
|
|
30
|
+
username: str
|
|
31
|
+
password: str
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class HTTPChannel(PikaChannel):
|
|
35
|
+
def __init__(
|
|
36
|
+
self,
|
|
37
|
+
url: str,
|
|
38
|
+
credentials: Tuple[str, str],
|
|
39
|
+
virtual_host: str,
|
|
40
|
+
retries=None,
|
|
41
|
+
retry_delay=None,
|
|
42
|
+
):
|
|
43
|
+
if retries is None:
|
|
44
|
+
retries = DEFAULT_NUM_CONNECTION_RETRIES
|
|
45
|
+
if retry_delay is None:
|
|
46
|
+
retry_delay = DEFAULT_CONNECTION_RETRY_DELAY
|
|
47
|
+
self.client = RequestsSession()
|
|
48
|
+
self.client.verify = False
|
|
49
|
+
self.client.auth = credentials
|
|
50
|
+
self.client.mount(
|
|
51
|
+
"https://",
|
|
52
|
+
HTTPAdapter(
|
|
53
|
+
max_retries=Retry(
|
|
54
|
+
total=retries,
|
|
55
|
+
backoff_factor=retry_delay,
|
|
56
|
+
status_forcelist=[404, 429, 500, 502, 503, 504],
|
|
57
|
+
)
|
|
58
|
+
),
|
|
59
|
+
)
|
|
60
|
+
self.url = url
|
|
61
|
+
self.virtual_host = virtual_host
|
|
62
|
+
if self.virtual_host == "/":
|
|
63
|
+
self.virtual_host = "%2F"
|
|
64
|
+
|
|
65
|
+
@classmethod
|
|
66
|
+
def basic_properties_to_json(cls, properties: Optional[BasicProperties]) -> dict:
|
|
67
|
+
props_json = {}
|
|
68
|
+
if properties:
|
|
69
|
+
for key in [
|
|
70
|
+
"app_id",
|
|
71
|
+
"content_type",
|
|
72
|
+
"content_encoding",
|
|
73
|
+
"correlation_id",
|
|
74
|
+
"cluster_id",
|
|
75
|
+
"delivery_mode",
|
|
76
|
+
"expiration",
|
|
77
|
+
"headers",
|
|
78
|
+
"message_id",
|
|
79
|
+
"priority",
|
|
80
|
+
"reply_to",
|
|
81
|
+
"timestamp",
|
|
82
|
+
"type",
|
|
83
|
+
"user_id",
|
|
84
|
+
]:
|
|
85
|
+
val = getattr(properties, key)
|
|
86
|
+
if val is not None:
|
|
87
|
+
props_json[key] = val
|
|
88
|
+
return props_json
|
|
89
|
+
|
|
90
|
+
def basic_publish(
|
|
91
|
+
self,
|
|
92
|
+
exchange: str,
|
|
93
|
+
routing_key: str,
|
|
94
|
+
body: bytes,
|
|
95
|
+
properties: Optional[BasicProperties] = None,
|
|
96
|
+
mandatory: bool = False,
|
|
97
|
+
):
|
|
98
|
+
if not exchange:
|
|
99
|
+
exchange = "amq.default"
|
|
100
|
+
publish_url = f"{self.url}/api/exchanges/{self.virtual_host}/{exchange}/publish"
|
|
101
|
+
try:
|
|
102
|
+
r = self.client.post(
|
|
103
|
+
publish_url,
|
|
104
|
+
json={
|
|
105
|
+
"properties": self.basic_properties_to_json(properties),
|
|
106
|
+
"routing_key": routing_key,
|
|
107
|
+
"payload": base64.b64encode(body).decode("utf-8"),
|
|
108
|
+
"payload_encoding": "base64",
|
|
109
|
+
},
|
|
110
|
+
)
|
|
111
|
+
r.raise_for_status()
|
|
112
|
+
except RequestException as e:
|
|
113
|
+
raise ConnectionError(e)
|
|
114
|
+
|
|
115
|
+
def queue_declare(
|
|
116
|
+
self,
|
|
117
|
+
queue,
|
|
118
|
+
passive=False,
|
|
119
|
+
durable=False,
|
|
120
|
+
exclusive=False,
|
|
121
|
+
auto_delete=False,
|
|
122
|
+
arguments=None,
|
|
123
|
+
callback=None,
|
|
124
|
+
):
|
|
125
|
+
declare_url = f"{self.url}/api/queues/{self.virtual_host}/{queue}"
|
|
126
|
+
try:
|
|
127
|
+
r = self.client.put(
|
|
128
|
+
declare_url, json={"auto_delete": auto_delete, "durable": durable}
|
|
129
|
+
)
|
|
130
|
+
r.raise_for_status()
|
|
131
|
+
except RequestException as e:
|
|
132
|
+
raise ConnectionError(e)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class HTTPProducer(BaseProducer):
|
|
136
|
+
def __init__(
|
|
137
|
+
self,
|
|
138
|
+
url_info: UrlInfo,
|
|
139
|
+
producer_app_id: Optional[str] = None,
|
|
140
|
+
):
|
|
141
|
+
super().__init__(url_info.amqp_url, producer_app_id)
|
|
142
|
+
self._channel = HTTPChannel(
|
|
143
|
+
url_info.http_url,
|
|
144
|
+
(url_info.username, url_info.password),
|
|
145
|
+
url_info.virtual_host,
|
|
146
|
+
)
|
|
147
|
+
self.hostname = urlparse(url_info.http_url).hostname
|
|
148
|
+
|
|
149
|
+
def _connect(self):
|
|
150
|
+
# no long-lived connection
|
|
151
|
+
pass
|
|
152
|
+
|
|
153
|
+
def is_connected(self):
|
|
154
|
+
return True # pragma: nocover
|
|
155
|
+
|
|
156
|
+
@property
|
|
157
|
+
def channel(self) -> HTTPChannel:
|
|
158
|
+
return self._channel
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class CompositeHTTPProducer(Loggable):
|
|
162
|
+
def __init__(self, url: str, producer_app_id: str = None):
|
|
163
|
+
url_list = get_queue_url_list(url)
|
|
164
|
+
self.producers = [
|
|
165
|
+
HTTPProducer(self.convert_amqp_url_to_http_url(alt_url), producer_app_id)
|
|
166
|
+
for alt_url in url_list
|
|
167
|
+
]
|
|
168
|
+
|
|
169
|
+
@classmethod
|
|
170
|
+
def convert_amqp_url_to_http_url(cls, amqp_url: str) -> UrlInfo:
|
|
171
|
+
url_params = pika.URLParameters(amqp_url)
|
|
172
|
+
host, delim, domain = url_params.host.partition(".")
|
|
173
|
+
hostname = f"{host}-admin{delim}{domain}:{url_params.port}"
|
|
174
|
+
scheme = "https" if url_params.ssl_options else "http"
|
|
175
|
+
return UrlInfo(
|
|
176
|
+
amqp_url=amqp_url,
|
|
177
|
+
http_url=f"{scheme}://{hostname}",
|
|
178
|
+
virtual_host=url_params.virtual_host,
|
|
179
|
+
username=url_params.credentials.username,
|
|
180
|
+
password=url_params.credentials.password,
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
def queue_declare(self, queue, **kwargs):
|
|
184
|
+
for index, producer in enumerate(self.producers):
|
|
185
|
+
if index:
|
|
186
|
+
self.logger.warning(
|
|
187
|
+
f"Failed to execute queue_declare for host {self.producers[index-1].hostname}, retrying with {producer.hostname}..."
|
|
188
|
+
)
|
|
189
|
+
try:
|
|
190
|
+
return producer.queue_declare(queue, **kwargs)
|
|
191
|
+
except ConnectionError:
|
|
192
|
+
if index == len(self.producers) - 1:
|
|
193
|
+
raise
|
|
194
|
+
continue
|
|
195
|
+
|
|
196
|
+
def publish(self, exchange: str, routing_key: str, payload: dict, persistent: bool):
|
|
197
|
+
for index, producer in enumerate(self.producers):
|
|
198
|
+
if index:
|
|
199
|
+
self.logger.warning(
|
|
200
|
+
f"Failed to execute publish for host {self.producers[index-1].hostname}, retrying with {producer.hostname}..."
|
|
201
|
+
)
|
|
202
|
+
try:
|
|
203
|
+
return producer.publish(exchange, routing_key, payload, persistent)
|
|
204
|
+
except ConnectionError:
|
|
205
|
+
if index == len(self.producers) - 1:
|
|
206
|
+
raise
|
|
207
|
+
continue
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import os
|
|
2
|
-
|
|
3
|
-
from .base_producer import BlockingProducer, ConnectionError
|
|
2
|
+
import re
|
|
3
|
+
from .base_producer import BlockingProducer as AMQPBlockingProducer, ConnectionError
|
|
4
|
+
from .http_producer import CompositeHTTPProducer
|
|
5
|
+
from ..utils import UTF_QUEUE_SERVER_PATTERN
|
|
4
6
|
from ..models import SqaAppBuildResult
|
|
5
7
|
from ..models import SqaTestResult
|
|
6
8
|
from ..models import SqaTestSession
|
|
@@ -10,6 +12,8 @@ from socket import gethostname
|
|
|
10
12
|
from datetime import datetime
|
|
11
13
|
from abc import ABC, abstractmethod
|
|
12
14
|
from otel_extensions import instrumented
|
|
15
|
+
from typing import Optional
|
|
16
|
+
|
|
13
17
|
|
|
14
18
|
__all__ = [
|
|
15
19
|
"SqaTestResultProducer",
|
|
@@ -43,9 +47,16 @@ class BaseSqaTestResultProducer(ABC):
|
|
|
43
47
|
class SqaTestResultProducer(BaseSqaTestResultProducer):
|
|
44
48
|
RECORD_TYPE = "UTF_TEST_EVENT"
|
|
45
49
|
|
|
46
|
-
def __init__(
|
|
50
|
+
def __init__(
|
|
51
|
+
self,
|
|
52
|
+
url=None,
|
|
53
|
+
producer_app_id: str = None,
|
|
54
|
+
producer_class: Optional[type] = None,
|
|
55
|
+
):
|
|
56
|
+
if producer_class is None:
|
|
57
|
+
producer_class = AMQPBlockingProducer
|
|
47
58
|
self.queue_name = "default"
|
|
48
|
-
self.__client =
|
|
59
|
+
self.__client = producer_class(url, producer_app_id)
|
|
49
60
|
self.__client.queue_declare(queue=self.queue_name, durable=True)
|
|
50
61
|
self.producer_app_id = producer_app_id
|
|
51
62
|
|
|
@@ -136,13 +147,20 @@ class DummySqaTestResultProducer(BaseSqaTestResultProducer):
|
|
|
136
147
|
|
|
137
148
|
class SqaTestResultProducerFactory(Loggable):
|
|
138
149
|
@classmethod
|
|
139
|
-
def create_producer(cls, raise_on_connection_error=False):
|
|
150
|
+
def create_producer(cls, raise_on_connection_error=False, use_http_producer=True):
|
|
140
151
|
try:
|
|
141
152
|
queue_server_url = os.environ.get("UTF_QUEUE_SERVER_URL")
|
|
142
153
|
if queue_server_url is not None:
|
|
154
|
+
producer_class = None
|
|
155
|
+
use_http_producer = use_http_producer and os.environ.get("UTF_QUEUE_USE_HTTP_PRODUCER") == "true"
|
|
156
|
+
if use_http_producer and re.match(
|
|
157
|
+
UTF_QUEUE_SERVER_PATTERN, queue_server_url
|
|
158
|
+
):
|
|
159
|
+
producer_class = CompositeHTTPProducer
|
|
143
160
|
return SqaTestResultProducer(
|
|
144
161
|
url=queue_server_url,
|
|
145
162
|
producer_app_id=os.environ.get("UTF_PRODUCER_APP_ID"),
|
|
163
|
+
producer_class=producer_class,
|
|
146
164
|
)
|
|
147
165
|
else:
|
|
148
166
|
return LocalSqaTestResultProducer()
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from urllib import parse
|
|
3
|
+
from typing import List
|
|
4
|
+
import re
|
|
5
|
+
|
|
6
|
+
QUEUE_APP_ID = "app_queue_client"
|
|
7
|
+
UTF_QUEUE_REGION_MAP = {
|
|
8
|
+
"aws-us-east-1": "central",
|
|
9
|
+
"vmw-us-aus-1": "aus",
|
|
10
|
+
"vmw-us-bos-1": "bos",
|
|
11
|
+
"vmw-eu-bud-1": "bud",
|
|
12
|
+
"vmw-ap-hyd-1": "hyd",
|
|
13
|
+
"vmw-ca-yul-1": "yul",
|
|
14
|
+
}
|
|
15
|
+
UTF_QUEUE_SERVER_PREFIX = "utf-queue-"
|
|
16
|
+
UTF_QUEUE_SERVER_PATTERN = (
|
|
17
|
+
"amqps://(.*):(.*)@"
|
|
18
|
+
+ rf"({UTF_QUEUE_SERVER_PREFIX}({'|'.join(UTF_QUEUE_REGION_MAP.values())}))"
|
|
19
|
+
+ r"\.(dev\.silabs\.net|silabs\.net):443"
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def prepare_queue_central_url(username, password):
|
|
24
|
+
if username is None or password is None:
|
|
25
|
+
raise RuntimeError("username or password must be provided if using queue")
|
|
26
|
+
hostname = os.environ.get("UTF_QUEUE_HOSTNAME", "utf-queue-central.silabs.net")
|
|
27
|
+
scheme = os.environ.get("UTF_QUEUE_SCHEME", "amqps")
|
|
28
|
+
port = os.environ.get("UTF_QUEUE_PORT", "443")
|
|
29
|
+
virtual_host = os.environ.get("UTF_QUEUE_VIRTUAL_HOST", "%2f")
|
|
30
|
+
return f"{scheme}://{username}:{parse.quote(password)}@{hostname}:{port}/{virtual_host}"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def get_closest_queue_region():
|
|
34
|
+
"""
|
|
35
|
+
Return the correct value from the UTF_QUEUE_REGION_MAP dictionary
|
|
36
|
+
based on the key contained in the NOMAD_DC environment variable (if it's defined).
|
|
37
|
+
If NOMAD_DC is not defined or the key is not in UTF_QUEUE_REGION_MAP, we'll just
|
|
38
|
+
return "central"
|
|
39
|
+
"""
|
|
40
|
+
if "NOMAD_DC" in os.environ:
|
|
41
|
+
dc = os.environ["NOMAD_DC"]
|
|
42
|
+
if dc in UTF_QUEUE_REGION_MAP:
|
|
43
|
+
return UTF_QUEUE_REGION_MAP[dc]
|
|
44
|
+
|
|
45
|
+
return "central"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def get_queue_url_list(original_url: str) -> List[str]:
|
|
49
|
+
"""
|
|
50
|
+
The purpose of this function is to return a list of alternative
|
|
51
|
+
utf queue urls given a single original url.
|
|
52
|
+
If 'original_url' matches the UTF_QUEUE_SERVER_PATTERN regex,
|
|
53
|
+
return a list of urls based on the values in the UTF_QUEUE_REGION_MAP dict,
|
|
54
|
+
where the order is determined by the NOMAD_DC environment variable.
|
|
55
|
+
|
|
56
|
+
Example:
|
|
57
|
+
1) if original_url is "amqps://xxxx:yyyy@utf-queue-central.silabs.net:443/%2F"
|
|
58
|
+
and NOMAD_DC is "vmw-ap-hyd-1",
|
|
59
|
+
we'll return
|
|
60
|
+
- amqps://xxxx:yyyy@utf-queue-hyd.silabs.net:443/%2F
|
|
61
|
+
- amqps://xxxx:yyyy@utf-queue-central.silabs.net:443/%2F
|
|
62
|
+
- amqps://xxxx:yyyy@utf-queue-aus.silabs.net:443/%2F
|
|
63
|
+
- amqps://xxxx:yyyy@utf-queue-bos.silabs.net:443/%2F
|
|
64
|
+
etc.
|
|
65
|
+
2) if original_url is "amqps://xxxx:yyyy@utf-queue-central.silabs.net:443/%2F"
|
|
66
|
+
and NOMAD_DC is not defined,
|
|
67
|
+
we'll return
|
|
68
|
+
- amqps://xxxx:yyyy@utf-queue-central.silabs.net:443/%2F
|
|
69
|
+
- amqps://xxxx:yyyy@utf-queue-aus.silabs.net:443/%2F
|
|
70
|
+
- amqps://xxxx:yyyy@utf-queue-bos.silabs.net:443/%2F
|
|
71
|
+
etc.
|
|
72
|
+
3) if original_url doesn't match the UTF_QUEUE_SERVER_PATTERN pattern
|
|
73
|
+
we'll return a list containing original_url as the only item.
|
|
74
|
+
"""
|
|
75
|
+
m = re.match(UTF_QUEUE_SERVER_PATTERN, original_url)
|
|
76
|
+
if not m:
|
|
77
|
+
return [original_url]
|
|
78
|
+
|
|
79
|
+
hostname_base = m.group(3)
|
|
80
|
+
domain = m.group(5)
|
|
81
|
+
|
|
82
|
+
url_list = []
|
|
83
|
+
region_list = [get_closest_queue_region()] + list(UTF_QUEUE_REGION_MAP.values())
|
|
84
|
+
for region in region_list:
|
|
85
|
+
url = original_url.replace(
|
|
86
|
+
f"{hostname_base}.{domain}", f"{UTF_QUEUE_SERVER_PREFIX}{region}.{domain}"
|
|
87
|
+
)
|
|
88
|
+
url_list.append(url)
|
|
89
|
+
return list(dict.fromkeys(url_list))
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
README.md
|
|
2
2
|
setup.cfg
|
|
3
3
|
setup.py
|
|
4
|
+
test/test_amqp_connections.py
|
|
4
5
|
test/test_async_consumer.py
|
|
5
6
|
test/test_connection_retry.py
|
|
6
7
|
test/test_data_lake_data_producer.py
|
|
8
|
+
test/test_http_producer.py
|
|
7
9
|
test/test_legacy_base_model.py
|
|
8
10
|
test/test_models.py
|
|
9
11
|
test/test_opentelemetry_data_producer.py
|
|
@@ -14,6 +16,7 @@ test/test_ubai_copy.py
|
|
|
14
16
|
test/test_ubai_search.py
|
|
15
17
|
test/test_ubai_update_retention.py
|
|
16
18
|
test/test_ubai_upload.py
|
|
19
|
+
test/test_utils.py
|
|
17
20
|
utf_queue_client/__init__.py
|
|
18
21
|
utf_queue_client.egg-info/PKG-INFO
|
|
19
22
|
utf_queue_client.egg-info/SOURCES.txt
|
|
@@ -25,6 +28,7 @@ utf_queue_client/clients/__init__.py
|
|
|
25
28
|
utf_queue_client/clients/async_consumer.py
|
|
26
29
|
utf_queue_client/clients/base_producer.py
|
|
27
30
|
utf_queue_client/clients/data_lake_data_producer.py
|
|
31
|
+
utf_queue_client/clients/http_producer.py
|
|
28
32
|
utf_queue_client/clients/opentelemetry_data_producer.py
|
|
29
33
|
utf_queue_client/clients/sqa_test_result_producer.py
|
|
30
34
|
utf_queue_client/clients/ubai_artifact_upload_request_producer.py
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import os
|
|
2
|
-
from urllib import parse
|
|
3
|
-
|
|
4
|
-
QUEUE_APP_ID = "app_queue_client"
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
def prepare_queue_central_url(username, password):
|
|
8
|
-
if username is None or password is None:
|
|
9
|
-
raise RuntimeError("username or password must be provided if using queue")
|
|
10
|
-
hostname = os.environ.get("UTF_QUEUE_HOSTNAME", "utf-queue-central.silabs.net")
|
|
11
|
-
scheme = os.environ.get("UTF_QUEUE_SCHEME", "amqps")
|
|
12
|
-
port = os.environ.get("UTF_QUEUE_PORT", "443")
|
|
13
|
-
virtual_host = os.environ.get("UTF_QUEUE_VIRTUAL_HOST", "%2f")
|
|
14
|
-
return f"{scheme}://{username}:{parse.quote(password)}@{hostname}:{port}/{virtual_host}"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/test/test_opentelemetry_data_producer.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/clients/async_consumer.py
RENAMED
|
File without changes
|
{utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/clients/base_producer.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/models/model_factory.py
RENAMED
|
File without changes
|
{utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/models/schemas/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/scripts/ubai_copy_cli.py
RENAMED
|
File without changes
|
{utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/scripts/ubai_search_cli.py
RENAMED
|
File without changes
|
|
File without changes
|
{utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/scripts/ubai_upload_cli.py
RENAMED
|
File without changes
|
{utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/utils/queue_metrics_exporter.py
RENAMED
|
File without changes
|
{utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client/utils/queue_span_exporter.py
RENAMED
|
File without changes
|
{utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
{utf-queue-client-1.15.1 → utf-queue-client-1.16.0}/utf_queue_client.egg-info/entry_points.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|