utf-queue-client 1.17.0__tar.gz → 1.17.0b1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/PKG-INFO +15 -13
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/setup.cfg +7 -7
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/setup.py +64 -64
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_amqp_connections.py +46 -46
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_async_consumer.py +11 -11
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_connection_retry.py +40 -40
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_data_lake_data_producer.py +57 -57
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_http_producer.py +90 -90
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_legacy_base_model.py +47 -47
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_models.py +125 -125
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_opentelemetry_data_producer.py +168 -168
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_publish_test_results.py +160 -160
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_rail_data_producer_tests.py +64 -64
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_sqa_test_result_producer.py +48 -48
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_ubai_search.py +81 -81
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_ubai_update_retention.py +61 -61
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_ubai_upload.py +113 -113
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_utils.py +77 -77
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/__init__.py +1 -1
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/clients/__init__.py +48 -48
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/clients/async_consumer.py +579 -579
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/clients/base_producer.py +164 -164
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/clients/data_lake_data_producer.py +135 -135
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/clients/http_producer.py +207 -207
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/clients/opentelemetry_data_producer.py +53 -53
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/clients/sqa_test_result_producer.py +171 -171
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/clients/test_rail_data_producer.py +95 -95
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/clients/ubai_artifact_upload_request_producer.py +123 -123
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/exceptions/__init__.py +20 -20
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/models/__init__.py +1 -1
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/models/base_model.py +164 -164
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/models/model_factory.py +19 -19
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/models/schemas/__init__.py +22 -22
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/scripts/__init__.py +86 -86
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/scripts/publish_test_results_cli.py +146 -146
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/scripts/ubai_search_cli.py +78 -78
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/scripts/ubai_update_retention_cli.py +73 -73
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/scripts/ubai_upload_cli.py +135 -135
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/utils/__init__.py +89 -89
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/utils/queue_metrics_exporter.py +85 -85
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/utils/queue_span_exporter.py +86 -86
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client.egg-info/PKG-INFO +15 -13
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/README.md +0 -0
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_publish_release_info.py +0 -0
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_ubai_copy.py +0 -0
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/scripts/publish_release_info_cli.py +0 -0
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/scripts/ubai_copy_cli.py +0 -0
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client.egg-info/SOURCES.txt +0 -0
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client.egg-info/dependency_links.txt +0 -0
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client.egg-info/entry_points.txt +0 -0
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client.egg-info/requires.txt +7 -7
- {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client.egg-info/top_level.txt +0 -0
|
@@ -1,13 +1,15 @@
|
|
|
1
|
-
Metadata-Version: 1
|
|
2
|
-
Name: utf-queue-client
|
|
3
|
-
Version: 1.17.
|
|
4
|
-
Summary: UNKNOWN
|
|
5
|
-
Home-page: UNKNOWN
|
|
6
|
-
Author: Joe Savage
|
|
7
|
-
Author-email: joe.savage@silabs.com
|
|
8
|
-
License: UNKNOWN
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: utf-queue-client
|
|
3
|
+
Version: 1.17.0b1
|
|
4
|
+
Summary: UNKNOWN
|
|
5
|
+
Home-page: UNKNOWN
|
|
6
|
+
Author: Joe Savage
|
|
7
|
+
Author-email: joe.savage@silabs.com
|
|
8
|
+
License: UNKNOWN
|
|
9
|
+
Keywords: UTF,
|
|
10
|
+
Platform: UNKNOWN
|
|
11
|
+
Requires-Python: >=3.6
|
|
12
|
+
|
|
13
|
+
No description provided
|
|
14
|
+
|
|
15
|
+
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
[flake8]
|
|
2
|
-
max-line-length = 99
|
|
3
|
-
|
|
4
|
-
[egg_info]
|
|
5
|
-
tag_build =
|
|
6
|
-
tag_date = 0
|
|
7
|
-
|
|
1
|
+
[flake8]
|
|
2
|
+
max-line-length = 99
|
|
3
|
+
|
|
4
|
+
[egg_info]
|
|
5
|
+
tag_build =
|
|
6
|
+
tag_date = 0
|
|
7
|
+
|
|
@@ -1,64 +1,64 @@
|
|
|
1
|
-
"""
|
|
2
|
-
|
|
3
|
-
"""
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
from setuptools import setup, find_packages # noqa: H301
|
|
7
|
-
|
|
8
|
-
NAME = "utf-queue-client"
|
|
9
|
-
VERSION = "1.17.
|
|
10
|
-
# To install the library, run the following
|
|
11
|
-
#
|
|
12
|
-
# python setup.py install
|
|
13
|
-
#
|
|
14
|
-
# prerequisite: setuptools
|
|
15
|
-
# http://pypi.python.org/pypi/setuptools
|
|
16
|
-
|
|
17
|
-
REQUIRES = [
|
|
18
|
-
"pika >= 1.2.0",
|
|
19
|
-
"jsonschema >= 3.2.0",
|
|
20
|
-
"msgpack >= 1.0.2",
|
|
21
|
-
"pydantic == 1.*",
|
|
22
|
-
"regex",
|
|
23
|
-
"typing_extensions; python_version < '3.8'",
|
|
24
|
-
"click",
|
|
25
|
-
"ubai-client >= 1.2.0",
|
|
26
|
-
"opentelemetry-distro[otlp]",
|
|
27
|
-
"otel-extensions >= 0.1.2",
|
|
28
|
-
"retry>=0.9.2",
|
|
29
|
-
"requests",
|
|
30
|
-
"ndjson~=0.3",
|
|
31
|
-
]
|
|
32
|
-
|
|
33
|
-
setup(
|
|
34
|
-
name=NAME,
|
|
35
|
-
version=VERSION,
|
|
36
|
-
description="",
|
|
37
|
-
author="Joe Savage",
|
|
38
|
-
author_email="joe.savage@silabs.com",
|
|
39
|
-
url="",
|
|
40
|
-
keywords=["UTF", ""],
|
|
41
|
-
python_requires=">=3.6",
|
|
42
|
-
install_requires=REQUIRES,
|
|
43
|
-
packages=find_packages(exclude=["test", "tests"]),
|
|
44
|
-
package_data={
|
|
45
|
-
"utf_queue_client": [
|
|
46
|
-
"models/schemas/utf_queue_models/models/python/*.py",
|
|
47
|
-
"models/schemas/utf_queue_models/schema/*.json",
|
|
48
|
-
]
|
|
49
|
-
},
|
|
50
|
-
entry_points={
|
|
51
|
-
"console_scripts": [
|
|
52
|
-
"ubai_upload_cli = utf_queue_client.scripts.ubai_upload_cli:cli_entrypoint",
|
|
53
|
-
"ubai_search_cli = utf_queue_client.scripts.ubai_search_cli:cli_entrypoint",
|
|
54
|
-
"publish_test_results_cli = utf_queue_client.scripts.publish_test_results_cli:cli_entrypoint",
|
|
55
|
-
"ubai_update_retention_cli = utf_queue_client.scripts.ubai_update_retention_cli:cli_entrypoint",
|
|
56
|
-
"publish_release_info_cli = utf_queue_client.scripts.publish_release_info_cli:cli_entrypoint",
|
|
57
|
-
"ubai_copy_cli = utf_queue_client.scripts.ubai_copy_cli:cli_entrypoint"
|
|
58
|
-
],
|
|
59
|
-
},
|
|
60
|
-
include_package_data=True,
|
|
61
|
-
long_description="""\
|
|
62
|
-
No description provided
|
|
63
|
-
""",
|
|
64
|
-
)
|
|
1
|
+
"""
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
from setuptools import setup, find_packages # noqa: H301
|
|
7
|
+
|
|
8
|
+
NAME = "utf-queue-client"
|
|
9
|
+
VERSION = "1.17.0b1"
|
|
10
|
+
# To install the library, run the following
|
|
11
|
+
#
|
|
12
|
+
# python setup.py install
|
|
13
|
+
#
|
|
14
|
+
# prerequisite: setuptools
|
|
15
|
+
# http://pypi.python.org/pypi/setuptools
|
|
16
|
+
|
|
17
|
+
REQUIRES = [
|
|
18
|
+
"pika >= 1.2.0",
|
|
19
|
+
"jsonschema >= 3.2.0",
|
|
20
|
+
"msgpack >= 1.0.2",
|
|
21
|
+
"pydantic == 1.*",
|
|
22
|
+
"regex",
|
|
23
|
+
"typing_extensions; python_version < '3.8'",
|
|
24
|
+
"click",
|
|
25
|
+
"ubai-client >= 1.2.0",
|
|
26
|
+
"opentelemetry-distro[otlp]",
|
|
27
|
+
"otel-extensions >= 0.1.2",
|
|
28
|
+
"retry>=0.9.2",
|
|
29
|
+
"requests",
|
|
30
|
+
"ndjson~=0.3",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
setup(
|
|
34
|
+
name=NAME,
|
|
35
|
+
version=VERSION,
|
|
36
|
+
description="",
|
|
37
|
+
author="Joe Savage",
|
|
38
|
+
author_email="joe.savage@silabs.com",
|
|
39
|
+
url="",
|
|
40
|
+
keywords=["UTF", ""],
|
|
41
|
+
python_requires=">=3.6",
|
|
42
|
+
install_requires=REQUIRES,
|
|
43
|
+
packages=find_packages(exclude=["test", "tests"]),
|
|
44
|
+
package_data={
|
|
45
|
+
"utf_queue_client": [
|
|
46
|
+
"models/schemas/utf_queue_models/models/python/*.py",
|
|
47
|
+
"models/schemas/utf_queue_models/schema/*.json",
|
|
48
|
+
]
|
|
49
|
+
},
|
|
50
|
+
entry_points={
|
|
51
|
+
"console_scripts": [
|
|
52
|
+
"ubai_upload_cli = utf_queue_client.scripts.ubai_upload_cli:cli_entrypoint",
|
|
53
|
+
"ubai_search_cli = utf_queue_client.scripts.ubai_search_cli:cli_entrypoint",
|
|
54
|
+
"publish_test_results_cli = utf_queue_client.scripts.publish_test_results_cli:cli_entrypoint",
|
|
55
|
+
"ubai_update_retention_cli = utf_queue_client.scripts.ubai_update_retention_cli:cli_entrypoint",
|
|
56
|
+
"publish_release_info_cli = utf_queue_client.scripts.publish_release_info_cli:cli_entrypoint",
|
|
57
|
+
"ubai_copy_cli = utf_queue_client.scripts.ubai_copy_cli:cli_entrypoint"
|
|
58
|
+
],
|
|
59
|
+
},
|
|
60
|
+
include_package_data=True,
|
|
61
|
+
long_description="""\
|
|
62
|
+
No description provided
|
|
63
|
+
""",
|
|
64
|
+
)
|
|
@@ -1,46 +1,46 @@
|
|
|
1
|
-
import os
|
|
2
|
-
from utf_queue_client.clients.base_producer import BlockingProducer
|
|
3
|
-
import pytest
|
|
4
|
-
import logging
|
|
5
|
-
import pika
|
|
6
|
-
import time
|
|
7
|
-
from utf_queue_client.utils import UTF_QUEUE_REGION_MAP
|
|
8
|
-
from urllib.parse import quote
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
regions = list(UTF_QUEUE_REGION_MAP.values())
|
|
12
|
-
servers = [f"utf-queue-{region}" for region in regions]
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
@pytest.fixture(params=servers)
|
|
16
|
-
def queue_server(request):
|
|
17
|
-
yield request.param
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
@pytest.mark.parametrize(
|
|
21
|
-
"username, password",
|
|
22
|
-
[
|
|
23
|
-
(os.environ["UTF_QUEUE_USERNAME_LDAP"], os.environ["UTF_QUEUE_PASSWORD_LDAP"]),
|
|
24
|
-
(os.environ["UTF_QUEUE_USERNAME"], os.environ["UTF_QUEUE_PASSWORD"]),
|
|
25
|
-
],
|
|
26
|
-
)
|
|
27
|
-
def test_producer(queue_server, username, password):
|
|
28
|
-
suffix = ".dev.silabs.net"
|
|
29
|
-
queue_server = f"{queue_server}{suffix}"
|
|
30
|
-
l = logging.getLogger("test_producer")
|
|
31
|
-
l.info("Creating producer...")
|
|
32
|
-
p = BlockingProducer(
|
|
33
|
-
url=f"amqps://{username}:{quote(password)}@{queue_server}:443/testing?stack_timeout=30",
|
|
34
|
-
)
|
|
35
|
-
l.info(f"{queue_server}: Created producer")
|
|
36
|
-
params = pika.URLParameters(p.url)
|
|
37
|
-
l.info(f"stack timeout: {params.stack_timeout}")
|
|
38
|
-
begin = time.time()
|
|
39
|
-
for _ in range(0, 10):
|
|
40
|
-
l.info(f"{queue_server}: declaring queue...")
|
|
41
|
-
start = time.time()
|
|
42
|
-
p.queue_declare("default", durable=True)
|
|
43
|
-
stop = time.time()
|
|
44
|
-
l.info(
|
|
45
|
-
f"{queue_server}: done declaring queue in {stop - start} sec. Total time: {stop - begin}"
|
|
46
|
-
)
|
|
1
|
+
import os
|
|
2
|
+
from utf_queue_client.clients.base_producer import BlockingProducer
|
|
3
|
+
import pytest
|
|
4
|
+
import logging
|
|
5
|
+
import pika
|
|
6
|
+
import time
|
|
7
|
+
from utf_queue_client.utils import UTF_QUEUE_REGION_MAP
|
|
8
|
+
from urllib.parse import quote
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
regions = list(UTF_QUEUE_REGION_MAP.values())
|
|
12
|
+
servers = [f"utf-queue-{region}" for region in regions]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@pytest.fixture(params=servers)
|
|
16
|
+
def queue_server(request):
|
|
17
|
+
yield request.param
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@pytest.mark.parametrize(
|
|
21
|
+
"username, password",
|
|
22
|
+
[
|
|
23
|
+
(os.environ["UTF_QUEUE_USERNAME_LDAP"], os.environ["UTF_QUEUE_PASSWORD_LDAP"]),
|
|
24
|
+
(os.environ["UTF_QUEUE_USERNAME"], os.environ["UTF_QUEUE_PASSWORD"]),
|
|
25
|
+
],
|
|
26
|
+
)
|
|
27
|
+
def test_producer(queue_server, username, password):
|
|
28
|
+
suffix = ".dev.silabs.net"
|
|
29
|
+
queue_server = f"{queue_server}{suffix}"
|
|
30
|
+
l = logging.getLogger("test_producer")
|
|
31
|
+
l.info("Creating producer...")
|
|
32
|
+
p = BlockingProducer(
|
|
33
|
+
url=f"amqps://{username}:{quote(password)}@{queue_server}:443/testing?stack_timeout=30",
|
|
34
|
+
)
|
|
35
|
+
l.info(f"{queue_server}: Created producer")
|
|
36
|
+
params = pika.URLParameters(p.url)
|
|
37
|
+
l.info(f"stack timeout: {params.stack_timeout}")
|
|
38
|
+
begin = time.time()
|
|
39
|
+
for _ in range(0, 10):
|
|
40
|
+
l.info(f"{queue_server}: declaring queue...")
|
|
41
|
+
start = time.time()
|
|
42
|
+
p.queue_declare("default", durable=True)
|
|
43
|
+
stop = time.time()
|
|
44
|
+
l.info(
|
|
45
|
+
f"{queue_server}: done declaring queue in {stop - start} sec. Total time: {stop - begin}"
|
|
46
|
+
)
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
from utf_queue_client.clients.async_consumer import AsyncConsumerBase
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
def test_sanitize_url():
|
|
6
|
-
params = [
|
|
7
|
-
("amqp://www.ni.com", "amqp://www.ni.com"),
|
|
8
|
-
("amqps://utfsilabs:password1234@www.ni.com", "amqps://utfsilabs:******@www.ni.com"),
|
|
9
|
-
]
|
|
10
|
-
for url, display_url in params:
|
|
11
|
-
sanitized_url = AsyncConsumerBase.sanitize_url_for_display(url)
|
|
1
|
+
from utf_queue_client.clients.async_consumer import AsyncConsumerBase
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def test_sanitize_url():
|
|
6
|
+
params = [
|
|
7
|
+
("amqp://www.ni.com", "amqp://www.ni.com"),
|
|
8
|
+
("amqps://utfsilabs:password1234@www.ni.com", "amqps://utfsilabs:******@www.ni.com"),
|
|
9
|
+
]
|
|
10
|
+
for url, display_url in params:
|
|
11
|
+
sanitized_url = AsyncConsumerBase.sanitize_url_for_display(url)
|
|
12
12
|
assert sanitized_url == display_url
|
|
@@ -1,40 +1,40 @@
|
|
|
1
|
-
import logging
|
|
2
|
-
import logging.handlers
|
|
3
|
-
|
|
4
|
-
import pytest
|
|
5
|
-
from utf_queue_client.clients.sqa_test_result_producer import (
|
|
6
|
-
SqaTestResultProducer,
|
|
7
|
-
)
|
|
8
|
-
from utf_queue_client.clients.base_producer import ConnectionError
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
@pytest.fixture()
|
|
12
|
-
def pika_logging_handler():
|
|
13
|
-
"""Add a handler to the pika logger to capture its output."""
|
|
14
|
-
pika_logger = logging.getLogger("pika.adapters.utils.connection_workflow")
|
|
15
|
-
handler = logging.handlers.BufferingHandler(capacity=100)
|
|
16
|
-
pika_logger.addHandler(handler)
|
|
17
|
-
yield handler
|
|
18
|
-
pika_logger.removeHandler(handler)
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
@pytest.mark.parametrize(
|
|
22
|
-
"set_environment_variable",
|
|
23
|
-
[("UTF_QUEUE_SERVER_URL", "amqps://utfsilabs:utfsilabs@12.34.56.78:5671/%2f")],
|
|
24
|
-
indirect=True,
|
|
25
|
-
)
|
|
26
|
-
def test_result_producer_connection_retry(
|
|
27
|
-
set_environment_variable, mocker, pika_logging_handler
|
|
28
|
-
):
|
|
29
|
-
"""Test that the connection retry logic works as expected."""
|
|
30
|
-
mocker.patch("utf_queue_client.clients.base_producer.NUM_CONNECTION_TRIES", 2)
|
|
31
|
-
with pytest.raises(ConnectionError):
|
|
32
|
-
_ = SqaTestResultProducer()
|
|
33
|
-
error_records = [
|
|
34
|
-
record
|
|
35
|
-
for record in pika_logging_handler.buffer
|
|
36
|
-
if record.levelno == logging.ERROR
|
|
37
|
-
]
|
|
38
|
-
# The assertion below depends on the logging output of the pika library (version 1.3.2 as of this writing)
|
|
39
|
-
# so if that changes, the message below may need to be updated.
|
|
40
|
-
assert "2 exceptions in all" in error_records[-1].message
|
|
1
|
+
import logging
|
|
2
|
+
import logging.handlers
|
|
3
|
+
|
|
4
|
+
import pytest
|
|
5
|
+
from utf_queue_client.clients.sqa_test_result_producer import (
|
|
6
|
+
SqaTestResultProducer,
|
|
7
|
+
)
|
|
8
|
+
from utf_queue_client.clients.base_producer import ConnectionError
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@pytest.fixture()
|
|
12
|
+
def pika_logging_handler():
|
|
13
|
+
"""Add a handler to the pika logger to capture its output."""
|
|
14
|
+
pika_logger = logging.getLogger("pika.adapters.utils.connection_workflow")
|
|
15
|
+
handler = logging.handlers.BufferingHandler(capacity=100)
|
|
16
|
+
pika_logger.addHandler(handler)
|
|
17
|
+
yield handler
|
|
18
|
+
pika_logger.removeHandler(handler)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@pytest.mark.parametrize(
|
|
22
|
+
"set_environment_variable",
|
|
23
|
+
[("UTF_QUEUE_SERVER_URL", "amqps://utfsilabs:utfsilabs@12.34.56.78:5671/%2f")],
|
|
24
|
+
indirect=True,
|
|
25
|
+
)
|
|
26
|
+
def test_result_producer_connection_retry(
|
|
27
|
+
set_environment_variable, mocker, pika_logging_handler
|
|
28
|
+
):
|
|
29
|
+
"""Test that the connection retry logic works as expected."""
|
|
30
|
+
mocker.patch("utf_queue_client.clients.base_producer.NUM_CONNECTION_TRIES", 2)
|
|
31
|
+
with pytest.raises(ConnectionError):
|
|
32
|
+
_ = SqaTestResultProducer()
|
|
33
|
+
error_records = [
|
|
34
|
+
record
|
|
35
|
+
for record in pika_logging_handler.buffer
|
|
36
|
+
if record.levelno == logging.ERROR
|
|
37
|
+
]
|
|
38
|
+
# The assertion below depends on the logging output of the pika library (version 1.3.2 as of this writing)
|
|
39
|
+
# so if that changes, the message below may need to be updated.
|
|
40
|
+
assert "2 exceptions in all" in error_records[-1].message
|
|
@@ -1,57 +1,57 @@
|
|
|
1
|
-
import json
|
|
2
|
-
import uuid
|
|
3
|
-
|
|
4
|
-
from utf_queue_client.clients.data_lake_data_producer import (
|
|
5
|
-
DataLakeDataProducer,
|
|
6
|
-
DataLakeDataProducerFactory,
|
|
7
|
-
)
|
|
8
|
-
import pytest
|
|
9
|
-
import os
|
|
10
|
-
import ndjson
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
def test_data_lake_producer_empty_url():
|
|
14
|
-
with pytest.raises(RuntimeError):
|
|
15
|
-
_ = DataLakeDataProducer("dummy tenant")
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
@pytest.mark.parametrize("use_object_id", [True, False])
|
|
19
|
-
def test_data_lake_producer_central_queue(
|
|
20
|
-
request,
|
|
21
|
-
data_lake_tenant_key,
|
|
22
|
-
csv_file_path,
|
|
23
|
-
json_file_path,
|
|
24
|
-
parquet_file_path,
|
|
25
|
-
ndjson_file_path,
|
|
26
|
-
amqp_url,
|
|
27
|
-
queue_consumer,
|
|
28
|
-
use_object_id,
|
|
29
|
-
):
|
|
30
|
-
os.environ["UTF_PRODUCER_APP_ID"] = request.node.name
|
|
31
|
-
producer = DataLakeDataProducerFactory.create_producer(
|
|
32
|
-
data_lake_tenant_key, amqp_url
|
|
33
|
-
)
|
|
34
|
-
producer.publish_data_lake_csv_file(
|
|
35
|
-
csv_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
|
|
36
|
-
)
|
|
37
|
-
queue_consumer.expect_messages(1)
|
|
38
|
-
producer.publish_data_lake_json_file(
|
|
39
|
-
json_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
|
|
40
|
-
)
|
|
41
|
-
queue_consumer.expect_messages(2)
|
|
42
|
-
with open(json_file_path, "rb") as f:
|
|
43
|
-
producer.publish_data_lake_json(json.load(f))
|
|
44
|
-
producer.publish_data_lake_parquet_file(
|
|
45
|
-
parquet_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
|
|
46
|
-
)
|
|
47
|
-
queue_consumer.expect_messages(3)
|
|
48
|
-
producer.publish_data_lake_ndjson_file(
|
|
49
|
-
ndjson_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
|
|
50
|
-
)
|
|
51
|
-
queue_consumer.expect_messages(4)
|
|
52
|
-
with open(ndjson_file_path) as f:
|
|
53
|
-
data = ndjson.load(f)
|
|
54
|
-
producer.publish_data_lake_ndjson(
|
|
55
|
-
data, object_id=str(uuid.uuid4()) if use_object_id else None
|
|
56
|
-
)
|
|
57
|
-
queue_consumer.expect_messages(5)
|
|
1
|
+
import json
|
|
2
|
+
import uuid
|
|
3
|
+
|
|
4
|
+
from utf_queue_client.clients.data_lake_data_producer import (
|
|
5
|
+
DataLakeDataProducer,
|
|
6
|
+
DataLakeDataProducerFactory,
|
|
7
|
+
)
|
|
8
|
+
import pytest
|
|
9
|
+
import os
|
|
10
|
+
import ndjson
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def test_data_lake_producer_empty_url():
|
|
14
|
+
with pytest.raises(RuntimeError):
|
|
15
|
+
_ = DataLakeDataProducer("dummy tenant")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@pytest.mark.parametrize("use_object_id", [True, False])
|
|
19
|
+
def test_data_lake_producer_central_queue(
|
|
20
|
+
request,
|
|
21
|
+
data_lake_tenant_key,
|
|
22
|
+
csv_file_path,
|
|
23
|
+
json_file_path,
|
|
24
|
+
parquet_file_path,
|
|
25
|
+
ndjson_file_path,
|
|
26
|
+
amqp_url,
|
|
27
|
+
queue_consumer,
|
|
28
|
+
use_object_id,
|
|
29
|
+
):
|
|
30
|
+
os.environ["UTF_PRODUCER_APP_ID"] = request.node.name
|
|
31
|
+
producer = DataLakeDataProducerFactory.create_producer(
|
|
32
|
+
data_lake_tenant_key, amqp_url
|
|
33
|
+
)
|
|
34
|
+
producer.publish_data_lake_csv_file(
|
|
35
|
+
csv_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
|
|
36
|
+
)
|
|
37
|
+
queue_consumer.expect_messages(1)
|
|
38
|
+
producer.publish_data_lake_json_file(
|
|
39
|
+
json_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
|
|
40
|
+
)
|
|
41
|
+
queue_consumer.expect_messages(2)
|
|
42
|
+
with open(json_file_path, "rb") as f:
|
|
43
|
+
producer.publish_data_lake_json(json.load(f))
|
|
44
|
+
producer.publish_data_lake_parquet_file(
|
|
45
|
+
parquet_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
|
|
46
|
+
)
|
|
47
|
+
queue_consumer.expect_messages(3)
|
|
48
|
+
producer.publish_data_lake_ndjson_file(
|
|
49
|
+
ndjson_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
|
|
50
|
+
)
|
|
51
|
+
queue_consumer.expect_messages(4)
|
|
52
|
+
with open(ndjson_file_path) as f:
|
|
53
|
+
data = ndjson.load(f)
|
|
54
|
+
producer.publish_data_lake_ndjson(
|
|
55
|
+
data, object_id=str(uuid.uuid4()) if use_object_id else None
|
|
56
|
+
)
|
|
57
|
+
queue_consumer.expect_messages(5)
|