utf-queue-client 1.17.2__tar.gz → 1.17.2b1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. utf-queue-client-1.17.2b1/MANIFEST.in +20 -0
  2. utf-queue-client-1.17.2b1/PKG-INFO +24 -0
  3. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/setup.cfg +7 -7
  4. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/setup.py +63 -63
  5. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/test/test_amqp_connections.py +46 -46
  6. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/test/test_async_consumer.py +11 -11
  7. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/test/test_connection_retry.py +40 -40
  8. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/test/test_data_lake_data_producer.py +57 -57
  9. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/test/test_http_producer.py +90 -90
  10. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/test/test_legacy_base_model.py +47 -47
  11. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/test/test_models.py +125 -125
  12. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/test/test_opentelemetry_data_producer.py +168 -168
  13. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/test/test_publish_test_results.py +160 -160
  14. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/test/test_rail_data_producer_tests.py +64 -64
  15. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/test/test_sqa_test_result_producer.py +48 -48
  16. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/test/test_ubai_search.py +81 -81
  17. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/test/test_ubai_update_retention.py +61 -61
  18. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/test/test_ubai_upload.py +113 -113
  19. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/test/test_utils.py +77 -77
  20. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/__init__.py +1 -1
  21. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/clients/__init__.py +48 -48
  22. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/clients/async_consumer.py +579 -579
  23. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/clients/base_producer.py +164 -164
  24. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/clients/data_lake_data_producer.py +135 -135
  25. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/clients/http_producer.py +207 -207
  26. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/clients/opentelemetry_data_producer.py +53 -53
  27. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/clients/sqa_test_result_producer.py +171 -171
  28. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/clients/test_rail_data_producer.py +95 -95
  29. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/clients/ubai_artifact_upload_request_producer.py +123 -123
  30. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/exceptions/__init__.py +20 -20
  31. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/models/__init__.py +1 -1
  32. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/models/base_model.py +164 -168
  33. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/models/model_factory.py +19 -19
  34. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/models/schemas/__init__.py +22 -22
  35. utf-queue-client-1.17.2b1/utf_queue_client/models/schemas/utf_queue_models/schema/queue_message.ts +568 -0
  36. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/scripts/__init__.py +86 -86
  37. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/scripts/publish_test_results_cli.py +146 -146
  38. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/scripts/ubai_search_cli.py +78 -78
  39. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/scripts/ubai_update_retention_cli.py +73 -73
  40. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/scripts/ubai_upload_cli.py +135 -135
  41. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/utils/__init__.py +89 -89
  42. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/utils/queue_metrics_exporter.py +85 -85
  43. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/utils/queue_span_exporter.py +86 -86
  44. utf-queue-client-1.17.2b1/utf_queue_client.egg-info/PKG-INFO +24 -0
  45. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client.egg-info/SOURCES.txt +2 -0
  46. utf-queue-client-1.17.2/PKG-INFO +0 -11
  47. utf-queue-client-1.17.2/utf_queue_client.egg-info/PKG-INFO +0 -11
  48. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/README.md +0 -0
  49. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/test/test_publish_release_info.py +0 -0
  50. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/test/test_ubai_copy.py +0 -0
  51. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/models/schemas/utf_queue_models/models/python/__init__.py +0 -0
  52. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/models/schemas/utf_queue_models/models/python/generated_models.py +0 -0
  53. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/models/schemas/utf_queue_models/schema/queue_message.json +0 -0
  54. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/scripts/publish_release_info_cli.py +0 -0
  55. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client/scripts/ubai_copy_cli.py +0 -0
  56. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client.egg-info/dependency_links.txt +0 -0
  57. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client.egg-info/entry_points.txt +0 -0
  58. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client.egg-info/requires.txt +7 -7
  59. {utf-queue-client-1.17.2 → utf-queue-client-1.17.2b1}/utf_queue_client.egg-info/top_level.txt +0 -0
@@ -0,0 +1,20 @@
1
+ # Include all Python files
2
+ include *.py
3
+
4
+ # Include all JSON files in a specific folder
5
+ graft utf_queue_client/models/schemas/utf_queue_models/schema
6
+
7
+ # Exclude all compiled Python files
8
+ # global-exclude *.pyc
9
+ # global-exclude *.pyo
10
+
11
+ # Exclude specific directories
12
+ prune __pycache__
13
+ prune tests/
14
+ prune test/
15
+
16
+ # Include specific text files
17
+ # include *.txt
18
+
19
+ # Exclude all files in a particular directory
20
+ # recursive-exclude unwanted_directory *
@@ -0,0 +1,24 @@
1
+ Metadata-Version: 2.1
2
+ Name: utf-queue-client
3
+ Version: 1.17.2b1
4
+ Home-page:
5
+ Author: Joe Savage
6
+ Author-email: joe.savage@silabs.com
7
+ Keywords: UTF,
8
+ Requires-Python: >=3.6
9
+ Requires-Dist: pika>=1.2.0
10
+ Requires-Dist: jsonschema>=3.2.0
11
+ Requires-Dist: msgpack>=1.0.2
12
+ Requires-Dist: pydantic==1.*
13
+ Requires-Dist: regex
14
+ Requires-Dist: typing_extensions; python_version < "3.8"
15
+ Requires-Dist: click
16
+ Requires-Dist: ubai-client>=1.2.0
17
+ Requires-Dist: opentelemetry-distro[otlp]
18
+ Requires-Dist: otel-extensions>=0.1.2
19
+ Requires-Dist: retry>=0.9.2
20
+ Requires-Dist: requests
21
+ Requires-Dist: ndjson~=0.3
22
+
23
+ No description provided
24
+
@@ -1,7 +1,7 @@
1
- [flake8]
2
- max-line-length = 99
3
-
4
- [egg_info]
5
- tag_build =
6
- tag_date = 0
7
-
1
+ [flake8]
2
+ max-line-length = 99
3
+
4
+ [egg_info]
5
+ tag_build =
6
+ tag_date = 0
7
+
@@ -1,63 +1,63 @@
1
- """
2
-
3
- """
4
-
5
- from setuptools import setup, find_packages # noqa: H301
6
-
7
- NAME = "utf-queue-client"
8
- VERSION = "1.17.2"
9
- # To install the library, run the following
10
- #
11
- # python setup.py install
12
- #
13
- # prerequisite: setuptools
14
- # http://pypi.python.org/pypi/setuptools
15
-
16
- REQUIRES = [
17
- "pika >= 1.2.0",
18
- "jsonschema >= 3.2.0",
19
- "msgpack >= 1.0.2",
20
- "pydantic == 1.*",
21
- "regex",
22
- "typing_extensions; python_version < '3.8'",
23
- "click",
24
- "ubai-client >= 1.2.0",
25
- "opentelemetry-distro[otlp]",
26
- "otel-extensions >= 0.1.2",
27
- "retry>=0.9.2",
28
- "requests",
29
- "ndjson~=0.3",
30
- ]
31
-
32
- setup(
33
- name=NAME,
34
- version=VERSION,
35
- description="",
36
- author="Joe Savage",
37
- author_email="joe.savage@silabs.com",
38
- url="",
39
- keywords=["UTF", ""],
40
- python_requires=">=3.6",
41
- install_requires=REQUIRES,
42
- packages=find_packages(exclude=["test", "tests"]),
43
- package_data={
44
- "utf_queue_client": [
45
- "models/schemas/utf_queue_models/models/python/*.py",
46
- "models/schemas/utf_queue_models/schema/*.json",
47
- ]
48
- },
49
- entry_points={
50
- "console_scripts": [
51
- "ubai_upload_cli = utf_queue_client.scripts.ubai_upload_cli:cli_entrypoint",
52
- "ubai_search_cli = utf_queue_client.scripts.ubai_search_cli:cli_entrypoint",
53
- "publish_test_results_cli = utf_queue_client.scripts.publish_test_results_cli:cli_entrypoint",
54
- "ubai_update_retention_cli = utf_queue_client.scripts.ubai_update_retention_cli:cli_entrypoint",
55
- "publish_release_info_cli = utf_queue_client.scripts.publish_release_info_cli:cli_entrypoint",
56
- "ubai_copy_cli = utf_queue_client.scripts.ubai_copy_cli:cli_entrypoint",
57
- ],
58
- },
59
- include_package_data=True,
60
- long_description="""\
61
- No description provided
62
- """,
63
- )
1
+ """
2
+
3
+ """
4
+
5
+ from setuptools import setup, find_packages # noqa: H301
6
+
7
+ NAME = "utf-queue-client"
8
+ VERSION = "1.17.2b1"
9
+ # To install the library, run the following
10
+ #
11
+ # python setup.py install
12
+ #
13
+ # prerequisite: setuptools
14
+ # http://pypi.python.org/pypi/setuptools
15
+
16
+ REQUIRES = [
17
+ "pika >= 1.2.0",
18
+ "jsonschema >= 3.2.0",
19
+ "msgpack >= 1.0.2",
20
+ "pydantic == 1.*",
21
+ "regex",
22
+ "typing_extensions; python_version < '3.8'",
23
+ "click",
24
+ "ubai-client >= 1.2.0",
25
+ "opentelemetry-distro[otlp]",
26
+ "otel-extensions >= 0.1.2",
27
+ "retry>=0.9.2",
28
+ "requests",
29
+ "ndjson~=0.3",
30
+ ]
31
+
32
+ setup(
33
+ name=NAME,
34
+ version=VERSION,
35
+ description="",
36
+ author="Joe Savage",
37
+ author_email="joe.savage@silabs.com",
38
+ url="",
39
+ keywords=["UTF", ""],
40
+ python_requires=">=3.6",
41
+ install_requires=REQUIRES,
42
+ packages=find_packages(),
43
+ package_data={
44
+ "utf_queue_client": [
45
+ "models/schemas/utf_queue_models/models/python/*.py",
46
+ "models/schemas/utf_queue_models/schema/*.json",
47
+ ]
48
+ },
49
+ entry_points={
50
+ "console_scripts": [
51
+ "ubai_upload_cli = utf_queue_client.scripts.ubai_upload_cli:cli_entrypoint",
52
+ "ubai_search_cli = utf_queue_client.scripts.ubai_search_cli:cli_entrypoint",
53
+ "publish_test_results_cli = utf_queue_client.scripts.publish_test_results_cli:cli_entrypoint",
54
+ "ubai_update_retention_cli = utf_queue_client.scripts.ubai_update_retention_cli:cli_entrypoint",
55
+ "publish_release_info_cli = utf_queue_client.scripts.publish_release_info_cli:cli_entrypoint",
56
+ "ubai_copy_cli = utf_queue_client.scripts.ubai_copy_cli:cli_entrypoint"
57
+ ],
58
+ },
59
+ include_package_data=True,
60
+ long_description="""\
61
+ No description provided
62
+ """,
63
+ )
@@ -1,46 +1,46 @@
1
- import os
2
- from utf_queue_client.clients.base_producer import BlockingProducer
3
- import pytest
4
- import logging
5
- import pika
6
- import time
7
- from utf_queue_client.utils import UTF_QUEUE_REGION_MAP
8
- from urllib.parse import quote
9
-
10
-
11
- regions = list(UTF_QUEUE_REGION_MAP.values())
12
- servers = [f"utf-queue-{region}" for region in regions]
13
-
14
-
15
- @pytest.fixture(params=servers)
16
- def queue_server(request):
17
- yield request.param
18
-
19
-
20
- @pytest.mark.parametrize(
21
- "username, password",
22
- [
23
- (os.environ["UTF_QUEUE_USERNAME_LDAP"], os.environ["UTF_QUEUE_PASSWORD_LDAP"]),
24
- (os.environ["UTF_QUEUE_USERNAME"], os.environ["UTF_QUEUE_PASSWORD"]),
25
- ],
26
- )
27
- def test_producer(queue_server, username, password):
28
- suffix = ".silabs.net"
29
- queue_server = f"{queue_server}{suffix}"
30
- l = logging.getLogger("test_producer")
31
- l.info("Creating producer...")
32
- p = BlockingProducer(
33
- url=f"amqps://{username}:{quote(password)}@{queue_server}:443/testing?stack_timeout=30",
34
- )
35
- l.info(f"{queue_server}: Created producer")
36
- params = pika.URLParameters(p.url)
37
- l.info(f"stack timeout: {params.stack_timeout}")
38
- begin = time.time()
39
- for _ in range(0, 10):
40
- l.info(f"{queue_server}: declaring queue...")
41
- start = time.time()
42
- p.queue_declare("default", durable=True)
43
- stop = time.time()
44
- l.info(
45
- f"{queue_server}: done declaring queue in {stop - start} sec. Total time: {stop - begin}"
46
- )
1
+ import os
2
+ from utf_queue_client.clients.base_producer import BlockingProducer
3
+ import pytest
4
+ import logging
5
+ import pika
6
+ import time
7
+ from utf_queue_client.utils import UTF_QUEUE_REGION_MAP
8
+ from urllib.parse import quote
9
+
10
+
11
+ regions = list(UTF_QUEUE_REGION_MAP.values())
12
+ servers = [f"utf-queue-{region}" for region in regions]
13
+
14
+
15
+ @pytest.fixture(params=servers)
16
+ def queue_server(request):
17
+ yield request.param
18
+
19
+
20
+ @pytest.mark.parametrize(
21
+ "username, password",
22
+ [
23
+ (os.environ["UTF_QUEUE_USERNAME_LDAP"], os.environ["UTF_QUEUE_PASSWORD_LDAP"]),
24
+ (os.environ["UTF_QUEUE_USERNAME"], os.environ["UTF_QUEUE_PASSWORD"]),
25
+ ],
26
+ )
27
+ def test_producer(queue_server, username, password):
28
+ suffix = ".silabs.net"
29
+ queue_server = f"{queue_server}{suffix}"
30
+ l = logging.getLogger("test_producer")
31
+ l.info("Creating producer...")
32
+ p = BlockingProducer(
33
+ url=f"amqps://{username}:{quote(password)}@{queue_server}:443/testing?stack_timeout=30",
34
+ )
35
+ l.info(f"{queue_server}: Created producer")
36
+ params = pika.URLParameters(p.url)
37
+ l.info(f"stack timeout: {params.stack_timeout}")
38
+ begin = time.time()
39
+ for _ in range(0, 10):
40
+ l.info(f"{queue_server}: declaring queue...")
41
+ start = time.time()
42
+ p.queue_declare("default", durable=True)
43
+ stop = time.time()
44
+ l.info(
45
+ f"{queue_server}: done declaring queue in {stop - start} sec. Total time: {stop - begin}"
46
+ )
@@ -1,12 +1,12 @@
1
- from utf_queue_client.clients.async_consumer import AsyncConsumerBase
2
-
3
-
4
-
5
- def test_sanitize_url():
6
- params = [
7
- ("amqp://www.ni.com", "amqp://www.ni.com"),
8
- ("amqps://utfsilabs:password1234@www.ni.com", "amqps://utfsilabs:******@www.ni.com"),
9
- ]
10
- for url, display_url in params:
11
- sanitized_url = AsyncConsumerBase.sanitize_url_for_display(url)
1
+ from utf_queue_client.clients.async_consumer import AsyncConsumerBase
2
+
3
+
4
+
5
+ def test_sanitize_url():
6
+ params = [
7
+ ("amqp://www.ni.com", "amqp://www.ni.com"),
8
+ ("amqps://utfsilabs:password1234@www.ni.com", "amqps://utfsilabs:******@www.ni.com"),
9
+ ]
10
+ for url, display_url in params:
11
+ sanitized_url = AsyncConsumerBase.sanitize_url_for_display(url)
12
12
  assert sanitized_url == display_url
@@ -1,40 +1,40 @@
1
- import logging
2
- import logging.handlers
3
-
4
- import pytest
5
- from utf_queue_client.clients.sqa_test_result_producer import (
6
- SqaTestResultProducer,
7
- )
8
- from utf_queue_client.clients.base_producer import ConnectionError
9
-
10
-
11
- @pytest.fixture()
12
- def pika_logging_handler():
13
- """Add a handler to the pika logger to capture its output."""
14
- pika_logger = logging.getLogger("pika.adapters.utils.connection_workflow")
15
- handler = logging.handlers.BufferingHandler(capacity=100)
16
- pika_logger.addHandler(handler)
17
- yield handler
18
- pika_logger.removeHandler(handler)
19
-
20
-
21
- @pytest.mark.parametrize(
22
- "set_environment_variable",
23
- [("UTF_QUEUE_SERVER_URL", "amqps://utfsilabs:utfsilabs@12.34.56.78:5671/%2f")],
24
- indirect=True,
25
- )
26
- def test_result_producer_connection_retry(
27
- set_environment_variable, mocker, pika_logging_handler
28
- ):
29
- """Test that the connection retry logic works as expected."""
30
- mocker.patch("utf_queue_client.clients.base_producer.NUM_CONNECTION_TRIES", 2)
31
- with pytest.raises(ConnectionError):
32
- _ = SqaTestResultProducer()
33
- error_records = [
34
- record
35
- for record in pika_logging_handler.buffer
36
- if record.levelno == logging.ERROR
37
- ]
38
- # The assertion below depends on the logging output of the pika library (version 1.3.2 as of this writing)
39
- # so if that changes, the message below may need to be updated.
40
- assert "2 exceptions in all" in error_records[-1].message
1
+ import logging
2
+ import logging.handlers
3
+
4
+ import pytest
5
+ from utf_queue_client.clients.sqa_test_result_producer import (
6
+ SqaTestResultProducer,
7
+ )
8
+ from utf_queue_client.clients.base_producer import ConnectionError
9
+
10
+
11
+ @pytest.fixture()
12
+ def pika_logging_handler():
13
+ """Add a handler to the pika logger to capture its output."""
14
+ pika_logger = logging.getLogger("pika.adapters.utils.connection_workflow")
15
+ handler = logging.handlers.BufferingHandler(capacity=100)
16
+ pika_logger.addHandler(handler)
17
+ yield handler
18
+ pika_logger.removeHandler(handler)
19
+
20
+
21
+ @pytest.mark.parametrize(
22
+ "set_environment_variable",
23
+ [("UTF_QUEUE_SERVER_URL", "amqps://utfsilabs:utfsilabs@12.34.56.78:5671/%2f")],
24
+ indirect=True,
25
+ )
26
+ def test_result_producer_connection_retry(
27
+ set_environment_variable, mocker, pika_logging_handler
28
+ ):
29
+ """Test that the connection retry logic works as expected."""
30
+ mocker.patch("utf_queue_client.clients.base_producer.NUM_CONNECTION_TRIES", 2)
31
+ with pytest.raises(ConnectionError):
32
+ _ = SqaTestResultProducer()
33
+ error_records = [
34
+ record
35
+ for record in pika_logging_handler.buffer
36
+ if record.levelno == logging.ERROR
37
+ ]
38
+ # The assertion below depends on the logging output of the pika library (version 1.3.2 as of this writing)
39
+ # so if that changes, the message below may need to be updated.
40
+ assert "2 exceptions in all" in error_records[-1].message
@@ -1,57 +1,57 @@
1
- import json
2
- import uuid
3
-
4
- from utf_queue_client.clients.data_lake_data_producer import (
5
- DataLakeDataProducer,
6
- DataLakeDataProducerFactory,
7
- )
8
- import pytest
9
- import os
10
- import ndjson
11
-
12
-
13
- def test_data_lake_producer_empty_url():
14
- with pytest.raises(RuntimeError):
15
- _ = DataLakeDataProducer("dummy tenant")
16
-
17
-
18
- @pytest.mark.parametrize("use_object_id", [True, False])
19
- def test_data_lake_producer_central_queue(
20
- request,
21
- data_lake_tenant_key,
22
- csv_file_path,
23
- json_file_path,
24
- parquet_file_path,
25
- ndjson_file_path,
26
- amqp_url,
27
- queue_consumer,
28
- use_object_id,
29
- ):
30
- os.environ["UTF_PRODUCER_APP_ID"] = request.node.name
31
- producer = DataLakeDataProducerFactory.create_producer(
32
- data_lake_tenant_key, amqp_url
33
- )
34
- producer.publish_data_lake_csv_file(
35
- csv_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
36
- )
37
- queue_consumer.expect_messages(1)
38
- producer.publish_data_lake_json_file(
39
- json_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
40
- )
41
- queue_consumer.expect_messages(2)
42
- with open(json_file_path, "rb") as f:
43
- producer.publish_data_lake_json(json.load(f))
44
- producer.publish_data_lake_parquet_file(
45
- parquet_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
46
- )
47
- queue_consumer.expect_messages(3)
48
- producer.publish_data_lake_ndjson_file(
49
- ndjson_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
50
- )
51
- queue_consumer.expect_messages(4)
52
- with open(ndjson_file_path) as f:
53
- data = ndjson.load(f)
54
- producer.publish_data_lake_ndjson(
55
- data, object_id=str(uuid.uuid4()) if use_object_id else None
56
- )
57
- queue_consumer.expect_messages(5)
1
+ import json
2
+ import uuid
3
+
4
+ from utf_queue_client.clients.data_lake_data_producer import (
5
+ DataLakeDataProducer,
6
+ DataLakeDataProducerFactory,
7
+ )
8
+ import pytest
9
+ import os
10
+ import ndjson
11
+
12
+
13
+ def test_data_lake_producer_empty_url():
14
+ with pytest.raises(RuntimeError):
15
+ _ = DataLakeDataProducer("dummy tenant")
16
+
17
+
18
+ @pytest.mark.parametrize("use_object_id", [True, False])
19
+ def test_data_lake_producer_central_queue(
20
+ request,
21
+ data_lake_tenant_key,
22
+ csv_file_path,
23
+ json_file_path,
24
+ parquet_file_path,
25
+ ndjson_file_path,
26
+ amqp_url,
27
+ queue_consumer,
28
+ use_object_id,
29
+ ):
30
+ os.environ["UTF_PRODUCER_APP_ID"] = request.node.name
31
+ producer = DataLakeDataProducerFactory.create_producer(
32
+ data_lake_tenant_key, amqp_url
33
+ )
34
+ producer.publish_data_lake_csv_file(
35
+ csv_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
36
+ )
37
+ queue_consumer.expect_messages(1)
38
+ producer.publish_data_lake_json_file(
39
+ json_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
40
+ )
41
+ queue_consumer.expect_messages(2)
42
+ with open(json_file_path, "rb") as f:
43
+ producer.publish_data_lake_json(json.load(f))
44
+ producer.publish_data_lake_parquet_file(
45
+ parquet_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
46
+ )
47
+ queue_consumer.expect_messages(3)
48
+ producer.publish_data_lake_ndjson_file(
49
+ ndjson_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
50
+ )
51
+ queue_consumer.expect_messages(4)
52
+ with open(ndjson_file_path) as f:
53
+ data = ndjson.load(f)
54
+ producer.publish_data_lake_ndjson(
55
+ data, object_id=str(uuid.uuid4()) if use_object_id else None
56
+ )
57
+ queue_consumer.expect_messages(5)