utf-queue-client 1.17.0__tar.gz → 1.17.0b1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/PKG-INFO +15 -13
  2. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/setup.cfg +7 -7
  3. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/setup.py +64 -64
  4. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_amqp_connections.py +46 -46
  5. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_async_consumer.py +11 -11
  6. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_connection_retry.py +40 -40
  7. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_data_lake_data_producer.py +57 -57
  8. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_http_producer.py +90 -90
  9. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_legacy_base_model.py +47 -47
  10. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_models.py +125 -125
  11. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_opentelemetry_data_producer.py +168 -168
  12. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_publish_test_results.py +160 -160
  13. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_rail_data_producer_tests.py +64 -64
  14. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_sqa_test_result_producer.py +48 -48
  15. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_ubai_search.py +81 -81
  16. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_ubai_update_retention.py +61 -61
  17. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_ubai_upload.py +113 -113
  18. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_utils.py +77 -77
  19. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/__init__.py +1 -1
  20. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/clients/__init__.py +48 -48
  21. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/clients/async_consumer.py +579 -579
  22. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/clients/base_producer.py +164 -164
  23. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/clients/data_lake_data_producer.py +135 -135
  24. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/clients/http_producer.py +207 -207
  25. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/clients/opentelemetry_data_producer.py +53 -53
  26. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/clients/sqa_test_result_producer.py +171 -171
  27. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/clients/test_rail_data_producer.py +95 -95
  28. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/clients/ubai_artifact_upload_request_producer.py +123 -123
  29. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/exceptions/__init__.py +20 -20
  30. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/models/__init__.py +1 -1
  31. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/models/base_model.py +164 -164
  32. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/models/model_factory.py +19 -19
  33. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/models/schemas/__init__.py +22 -22
  34. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/scripts/__init__.py +86 -86
  35. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/scripts/publish_test_results_cli.py +146 -146
  36. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/scripts/ubai_search_cli.py +78 -78
  37. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/scripts/ubai_update_retention_cli.py +73 -73
  38. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/scripts/ubai_upload_cli.py +135 -135
  39. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/utils/__init__.py +89 -89
  40. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/utils/queue_metrics_exporter.py +85 -85
  41. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/utils/queue_span_exporter.py +86 -86
  42. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client.egg-info/PKG-INFO +15 -13
  43. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/README.md +0 -0
  44. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_publish_release_info.py +0 -0
  45. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/test/test_ubai_copy.py +0 -0
  46. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/scripts/publish_release_info_cli.py +0 -0
  47. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client/scripts/ubai_copy_cli.py +0 -0
  48. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client.egg-info/SOURCES.txt +0 -0
  49. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client.egg-info/dependency_links.txt +0 -0
  50. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client.egg-info/entry_points.txt +0 -0
  51. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client.egg-info/requires.txt +7 -7
  52. {utf-queue-client-1.17.0 → utf-queue-client-1.17.0b1}/utf_queue_client.egg-info/top_level.txt +0 -0
@@ -1,13 +1,15 @@
1
- Metadata-Version: 1.2
2
- Name: utf-queue-client
3
- Version: 1.17.0
4
- Summary: UNKNOWN
5
- Home-page: UNKNOWN
6
- Author: Joe Savage
7
- Author-email: joe.savage@silabs.com
8
- License: UNKNOWN
9
- Description: No description provided
10
-
11
- Keywords: UTF,
12
- Platform: UNKNOWN
13
- Requires-Python: >=3.6
1
+ Metadata-Version: 2.1
2
+ Name: utf-queue-client
3
+ Version: 1.17.0b1
4
+ Summary: UNKNOWN
5
+ Home-page: UNKNOWN
6
+ Author: Joe Savage
7
+ Author-email: joe.savage@silabs.com
8
+ License: UNKNOWN
9
+ Keywords: UTF,
10
+ Platform: UNKNOWN
11
+ Requires-Python: >=3.6
12
+
13
+ No description provided
14
+
15
+
@@ -1,7 +1,7 @@
1
- [flake8]
2
- max-line-length = 99
3
-
4
- [egg_info]
5
- tag_build =
6
- tag_date = 0
7
-
1
+ [flake8]
2
+ max-line-length = 99
3
+
4
+ [egg_info]
5
+ tag_build =
6
+ tag_date = 0
7
+
@@ -1,64 +1,64 @@
1
- """
2
-
3
- """
4
-
5
-
6
- from setuptools import setup, find_packages # noqa: H301
7
-
8
- NAME = "utf-queue-client"
9
- VERSION = "1.17.0"
10
- # To install the library, run the following
11
- #
12
- # python setup.py install
13
- #
14
- # prerequisite: setuptools
15
- # http://pypi.python.org/pypi/setuptools
16
-
17
- REQUIRES = [
18
- "pika >= 1.2.0",
19
- "jsonschema >= 3.2.0",
20
- "msgpack >= 1.0.2",
21
- "pydantic == 1.*",
22
- "regex",
23
- "typing_extensions; python_version < '3.8'",
24
- "click",
25
- "ubai-client >= 1.2.0",
26
- "opentelemetry-distro[otlp]",
27
- "otel-extensions >= 0.1.2",
28
- "retry>=0.9.2",
29
- "requests",
30
- "ndjson~=0.3",
31
- ]
32
-
33
- setup(
34
- name=NAME,
35
- version=VERSION,
36
- description="",
37
- author="Joe Savage",
38
- author_email="joe.savage@silabs.com",
39
- url="",
40
- keywords=["UTF", ""],
41
- python_requires=">=3.6",
42
- install_requires=REQUIRES,
43
- packages=find_packages(exclude=["test", "tests"]),
44
- package_data={
45
- "utf_queue_client": [
46
- "models/schemas/utf_queue_models/models/python/*.py",
47
- "models/schemas/utf_queue_models/schema/*.json",
48
- ]
49
- },
50
- entry_points={
51
- "console_scripts": [
52
- "ubai_upload_cli = utf_queue_client.scripts.ubai_upload_cli:cli_entrypoint",
53
- "ubai_search_cli = utf_queue_client.scripts.ubai_search_cli:cli_entrypoint",
54
- "publish_test_results_cli = utf_queue_client.scripts.publish_test_results_cli:cli_entrypoint",
55
- "ubai_update_retention_cli = utf_queue_client.scripts.ubai_update_retention_cli:cli_entrypoint",
56
- "publish_release_info_cli = utf_queue_client.scripts.publish_release_info_cli:cli_entrypoint",
57
- "ubai_copy_cli = utf_queue_client.scripts.ubai_copy_cli:cli_entrypoint"
58
- ],
59
- },
60
- include_package_data=True,
61
- long_description="""\
62
- No description provided
63
- """,
64
- )
1
+ """
2
+
3
+ """
4
+
5
+
6
+ from setuptools import setup, find_packages # noqa: H301
7
+
8
+ NAME = "utf-queue-client"
9
+ VERSION = "1.17.0b1"
10
+ # To install the library, run the following
11
+ #
12
+ # python setup.py install
13
+ #
14
+ # prerequisite: setuptools
15
+ # http://pypi.python.org/pypi/setuptools
16
+
17
+ REQUIRES = [
18
+ "pika >= 1.2.0",
19
+ "jsonschema >= 3.2.0",
20
+ "msgpack >= 1.0.2",
21
+ "pydantic == 1.*",
22
+ "regex",
23
+ "typing_extensions; python_version < '3.8'",
24
+ "click",
25
+ "ubai-client >= 1.2.0",
26
+ "opentelemetry-distro[otlp]",
27
+ "otel-extensions >= 0.1.2",
28
+ "retry>=0.9.2",
29
+ "requests",
30
+ "ndjson~=0.3",
31
+ ]
32
+
33
+ setup(
34
+ name=NAME,
35
+ version=VERSION,
36
+ description="",
37
+ author="Joe Savage",
38
+ author_email="joe.savage@silabs.com",
39
+ url="",
40
+ keywords=["UTF", ""],
41
+ python_requires=">=3.6",
42
+ install_requires=REQUIRES,
43
+ packages=find_packages(exclude=["test", "tests"]),
44
+ package_data={
45
+ "utf_queue_client": [
46
+ "models/schemas/utf_queue_models/models/python/*.py",
47
+ "models/schemas/utf_queue_models/schema/*.json",
48
+ ]
49
+ },
50
+ entry_points={
51
+ "console_scripts": [
52
+ "ubai_upload_cli = utf_queue_client.scripts.ubai_upload_cli:cli_entrypoint",
53
+ "ubai_search_cli = utf_queue_client.scripts.ubai_search_cli:cli_entrypoint",
54
+ "publish_test_results_cli = utf_queue_client.scripts.publish_test_results_cli:cli_entrypoint",
55
+ "ubai_update_retention_cli = utf_queue_client.scripts.ubai_update_retention_cli:cli_entrypoint",
56
+ "publish_release_info_cli = utf_queue_client.scripts.publish_release_info_cli:cli_entrypoint",
57
+ "ubai_copy_cli = utf_queue_client.scripts.ubai_copy_cli:cli_entrypoint"
58
+ ],
59
+ },
60
+ include_package_data=True,
61
+ long_description="""\
62
+ No description provided
63
+ """,
64
+ )
@@ -1,46 +1,46 @@
1
- import os
2
- from utf_queue_client.clients.base_producer import BlockingProducer
3
- import pytest
4
- import logging
5
- import pika
6
- import time
7
- from utf_queue_client.utils import UTF_QUEUE_REGION_MAP
8
- from urllib.parse import quote
9
-
10
-
11
- regions = list(UTF_QUEUE_REGION_MAP.values())
12
- servers = [f"utf-queue-{region}" for region in regions]
13
-
14
-
15
- @pytest.fixture(params=servers)
16
- def queue_server(request):
17
- yield request.param
18
-
19
-
20
- @pytest.mark.parametrize(
21
- "username, password",
22
- [
23
- (os.environ["UTF_QUEUE_USERNAME_LDAP"], os.environ["UTF_QUEUE_PASSWORD_LDAP"]),
24
- (os.environ["UTF_QUEUE_USERNAME"], os.environ["UTF_QUEUE_PASSWORD"]),
25
- ],
26
- )
27
- def test_producer(queue_server, username, password):
28
- suffix = ".dev.silabs.net"
29
- queue_server = f"{queue_server}{suffix}"
30
- l = logging.getLogger("test_producer")
31
- l.info("Creating producer...")
32
- p = BlockingProducer(
33
- url=f"amqps://{username}:{quote(password)}@{queue_server}:443/testing?stack_timeout=30",
34
- )
35
- l.info(f"{queue_server}: Created producer")
36
- params = pika.URLParameters(p.url)
37
- l.info(f"stack timeout: {params.stack_timeout}")
38
- begin = time.time()
39
- for _ in range(0, 10):
40
- l.info(f"{queue_server}: declaring queue...")
41
- start = time.time()
42
- p.queue_declare("default", durable=True)
43
- stop = time.time()
44
- l.info(
45
- f"{queue_server}: done declaring queue in {stop - start} sec. Total time: {stop - begin}"
46
- )
1
+ import os
2
+ from utf_queue_client.clients.base_producer import BlockingProducer
3
+ import pytest
4
+ import logging
5
+ import pika
6
+ import time
7
+ from utf_queue_client.utils import UTF_QUEUE_REGION_MAP
8
+ from urllib.parse import quote
9
+
10
+
11
+ regions = list(UTF_QUEUE_REGION_MAP.values())
12
+ servers = [f"utf-queue-{region}" for region in regions]
13
+
14
+
15
+ @pytest.fixture(params=servers)
16
+ def queue_server(request):
17
+ yield request.param
18
+
19
+
20
+ @pytest.mark.parametrize(
21
+ "username, password",
22
+ [
23
+ (os.environ["UTF_QUEUE_USERNAME_LDAP"], os.environ["UTF_QUEUE_PASSWORD_LDAP"]),
24
+ (os.environ["UTF_QUEUE_USERNAME"], os.environ["UTF_QUEUE_PASSWORD"]),
25
+ ],
26
+ )
27
+ def test_producer(queue_server, username, password):
28
+ suffix = ".dev.silabs.net"
29
+ queue_server = f"{queue_server}{suffix}"
30
+ l = logging.getLogger("test_producer")
31
+ l.info("Creating producer...")
32
+ p = BlockingProducer(
33
+ url=f"amqps://{username}:{quote(password)}@{queue_server}:443/testing?stack_timeout=30",
34
+ )
35
+ l.info(f"{queue_server}: Created producer")
36
+ params = pika.URLParameters(p.url)
37
+ l.info(f"stack timeout: {params.stack_timeout}")
38
+ begin = time.time()
39
+ for _ in range(0, 10):
40
+ l.info(f"{queue_server}: declaring queue...")
41
+ start = time.time()
42
+ p.queue_declare("default", durable=True)
43
+ stop = time.time()
44
+ l.info(
45
+ f"{queue_server}: done declaring queue in {stop - start} sec. Total time: {stop - begin}"
46
+ )
@@ -1,12 +1,12 @@
1
- from utf_queue_client.clients.async_consumer import AsyncConsumerBase
2
-
3
-
4
-
5
- def test_sanitize_url():
6
- params = [
7
- ("amqp://www.ni.com", "amqp://www.ni.com"),
8
- ("amqps://utfsilabs:password1234@www.ni.com", "amqps://utfsilabs:******@www.ni.com"),
9
- ]
10
- for url, display_url in params:
11
- sanitized_url = AsyncConsumerBase.sanitize_url_for_display(url)
1
+ from utf_queue_client.clients.async_consumer import AsyncConsumerBase
2
+
3
+
4
+
5
+ def test_sanitize_url():
6
+ params = [
7
+ ("amqp://www.ni.com", "amqp://www.ni.com"),
8
+ ("amqps://utfsilabs:password1234@www.ni.com", "amqps://utfsilabs:******@www.ni.com"),
9
+ ]
10
+ for url, display_url in params:
11
+ sanitized_url = AsyncConsumerBase.sanitize_url_for_display(url)
12
12
  assert sanitized_url == display_url
@@ -1,40 +1,40 @@
1
- import logging
2
- import logging.handlers
3
-
4
- import pytest
5
- from utf_queue_client.clients.sqa_test_result_producer import (
6
- SqaTestResultProducer,
7
- )
8
- from utf_queue_client.clients.base_producer import ConnectionError
9
-
10
-
11
- @pytest.fixture()
12
- def pika_logging_handler():
13
- """Add a handler to the pika logger to capture its output."""
14
- pika_logger = logging.getLogger("pika.adapters.utils.connection_workflow")
15
- handler = logging.handlers.BufferingHandler(capacity=100)
16
- pika_logger.addHandler(handler)
17
- yield handler
18
- pika_logger.removeHandler(handler)
19
-
20
-
21
- @pytest.mark.parametrize(
22
- "set_environment_variable",
23
- [("UTF_QUEUE_SERVER_URL", "amqps://utfsilabs:utfsilabs@12.34.56.78:5671/%2f")],
24
- indirect=True,
25
- )
26
- def test_result_producer_connection_retry(
27
- set_environment_variable, mocker, pika_logging_handler
28
- ):
29
- """Test that the connection retry logic works as expected."""
30
- mocker.patch("utf_queue_client.clients.base_producer.NUM_CONNECTION_TRIES", 2)
31
- with pytest.raises(ConnectionError):
32
- _ = SqaTestResultProducer()
33
- error_records = [
34
- record
35
- for record in pika_logging_handler.buffer
36
- if record.levelno == logging.ERROR
37
- ]
38
- # The assertion below depends on the logging output of the pika library (version 1.3.2 as of this writing)
39
- # so if that changes, the message below may need to be updated.
40
- assert "2 exceptions in all" in error_records[-1].message
1
+ import logging
2
+ import logging.handlers
3
+
4
+ import pytest
5
+ from utf_queue_client.clients.sqa_test_result_producer import (
6
+ SqaTestResultProducer,
7
+ )
8
+ from utf_queue_client.clients.base_producer import ConnectionError
9
+
10
+
11
+ @pytest.fixture()
12
+ def pika_logging_handler():
13
+ """Add a handler to the pika logger to capture its output."""
14
+ pika_logger = logging.getLogger("pika.adapters.utils.connection_workflow")
15
+ handler = logging.handlers.BufferingHandler(capacity=100)
16
+ pika_logger.addHandler(handler)
17
+ yield handler
18
+ pika_logger.removeHandler(handler)
19
+
20
+
21
+ @pytest.mark.parametrize(
22
+ "set_environment_variable",
23
+ [("UTF_QUEUE_SERVER_URL", "amqps://utfsilabs:utfsilabs@12.34.56.78:5671/%2f")],
24
+ indirect=True,
25
+ )
26
+ def test_result_producer_connection_retry(
27
+ set_environment_variable, mocker, pika_logging_handler
28
+ ):
29
+ """Test that the connection retry logic works as expected."""
30
+ mocker.patch("utf_queue_client.clients.base_producer.NUM_CONNECTION_TRIES", 2)
31
+ with pytest.raises(ConnectionError):
32
+ _ = SqaTestResultProducer()
33
+ error_records = [
34
+ record
35
+ for record in pika_logging_handler.buffer
36
+ if record.levelno == logging.ERROR
37
+ ]
38
+ # The assertion below depends on the logging output of the pika library (version 1.3.2 as of this writing)
39
+ # so if that changes, the message below may need to be updated.
40
+ assert "2 exceptions in all" in error_records[-1].message
@@ -1,57 +1,57 @@
1
- import json
2
- import uuid
3
-
4
- from utf_queue_client.clients.data_lake_data_producer import (
5
- DataLakeDataProducer,
6
- DataLakeDataProducerFactory,
7
- )
8
- import pytest
9
- import os
10
- import ndjson
11
-
12
-
13
- def test_data_lake_producer_empty_url():
14
- with pytest.raises(RuntimeError):
15
- _ = DataLakeDataProducer("dummy tenant")
16
-
17
-
18
- @pytest.mark.parametrize("use_object_id", [True, False])
19
- def test_data_lake_producer_central_queue(
20
- request,
21
- data_lake_tenant_key,
22
- csv_file_path,
23
- json_file_path,
24
- parquet_file_path,
25
- ndjson_file_path,
26
- amqp_url,
27
- queue_consumer,
28
- use_object_id,
29
- ):
30
- os.environ["UTF_PRODUCER_APP_ID"] = request.node.name
31
- producer = DataLakeDataProducerFactory.create_producer(
32
- data_lake_tenant_key, amqp_url
33
- )
34
- producer.publish_data_lake_csv_file(
35
- csv_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
36
- )
37
- queue_consumer.expect_messages(1)
38
- producer.publish_data_lake_json_file(
39
- json_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
40
- )
41
- queue_consumer.expect_messages(2)
42
- with open(json_file_path, "rb") as f:
43
- producer.publish_data_lake_json(json.load(f))
44
- producer.publish_data_lake_parquet_file(
45
- parquet_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
46
- )
47
- queue_consumer.expect_messages(3)
48
- producer.publish_data_lake_ndjson_file(
49
- ndjson_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
50
- )
51
- queue_consumer.expect_messages(4)
52
- with open(ndjson_file_path) as f:
53
- data = ndjson.load(f)
54
- producer.publish_data_lake_ndjson(
55
- data, object_id=str(uuid.uuid4()) if use_object_id else None
56
- )
57
- queue_consumer.expect_messages(5)
1
+ import json
2
+ import uuid
3
+
4
+ from utf_queue_client.clients.data_lake_data_producer import (
5
+ DataLakeDataProducer,
6
+ DataLakeDataProducerFactory,
7
+ )
8
+ import pytest
9
+ import os
10
+ import ndjson
11
+
12
+
13
+ def test_data_lake_producer_empty_url():
14
+ with pytest.raises(RuntimeError):
15
+ _ = DataLakeDataProducer("dummy tenant")
16
+
17
+
18
+ @pytest.mark.parametrize("use_object_id", [True, False])
19
+ def test_data_lake_producer_central_queue(
20
+ request,
21
+ data_lake_tenant_key,
22
+ csv_file_path,
23
+ json_file_path,
24
+ parquet_file_path,
25
+ ndjson_file_path,
26
+ amqp_url,
27
+ queue_consumer,
28
+ use_object_id,
29
+ ):
30
+ os.environ["UTF_PRODUCER_APP_ID"] = request.node.name
31
+ producer = DataLakeDataProducerFactory.create_producer(
32
+ data_lake_tenant_key, amqp_url
33
+ )
34
+ producer.publish_data_lake_csv_file(
35
+ csv_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
36
+ )
37
+ queue_consumer.expect_messages(1)
38
+ producer.publish_data_lake_json_file(
39
+ json_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
40
+ )
41
+ queue_consumer.expect_messages(2)
42
+ with open(json_file_path, "rb") as f:
43
+ producer.publish_data_lake_json(json.load(f))
44
+ producer.publish_data_lake_parquet_file(
45
+ parquet_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
46
+ )
47
+ queue_consumer.expect_messages(3)
48
+ producer.publish_data_lake_ndjson_file(
49
+ ndjson_file_path, object_id=str(uuid.uuid4()) if use_object_id else None
50
+ )
51
+ queue_consumer.expect_messages(4)
52
+ with open(ndjson_file_path) as f:
53
+ data = ndjson.load(f)
54
+ producer.publish_data_lake_ndjson(
55
+ data, object_id=str(uuid.uuid4()) if use_object_id else None
56
+ )
57
+ queue_consumer.expect_messages(5)