dataforge-sdk 11.0.0rc66__tar.gz → 11.0.0rc67__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dataforge-sdk might be problematic. Click here for more details.
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/PKG-INFO +6 -3
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/README.md +5 -2
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge/__init__.py +1 -1
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge/_base_session.py +3 -1
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge/post_output_session.py +18 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge_sdk.egg-info/PKG-INFO +6 -3
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/pyproject.toml +1 -1
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/tests/test_post_output_enqueue.py +72 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge/_session.py +0 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge/databricks/__init__.py +0 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge/databricks/_databricks_base_session.py +0 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge/databricks/_databricks_ingestion_session.py +0 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge/databricks/_databricks_parsing_session.py +0 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge/databricks/_databricks_pg.py +0 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge/ingestion_session.py +0 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge/parsing_session.py +0 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge/pg.py +0 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge/postgres_connection.py +0 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge/process_record.py +0 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge/snowflake/__init__.py +0 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge/snowflake/_snowflake_base_session.py +0 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge/snowflake/_snowflake_ingestion_session.py +0 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge/snowflake/_snowflake_parsing_session.py +0 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge/system_configuration.py +0 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge/utils.py +0 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge_sdk.egg-info/SOURCES.txt +0 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge_sdk.egg-info/dependency_links.txt +0 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge_sdk.egg-info/requires.txt +0 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge_sdk.egg-info/top_level.txt +0 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/setup.cfg +0 -0
- {dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/tests/test_error_logging.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dataforge-sdk
|
|
3
|
-
Version: 11.0.
|
|
3
|
+
Version: 11.0.0rc67
|
|
4
4
|
Summary: SDK for creating DataForge extensions
|
|
5
5
|
Author-email: Vadim Orlov <vorlov@dataforgelabs.com>
|
|
6
6
|
Project-URL: Homepage, https://docs.dataforgelabs.com
|
|
@@ -126,8 +126,11 @@ session = PostOutputSession(output_name="report", output_source_name="my_source"
|
|
|
126
126
|
# Get the path of file generated by preceding output process
|
|
127
127
|
path = session.file_path()
|
|
128
128
|
|
|
129
|
-
#
|
|
130
|
-
|
|
129
|
+
# Read a Custom connection created with Direction = Output (exact connection name).
|
|
130
|
+
# Private parameters are decrypted; do not print or log the returned values.
|
|
131
|
+
connection_parameters = session.connection_parameters("notification_service")
|
|
132
|
+
public_parameters = connection_parameters["public_connection_parameters"]
|
|
133
|
+
private_parameters = connection_parameters["private_connection_parameters"]
|
|
131
134
|
|
|
132
135
|
# Retrieve custom parameters for the current source
|
|
133
136
|
custom_parameters = session.custom_parameters()
|
|
@@ -114,8 +114,11 @@ session = PostOutputSession(output_name="report", output_source_name="my_source"
|
|
|
114
114
|
# Get the path of file generated by preceding output process
|
|
115
115
|
path = session.file_path()
|
|
116
116
|
|
|
117
|
-
#
|
|
118
|
-
|
|
117
|
+
# Read a Custom connection created with Direction = Output (exact connection name).
|
|
118
|
+
# Private parameters are decrypted; do not print or log the returned values.
|
|
119
|
+
connection_parameters = session.connection_parameters("notification_service")
|
|
120
|
+
public_parameters = connection_parameters["public_connection_parameters"]
|
|
121
|
+
private_parameters = connection_parameters["private_connection_parameters"]
|
|
119
122
|
|
|
120
123
|
# Retrieve custom parameters for the current source
|
|
121
124
|
custom_parameters = session.custom_parameters()
|
|
@@ -2,5 +2,5 @@ from .post_output_session import PostOutputSession
|
|
|
2
2
|
from .ingestion_session import IngestionSession
|
|
3
3
|
from .parsing_session import ParsingSession
|
|
4
4
|
|
|
5
|
-
__version__ = "11.0.
|
|
5
|
+
__version__ = "11.0.0-rc.67"
|
|
6
6
|
__all__ = ['PostOutputSession','IngestionSession', 'ParsingSession']
|
|
@@ -132,7 +132,9 @@ class _Base_Session:
|
|
|
132
132
|
Returns:
|
|
133
133
|
dict: Dictionary containing private and public connection parameters.
|
|
134
134
|
"""
|
|
135
|
-
|
|
135
|
+
return self._read_connection_parameters(self.process.connectionId)
|
|
136
|
+
|
|
137
|
+
def _read_connection_parameters(self, connection_id: int):
|
|
136
138
|
self.log(f"Reading connection parameters of connection id {connection_id}")
|
|
137
139
|
params = self._core_api_call(f"core/connection/{connection_id}").get("parameters")
|
|
138
140
|
return {"private_connection_parameters": params.get("private_connection_parameters"),
|
|
@@ -32,6 +32,24 @@ class PostOutputSession(_Session):
|
|
|
32
32
|
"""Returns file path for the file generated by output process"""
|
|
33
33
|
return self.process.parameters.get("file_path")
|
|
34
34
|
|
|
35
|
+
def connection_parameters(self, connection_name: str | None = None):
|
|
36
|
+
"""Read decrypted public/private parameters of a named output Custom connection.
|
|
37
|
+
|
|
38
|
+
Names match exactly across the environment. Call while the session is open.
|
|
39
|
+
Omitting the name retains the existing process-connection lookup.
|
|
40
|
+
"""
|
|
41
|
+
if connection_name is None:
|
|
42
|
+
return super().connection_parameters()
|
|
43
|
+
if not self._is_open:
|
|
44
|
+
raise RuntimeError("Session is closed")
|
|
45
|
+
if not isinstance(connection_name, str) or not connection_name.strip():
|
|
46
|
+
raise ValueError("connection_name must be a non-empty output Custom connection name")
|
|
47
|
+
connection_id = self._pg.sql(
|
|
48
|
+
"select sparky.sdk_get_output_connection_id(%s, %s)",
|
|
49
|
+
(self.process.processId, connection_name),
|
|
50
|
+
)
|
|
51
|
+
return self._read_connection_parameters(connection_id)
|
|
52
|
+
|
|
35
53
|
def enqueue_ingestion(self, source_name: str) -> dict:
|
|
36
54
|
"""Immediately enqueue a separate custom ingestion in this session's project.
|
|
37
55
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dataforge-sdk
|
|
3
|
-
Version: 11.0.
|
|
3
|
+
Version: 11.0.0rc67
|
|
4
4
|
Summary: SDK for creating DataForge extensions
|
|
5
5
|
Author-email: Vadim Orlov <vorlov@dataforgelabs.com>
|
|
6
6
|
Project-URL: Homepage, https://docs.dataforgelabs.com
|
|
@@ -126,8 +126,11 @@ session = PostOutputSession(output_name="report", output_source_name="my_source"
|
|
|
126
126
|
# Get the path of file generated by preceding output process
|
|
127
127
|
path = session.file_path()
|
|
128
128
|
|
|
129
|
-
#
|
|
130
|
-
|
|
129
|
+
# Read a Custom connection created with Direction = Output (exact connection name).
|
|
130
|
+
# Private parameters are decrypted; do not print or log the returned values.
|
|
131
|
+
connection_parameters = session.connection_parameters("notification_service")
|
|
132
|
+
public_parameters = connection_parameters["public_connection_parameters"]
|
|
133
|
+
private_parameters = connection_parameters["private_connection_parameters"]
|
|
131
134
|
|
|
132
135
|
# Retrieve custom parameters for the current source
|
|
133
136
|
custom_parameters = session.custom_parameters()
|
|
@@ -21,9 +21,48 @@ def load_session():
|
|
|
21
21
|
return module.PostOutputSession
|
|
22
22
|
|
|
23
23
|
|
|
24
|
+
def load_base_session():
|
|
25
|
+
root = Path(__file__).resolve().parents[1] / "dataforge"
|
|
26
|
+
package = ModuleType("dataforge")
|
|
27
|
+
package.__path__ = [str(root)]
|
|
28
|
+
modules = {package.__name__: package, "requests": Mock()}
|
|
29
|
+
for suffix, attribute in (("system_configuration", "SystemConfiguration"),
|
|
30
|
+
("postgres_connection", "PostgresConnection"),
|
|
31
|
+
("utils", "_setup_logger")):
|
|
32
|
+
dependency = ModuleType(f"{package.__name__}.{suffix}")
|
|
33
|
+
setattr(dependency, attribute, Mock())
|
|
34
|
+
modules[dependency.__name__] = dependency
|
|
35
|
+
name = f"{package.__name__}._base_session"
|
|
36
|
+
spec = importlib.util.spec_from_file_location(name, root / "_base_session.py")
|
|
37
|
+
module = importlib.util.module_from_spec(spec)
|
|
38
|
+
with patch.dict(sys.modules, modules), patch("importlib.metadata.version", return_value="test"):
|
|
39
|
+
spec.loader.exec_module(module)
|
|
40
|
+
return module._Base_Session
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
BaseSession = load_base_session()
|
|
24
44
|
PostOutputSession = load_session()
|
|
25
45
|
|
|
26
46
|
|
|
47
|
+
class ConnectionParameterReadTests(unittest.TestCase):
|
|
48
|
+
def test_resolved_id_uses_core_and_returns_both_parameter_objects(self):
|
|
49
|
+
session = BaseSession.__new__(BaseSession)
|
|
50
|
+
session.log = Mock()
|
|
51
|
+
parameters = {"public_connection_parameters": {"region": "west"},
|
|
52
|
+
"private_connection_parameters": {"key": "test-value"}}
|
|
53
|
+
session._core_api_call = Mock(return_value={"parameters": parameters})
|
|
54
|
+
self.assertEqual(session._read_connection_parameters(17), parameters)
|
|
55
|
+
session._core_api_call.assert_called_once_with("core/connection/17")
|
|
56
|
+
session.log.assert_called_once_with("Reading connection parameters of connection id 17")
|
|
57
|
+
|
|
58
|
+
def test_existing_no_name_accessor_keeps_process_connection(self):
|
|
59
|
+
session = BaseSession.__new__(BaseSession)
|
|
60
|
+
session.process = SimpleNamespace(connectionId=19)
|
|
61
|
+
session._read_connection_parameters = Mock(return_value={"result": True})
|
|
62
|
+
self.assertEqual(session.connection_parameters(), {"result": True})
|
|
63
|
+
session._read_connection_parameters.assert_called_once_with(19)
|
|
64
|
+
|
|
65
|
+
|
|
27
66
|
class PostOutputEnqueueTests(unittest.TestCase):
|
|
28
67
|
def setUp(self):
|
|
29
68
|
self.session = PostOutputSession.__new__(PostOutputSession)
|
|
@@ -35,6 +74,39 @@ class PostOutputEnqueueTests(unittest.TestCase):
|
|
|
35
74
|
self.session._end_process = Mock()
|
|
36
75
|
self.session._log_fail = Mock()
|
|
37
76
|
|
|
77
|
+
def test_named_connection_uses_resolved_id_without_changing_process(self):
|
|
78
|
+
name = "notification's service / 東京; --"
|
|
79
|
+
parameters = {"public_connection_parameters": {"region": "west"},
|
|
80
|
+
"private_connection_parameters": {"key": "test-value"}}
|
|
81
|
+
self.session._pg.sql.return_value = 17
|
|
82
|
+
self.session._read_connection_parameters = Mock(return_value=parameters)
|
|
83
|
+
self.assertEqual(self.session.connection_parameters(name), parameters)
|
|
84
|
+
self.session._pg.sql.assert_called_once_with(
|
|
85
|
+
"select sparky.sdk_get_output_connection_id(%s, %s)", (41, name)
|
|
86
|
+
)
|
|
87
|
+
self.session._read_connection_parameters.assert_called_once_with(17)
|
|
88
|
+
self.assertEqual(self.session.process.processId, 41)
|
|
89
|
+
self.session.log.assert_not_called()
|
|
90
|
+
|
|
91
|
+
def test_invalid_connection_names_do_not_lookup(self):
|
|
92
|
+
for name in ("", " \t\n", 123):
|
|
93
|
+
with self.subTest(name=name), self.assertRaises(ValueError):
|
|
94
|
+
self.session.connection_parameters(name)
|
|
95
|
+
self.session._pg.sql.assert_not_called()
|
|
96
|
+
|
|
97
|
+
def test_closed_connection_lookup_is_rejected(self):
|
|
98
|
+
self.session._is_open = False
|
|
99
|
+
with self.assertRaisesRegex(RuntimeError, "Session is closed"):
|
|
100
|
+
self.session.connection_parameters("notification_service")
|
|
101
|
+
self.session._pg.sql.assert_not_called()
|
|
102
|
+
|
|
103
|
+
def test_missing_connection_does_not_fetch_credentials(self):
|
|
104
|
+
self.session._pg.sql.side_effect = RuntimeError("Active output Custom connection was not found")
|
|
105
|
+
self.session._read_connection_parameters = Mock()
|
|
106
|
+
with self.assertRaisesRegex(RuntimeError, "not found"):
|
|
107
|
+
self.session.connection_parameters("missing")
|
|
108
|
+
self.session._read_connection_parameters.assert_not_called()
|
|
109
|
+
|
|
38
110
|
def test_name_is_bound_as_data_and_receipt_is_returned(self):
|
|
39
111
|
name = "destination's source; --"
|
|
40
112
|
receipt = {"status": "queued", "ingestion_queue_id": 100, "process_id": 41, "source_id": 8}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge/databricks/_databricks_pg.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge/snowflake/_snowflake_base_session.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dataforge_sdk-11.0.0rc66 → dataforge_sdk-11.0.0rc67}/dataforge_sdk.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|