dataforge-sdk 11.0.0rc65__tar.gz → 11.0.0rc67__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dataforge-sdk might be problematic. Click here for more details.

Files changed (31) hide show
  1. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/PKG-INFO +6 -3
  2. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/README.md +5 -2
  3. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge/__init__.py +1 -1
  4. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge/_base_session.py +3 -1
  5. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge/post_output_session.py +18 -0
  6. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge_sdk.egg-info/PKG-INFO +6 -3
  7. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/pyproject.toml +1 -1
  8. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/tests/test_post_output_enqueue.py +72 -0
  9. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge/_session.py +0 -0
  10. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge/databricks/__init__.py +0 -0
  11. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge/databricks/_databricks_base_session.py +0 -0
  12. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge/databricks/_databricks_ingestion_session.py +0 -0
  13. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge/databricks/_databricks_parsing_session.py +0 -0
  14. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge/databricks/_databricks_pg.py +0 -0
  15. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge/ingestion_session.py +0 -0
  16. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge/parsing_session.py +0 -0
  17. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge/pg.py +0 -0
  18. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge/postgres_connection.py +0 -0
  19. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge/process_record.py +0 -0
  20. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge/snowflake/__init__.py +0 -0
  21. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge/snowflake/_snowflake_base_session.py +0 -0
  22. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge/snowflake/_snowflake_ingestion_session.py +0 -0
  23. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge/snowflake/_snowflake_parsing_session.py +0 -0
  24. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge/system_configuration.py +0 -0
  25. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge/utils.py +0 -0
  26. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge_sdk.egg-info/SOURCES.txt +0 -0
  27. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge_sdk.egg-info/dependency_links.txt +0 -0
  28. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge_sdk.egg-info/requires.txt +0 -0
  29. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/dataforge_sdk.egg-info/top_level.txt +0 -0
  30. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/setup.cfg +0 -0
  31. {dataforge_sdk-11.0.0rc65 → dataforge_sdk-11.0.0rc67}/tests/test_error_logging.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dataforge-sdk
3
- Version: 11.0.0rc65
3
+ Version: 11.0.0rc67
4
4
  Summary: SDK for creating DataForge extensions
5
5
  Author-email: Vadim Orlov <vorlov@dataforgelabs.com>
6
6
  Project-URL: Homepage, https://docs.dataforgelabs.com
@@ -126,8 +126,11 @@ session = PostOutputSession(output_name="report", output_source_name="my_source"
126
126
  # Get the path of file generated by preceding output process
127
127
  path = session.file_path()
128
128
 
129
- # Retrieve connection parameters for the current output
130
- connection_parameters = session.connection_parameters()
129
+ # Read a Custom connection created with Direction = Output (exact connection name).
130
+ # Private parameters are decrypted; do not print or log the returned values.
131
+ connection_parameters = session.connection_parameters("notification_service")
132
+ public_parameters = connection_parameters["public_connection_parameters"]
133
+ private_parameters = connection_parameters["private_connection_parameters"]
131
134
 
132
135
  # Retrieve custom parameters for the current source
133
136
  custom_parameters = session.custom_parameters()
@@ -114,8 +114,11 @@ session = PostOutputSession(output_name="report", output_source_name="my_source"
114
114
  # Get the path of file generated by preceding output process
115
115
  path = session.file_path()
116
116
 
117
- # Retrieve connection parameters for the current output
118
- connection_parameters = session.connection_parameters()
117
+ # Read a Custom connection created with Direction = Output (exact connection name).
118
+ # Private parameters are decrypted; do not print or log the returned values.
119
+ connection_parameters = session.connection_parameters("notification_service")
120
+ public_parameters = connection_parameters["public_connection_parameters"]
121
+ private_parameters = connection_parameters["private_connection_parameters"]
119
122
 
120
123
  # Retrieve custom parameters for the current source
121
124
  custom_parameters = session.custom_parameters()
@@ -2,5 +2,5 @@ from .post_output_session import PostOutputSession
2
2
  from .ingestion_session import IngestionSession
3
3
  from .parsing_session import ParsingSession
4
4
 
5
- __version__ = "11.0.0rc65"
5
+ __version__ = "11.0.0-rc.67"
6
6
  __all__ = ['PostOutputSession','IngestionSession', 'ParsingSession']
@@ -132,7 +132,9 @@ class _Base_Session:
132
132
  Returns:
133
133
  dict: Dictionary containing private and public connection parameters.
134
134
  """
135
- connection_id = self.process.connectionId
135
+ return self._read_connection_parameters(self.process.connectionId)
136
+
137
+ def _read_connection_parameters(self, connection_id: int):
136
138
  self.log(f"Reading connection parameters of connection id {connection_id}")
137
139
  params = self._core_api_call(f"core/connection/{connection_id}").get("parameters")
138
140
  return {"private_connection_parameters": params.get("private_connection_parameters"),
@@ -32,6 +32,24 @@ class PostOutputSession(_Session):
32
32
  """Returns file path for the file generated by output process"""
33
33
  return self.process.parameters.get("file_path")
34
34
 
35
+ def connection_parameters(self, connection_name: str | None = None):
36
+ """Read decrypted public/private parameters of a named output Custom connection.
37
+
38
+ Names match exactly across the environment. Call while the session is open.
39
+ Omitting the name retains the existing process-connection lookup.
40
+ """
41
+ if connection_name is None:
42
+ return super().connection_parameters()
43
+ if not self._is_open:
44
+ raise RuntimeError("Session is closed")
45
+ if not isinstance(connection_name, str) or not connection_name.strip():
46
+ raise ValueError("connection_name must be a non-empty output Custom connection name")
47
+ connection_id = self._pg.sql(
48
+ "select sparky.sdk_get_output_connection_id(%s, %s)",
49
+ (self.process.processId, connection_name),
50
+ )
51
+ return self._read_connection_parameters(connection_id)
52
+
35
53
  def enqueue_ingestion(self, source_name: str) -> dict:
36
54
  """Immediately enqueue a separate custom ingestion in this session's project.
37
55
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dataforge-sdk
3
- Version: 11.0.0rc65
3
+ Version: 11.0.0rc67
4
4
  Summary: SDK for creating DataForge extensions
5
5
  Author-email: Vadim Orlov <vorlov@dataforgelabs.com>
6
6
  Project-URL: Homepage, https://docs.dataforgelabs.com
@@ -126,8 +126,11 @@ session = PostOutputSession(output_name="report", output_source_name="my_source"
126
126
  # Get the path of file generated by preceding output process
127
127
  path = session.file_path()
128
128
 
129
- # Retrieve connection parameters for the current output
130
- connection_parameters = session.connection_parameters()
129
+ # Read a Custom connection created with Direction = Output (exact connection name).
130
+ # Private parameters are decrypted; do not print or log the returned values.
131
+ connection_parameters = session.connection_parameters("notification_service")
132
+ public_parameters = connection_parameters["public_connection_parameters"]
133
+ private_parameters = connection_parameters["private_connection_parameters"]
131
134
 
132
135
  # Retrieve custom parameters for the current source
133
136
  custom_parameters = session.custom_parameters()
@@ -6,7 +6,7 @@ requires = [
6
6
  build-backend = "setuptools.build_meta"
7
7
  [project]
8
8
  name = "dataforge-sdk"
9
- version = "11.0.0rc65"
9
+ version = "11.0.0-rc.67"
10
10
  authors = [
11
11
  {name="Vadim Orlov", email="vorlov@dataforgelabs.com"}
12
12
  ]
@@ -21,9 +21,48 @@ def load_session():
21
21
  return module.PostOutputSession
22
22
 
23
23
 
24
+ def load_base_session():
25
+ root = Path(__file__).resolve().parents[1] / "dataforge"
26
+ package = ModuleType("dataforge")
27
+ package.__path__ = [str(root)]
28
+ modules = {package.__name__: package, "requests": Mock()}
29
+ for suffix, attribute in (("system_configuration", "SystemConfiguration"),
30
+ ("postgres_connection", "PostgresConnection"),
31
+ ("utils", "_setup_logger")):
32
+ dependency = ModuleType(f"{package.__name__}.{suffix}")
33
+ setattr(dependency, attribute, Mock())
34
+ modules[dependency.__name__] = dependency
35
+ name = f"{package.__name__}._base_session"
36
+ spec = importlib.util.spec_from_file_location(name, root / "_base_session.py")
37
+ module = importlib.util.module_from_spec(spec)
38
+ with patch.dict(sys.modules, modules), patch("importlib.metadata.version", return_value="test"):
39
+ spec.loader.exec_module(module)
40
+ return module._Base_Session
41
+
42
+
43
+ BaseSession = load_base_session()
24
44
  PostOutputSession = load_session()
25
45
 
26
46
 
47
+ class ConnectionParameterReadTests(unittest.TestCase):
48
+ def test_resolved_id_uses_core_and_returns_both_parameter_objects(self):
49
+ session = BaseSession.__new__(BaseSession)
50
+ session.log = Mock()
51
+ parameters = {"public_connection_parameters": {"region": "west"},
52
+ "private_connection_parameters": {"key": "test-value"}}
53
+ session._core_api_call = Mock(return_value={"parameters": parameters})
54
+ self.assertEqual(session._read_connection_parameters(17), parameters)
55
+ session._core_api_call.assert_called_once_with("core/connection/17")
56
+ session.log.assert_called_once_with("Reading connection parameters of connection id 17")
57
+
58
+ def test_existing_no_name_accessor_keeps_process_connection(self):
59
+ session = BaseSession.__new__(BaseSession)
60
+ session.process = SimpleNamespace(connectionId=19)
61
+ session._read_connection_parameters = Mock(return_value={"result": True})
62
+ self.assertEqual(session.connection_parameters(), {"result": True})
63
+ session._read_connection_parameters.assert_called_once_with(19)
64
+
65
+
27
66
  class PostOutputEnqueueTests(unittest.TestCase):
28
67
  def setUp(self):
29
68
  self.session = PostOutputSession.__new__(PostOutputSession)
@@ -35,6 +74,39 @@ class PostOutputEnqueueTests(unittest.TestCase):
35
74
  self.session._end_process = Mock()
36
75
  self.session._log_fail = Mock()
37
76
 
77
+ def test_named_connection_uses_resolved_id_without_changing_process(self):
78
+ name = "notification's service / 東京; --"
79
+ parameters = {"public_connection_parameters": {"region": "west"},
80
+ "private_connection_parameters": {"key": "test-value"}}
81
+ self.session._pg.sql.return_value = 17
82
+ self.session._read_connection_parameters = Mock(return_value=parameters)
83
+ self.assertEqual(self.session.connection_parameters(name), parameters)
84
+ self.session._pg.sql.assert_called_once_with(
85
+ "select sparky.sdk_get_output_connection_id(%s, %s)", (41, name)
86
+ )
87
+ self.session._read_connection_parameters.assert_called_once_with(17)
88
+ self.assertEqual(self.session.process.processId, 41)
89
+ self.session.log.assert_not_called()
90
+
91
+ def test_invalid_connection_names_do_not_lookup(self):
92
+ for name in ("", " \t\n", 123):
93
+ with self.subTest(name=name), self.assertRaises(ValueError):
94
+ self.session.connection_parameters(name)
95
+ self.session._pg.sql.assert_not_called()
96
+
97
+ def test_closed_connection_lookup_is_rejected(self):
98
+ self.session._is_open = False
99
+ with self.assertRaisesRegex(RuntimeError, "Session is closed"):
100
+ self.session.connection_parameters("notification_service")
101
+ self.session._pg.sql.assert_not_called()
102
+
103
+ def test_missing_connection_does_not_fetch_credentials(self):
104
+ self.session._pg.sql.side_effect = RuntimeError("Active output Custom connection was not found")
105
+ self.session._read_connection_parameters = Mock()
106
+ with self.assertRaisesRegex(RuntimeError, "not found"):
107
+ self.session.connection_parameters("missing")
108
+ self.session._read_connection_parameters.assert_not_called()
109
+
38
110
  def test_name_is_bound_as_data_and_receipt_is_returned(self):
39
111
  name = "destination's source; --"
40
112
  receipt = {"status": "queued", "ingestion_queue_id": 100, "process_id": 41, "source_id": 8}