dataforge-sdk 10.3.1__tar.gz → 11.0.1rc1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/PKG-INFO +60 -4
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/README.md +58 -2
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/__init__.py +1 -1
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/_base_session.py +28 -11
- dataforge_sdk-11.0.1rc1/dataforge/databricks/_databricks_base_session.py +64 -0
- dataforge_sdk-11.0.1rc1/dataforge/databricks/_databricks_ingestion_session.py +64 -0
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/databricks/_databricks_parsing_session.py +6 -13
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/databricks/_databricks_pg.py +3 -4
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/ingestion_session.py +6 -3
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/pg.py +3 -0
- dataforge_sdk-11.0.1rc1/dataforge/post_output_session.py +106 -0
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/snowflake/_snowflake_base_session.py +7 -3
- dataforge_sdk-11.0.1rc1/dataforge/snowflake/_snowflake_ingestion_session.py +70 -0
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/snowflake/_snowflake_parsing_session.py +5 -1
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge_sdk.egg-info/PKG-INFO +60 -4
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge_sdk.egg-info/SOURCES.txt +3 -1
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/pyproject.toml +3 -3
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/setup.cfg +2 -2
- dataforge_sdk-11.0.1rc1/tests/test_error_logging.py +121 -0
- dataforge_sdk-11.0.1rc1/tests/test_post_output_enqueue.py +172 -0
- dataforge_sdk-10.3.1/dataforge/databricks/_databricks_base_session.py +0 -146
- dataforge_sdk-10.3.1/dataforge/databricks/_databricks_ingestion_session.py +0 -70
- dataforge_sdk-10.3.1/dataforge/post_output_session.py +0 -59
- dataforge_sdk-10.3.1/dataforge/snowflake/_snowflake_ingestion_session.py +0 -39
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/_session.py +0 -0
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/databricks/__init__.py +0 -0
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/parsing_session.py +0 -0
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/postgres_connection.py +0 -0
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/process_record.py +0 -0
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/snowflake/__init__.py +0 -0
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/system_configuration.py +0 -0
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/utils.py +0 -0
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge_sdk.egg-info/dependency_links.txt +0 -0
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge_sdk.egg-info/requires.txt +0 -0
- {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge_sdk.egg-info/top_level.txt +0 -0
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dataforge-sdk
|
|
3
|
-
Version:
|
|
3
|
+
Version: 11.0.1rc1
|
|
4
4
|
Summary: SDK for creating DataForge extensions
|
|
5
5
|
Author-email: Vadim Orlov <vorlov@dataforgelabs.com>
|
|
6
6
|
Project-URL: Homepage, https://docs.dataforgelabs.com
|
|
7
7
|
Project-URL: Issues, https://docs.dataforgelabs.com/hc/en-us/requests/new
|
|
8
|
-
Requires-Python: >=3.
|
|
8
|
+
Requires-Python: >=3.12
|
|
9
9
|
Description-Content-Type: text/markdown
|
|
10
10
|
Provides-Extra: psycopg2
|
|
11
11
|
Requires-Dist: psycopg2-binary>=2.9; extra == "psycopg2"
|
|
@@ -53,6 +53,11 @@ session.ingest(lambda: spark.read.csv("s3://bucket/path/input.csv"))
|
|
|
53
53
|
df = spark.read.csv("s3://bucket/path/input.csv")
|
|
54
54
|
session.ingest(df)
|
|
55
55
|
|
|
56
|
+
# Optional complete key snapshot for incremental custom ingestion.
|
|
57
|
+
# When omitted, df is treated as the complete current dataset for delete tracking.
|
|
58
|
+
# It must contain exactly the configured key columns and at least one row.
|
|
59
|
+
session.ingest(df, all_keys_df=all_current_keys_df)
|
|
60
|
+
|
|
56
61
|
# ingest empty dataframe to create 0-record input
|
|
57
62
|
session.ingest()
|
|
58
63
|
|
|
@@ -121,8 +126,11 @@ session = PostOutputSession(output_name="report", output_source_name="my_source"
|
|
|
121
126
|
# Get the path of file generated by preceding output process
|
|
122
127
|
path = session.file_path()
|
|
123
128
|
|
|
124
|
-
#
|
|
125
|
-
|
|
129
|
+
# Read a Custom connection created with Direction = Output (exact connection name).
|
|
130
|
+
# Private parameters are decrypted; do not print or log the returned values.
|
|
131
|
+
connection_parameters = session.connection_parameters("notification_service")
|
|
132
|
+
public_parameters = connection_parameters["public_connection_parameters"]
|
|
133
|
+
private_parameters = connection_parameters["private_connection_parameters"]
|
|
126
134
|
|
|
127
135
|
# Retrieve custom parameters for the current source
|
|
128
136
|
custom_parameters = session.custom_parameters()
|
|
@@ -136,3 +144,51 @@ session.run(lambda: print(f"Uploading file from {path}"))
|
|
|
136
144
|
# Fail the process with error message
|
|
137
145
|
session.fail("Error message")
|
|
138
146
|
```
|
|
147
|
+
|
|
148
|
+
### Enqueue a separate custom ingestion
|
|
149
|
+
|
|
150
|
+
Configure the destination as an active **batch custom ingestion** source with its
|
|
151
|
+
own notebook. In the orchestrated post-output notebook:
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
from dataforge import PostOutputSession
|
|
155
|
+
|
|
156
|
+
session = PostOutputSession()
|
|
157
|
+
|
|
158
|
+
def post_output():
|
|
159
|
+
# Complete any writes needed by the destination before enqueueing.
|
|
160
|
+
request = session.enqueue_ingestion("downstream_source")
|
|
161
|
+
# {"status": "queued", "ingestion_queue_id": ..., "process_id": ..., "source_id": ...}
|
|
162
|
+
|
|
163
|
+
session.run(post_output)
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
The destination notebook runs independently with `IngestionSession()` and reads
|
|
167
|
+
its own configured data. No nested ingestion session, project name, DataFrame,
|
|
168
|
+
output view parameters, or snapshot is passed by `enqueue_ingestion`.
|
|
169
|
+
|
|
170
|
+
The source name is resolved in the post-output session's project (the same
|
|
171
|
+
project as `session.process.parameters['project_name']`), without a fallback to
|
|
172
|
+
another project. Missing, inactive, non-custom, non-batch, or unconfigured
|
|
173
|
+
destinations raise an error. Call before the session closes, normally inside
|
|
174
|
+
`run`'s callback.
|
|
175
|
+
|
|
176
|
+
Every call immediately creates a queue record. Ingestion may start before
|
|
177
|
+
post-output finishes, and later post-output failure or cancellation does not
|
|
178
|
+
cancel the request. Repeated calls and notebook retries create additional queue
|
|
179
|
+
records. Returning from this method means acceptance, not that ingestion has
|
|
180
|
+
started or finished. Neither notebook waits for the other to finish. Complete
|
|
181
|
+
the destination's data writes before calling `enqueue_ingestion`.
|
|
182
|
+
|
|
183
|
+
Queued requests wait until source, project, and environment initiation
|
|
184
|
+
are enabled. They survive schedule reloads/restarts and never clear disable flags.
|
|
185
|
+
If a destination is deactivated or loses its custom notebook configuration while
|
|
186
|
+
waiting, it remains pending until that configuration is restored. Deleting the
|
|
187
|
+
destination cancels its pending requests. Queue history records the originating
|
|
188
|
+
process as `post_output:<process_id>`.
|
|
189
|
+
|
|
190
|
+
This uses the scheduler's **refresh-latest** behavior: pending requests for a
|
|
191
|
+
destination can be served by one ingestion. It does not promise one ingestion per
|
|
192
|
+
output event or preserve an output snapshot. Use durable, independently readable
|
|
193
|
+
data and handle incremental progress in the destination notebook. Existing
|
|
194
|
+
`pg.pull` retains its manual Pull Now behavior.
|
|
@@ -41,6 +41,11 @@ session.ingest(lambda: spark.read.csv("s3://bucket/path/input.csv"))
|
|
|
41
41
|
df = spark.read.csv("s3://bucket/path/input.csv")
|
|
42
42
|
session.ingest(df)
|
|
43
43
|
|
|
44
|
+
# Optional complete key snapshot for incremental custom ingestion.
|
|
45
|
+
# When omitted, df is treated as the complete current dataset for delete tracking.
|
|
46
|
+
# It must contain exactly the configured key columns and at least one row.
|
|
47
|
+
session.ingest(df, all_keys_df=all_current_keys_df)
|
|
48
|
+
|
|
44
49
|
# ingest empty dataframe to create 0-record input
|
|
45
50
|
session.ingest()
|
|
46
51
|
|
|
@@ -109,8 +114,11 @@ session = PostOutputSession(output_name="report", output_source_name="my_source"
|
|
|
109
114
|
# Get the path of file generated by preceding output process
|
|
110
115
|
path = session.file_path()
|
|
111
116
|
|
|
112
|
-
#
|
|
113
|
-
|
|
117
|
+
# Read a Custom connection created with Direction = Output (exact connection name).
|
|
118
|
+
# Private parameters are decrypted; do not print or log the returned values.
|
|
119
|
+
connection_parameters = session.connection_parameters("notification_service")
|
|
120
|
+
public_parameters = connection_parameters["public_connection_parameters"]
|
|
121
|
+
private_parameters = connection_parameters["private_connection_parameters"]
|
|
114
122
|
|
|
115
123
|
# Retrieve custom parameters for the current source
|
|
116
124
|
custom_parameters = session.custom_parameters()
|
|
@@ -124,3 +132,51 @@ session.run(lambda: print(f"Uploading file from {path}"))
|
|
|
124
132
|
# Fail the process with error message
|
|
125
133
|
session.fail("Error message")
|
|
126
134
|
```
|
|
135
|
+
|
|
136
|
+
### Enqueue a separate custom ingestion
|
|
137
|
+
|
|
138
|
+
Configure the destination as an active **batch custom ingestion** source with its
|
|
139
|
+
own notebook. In the orchestrated post-output notebook:
|
|
140
|
+
|
|
141
|
+
```python
|
|
142
|
+
from dataforge import PostOutputSession
|
|
143
|
+
|
|
144
|
+
session = PostOutputSession()
|
|
145
|
+
|
|
146
|
+
def post_output():
|
|
147
|
+
# Complete any writes needed by the destination before enqueueing.
|
|
148
|
+
request = session.enqueue_ingestion("downstream_source")
|
|
149
|
+
# {"status": "queued", "ingestion_queue_id": ..., "process_id": ..., "source_id": ...}
|
|
150
|
+
|
|
151
|
+
session.run(post_output)
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
The destination notebook runs independently with `IngestionSession()` and reads
|
|
155
|
+
its own configured data. No nested ingestion session, project name, DataFrame,
|
|
156
|
+
output view parameters, or snapshot is passed by `enqueue_ingestion`.
|
|
157
|
+
|
|
158
|
+
The source name is resolved in the post-output session's project (the same
|
|
159
|
+
project as `session.process.parameters['project_name']`), without a fallback to
|
|
160
|
+
another project. Missing, inactive, non-custom, non-batch, or unconfigured
|
|
161
|
+
destinations raise an error. Call before the session closes, normally inside
|
|
162
|
+
`run`'s callback.
|
|
163
|
+
|
|
164
|
+
Every call immediately creates a queue record. Ingestion may start before
|
|
165
|
+
post-output finishes, and later post-output failure or cancellation does not
|
|
166
|
+
cancel the request. Repeated calls and notebook retries create additional queue
|
|
167
|
+
records. Returning from this method means acceptance, not that ingestion has
|
|
168
|
+
started or finished. Neither notebook waits for the other to finish. Complete
|
|
169
|
+
the destination's data writes before calling `enqueue_ingestion`.
|
|
170
|
+
|
|
171
|
+
Queued requests wait until source, project, and environment initiation
|
|
172
|
+
are enabled. They survive schedule reloads/restarts and never clear disable flags.
|
|
173
|
+
If a destination is deactivated or loses its custom notebook configuration while
|
|
174
|
+
waiting, it remains pending until that configuration is restored. Deleting the
|
|
175
|
+
destination cancels its pending requests. Queue history records the originating
|
|
176
|
+
process as `post_output:<process_id>`.
|
|
177
|
+
|
|
178
|
+
This uses the scheduler's **refresh-latest** behavior: pending requests for a
|
|
179
|
+
destination can be served by one ingestion. It does not promise one ingestion per
|
|
180
|
+
output event or preserve an output snapshot. Use durable, independently readable
|
|
181
|
+
data and handle incremental progress in the destination notebook. Existing
|
|
182
|
+
`pg.pull` retains its manual Pull Now behavior.
|
|
@@ -2,5 +2,5 @@ from .post_output_session import PostOutputSession
|
|
|
2
2
|
from .ingestion_session import IngestionSession
|
|
3
3
|
from .parsing_session import ParsingSession
|
|
4
4
|
|
|
5
|
-
__version__ = "
|
|
5
|
+
__version__ = "11.0.1-rc.1"
|
|
6
6
|
__all__ = ['PostOutputSession','IngestionSession', 'ParsingSession']
|
|
@@ -9,6 +9,7 @@ Classes:
|
|
|
9
9
|
"""
|
|
10
10
|
import json
|
|
11
11
|
import traceback
|
|
12
|
+
from datetime import datetime, timezone
|
|
12
13
|
from typing import Optional, Literal
|
|
13
14
|
import requests
|
|
14
15
|
import logging
|
|
@@ -120,7 +121,9 @@ class _Base_Session:
|
|
|
120
121
|
case "E":
|
|
121
122
|
self.logger.error(message)
|
|
122
123
|
payload = {"log_id": self.process.logId, "operation_type": self.process.operationType,
|
|
123
|
-
"severity": severity, "message": message
|
|
124
|
+
"severity": severity, "message": message,
|
|
125
|
+
"insert_datetime": datetime.now(timezone.utc).isoformat(),
|
|
126
|
+
"job_run_id": getattr(self, "process_parameters", {}).get("job_run_id")}
|
|
124
127
|
self._pg.sql("SELECT sparky.write_log(%s)", [json.dumps(payload)], fetch=False)
|
|
125
128
|
|
|
126
129
|
def connection_parameters(self):
|
|
@@ -129,7 +132,9 @@ class _Base_Session:
|
|
|
129
132
|
Returns:
|
|
130
133
|
dict: Dictionary containing private and public connection parameters.
|
|
131
134
|
"""
|
|
132
|
-
|
|
135
|
+
return self._read_connection_parameters(self.process.connectionId)
|
|
136
|
+
|
|
137
|
+
def _read_connection_parameters(self, connection_id: int):
|
|
133
138
|
self.log(f"Reading connection parameters of connection id {connection_id}")
|
|
134
139
|
params = self._core_api_call(f"core/connection/{connection_id}").get("parameters")
|
|
135
140
|
return {"private_connection_parameters": params.get("private_connection_parameters"),
|
|
@@ -157,16 +162,30 @@ class _Base_Session:
|
|
|
157
162
|
message (str): The log message.
|
|
158
163
|
"""
|
|
159
164
|
self.log(f"Process failed with error: {message}", "E")
|
|
160
|
-
self.
|
|
165
|
+
if self.process.startProcessFlag:
|
|
166
|
+
self._end_process("F")
|
|
167
|
+
else:
|
|
168
|
+
self.close()
|
|
169
|
+
raise RuntimeError(message)
|
|
161
170
|
|
|
162
171
|
def _log_fail(self, e: Exception):
|
|
163
|
-
"""
|
|
172
|
+
"""Keep the full driver trace and persist caught errors for scheduled runs."""
|
|
173
|
+
traceback.print_exception(e, chain=True)
|
|
174
|
+
try:
|
|
175
|
+
if self.process.startProcessFlag:
|
|
176
|
+
return
|
|
177
|
+
message = "SDK caught exception:\n" + "".join(traceback.format_exception(e, chain=True))
|
|
178
|
+
limit = 32000
|
|
179
|
+
marker = "\n... [truncated; full traceback in notebook/driver logs] ...\n"
|
|
180
|
+
if len(message) > limit:
|
|
181
|
+
head = (limit - len(marker)) // 2
|
|
182
|
+
tail = limit - len(marker) - head
|
|
183
|
+
message = message[:head] + marker + message[-tail:]
|
|
184
|
+
self.log(message, "E")
|
|
185
|
+
except Exception:
|
|
186
|
+
# Diagnostics must not prevent the caller from rethrowing the failure.
|
|
187
|
+
traceback.print_exc()
|
|
164
188
|
|
|
165
|
-
Args:
|
|
166
|
-
e (Exception): The exception that caused the failure.
|
|
167
|
-
"""
|
|
168
|
-
traceback.print_exception(e)
|
|
169
|
-
self.log(f"Process failed with error: {e}", "E")
|
|
170
189
|
|
|
171
190
|
def close(self):
|
|
172
191
|
"""Close the session."""
|
|
@@ -186,5 +205,3 @@ class _Base_Session:
|
|
|
186
205
|
def system_configuration(self) -> SystemConfiguration:
|
|
187
206
|
"""Public system configuration for the current session."""
|
|
188
207
|
return self._systemConfiguration
|
|
189
|
-
|
|
190
|
-
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
from dataforge._base_session import _Base_Session
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class _Databricks_Base_Session(_Base_Session):
|
|
5
|
+
"""Base session class for Databricks platform.
|
|
6
|
+
Class should not be instantiated by user directly: use process-specific Session classes instead
|
|
7
|
+
Adds Spark session, DBUtilsto Base_Session
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def __init__(self):
|
|
12
|
+
from pyspark.sql import SparkSession
|
|
13
|
+
self.spark = SparkSession.builder.getOrCreate()
|
|
14
|
+
self.dbutils = self._get_dbutils()
|
|
15
|
+
pg_connection_string_read = self.dbutils.secrets.get("sparky", "pg_read")
|
|
16
|
+
core_jwt_token = self.dbutils.secrets.get("sparky", "coreJWT")
|
|
17
|
+
try:
|
|
18
|
+
process_id = self.dbutils.widgets.get("process_id")
|
|
19
|
+
except Exception:
|
|
20
|
+
process_id = None
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
super().__init__(pg_connection_string_read, core_jwt_token, process_id)
|
|
24
|
+
try:
|
|
25
|
+
self.process_parameters["job_run_id"] = self.dbutils.widgets.get("job_run_id")
|
|
26
|
+
except Exception:
|
|
27
|
+
pass
|
|
28
|
+
self.process_parameters["start_process_flag"] = process_id is None
|
|
29
|
+
self.logger.info(f"Initialized databricks base session for {self.__class__.__name__} with parameters {self.process_parameters}")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _get_dbutils(self):
|
|
33
|
+
from pyspark.dbutils import DBUtils
|
|
34
|
+
return DBUtils(self.spark)
|
|
35
|
+
|
|
36
|
+
def _write_input_table(self, in_df):
|
|
37
|
+
"""Publish notebook output directly to the raw-table handoff."""
|
|
38
|
+
from pyspark.sql.types import StructField, StructType, StringType
|
|
39
|
+
|
|
40
|
+
if not self._is_open:
|
|
41
|
+
raise Exception("Session is closed")
|
|
42
|
+
result_df = in_df() if callable(in_df) else in_df
|
|
43
|
+
if result_df is None:
|
|
44
|
+
result_df = self.spark.createDataFrame([], StructType([StructField("id", StringType())]))
|
|
45
|
+
table = f"{self._systemConfiguration.dataLakeDbName}.ORE.RAW_INPUT_{self.process.inputId}"
|
|
46
|
+
self.log(f"Writing dataframe to Delta table {table}")
|
|
47
|
+
# Delta overwrite commits atomically; failed writes propagate to Sparky.
|
|
48
|
+
result_df.write.format("delta").mode("overwrite").option("overwriteSchema", "true").option("delta.columnMapping.mode", "name").saveAsTable(table)
|
|
49
|
+
self.log(f"Table {table} written")
|
|
50
|
+
|
|
51
|
+
def _release_manual_process(self, key_source_mode=None):
|
|
52
|
+
self._pg.sql(
|
|
53
|
+
"SELECT sparky.sdk_complete_manual_process(%s, %s, %s)",
|
|
54
|
+
[self.process.processId, key_source_mode, self.process.startProcessFlag],
|
|
55
|
+
fetch=False,
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
def _fail_manual_process(self, error):
|
|
59
|
+
if self.process.startProcessFlag:
|
|
60
|
+
self._pg.sql(
|
|
61
|
+
"SELECT sparky.sdk_fail_manual_process(%s, %s)",
|
|
62
|
+
[self.process.processId, str(error)],
|
|
63
|
+
fetch=False,
|
|
64
|
+
)
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
import json
|
|
3
|
+
import re
|
|
4
|
+
from typing import Callable
|
|
5
|
+
from dataforge.databricks._databricks_base_session import _Databricks_Base_Session
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class _Databricks_Ingestion_Session(_Databricks_Base_Session):
|
|
9
|
+
"""Base session class for Databricks platform.
|
|
10
|
+
Class should not be instantiated by user directly: use process-specific Session classes instead
|
|
11
|
+
Adds Spark session, DBUtilsto Base_Session
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
def __init__(self):
|
|
15
|
+
super().__init__()
|
|
16
|
+
|
|
17
|
+
def ingest(self,df: pyspark.sql.DataFrame | Callable[[], pyspark.sql.DataFrame] | None = None,
|
|
18
|
+
all_keys_df: pyspark.sql.DataFrame | Callable[[], pyspark.sql.DataFrame] | None = None):
|
|
19
|
+
"""Ingest the provided DataFrame into the DataForge and update input record.
|
|
20
|
+
|
|
21
|
+
Writes the DataFrame to raw Parquet file,
|
|
22
|
+
updates the input record with status, file size, record count, and notifies
|
|
23
|
+
the Core API of process completion. On failure, updates logs and flags the input and process
|
|
24
|
+
records as failed.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
df (Callable[[], DataFrame] | DataFrame): parameterless def that you defined, returning the Spark DataFrame to ingest (recommended),
|
|
28
|
+
or spark DataFrame
|
|
29
|
+
"""
|
|
30
|
+
try:
|
|
31
|
+
if not self._is_open:
|
|
32
|
+
raise Exception("Session is closed")
|
|
33
|
+
self._write_input_table(df)
|
|
34
|
+
track_deletes = self.process.parameters.get("track_deletes", False)
|
|
35
|
+
key_source_mode = "main" if track_deletes else None
|
|
36
|
+
if track_deletes and all_keys_df is not None:
|
|
37
|
+
keys_df = (all_keys_df() if callable(all_keys_df) else all_keys_df).cache()
|
|
38
|
+
try:
|
|
39
|
+
if len(keys_df.take(1)) == 0:
|
|
40
|
+
raise Exception("all_keys_df returned zero rows. Delete tracking was stopped to protect existing data.")
|
|
41
|
+
key_columns = self.process.parameters.get("key_columns", [])
|
|
42
|
+
if len(keys_df.columns) != len(key_columns):
|
|
43
|
+
raise Exception("all_keys_df must contain exactly the configured key columns.")
|
|
44
|
+
from functools import reduce
|
|
45
|
+
from pyspark.sql.functions import col
|
|
46
|
+
if len(keys_df.filter(reduce(
|
|
47
|
+
lambda left, right: left | right,
|
|
48
|
+
[col(name).isNull() for name in key_columns]
|
|
49
|
+
)).take(1)) > 0:
|
|
50
|
+
raise Exception("all_keys_df contains a null key.")
|
|
51
|
+
keys_path = f"{self._systemConfiguration.datalakePath}/source_{self.process.sourceId}/keys/keys_input_{self.process.inputId}"
|
|
52
|
+
keys_df.write.format("parquet").mode("overwrite").save(keys_path)
|
|
53
|
+
finally:
|
|
54
|
+
keys_df.unpersist()
|
|
55
|
+
key_source_mode = "all_keys"
|
|
56
|
+
self._release_manual_process(key_source_mode)
|
|
57
|
+
self.logger.info("Ingestion completed successfully")
|
|
58
|
+
|
|
59
|
+
except Exception as e:
|
|
60
|
+
self._log_fail(e)
|
|
61
|
+
self._fail_manual_process(e)
|
|
62
|
+
raise
|
|
63
|
+
finally:
|
|
64
|
+
self.close()
|
{dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/databricks/_databricks_parsing_session.py
RENAMED
|
@@ -28,19 +28,12 @@ class _Databricks_Parsing_Session(_Databricks_Base_Session):
|
|
|
28
28
|
else:
|
|
29
29
|
result_df = df
|
|
30
30
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
else:
|
|
34
|
-
dest_file_path = f"{self._systemConfiguration.datalakePath}/source_{self.process.sourceId}/parsed/parsed_input_{self.process.inputId}"
|
|
35
|
-
file_size, row_count = self._write_parsed_data(result_df, dest_file_path)
|
|
36
|
-
input_update_json = {
|
|
37
|
-
"file_size": file_size,
|
|
38
|
-
"input_id": self.process.inputId,
|
|
39
|
-
"record_counts": {"Total": row_count}
|
|
40
|
-
}
|
|
41
|
-
self._end_process('P' if row_count > 0 else 'Z', input_update_json)
|
|
31
|
+
self._write_input_table(result_df)
|
|
32
|
+
self._release_manual_process()
|
|
42
33
|
|
|
43
34
|
except Exception as e:
|
|
44
35
|
self._log_fail(e)
|
|
45
|
-
self.
|
|
46
|
-
|
|
36
|
+
self._fail_manual_process(e)
|
|
37
|
+
raise
|
|
38
|
+
finally:
|
|
39
|
+
self.close()
|
|
@@ -13,11 +13,10 @@ class DataBricksPg:
|
|
|
13
13
|
def __init__(self):
|
|
14
14
|
from pyspark.dbutils import DBUtils
|
|
15
15
|
from pyspark.sql import SparkSession, DataFrame
|
|
16
|
-
from .postgres_connection import PostgresConnection
|
|
17
16
|
|
|
18
17
|
self.spark = SparkSession.builder.getOrCreate()
|
|
19
|
-
self.dbutils = DBUtils(spark)
|
|
20
|
-
self.pg_connection_string_read = dbutils.secrets.get("sparky", "pg_read")
|
|
18
|
+
self.dbutils = DBUtils(self.spark)
|
|
19
|
+
self.pg_connection_string_read = self.dbutils.secrets.get("sparky", "pg_read")
|
|
21
20
|
|
|
22
21
|
def update(self,query: str):
|
|
23
22
|
"""Execute an update SQL query on the DataForge metastore Postgres database.
|
|
@@ -92,4 +91,4 @@ class DataBricksPg:
|
|
|
92
91
|
conn_string = self.dbutils.secrets.get("sparky", "pg_write")
|
|
93
92
|
return PostgresConnection(conn_string + "&application_name=sdk-pg")
|
|
94
93
|
else:
|
|
95
|
-
raise Exception("pg_write secret is not defined in sparky scope")
|
|
94
|
+
raise Exception("pg_write secret is not defined in sparky scope")
|
|
@@ -59,6 +59,9 @@ class IngestionSession(_Ingestion_Session):
|
|
|
59
59
|
message (str): Custom error message explaining the failure.
|
|
60
60
|
"""
|
|
61
61
|
self.log(f"Custom Ingestion failed with error: {message}", "E")
|
|
62
|
-
self.
|
|
63
|
-
|
|
64
|
-
|
|
62
|
+
if self.process.startProcessFlag:
|
|
63
|
+
self._pg.sql("select meta.prc_iw_in_update_input_record(%s)",
|
|
64
|
+
[json.dumps({"process_id": self.process.processId, "ingestion_status_code" : "F"})], fetch=False)
|
|
65
|
+
self._core_api_call(f"process-complete/{self.process.processId}")
|
|
66
|
+
self.close()
|
|
67
|
+
raise RuntimeError(message)
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
|
|
2
|
+
import json
|
|
3
|
+
from typing import Optional, Callable
|
|
4
|
+
from .process_record import ProcessRecord
|
|
5
|
+
from ._session import _Session
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class PostOutputSession(_Session):
|
|
9
|
+
|
|
10
|
+
"""Session class to manage custom post-output process lifecycle.
|
|
11
|
+
|
|
12
|
+
Initializes post-output process record and provides run() method to execute custom logic post-output completion,
|
|
13
|
+
handle failures and update metadata.
|
|
14
|
+
"""
|
|
15
|
+
def __init__(self, output_name: Optional[str] = None, output_source_name: Optional[str] = None, project_name: Optional[str] = None):
|
|
16
|
+
"""Initialize custom post-output session and start a new post-output process.
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
Leave blank for production use.
|
|
20
|
+
"""
|
|
21
|
+
super().__init__()
|
|
22
|
+
# initialize process
|
|
23
|
+
if output_name is not None:
|
|
24
|
+
self.process_parameters["output_name"] = output_name
|
|
25
|
+
self.process_parameters["output_source_name"] = output_source_name
|
|
26
|
+
self.process_parameters["project_name"] = project_name
|
|
27
|
+
|
|
28
|
+
process = self._pg.sql("select sparky.sdk_new_post_output(%s)", (json.dumps(self.process_parameters),))
|
|
29
|
+
self.process = ProcessRecord(process)
|
|
30
|
+
|
|
31
|
+
def file_path(self):
|
|
32
|
+
"""Returns file path for the file generated by output process"""
|
|
33
|
+
return self.process.parameters.get("file_path")
|
|
34
|
+
|
|
35
|
+
def connection_parameters(self, connection_name: str | None = None):
|
|
36
|
+
"""Read decrypted public/private parameters of a named output Custom connection.
|
|
37
|
+
|
|
38
|
+
Names match exactly across the environment. Call while the session is open.
|
|
39
|
+
Omitting the name retains the existing process-connection lookup.
|
|
40
|
+
"""
|
|
41
|
+
if connection_name is None:
|
|
42
|
+
return super().connection_parameters()
|
|
43
|
+
if not self._is_open:
|
|
44
|
+
raise RuntimeError("Session is closed")
|
|
45
|
+
if not isinstance(connection_name, str) or not connection_name.strip():
|
|
46
|
+
raise ValueError("connection_name must be a non-empty output Custom connection name")
|
|
47
|
+
connection_id = self._pg.sql(
|
|
48
|
+
"select sparky.sdk_get_output_connection_id(%s, %s)",
|
|
49
|
+
(self.process.processId, connection_name),
|
|
50
|
+
)
|
|
51
|
+
return self._read_connection_parameters(connection_id)
|
|
52
|
+
|
|
53
|
+
def enqueue_ingestion(self, source_name: str) -> dict:
|
|
54
|
+
"""Immediately enqueue a separate custom ingestion in this session's project.
|
|
55
|
+
|
|
56
|
+
Resolve source_name in this session's project. The destination must be an
|
|
57
|
+
active batch custom ingestion source with a configured notebook. Requests
|
|
58
|
+
wait while source, project, or environment initiation is disabled, survive
|
|
59
|
+
schedule reloads, and never enable initiation. Each call creates a queue
|
|
60
|
+
record, independently of post-output success or failure. No data or view
|
|
61
|
+
context is passed. Pending records may be served by one ingestion.
|
|
62
|
+
|
|
63
|
+
Call before run() closes the session, normally inside its callback.
|
|
64
|
+
Returns status='queued', ingestion_queue_id, process_id, and source_id; this is
|
|
65
|
+
acceptance of the request, not completion of downstream ingestion.
|
|
66
|
+
"""
|
|
67
|
+
if not self._is_open:
|
|
68
|
+
raise RuntimeError("Session is closed")
|
|
69
|
+
if not isinstance(source_name, str) or not source_name.strip():
|
|
70
|
+
raise ValueError("source_name must be a non-empty custom ingestion source name")
|
|
71
|
+
return self._pg.sql(
|
|
72
|
+
"select sparky.sdk_enqueue_ingestion(%s, %s)",
|
|
73
|
+
(self.process.processId, source_name),
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
def run(self, po_commands: Callable[[],None]):
|
|
77
|
+
|
|
78
|
+
"""Save parsed file from the provided DataFrame, and upload it into the DataForge data lake.
|
|
79
|
+
|
|
80
|
+
Writes the DataFrame to parsed Parquet file,
|
|
81
|
+
updates the input record with status, file size, record count, and notifies
|
|
82
|
+
the Core API of process completion. On failure, updates logs and flags the input and process
|
|
83
|
+
records as failed.
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
po_commands (po_commands: Callable[[],None]): The custom-defined, parameterless def containing post-output logic.
|
|
87
|
+
Returns None
|
|
88
|
+
"""
|
|
89
|
+
try:
|
|
90
|
+
if not self._is_open:
|
|
91
|
+
raise Exception("Session is closed")
|
|
92
|
+
self.log(f"Running post output def {po_commands.__name__}")
|
|
93
|
+
po_commands()
|
|
94
|
+
self.log(f"Post output execution completed successfully")
|
|
95
|
+
if self.process.startProcessFlag:
|
|
96
|
+
self._end_process("P")
|
|
97
|
+
else:
|
|
98
|
+
self.close()
|
|
99
|
+
|
|
100
|
+
except Exception as e:
|
|
101
|
+
self._log_fail(e)
|
|
102
|
+
if self.process.startProcessFlag:
|
|
103
|
+
self._end_process("F")
|
|
104
|
+
else:
|
|
105
|
+
self.close()
|
|
106
|
+
raise
|
{dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/snowflake/_snowflake_base_session.py
RENAMED
|
@@ -46,9 +46,13 @@ class _Snowflake_Base_Session(_Base_Session):
|
|
|
46
46
|
table_type="transient"
|
|
47
47
|
)
|
|
48
48
|
self.log(f"Table {table} written")
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
49
|
+
|
|
50
|
+
def _complete_ingestion(self, key_source_mode: str | None):
|
|
51
|
+
# Record the key source only after both the data and optional key snapshot
|
|
52
|
+
# exist. Manually started processes are also released to Core here.
|
|
53
|
+
self._pg.sql("SELECT sparky.sdk_complete_manual_process(%s, %s, %s)",
|
|
54
|
+
[self.process.processId, key_source_mode, self.process.startProcessFlag],
|
|
55
|
+
fetch=False)
|
|
52
56
|
|
|
53
57
|
@staticmethod
|
|
54
58
|
def parse_key_value_args():
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
import json
|
|
3
|
+
from typing import Callable
|
|
4
|
+
from dataforge.snowflake._snowflake_base_session import _Snowflake_Base_Session
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class _Snowflake_Ingestion_Session(_Snowflake_Base_Session):
|
|
8
|
+
"""Base ingestion session class for Snowflake platform.
|
|
9
|
+
Class should not be instantiated by user directly: use process-specific Session classes instead
|
|
10
|
+
Adds Snowpark session
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
def __init__(self):
|
|
14
|
+
super().__init__()
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def ingest(self,df: snowflake.snowpark.dataframe.DataFrame | Callable[[], snowflake.snowpark.dataframeDataFrame] | None = None,
|
|
18
|
+
all_keys_df: snowflake.snowpark.dataframe.DataFrame | Callable[[], snowflake.snowpark.dataframeDataFrame] | None = None):
|
|
19
|
+
"""Ingest the provided DataFrame into the DataForge and update input record.
|
|
20
|
+
|
|
21
|
+
Writes the DataFrame to raw Snowflake table
|
|
22
|
+
|
|
23
|
+
Args:
|
|
24
|
+
df (Callable[[], DataFrame] | DataFrame): parameterless def that you defined, returning the Snowpark DataFrame to ingest (recommended),
|
|
25
|
+
or spark DataFrame
|
|
26
|
+
"""
|
|
27
|
+
try:
|
|
28
|
+
track_deletes = self.process.parameters.get("track_deletes", False)
|
|
29
|
+
key_source_mode = (
|
|
30
|
+
"all_keys" if track_deletes and all_keys_df is not None
|
|
31
|
+
else "main" if track_deletes
|
|
32
|
+
else None
|
|
33
|
+
)
|
|
34
|
+
self._write_input_table(df)
|
|
35
|
+
if track_deletes and all_keys_df is not None:
|
|
36
|
+
from snowflake.snowpark.functions import lit
|
|
37
|
+
keys_df = (all_keys_df() if callable(all_keys_df) else all_keys_df).cache_result()
|
|
38
|
+
if keys_df.limit(1).count() == 0:
|
|
39
|
+
raise Exception("all_keys_df returned zero rows. Delete tracking was stopped to protect existing data.")
|
|
40
|
+
key_columns = self.process.parameters.get("key_columns", [])
|
|
41
|
+
if len(keys_df.columns) != len(key_columns):
|
|
42
|
+
raise Exception("all_keys_df must contain exactly the configured key columns.")
|
|
43
|
+
null_predicate = " OR ".join(f"{name} IS NULL" for name in key_columns)
|
|
44
|
+
if keys_df.filter(null_predicate).limit(1).count() > 0:
|
|
45
|
+
raise Exception("all_keys_df contains a null key.")
|
|
46
|
+
stored_keys_df = keys_df.select_expr(
|
|
47
|
+
f"CONCAT_WS('|', {','.join(key_columns)}) AS S_KEY"
|
|
48
|
+
).with_column("S_INPUT_ID", lit(self.process.inputId))
|
|
49
|
+
keys_table = f"{self._systemConfiguration.dataLakeDbName}.ORE._{self.process.sourceId}_KEYS"
|
|
50
|
+
if self.snowpark_session.catalog.table_exists(keys_table):
|
|
51
|
+
self.snowpark_session.sql(
|
|
52
|
+
f"DELETE FROM {keys_table} WHERE S_INPUT_ID = {self.process.inputId}"
|
|
53
|
+
).collect()
|
|
54
|
+
stored_keys_df.write.save_as_table(
|
|
55
|
+
table_name=keys_table, mode="append", table_type="transient", column_order="name"
|
|
56
|
+
)
|
|
57
|
+
self._complete_ingestion(key_source_mode)
|
|
58
|
+
except Exception as e:
|
|
59
|
+
self._log_fail(e)
|
|
60
|
+
if self.process.startProcessFlag:
|
|
61
|
+
# Fail input and process to prevent core from executing it
|
|
62
|
+
failure_update_json = {
|
|
63
|
+
"process_id": self.process.processId,
|
|
64
|
+
"ingestion_status_code": "F"
|
|
65
|
+
}
|
|
66
|
+
self._pg.sql("SELECT meta.prc_iw_in_update_input_record(%s)",
|
|
67
|
+
(json.dumps(failure_update_json),), fetch=False)
|
|
68
|
+
raise
|
|
69
|
+
finally:
|
|
70
|
+
self.close()
|
{dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/snowflake/_snowflake_parsing_session.py
RENAMED
|
@@ -22,9 +22,13 @@ class _Snowflake_Parsing_Session(_Snowflake_Base_Session):
|
|
|
22
22
|
"""
|
|
23
23
|
try:
|
|
24
24
|
self._write_input_table(df)
|
|
25
|
+
self._complete_ingestion(None)
|
|
25
26
|
|
|
26
27
|
except Exception as e:
|
|
27
28
|
self._log_fail(e)
|
|
28
29
|
if self.process.startProcessFlag:
|
|
29
30
|
self._end_process("F")
|
|
30
|
-
|
|
31
|
+
raise
|
|
32
|
+
finally:
|
|
33
|
+
if self._is_open:
|
|
34
|
+
self.close()
|