dataforge-sdk 10.3.1__tar.gz → 11.0.1rc1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/PKG-INFO +60 -4
  2. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/README.md +58 -2
  3. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/__init__.py +1 -1
  4. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/_base_session.py +28 -11
  5. dataforge_sdk-11.0.1rc1/dataforge/databricks/_databricks_base_session.py +64 -0
  6. dataforge_sdk-11.0.1rc1/dataforge/databricks/_databricks_ingestion_session.py +64 -0
  7. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/databricks/_databricks_parsing_session.py +6 -13
  8. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/databricks/_databricks_pg.py +3 -4
  9. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/ingestion_session.py +6 -3
  10. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/pg.py +3 -0
  11. dataforge_sdk-11.0.1rc1/dataforge/post_output_session.py +106 -0
  12. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/snowflake/_snowflake_base_session.py +7 -3
  13. dataforge_sdk-11.0.1rc1/dataforge/snowflake/_snowflake_ingestion_session.py +70 -0
  14. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/snowflake/_snowflake_parsing_session.py +5 -1
  15. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge_sdk.egg-info/PKG-INFO +60 -4
  16. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge_sdk.egg-info/SOURCES.txt +3 -1
  17. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/pyproject.toml +3 -3
  18. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/setup.cfg +2 -2
  19. dataforge_sdk-11.0.1rc1/tests/test_error_logging.py +121 -0
  20. dataforge_sdk-11.0.1rc1/tests/test_post_output_enqueue.py +172 -0
  21. dataforge_sdk-10.3.1/dataforge/databricks/_databricks_base_session.py +0 -146
  22. dataforge_sdk-10.3.1/dataforge/databricks/_databricks_ingestion_session.py +0 -70
  23. dataforge_sdk-10.3.1/dataforge/post_output_session.py +0 -59
  24. dataforge_sdk-10.3.1/dataforge/snowflake/_snowflake_ingestion_session.py +0 -39
  25. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/_session.py +0 -0
  26. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/databricks/__init__.py +0 -0
  27. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/parsing_session.py +0 -0
  28. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/postgres_connection.py +0 -0
  29. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/process_record.py +0 -0
  30. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/snowflake/__init__.py +0 -0
  31. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/system_configuration.py +0 -0
  32. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge/utils.py +0 -0
  33. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge_sdk.egg-info/dependency_links.txt +0 -0
  34. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge_sdk.egg-info/requires.txt +0 -0
  35. {dataforge_sdk-10.3.1 → dataforge_sdk-11.0.1rc1}/dataforge_sdk.egg-info/top_level.txt +0 -0
@@ -1,11 +1,11 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dataforge-sdk
3
- Version: 10.3.1
3
+ Version: 11.0.1rc1
4
4
  Summary: SDK for creating DataForge extensions
5
5
  Author-email: Vadim Orlov <vorlov@dataforgelabs.com>
6
6
  Project-URL: Homepage, https://docs.dataforgelabs.com
7
7
  Project-URL: Issues, https://docs.dataforgelabs.com/hc/en-us/requests/new
8
- Requires-Python: >=3.10
8
+ Requires-Python: >=3.12
9
9
  Description-Content-Type: text/markdown
10
10
  Provides-Extra: psycopg2
11
11
  Requires-Dist: psycopg2-binary>=2.9; extra == "psycopg2"
@@ -53,6 +53,11 @@ session.ingest(lambda: spark.read.csv("s3://bucket/path/input.csv"))
53
53
  df = spark.read.csv("s3://bucket/path/input.csv")
54
54
  session.ingest(df)
55
55
 
56
+ # Optional complete key snapshot for incremental custom ingestion.
57
+ # When omitted, df is treated as the complete current dataset for delete tracking.
58
+ # It must contain exactly the configured key columns and at least one row.
59
+ session.ingest(df, all_keys_df=all_current_keys_df)
60
+
56
61
  # ingest empty dataframe to create 0-record input
57
62
  session.ingest()
58
63
 
@@ -121,8 +126,11 @@ session = PostOutputSession(output_name="report", output_source_name="my_source"
121
126
  # Get the path of file generated by preceding output process
122
127
  path = session.file_path()
123
128
 
124
- # Retrieve connection parameters for the current output
125
- connection_parameters = session.connection_parameters()
129
+ # Read a Custom connection created with Direction = Output (exact connection name).
130
+ # Private parameters are decrypted; do not print or log the returned values.
131
+ connection_parameters = session.connection_parameters("notification_service")
132
+ public_parameters = connection_parameters["public_connection_parameters"]
133
+ private_parameters = connection_parameters["private_connection_parameters"]
126
134
 
127
135
  # Retrieve custom parameters for the current source
128
136
  custom_parameters = session.custom_parameters()
@@ -136,3 +144,51 @@ session.run(lambda: print(f"Uploading file from {path}"))
136
144
  # Fail the process with error message
137
145
  session.fail("Error message")
138
146
  ```
147
+
148
+ ### Enqueue a separate custom ingestion
149
+
150
+ Configure the destination as an active **batch custom ingestion** source with its
151
+ own notebook. In the orchestrated post-output notebook:
152
+
153
+ ```python
154
+ from dataforge import PostOutputSession
155
+
156
+ session = PostOutputSession()
157
+
158
+ def post_output():
159
+ # Complete any writes needed by the destination before enqueueing.
160
+ request = session.enqueue_ingestion("downstream_source")
161
+ # {"status": "queued", "ingestion_queue_id": ..., "process_id": ..., "source_id": ...}
162
+
163
+ session.run(post_output)
164
+ ```
165
+
166
+ The destination notebook runs independently with `IngestionSession()` and reads
167
+ its own configured data. No nested ingestion session, project name, DataFrame,
168
+ output view parameters, or snapshot is passed by `enqueue_ingestion`.
169
+
170
+ The source name is resolved in the post-output session's project (the same
171
+ project as `session.process.parameters['project_name']`), without a fallback to
172
+ another project. Missing, inactive, non-custom, non-batch, or unconfigured
173
+ destinations raise an error. Call before the session closes, normally inside
174
+ `run`'s callback.
175
+
176
+ Every call immediately creates a queue record. Ingestion may start before
177
+ post-output finishes, and later post-output failure or cancellation does not
178
+ cancel the request. Repeated calls and notebook retries create additional queue
179
+ records. Returning from this method means acceptance, not that ingestion has
180
+ started or finished. Neither notebook waits for the other to finish. Complete
181
+ the destination's data writes before calling `enqueue_ingestion`.
182
+
183
+ Queued requests wait until source, project, and environment initiation
184
+ are enabled. They survive schedule reloads/restarts and never clear disable flags.
185
+ If a destination is deactivated or loses its custom notebook configuration while
186
+ waiting, it remains pending until that configuration is restored. Deleting the
187
+ destination cancels its pending requests. Queue history records the originating
188
+ process as `post_output:<process_id>`.
189
+
190
+ This uses the scheduler's **refresh-latest** behavior: pending requests for a
191
+ destination can be served by one ingestion. It does not promise one ingestion per
192
+ output event or preserve an output snapshot. Use durable, independently readable
193
+ data and handle incremental progress in the destination notebook. Existing
194
+ `pg.pull` retains its manual Pull Now behavior.
@@ -41,6 +41,11 @@ session.ingest(lambda: spark.read.csv("s3://bucket/path/input.csv"))
41
41
  df = spark.read.csv("s3://bucket/path/input.csv")
42
42
  session.ingest(df)
43
43
 
44
+ # Optional complete key snapshot for incremental custom ingestion.
45
+ # When omitted, df is treated as the complete current dataset for delete tracking.
46
+ # It must contain exactly the configured key columns and at least one row.
47
+ session.ingest(df, all_keys_df=all_current_keys_df)
48
+
44
49
  # ingest empty dataframe to create 0-record input
45
50
  session.ingest()
46
51
 
@@ -109,8 +114,11 @@ session = PostOutputSession(output_name="report", output_source_name="my_source"
109
114
  # Get the path of file generated by preceding output process
110
115
  path = session.file_path()
111
116
 
112
- # Retrieve connection parameters for the current output
113
- connection_parameters = session.connection_parameters()
117
+ # Read a Custom connection created with Direction = Output (exact connection name).
118
+ # Private parameters are decrypted; do not print or log the returned values.
119
+ connection_parameters = session.connection_parameters("notification_service")
120
+ public_parameters = connection_parameters["public_connection_parameters"]
121
+ private_parameters = connection_parameters["private_connection_parameters"]
114
122
 
115
123
  # Retrieve custom parameters for the current source
116
124
  custom_parameters = session.custom_parameters()
@@ -124,3 +132,51 @@ session.run(lambda: print(f"Uploading file from {path}"))
124
132
  # Fail the process with error message
125
133
  session.fail("Error message")
126
134
  ```
135
+
136
+ ### Enqueue a separate custom ingestion
137
+
138
+ Configure the destination as an active **batch custom ingestion** source with its
139
+ own notebook. In the orchestrated post-output notebook:
140
+
141
+ ```python
142
+ from dataforge import PostOutputSession
143
+
144
+ session = PostOutputSession()
145
+
146
+ def post_output():
147
+ # Complete any writes needed by the destination before enqueueing.
148
+ request = session.enqueue_ingestion("downstream_source")
149
+ # {"status": "queued", "ingestion_queue_id": ..., "process_id": ..., "source_id": ...}
150
+
151
+ session.run(post_output)
152
+ ```
153
+
154
+ The destination notebook runs independently with `IngestionSession()` and reads
155
+ its own configured data. No nested ingestion session, project name, DataFrame,
156
+ output view parameters, or snapshot is passed by `enqueue_ingestion`.
157
+
158
+ The source name is resolved in the post-output session's project (the same
159
+ project as `session.process.parameters['project_name']`), without a fallback to
160
+ another project. Missing, inactive, non-custom, non-batch, or unconfigured
161
+ destinations raise an error. Call before the session closes, normally inside
162
+ `run`'s callback.
163
+
164
+ Every call immediately creates a queue record. Ingestion may start before
165
+ post-output finishes, and later post-output failure or cancellation does not
166
+ cancel the request. Repeated calls and notebook retries create additional queue
167
+ records. Returning from this method means acceptance, not that ingestion has
168
+ started or finished. Neither notebook waits for the other to finish. Complete
169
+ the destination's data writes before calling `enqueue_ingestion`.
170
+
171
+ Queued requests wait until source, project, and environment initiation
172
+ are enabled. They survive schedule reloads/restarts and never clear disable flags.
173
+ If a destination is deactivated or loses its custom notebook configuration while
174
+ waiting, it remains pending until that configuration is restored. Deleting the
175
+ destination cancels its pending requests. Queue history records the originating
176
+ process as `post_output:<process_id>`.
177
+
178
+ This uses the scheduler's **refresh-latest** behavior: pending requests for a
179
+ destination can be served by one ingestion. It does not promise one ingestion per
180
+ output event or preserve an output snapshot. Use durable, independently readable
181
+ data and handle incremental progress in the destination notebook. Existing
182
+ `pg.pull` retains its manual Pull Now behavior.
@@ -2,5 +2,5 @@ from .post_output_session import PostOutputSession
2
2
  from .ingestion_session import IngestionSession
3
3
  from .parsing_session import ParsingSession
4
4
 
5
- __version__ = "10.3.1"
5
+ __version__ = "11.0.1-rc.1"
6
6
  __all__ = ['PostOutputSession','IngestionSession', 'ParsingSession']
@@ -9,6 +9,7 @@ Classes:
9
9
  """
10
10
  import json
11
11
  import traceback
12
+ from datetime import datetime, timezone
12
13
  from typing import Optional, Literal
13
14
  import requests
14
15
  import logging
@@ -120,7 +121,9 @@ class _Base_Session:
120
121
  case "E":
121
122
  self.logger.error(message)
122
123
  payload = {"log_id": self.process.logId, "operation_type": self.process.operationType,
123
- "severity": severity, "message": message}
124
+ "severity": severity, "message": message,
125
+ "insert_datetime": datetime.now(timezone.utc).isoformat(),
126
+ "job_run_id": getattr(self, "process_parameters", {}).get("job_run_id")}
124
127
  self._pg.sql("SELECT sparky.write_log(%s)", [json.dumps(payload)], fetch=False)
125
128
 
126
129
  def connection_parameters(self):
@@ -129,7 +132,9 @@ class _Base_Session:
129
132
  Returns:
130
133
  dict: Dictionary containing private and public connection parameters.
131
134
  """
132
- connection_id = self.process.connectionId
135
+ return self._read_connection_parameters(self.process.connectionId)
136
+
137
+ def _read_connection_parameters(self, connection_id: int):
133
138
  self.log(f"Reading connection parameters of connection id {connection_id}")
134
139
  params = self._core_api_call(f"core/connection/{connection_id}").get("parameters")
135
140
  return {"private_connection_parameters": params.get("private_connection_parameters"),
@@ -157,16 +162,30 @@ class _Base_Session:
157
162
  message (str): The log message.
158
163
  """
159
164
  self.log(f"Process failed with error: {message}", "E")
160
- self._end_process("F")
165
+ if self.process.startProcessFlag:
166
+ self._end_process("F")
167
+ else:
168
+ self.close()
169
+ raise RuntimeError(message)
161
170
 
162
171
  def _log_fail(self, e: Exception):
163
- """Log failure for the given exception.
172
+ """Keep the full driver trace and persist caught errors for scheduled runs."""
173
+ traceback.print_exception(e, chain=True)
174
+ try:
175
+ if self.process.startProcessFlag:
176
+ return
177
+ message = "SDK caught exception:\n" + "".join(traceback.format_exception(e, chain=True))
178
+ limit = 32000
179
+ marker = "\n... [truncated; full traceback in notebook/driver logs] ...\n"
180
+ if len(message) > limit:
181
+ head = (limit - len(marker)) // 2
182
+ tail = limit - len(marker) - head
183
+ message = message[:head] + marker + message[-tail:]
184
+ self.log(message, "E")
185
+ except Exception:
186
+ # Diagnostics must not prevent the caller from rethrowing the failure.
187
+ traceback.print_exc()
164
188
 
165
- Args:
166
- e (Exception): The exception that caused the failure.
167
- """
168
- traceback.print_exception(e)
169
- self.log(f"Process failed with error: {e}", "E")
170
189
 
171
190
  def close(self):
172
191
  """Close the session."""
@@ -186,5 +205,3 @@ class _Base_Session:
186
205
  def system_configuration(self) -> SystemConfiguration:
187
206
  """Public system configuration for the current session."""
188
207
  return self._systemConfiguration
189
-
190
-
@@ -0,0 +1,64 @@
1
+ from dataforge._base_session import _Base_Session
2
+
3
+
4
+ class _Databricks_Base_Session(_Base_Session):
5
+ """Base session class for Databricks platform.
6
+ Class should not be instantiated by user directly: use process-specific Session classes instead
7
+ Adds Spark session, DBUtilsto Base_Session
8
+ """
9
+
10
+
11
+ def __init__(self):
12
+ from pyspark.sql import SparkSession
13
+ self.spark = SparkSession.builder.getOrCreate()
14
+ self.dbutils = self._get_dbutils()
15
+ pg_connection_string_read = self.dbutils.secrets.get("sparky", "pg_read")
16
+ core_jwt_token = self.dbutils.secrets.get("sparky", "coreJWT")
17
+ try:
18
+ process_id = self.dbutils.widgets.get("process_id")
19
+ except Exception:
20
+ process_id = None
21
+
22
+
23
+ super().__init__(pg_connection_string_read, core_jwt_token, process_id)
24
+ try:
25
+ self.process_parameters["job_run_id"] = self.dbutils.widgets.get("job_run_id")
26
+ except Exception:
27
+ pass
28
+ self.process_parameters["start_process_flag"] = process_id is None
29
+ self.logger.info(f"Initialized databricks base session for {self.__class__.__name__} with parameters {self.process_parameters}")
30
+
31
+
32
+ def _get_dbutils(self):
33
+ from pyspark.dbutils import DBUtils
34
+ return DBUtils(self.spark)
35
+
36
+ def _write_input_table(self, in_df):
37
+ """Publish notebook output directly to the raw-table handoff."""
38
+ from pyspark.sql.types import StructField, StructType, StringType
39
+
40
+ if not self._is_open:
41
+ raise Exception("Session is closed")
42
+ result_df = in_df() if callable(in_df) else in_df
43
+ if result_df is None:
44
+ result_df = self.spark.createDataFrame([], StructType([StructField("id", StringType())]))
45
+ table = f"{self._systemConfiguration.dataLakeDbName}.ORE.RAW_INPUT_{self.process.inputId}"
46
+ self.log(f"Writing dataframe to Delta table {table}")
47
+ # Delta overwrite commits atomically; failed writes propagate to Sparky.
48
+ result_df.write.format("delta").mode("overwrite").option("overwriteSchema", "true").option("delta.columnMapping.mode", "name").saveAsTable(table)
49
+ self.log(f"Table {table} written")
50
+
51
+ def _release_manual_process(self, key_source_mode=None):
52
+ self._pg.sql(
53
+ "SELECT sparky.sdk_complete_manual_process(%s, %s, %s)",
54
+ [self.process.processId, key_source_mode, self.process.startProcessFlag],
55
+ fetch=False,
56
+ )
57
+
58
+ def _fail_manual_process(self, error):
59
+ if self.process.startProcessFlag:
60
+ self._pg.sql(
61
+ "SELECT sparky.sdk_fail_manual_process(%s, %s)",
62
+ [self.process.processId, str(error)],
63
+ fetch=False,
64
+ )
@@ -0,0 +1,64 @@
1
+ from __future__ import annotations
2
+ import json
3
+ import re
4
+ from typing import Callable
5
+ from dataforge.databricks._databricks_base_session import _Databricks_Base_Session
6
+
7
+
8
+ class _Databricks_Ingestion_Session(_Databricks_Base_Session):
9
+ """Base session class for Databricks platform.
10
+ Class should not be instantiated by user directly: use process-specific Session classes instead
11
+ Adds Spark session, DBUtilsto Base_Session
12
+ """
13
+
14
+ def __init__(self):
15
+ super().__init__()
16
+
17
+ def ingest(self,df: pyspark.sql.DataFrame | Callable[[], pyspark.sql.DataFrame] | None = None,
18
+ all_keys_df: pyspark.sql.DataFrame | Callable[[], pyspark.sql.DataFrame] | None = None):
19
+ """Ingest the provided DataFrame into the DataForge and update input record.
20
+
21
+ Writes the DataFrame to raw Parquet file,
22
+ updates the input record with status, file size, record count, and notifies
23
+ the Core API of process completion. On failure, updates logs and flags the input and process
24
+ records as failed.
25
+
26
+ Args:
27
+ df (Callable[[], DataFrame] | DataFrame): parameterless def that you defined, returning the Spark DataFrame to ingest (recommended),
28
+ or spark DataFrame
29
+ """
30
+ try:
31
+ if not self._is_open:
32
+ raise Exception("Session is closed")
33
+ self._write_input_table(df)
34
+ track_deletes = self.process.parameters.get("track_deletes", False)
35
+ key_source_mode = "main" if track_deletes else None
36
+ if track_deletes and all_keys_df is not None:
37
+ keys_df = (all_keys_df() if callable(all_keys_df) else all_keys_df).cache()
38
+ try:
39
+ if len(keys_df.take(1)) == 0:
40
+ raise Exception("all_keys_df returned zero rows. Delete tracking was stopped to protect existing data.")
41
+ key_columns = self.process.parameters.get("key_columns", [])
42
+ if len(keys_df.columns) != len(key_columns):
43
+ raise Exception("all_keys_df must contain exactly the configured key columns.")
44
+ from functools import reduce
45
+ from pyspark.sql.functions import col
46
+ if len(keys_df.filter(reduce(
47
+ lambda left, right: left | right,
48
+ [col(name).isNull() for name in key_columns]
49
+ )).take(1)) > 0:
50
+ raise Exception("all_keys_df contains a null key.")
51
+ keys_path = f"{self._systemConfiguration.datalakePath}/source_{self.process.sourceId}/keys/keys_input_{self.process.inputId}"
52
+ keys_df.write.format("parquet").mode("overwrite").save(keys_path)
53
+ finally:
54
+ keys_df.unpersist()
55
+ key_source_mode = "all_keys"
56
+ self._release_manual_process(key_source_mode)
57
+ self.logger.info("Ingestion completed successfully")
58
+
59
+ except Exception as e:
60
+ self._log_fail(e)
61
+ self._fail_manual_process(e)
62
+ raise
63
+ finally:
64
+ self.close()
@@ -28,19 +28,12 @@ class _Databricks_Parsing_Session(_Databricks_Base_Session):
28
28
  else:
29
29
  result_df = df
30
30
 
31
- if result_df is None or result_df.isEmpty():
32
- file_size, row_count = (0, 0)
33
- else:
34
- dest_file_path = f"{self._systemConfiguration.datalakePath}/source_{self.process.sourceId}/parsed/parsed_input_{self.process.inputId}"
35
- file_size, row_count = self._write_parsed_data(result_df, dest_file_path)
36
- input_update_json = {
37
- "file_size": file_size,
38
- "input_id": self.process.inputId,
39
- "record_counts": {"Total": row_count}
40
- }
41
- self._end_process('P' if row_count > 0 else 'Z', input_update_json)
31
+ self._write_input_table(result_df)
32
+ self._release_manual_process()
42
33
 
43
34
  except Exception as e:
44
35
  self._log_fail(e)
45
- self._end_process("F")
46
-
36
+ self._fail_manual_process(e)
37
+ raise
38
+ finally:
39
+ self.close()
@@ -13,11 +13,10 @@ class DataBricksPg:
13
13
  def __init__(self):
14
14
  from pyspark.dbutils import DBUtils
15
15
  from pyspark.sql import SparkSession, DataFrame
16
- from .postgres_connection import PostgresConnection
17
16
 
18
17
  self.spark = SparkSession.builder.getOrCreate()
19
- self.dbutils = DBUtils(spark)
20
- self.pg_connection_string_read = dbutils.secrets.get("sparky", "pg_read")
18
+ self.dbutils = DBUtils(self.spark)
19
+ self.pg_connection_string_read = self.dbutils.secrets.get("sparky", "pg_read")
21
20
 
22
21
  def update(self,query: str):
23
22
  """Execute an update SQL query on the DataForge metastore Postgres database.
@@ -92,4 +91,4 @@ class DataBricksPg:
92
91
  conn_string = self.dbutils.secrets.get("sparky", "pg_write")
93
92
  return PostgresConnection(conn_string + "&application_name=sdk-pg")
94
93
  else:
95
- raise Exception("pg_write secret is not defined in sparky scope")
94
+ raise Exception("pg_write secret is not defined in sparky scope")
@@ -59,6 +59,9 @@ class IngestionSession(_Ingestion_Session):
59
59
  message (str): Custom error message explaining the failure.
60
60
  """
61
61
  self.log(f"Custom Ingestion failed with error: {message}", "E")
62
- self._pg.sql("select meta.prc_iw_in_update_input_record(%s)",
63
- [json.dumps({"process_id": self.process.processId, "ingestion_status_code" : "F"})], fetch=False)
64
- self._core_api_call(f"process-complete/{self.process.processId}")
62
+ if self.process.startProcessFlag:
63
+ self._pg.sql("select meta.prc_iw_in_update_input_record(%s)",
64
+ [json.dumps({"process_id": self.process.processId, "ingestion_status_code" : "F"})], fetch=False)
65
+ self._core_api_call(f"process-complete/{self.process.processId}")
66
+ self.close()
67
+ raise RuntimeError(message)
@@ -8,3 +8,6 @@ from dataforge._session import _platform
8
8
  if _platform=='databricks':
9
9
  from dataforge.databricks._databricks_pg import DataBricksPg
10
10
  pg = DataBricksPg()
11
+ select = pg.select
12
+ update = pg.update
13
+ pull = pg.pull
@@ -0,0 +1,106 @@
1
+
2
+ import json
3
+ from typing import Optional, Callable
4
+ from .process_record import ProcessRecord
5
+ from ._session import _Session
6
+
7
+
8
+ class PostOutputSession(_Session):
9
+
10
+ """Session class to manage custom post-output process lifecycle.
11
+
12
+ Initializes post-output process record and provides run() method to execute custom logic post-output completion,
13
+ handle failures and update metadata.
14
+ """
15
+ def __init__(self, output_name: Optional[str] = None, output_source_name: Optional[str] = None, project_name: Optional[str] = None):
16
+ """Initialize custom post-output session and start a new post-output process.
17
+
18
+ Args:
19
+ Leave blank for production use.
20
+ """
21
+ super().__init__()
22
+ # initialize process
23
+ if output_name is not None:
24
+ self.process_parameters["output_name"] = output_name
25
+ self.process_parameters["output_source_name"] = output_source_name
26
+ self.process_parameters["project_name"] = project_name
27
+
28
+ process = self._pg.sql("select sparky.sdk_new_post_output(%s)", (json.dumps(self.process_parameters),))
29
+ self.process = ProcessRecord(process)
30
+
31
+ def file_path(self):
32
+ """Returns file path for the file generated by output process"""
33
+ return self.process.parameters.get("file_path")
34
+
35
+ def connection_parameters(self, connection_name: str | None = None):
36
+ """Read decrypted public/private parameters of a named output Custom connection.
37
+
38
+ Names match exactly across the environment. Call while the session is open.
39
+ Omitting the name retains the existing process-connection lookup.
40
+ """
41
+ if connection_name is None:
42
+ return super().connection_parameters()
43
+ if not self._is_open:
44
+ raise RuntimeError("Session is closed")
45
+ if not isinstance(connection_name, str) or not connection_name.strip():
46
+ raise ValueError("connection_name must be a non-empty output Custom connection name")
47
+ connection_id = self._pg.sql(
48
+ "select sparky.sdk_get_output_connection_id(%s, %s)",
49
+ (self.process.processId, connection_name),
50
+ )
51
+ return self._read_connection_parameters(connection_id)
52
+
53
+ def enqueue_ingestion(self, source_name: str) -> dict:
54
+ """Immediately enqueue a separate custom ingestion in this session's project.
55
+
56
+ Resolve source_name in this session's project. The destination must be an
57
+ active batch custom ingestion source with a configured notebook. Requests
58
+ wait while source, project, or environment initiation is disabled, survive
59
+ schedule reloads, and never enable initiation. Each call creates a queue
60
+ record, independently of post-output success or failure. No data or view
61
+ context is passed. Pending records may be served by one ingestion.
62
+
63
+ Call before run() closes the session, normally inside its callback.
64
+ Returns status='queued', ingestion_queue_id, process_id, and source_id; this is
65
+ acceptance of the request, not completion of downstream ingestion.
66
+ """
67
+ if not self._is_open:
68
+ raise RuntimeError("Session is closed")
69
+ if not isinstance(source_name, str) or not source_name.strip():
70
+ raise ValueError("source_name must be a non-empty custom ingestion source name")
71
+ return self._pg.sql(
72
+ "select sparky.sdk_enqueue_ingestion(%s, %s)",
73
+ (self.process.processId, source_name),
74
+ )
75
+
76
+ def run(self, po_commands: Callable[[],None]):
77
+
78
+ """Save parsed file from the provided DataFrame, and upload it into the DataForge data lake.
79
+
80
+ Writes the DataFrame to parsed Parquet file,
81
+ updates the input record with status, file size, record count, and notifies
82
+ the Core API of process completion. On failure, updates logs and flags the input and process
83
+ records as failed.
84
+
85
+ Args:
86
+ po_commands (po_commands: Callable[[],None]): The custom-defined, parameterless def containing post-output logic.
87
+ Returns None
88
+ """
89
+ try:
90
+ if not self._is_open:
91
+ raise Exception("Session is closed")
92
+ self.log(f"Running post output def {po_commands.__name__}")
93
+ po_commands()
94
+ self.log(f"Post output execution completed successfully")
95
+ if self.process.startProcessFlag:
96
+ self._end_process("P")
97
+ else:
98
+ self.close()
99
+
100
+ except Exception as e:
101
+ self._log_fail(e)
102
+ if self.process.startProcessFlag:
103
+ self._end_process("F")
104
+ else:
105
+ self.close()
106
+ raise
@@ -46,9 +46,13 @@ class _Snowflake_Base_Session(_Base_Session):
46
46
  table_type="transient"
47
47
  )
48
48
  self.log(f"Table {table} written")
49
- if self.process.startProcessFlag:
50
- # process started by IngestionSession, tell Core to continue and not run Notebook
51
- self._pg.sql("SELECT sparky.sdk_complete_manual_process(%s)", [self.process.processId], fetch=False)
49
+
50
+ def _complete_ingestion(self, key_source_mode: str | None):
51
+ # Record the key source only after both the data and optional key snapshot
52
+ # exist. Manually started processes are also released to Core here.
53
+ self._pg.sql("SELECT sparky.sdk_complete_manual_process(%s, %s, %s)",
54
+ [self.process.processId, key_source_mode, self.process.startProcessFlag],
55
+ fetch=False)
52
56
 
53
57
  @staticmethod
54
58
  def parse_key_value_args():
@@ -0,0 +1,70 @@
1
+ from __future__ import annotations
2
+ import json
3
+ from typing import Callable
4
+ from dataforge.snowflake._snowflake_base_session import _Snowflake_Base_Session
5
+
6
+
7
+ class _Snowflake_Ingestion_Session(_Snowflake_Base_Session):
8
+ """Base ingestion session class for Snowflake platform.
9
+ Class should not be instantiated by user directly: use process-specific Session classes instead
10
+ Adds Snowpark session
11
+ """
12
+
13
+ def __init__(self):
14
+ super().__init__()
15
+
16
+
17
+ def ingest(self,df: snowflake.snowpark.dataframe.DataFrame | Callable[[], snowflake.snowpark.dataframeDataFrame] | None = None,
18
+ all_keys_df: snowflake.snowpark.dataframe.DataFrame | Callable[[], snowflake.snowpark.dataframeDataFrame] | None = None):
19
+ """Ingest the provided DataFrame into the DataForge and update input record.
20
+
21
+ Writes the DataFrame to raw Snowflake table
22
+
23
+ Args:
24
+ df (Callable[[], DataFrame] | DataFrame): parameterless def that you defined, returning the Snowpark DataFrame to ingest (recommended),
25
+ or spark DataFrame
26
+ """
27
+ try:
28
+ track_deletes = self.process.parameters.get("track_deletes", False)
29
+ key_source_mode = (
30
+ "all_keys" if track_deletes and all_keys_df is not None
31
+ else "main" if track_deletes
32
+ else None
33
+ )
34
+ self._write_input_table(df)
35
+ if track_deletes and all_keys_df is not None:
36
+ from snowflake.snowpark.functions import lit
37
+ keys_df = (all_keys_df() if callable(all_keys_df) else all_keys_df).cache_result()
38
+ if keys_df.limit(1).count() == 0:
39
+ raise Exception("all_keys_df returned zero rows. Delete tracking was stopped to protect existing data.")
40
+ key_columns = self.process.parameters.get("key_columns", [])
41
+ if len(keys_df.columns) != len(key_columns):
42
+ raise Exception("all_keys_df must contain exactly the configured key columns.")
43
+ null_predicate = " OR ".join(f"{name} IS NULL" for name in key_columns)
44
+ if keys_df.filter(null_predicate).limit(1).count() > 0:
45
+ raise Exception("all_keys_df contains a null key.")
46
+ stored_keys_df = keys_df.select_expr(
47
+ f"CONCAT_WS('|', {','.join(key_columns)}) AS S_KEY"
48
+ ).with_column("S_INPUT_ID", lit(self.process.inputId))
49
+ keys_table = f"{self._systemConfiguration.dataLakeDbName}.ORE._{self.process.sourceId}_KEYS"
50
+ if self.snowpark_session.catalog.table_exists(keys_table):
51
+ self.snowpark_session.sql(
52
+ f"DELETE FROM {keys_table} WHERE S_INPUT_ID = {self.process.inputId}"
53
+ ).collect()
54
+ stored_keys_df.write.save_as_table(
55
+ table_name=keys_table, mode="append", table_type="transient", column_order="name"
56
+ )
57
+ self._complete_ingestion(key_source_mode)
58
+ except Exception as e:
59
+ self._log_fail(e)
60
+ if self.process.startProcessFlag:
61
+ # Fail input and process to prevent core from executing it
62
+ failure_update_json = {
63
+ "process_id": self.process.processId,
64
+ "ingestion_status_code": "F"
65
+ }
66
+ self._pg.sql("SELECT meta.prc_iw_in_update_input_record(%s)",
67
+ (json.dumps(failure_update_json),), fetch=False)
68
+ raise
69
+ finally:
70
+ self.close()
@@ -22,9 +22,13 @@ class _Snowflake_Parsing_Session(_Snowflake_Base_Session):
22
22
  """
23
23
  try:
24
24
  self._write_input_table(df)
25
+ self._complete_ingestion(None)
25
26
 
26
27
  except Exception as e:
27
28
  self._log_fail(e)
28
29
  if self.process.startProcessFlag:
29
30
  self._end_process("F")
30
-
31
+ raise
32
+ finally:
33
+ if self._is_open:
34
+ self.close()