databricks-modules-vy 1.3.0__tar.gz → 1.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/PKG-INFO +1 -1
  2. databricks_modules_vy-1.4.0/databricks_modules_vy/sap_rfc.py +531 -0
  3. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/databricks_modules_vy/sharepoint.py +5 -2
  4. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/databricks_modules_vy.egg-info/PKG-INFO +1 -1
  5. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/databricks_modules_vy.egg-info/SOURCES.txt +1 -0
  6. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/pyproject.toml +1 -1
  7. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/LICENSE +0 -0
  8. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/databricks_modules_vy/__init__.py +0 -0
  9. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/databricks_modules_vy/data_product.py +0 -0
  10. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/databricks_modules_vy/data_quality_tests.py +0 -0
  11. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/databricks_modules_vy/dataframe.py +0 -0
  12. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/databricks_modules_vy/extend_class.py +0 -0
  13. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/databricks_modules_vy/glue.py +0 -0
  14. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/databricks_modules_vy/logging.py +0 -0
  15. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/databricks_modules_vy/oracle.py +0 -0
  16. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/databricks_modules_vy/qlik.py +0 -0
  17. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/databricks_modules_vy/s3.py +0 -0
  18. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/databricks_modules_vy/service_platform.py +0 -0
  19. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/databricks_modules_vy/snaplogic.py +0 -0
  20. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/databricks_modules_vy/styringsinfo_postgres.py +0 -0
  21. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/databricks_modules_vy/unity_catalog.py +0 -0
  22. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/databricks_modules_vy.egg-info/dependency_links.txt +0 -0
  23. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/databricks_modules_vy.egg-info/top_level.txt +0 -0
  24. {databricks_modules_vy-1.3.0 → databricks_modules_vy-1.4.0}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: databricks_modules_vy
3
- Version: 1.3.0
3
+ Version: 1.4.0
4
4
  Summary: Databricks modules for Vy
5
5
  Author-email: Sindre Kjetså <sindre.kjetsa@vy.no>
6
6
  Project-URL: Homepage, https://github.com/nsbno/
@@ -0,0 +1,531 @@
1
+ from __future__ import annotations
2
+
3
+ import ctypes
4
+ import glob
5
+ import importlib
6
+ import importlib.util
7
+ import os
8
+ import re
9
+ import shutil
10
+ import subprocess
11
+ import sys
12
+ import tarfile
13
+ import tempfile
14
+ import zipfile
15
+ from datetime import datetime, timedelta
16
+
17
+ import pandas as pd
18
+ import pyspark.sql.functions as F
19
+ import pyspark.sql.types as T
20
+ from pyspark.dbutils import DBUtils
21
+ from pyspark.sql import DataFrame, SparkSession
22
+
23
+
24
+ from .logging import lp, utc_to_oslo
25
+ from .unity_catalog import upsert_to_uc, write_to_uc
26
+
27
+ spark = SparkSession.builder.appName("module").getOrCreate()
28
+ dbutils = DBUtils(spark)
29
+
30
+ # RFC_READ_TABLE limits
31
+ _DELIMITER = "^"
32
+ _MAX_ROW_WIDTH = 500 # the limit is 512 characters including delimiters
33
+ _OPTIONS_WIDTH = 72 # max length of one OPTIONS (WHERE) line
34
+
35
+ _SDK_VOLUME_SCHEMA_AND_NAME = "shared/sap_sdk"
36
+ _WORKSPACE_CATALOGS = {
37
+ "3034386312914956": "dataplattform_test",
38
+ "3366033470006123": "dataplattform",
39
+ }
40
+
41
+
42
+ def _sdk_volume() -> str:
43
+ from dbruntime.databricks_repl_context import get_context
44
+
45
+ workspace_id = str(get_context().workspaceId)
46
+ catalog = _WORKSPACE_CATALOGS.get(workspace_id)
47
+ if not catalog:
48
+ raise RuntimeError(f"No SAP SDK volume configured for workspace {workspace_id}. Pass sdk_source and wheel_dir.")
49
+ return f"/Volumes/{catalog}/{_SDK_VOLUME_SCHEMA_AND_NAME}"
50
+
51
+
52
+ # ----------------------------------------------------------------------------------------------------------------------
53
+ # SAP NW RFC SDK + pyrfc
54
+ # ----------------------------------------------------------------------------------------------------------------------
55
+ def _find_sdk_lib(root):
56
+ hits = glob.glob(os.path.join(root, "**", "libsapnwrfc.so"), recursive=True)
57
+ return os.path.dirname(hits[0]) if hits else None
58
+
59
+
60
+ def _copy_tree(src, dst):
61
+ """
62
+ Copies a folder with file contents only. shutil.copytree also copies permissions and timestamps, which fails
63
+ on volume paths (chmod/utime are not allowed there).
64
+ """
65
+ for root, _dirs, files in os.walk(src):
66
+ target = os.path.join(dst, os.path.relpath(root, src))
67
+ os.makedirs(target, exist_ok=True)
68
+ for f in files:
69
+ shutil.copyfile(os.path.join(root, f), os.path.join(target, f))
70
+ for root, _dirs, files in os.walk(dst):
71
+ os.chmod(root, 0o755)
72
+ for f in files:
73
+ os.chmod(os.path.join(root, f), 0o755)
74
+
75
+
76
+ def _copy_sdk(sdk_source, sdk_dir):
77
+ """Copies (or unpacks) the NW RFC SDK to local disk and returns the lib folder."""
78
+ lib_dir = _find_sdk_lib(sdk_dir)
79
+ if lib_dir:
80
+ return lib_dir
81
+
82
+ if os.path.exists(sdk_dir): # left over from a failed copy
83
+ shutil.rmtree(sdk_dir, ignore_errors=True)
84
+ try:
85
+ os.makedirs(sdk_dir, exist_ok=True)
86
+ if os.path.exists(sdk_dir) and not os.access(sdk_dir, os.W_OK):
87
+ raise PermissionError(sdk_dir)
88
+ except PermissionError:
89
+ sdk_dir = tempfile.mkdtemp(prefix="sapnwrfc_")
90
+ lp(f"Cannot write to the SDK folder, using {sdk_dir} instead")
91
+
92
+ if os.path.isdir(sdk_source):
93
+ lp(f"Copying SAP NW RFC SDK from {sdk_source} to {sdk_dir} ...")
94
+ _copy_tree(sdk_source, sdk_dir)
95
+ else:
96
+ if sdk_source.lower().endswith(".zip"):
97
+ with zipfile.ZipFile(sdk_source) as z:
98
+ z.extractall(sdk_dir)
99
+ elif sdk_source.lower().endswith((".tar.gz", ".tgz")):
100
+ with tarfile.open(sdk_source) as t:
101
+ t.extractall(sdk_dir)
102
+ else:
103
+ raise ValueError("sdk_source must be a folder, .zip or .tar.gz (a .SAR needs SAPCAR to unpack first)")
104
+
105
+ lib_dir = _find_sdk_lib(sdk_dir)
106
+ if not lib_dir:
107
+ raise FileNotFoundError(f"libsapnwrfc.so not found under {sdk_source}. Is it the Linux x86_64 SDK?")
108
+ return lib_dir
109
+
110
+
111
+ def _install_pyrfc(home, pyrfc_spec, wheel_dir):
112
+ """Installs pyrfc from a prebuilt wheel in wheel_dir, builds (and saves) the wheel if there is none."""
113
+ wheels = sorted(glob.glob(os.path.join(wheel_dir, "pyrfc-*.whl"))) if wheel_dir else []
114
+ if not wheels:
115
+ lp(f"No pyrfc wheel found, building {pyrfc_spec} ...")
116
+ build_dir = "/tmp/pyrfc_wheels"
117
+ os.makedirs(build_dir, exist_ok=True)
118
+ r = subprocess.run(
119
+ [sys.executable, "-m", "pip", "wheel", pyrfc_spec, "--no-deps", "-w", build_dir],
120
+ env=dict(os.environ, SAPNWRFC_HOME=home),
121
+ capture_output=True,
122
+ text=True,
123
+ )
124
+ if r.returncode != 0:
125
+ raise RuntimeError(f"pyrfc wheel build failed:\n{r.stderr[-1500:]}")
126
+ built = sorted(glob.glob(os.path.join(build_dir, "pyrfc-*.whl")))
127
+ if wheel_dir:
128
+ os.makedirs(wheel_dir, exist_ok=True)
129
+ for f in built:
130
+ shutil.copyfile(f, os.path.join(wheel_dir, os.path.basename(f))) # copy() also chmods, not allowed on volumes
131
+ wheels = sorted(glob.glob(os.path.join(wheel_dir, "pyrfc-*.whl")))
132
+ else:
133
+ wheels = built
134
+ lp(f"Installing {wheels[-1]}")
135
+ subprocess.check_call([sys.executable, "-m", "pip", "install", "-q", "--no-deps", wheels[-1]])
136
+ importlib.invalidate_caches()
137
+
138
+
139
+ def _patch_rpath(lib_dir):
140
+ """
141
+ LD_LIBRARY_PATH is only read when the Python process starts, so it cannot be set from a notebook.
142
+ Instead the compiled pyrfc module gets an RPATH to the SDK lib folder.
143
+ """
144
+ pkg_dir = list(importlib.util.find_spec("pyrfc").submodule_search_locations)[0]
145
+ so_files = glob.glob(os.path.join(pkg_dir, "_cyrfc*.so"))
146
+ if not so_files:
147
+ raise FileNotFoundError(f"No compiled _cyrfc*.so in {pkg_dir}")
148
+ subprocess.check_call([sys.executable, "-m", "pip", "install", "-q", "patchelf"])
149
+ patchelf = shutil.which("patchelf") or os.path.join(os.path.dirname(sys.executable), "patchelf")
150
+ for so in so_files:
151
+ # --force-rpath: DT_RPATH (not RUNPATH) also applies to the dependencies of libsapnwrfc.so
152
+ subprocess.check_call([patchelf, "--force-rpath", "--set-rpath", lib_dir, so])
153
+ lp(f"Patched RPATH of {so}")
154
+
155
+
156
+ def _import_connection():
157
+ for m in [m for m in sys.modules if m == "pyrfc" or m.startswith("pyrfc.")]:
158
+ del sys.modules[m] # drop a half-imported pyrfc from an earlier failed attempt
159
+ importlib.invalidate_caches()
160
+ # pyrfc/__init__.py swallows the real error of its compiled module, so import _cyrfc directly
161
+ from pyrfc._cyrfc import Connection
162
+
163
+ return Connection
164
+
165
+
166
+ def sap_setup_rfc(
167
+ sdk_source: str,
168
+ sdk_dir: str = "/tmp/sapnwrfc",
169
+ wheel_dir: str | None = None,
170
+ pyrfc_spec: str = "pyrfc==3.3.1",
171
+ ):
172
+ """
173
+ Makes the SAP NW RFC SDK and pyrfc usable on the cluster and returns the pyrfc Connection class.
174
+
175
+ The SDK is copied to local disk (/tmp), pyrfc is installed from a prebuilt wheel (built and saved the first time),
176
+ and the compiled pyrfc module gets an RPATH to the SDK libraries. Dedicated (single user) clusters are recommended.
177
+
178
+ Parameters:
179
+ sdk_source (str): The extracted nwrfcsdk folder (with lib/ and include/), or a .zip/.tar.gz of it, e.g. in a volume.
180
+ sdk_dir (str): Local copy of the SDK on the driver. Defaults to /tmp/sapnwrfc.
181
+ wheel_dir (str, optional): Folder with a prebuilt pyrfc wheel, e.g. in a volume. The wheel is saved here if missing.
182
+ pyrfc_spec (str, optional): What to build if there is no wheel. Defaults to pyrfc==3.3.1.
183
+
184
+ Returns:
185
+ The pyrfc Connection class.
186
+ """
187
+ lib_dir = _copy_sdk(sdk_source, sdk_dir)
188
+ home = os.path.dirname(lib_dir)
189
+ lp(f"SAP NW RFC SDK libs: {lib_dir}")
190
+
191
+ os.environ["SAPNWRFC_HOME"] = home
192
+ os.environ["LD_LIBRARY_PATH"] = lib_dir + ":" + os.environ.get("LD_LIBRARY_PATH", "")
193
+ for name in ["libicudata", "libicuuc", "libicui18n", "libsapucum", "libsapnwrfc"]:
194
+ found = sorted(glob.glob(os.path.join(lib_dir, name + ".so*")))
195
+ if found:
196
+ ctypes.CDLL(found[0], mode=ctypes.RTLD_GLOBAL)
197
+
198
+ if importlib.util.find_spec("pyrfc") is None:
199
+ _install_pyrfc(home, pyrfc_spec, wheel_dir)
200
+
201
+ try:
202
+ return _import_connection()
203
+ except ImportError as e:
204
+ lp(f"pyrfc failed to load ({e}), patching RPATH ...")
205
+ _patch_rpath(lib_dir)
206
+ return _import_connection()
207
+
208
+
209
+ def sap_connect(
210
+ scope: str = "VL_SAPMRO",
211
+ *,
212
+ host: str | None = None,
213
+ sysnr: str | None = None,
214
+ client: str | None = None,
215
+ user: str | None = None,
216
+ password: str | None = None,
217
+ sdk_source: str | None = None,
218
+ sdk_dir: str = "/tmp/sapnwrfc",
219
+ wheel_dir: str | None = None,
220
+ ):
221
+ """
222
+ Opens an RFC connection to an SAP system.
223
+
224
+ Values that are not given are read from the Databricks secret scope VL_SAPMRO
225
+ with the keys: hostname, sysnr, client, username, password.
226
+
227
+ Parameters:
228
+ scope (str): Databricks secret scope with the SAP credentials.
229
+ host, sysnr, client, user, password (str, optional): Override the values from the secret scope.
230
+ sdk_source (str, optional): Override the SDK location. See sap_setup_rfc.
231
+ sdk_dir (str, optional): See sap_setup_rfc.
232
+ wheel_dir (str, optional): Override the wheel location. See sap_setup_rfc.
233
+
234
+ Returns:
235
+ A pyrfc Connection. Close it with .close() when done.
236
+ """
237
+ host = host or dbutils.secrets.get(scope, "hostname")
238
+ sysnr = sysnr or dbutils.secrets.get(scope, "sysnr")
239
+ client = client or dbutils.secrets.get(scope, "client")
240
+ user = user or dbutils.secrets.get(scope, "username")
241
+ password = password or dbutils.secrets.get(scope, "password")
242
+
243
+ if not (sdk_source and wheel_dir):
244
+ volume = _sdk_volume()
245
+ sdk_source = sdk_source or f"{volume}/nwrfcsdk"
246
+ wheel_dir = wheel_dir or f"{volume}/wheels"
247
+ Connection = sap_setup_rfc(sdk_source, sdk_dir=sdk_dir, wheel_dir=wheel_dir)
248
+
249
+ lp(f"Connecting to SAP ({host}, sysnr {sysnr}, client {client}) as {user}")
250
+ return Connection(ashost=host, sysnr=sysnr, client=client, user=user, passwd=password, lang="EN")
251
+
252
+
253
+ # ----------------------------------------------------------------------------------------------------------------------
254
+ # RFC_READ_TABLE helpers
255
+ # ----------------------------------------------------------------------------------------------------------------------
256
+ def _window_condition(
257
+ lookback_days: int | None,
258
+ change_dates: list[str] | None,
259
+ delta_period: dict | None,
260
+ ) -> str | None:
261
+ """
262
+ The delta window as a WHERE condition, or None for full reads.
263
+
264
+ delta_period: {year: LFGJA, month: LFMON, mode: current_month} this year and month
265
+ {year: LFGJA, month: LFMON, mode: lookback, days: 70} from that month and later
266
+ otherwise: change_dates >= today - lookback_days
267
+ """
268
+ today = utc_to_oslo(datetime.now()).date()
269
+
270
+ if delta_period:
271
+ y, m = delta_period["year"], delta_period["month"]
272
+ if delta_period.get("mode", "current_month") == "current_month":
273
+ return f"{y} = '{today:%Y}' AND {m} = '{today:%m}'"
274
+ cutoff = today - timedelta(days=int(delta_period["days"]))
275
+ return f"{y} > '{cutoff:%Y}' OR ({y} = '{cutoff:%Y}' AND {m} >= '{cutoff:%m}')"
276
+
277
+ if lookback_days and change_dates:
278
+ cutoff = today - timedelta(days=int(lookback_days))
279
+ return " OR ".join(f"{c} >= '{cutoff:%Y%m%d}'" for c in change_dates)
280
+
281
+ return None
282
+
283
+
284
+ def _expand_in(condition: str) -> str:
285
+ """The dynamic WHERE parser of this SAP rejects IN ('a', 'b'). Rewrites it to (f = 'a' OR f = 'b')."""
286
+
287
+ def repl(m):
288
+ values = re.findall(r"'[^']*'", m.group(2))
289
+ return "(" + " OR ".join(f"{m.group(1)} = {v}" for v in values) + ")"
290
+
291
+ return re.sub(r"\b(\w+)\s+IN\s*\(([^)]*)\)", repl, condition)
292
+
293
+
294
+ def _build_options(conditions: list[str | None]) -> list[dict]:
295
+ """
296
+ ANDs the conditions and splits them into RFC_READ_TABLE OPTIONS lines of max 72 characters.
297
+ Continuation lines start with a space because trailing spaces are trimmed in transit.
298
+ """
299
+ conds = [_expand_in(c) for c in conditions if c]
300
+ if len(conds) > 1:
301
+ conds = [f"({c})" for c in conds]
302
+ text = " AND ".join(conds)
303
+ if not text:
304
+ return []
305
+
306
+ lines, cur = [], ""
307
+ for word in text.split(" "):
308
+ candidate = f"{cur} {word}" if cur else word
309
+ if len(candidate) > _OPTIONS_WIDTH:
310
+ lines.append(cur)
311
+ cur = f" {word}"
312
+ else:
313
+ cur = candidate
314
+ lines.append(cur)
315
+ return [{"TEXT": line} for line in lines]
316
+
317
+
318
+ def _field_lengths(conn, table: str, fields: list[str]) -> dict:
319
+ """Field lengths from the RFC_READ_TABLE metadata (NO_DATA). DDIF_FIELDS_GET does not exist on this SAP."""
320
+ r = conn.call(
321
+ "RFC_READ_TABLE",
322
+ QUERY_TABLE=table.upper(),
323
+ DELIMITER=_DELIMITER,
324
+ FIELDS=[{"FIELDNAME": f} for f in fields],
325
+ NO_DATA="X",
326
+ )
327
+ meta = {f["FIELDNAME"]: int(f["LENGTH"]) for f in r["FIELDS"]}
328
+ return {f: meta[f] for f in fields}
329
+
330
+
331
+ def _make_groups(fields: list[str], keys: list[str], lengths: dict) -> list[list[str]]:
332
+ """Splits the fields into groups that fit in one RFC_READ_TABLE row. The keys are repeated in every group."""
333
+
334
+ def width(fs):
335
+ return sum(lengths[f] + 1 for f in fs)
336
+
337
+ if width(fields) <= _MAX_ROW_WIDTH:
338
+ return [fields]
339
+
340
+ groups, cur = [], list(keys)
341
+ for f in fields:
342
+ if f in keys:
343
+ continue
344
+ if width(cur + [f]) > _MAX_ROW_WIDTH:
345
+ groups.append(cur)
346
+ cur = list(keys)
347
+ cur.append(f)
348
+ groups.append(cur)
349
+ return groups
350
+
351
+
352
+ def _read_rfc(conn, table: str, fields: list[str], options: list[dict], row_limit: int | None):
353
+ res = conn.call(
354
+ "RFC_READ_TABLE",
355
+ QUERY_TABLE=table.upper(),
356
+ DELIMITER=_DELIMITER,
357
+ FIELDS=[{"FIELDNAME": f} for f in fields],
358
+ OPTIONS=options,
359
+ ROWCOUNT=row_limit or 0,
360
+ )
361
+ rows, bad = [], 0
362
+ for r in res["DATA"]:
363
+ parts = r["WA"].split(_DELIMITER)
364
+ if len(parts) != len(fields): # e.g. a value that contains the delimiter
365
+ bad += 1
366
+ continue
367
+ rows.append(parts)
368
+ return pd.DataFrame(rows, columns=fields, dtype=str), bad
369
+
370
+
371
+ # ----------------------------------------------------------------------------------------------------------------------
372
+ # Public functions
373
+ # ----------------------------------------------------------------------------------------------------------------------
374
+ def sap_table(
375
+ conn,
376
+ table: str,
377
+ *,
378
+ fields: list[str],
379
+ keys: list[str],
380
+ where: str | None = None,
381
+ lookback_days: int | None = None,
382
+ change_dates: list[str] | None = None,
383
+ delta_period: dict | None = None,
384
+ outputs: list[dict | str] | None = None,
385
+ row_limit: int | None = None,
386
+ clean: bool = True,
387
+ ) -> DataFrame | None:
388
+ """
389
+ Reads a SAP table with RFC_READ_TABLE and returns it as a Spark DataFrame with all columns as string
390
+ (dates as YYYYMMDD, 00000000 = unset), like the SnapLogic landing files.
391
+
392
+ Wide tables are read in several field groups (max 512 characters per row) and joined on the keys.
393
+
394
+ Parameters:
395
+ conn: pyrfc Connection from sap_connect.
396
+ table (str): SAP table name, e.g. 'caufv'.
397
+ fields (list): Columns to read.
398
+ keys (list): Key columns. Must be in fields. Used to join field groups.
399
+ where (str, optional): Static filter, ANDed with the delta window.
400
+ lookback_days (int, optional): Delta window in days. Without it (and without delta_period) the table is read in full.
401
+ change_dates (list, optional): Date columns the window is applied to, OR'ed.
402
+ delta_period (dict, optional): Posting period window, see _window_condition.
403
+ outputs (list, optional): One query per entry (extra WHERE), all results are appended.
404
+ Entries are strings or dicts with the key 'where'.
405
+ row_limit (int, optional): Max rows per query, for testing. Defaults to no limit.
406
+ clean (bool, optional): Trim strings and convert empty strings to null. Defaults to True.
407
+
408
+ Returns:
409
+ A Spark DataFrame, or None if no rows were returned (nothing should be written then, same as SnapLogic).
410
+ """
411
+ missing = set(keys) - set(fields)
412
+ if missing:
413
+ raise ValueError(f"{table}: keys not in fields: {sorted(missing)}")
414
+
415
+ groups = _make_groups(fields, keys, _field_lengths(conn, table, fields))
416
+ window = _window_condition(lookback_days, change_dates, delta_period)
417
+ lp(f"{table}: {len(groups)} field group(s), window: {window}")
418
+
419
+ frames = []
420
+ for output in outputs or [None]:
421
+ output_where = output.get("where") if isinstance(output, dict) else output
422
+ options = _build_options([where, window, output_where])
423
+ lp(f"{table}: WHERE {' '.join(o['TEXT'] for o in options) or '(none)'}")
424
+
425
+ parts = []
426
+ for g in groups:
427
+ pdf, bad = _read_rfc(conn, table, g, options, row_limit)
428
+ lp(f"{table}: {len(pdf)} rows, {len(g)} fields" + (f", {bad} bad rows skipped" if bad else ""))
429
+ parts.append(pdf)
430
+
431
+ merged = parts[0]
432
+ for p in parts[1:]:
433
+ merged = merged.merge(p, on=keys, how="inner")
434
+ if len(parts) > 1 and len(merged) < min(len(p) for p in parts):
435
+ lp(f"{table}: WARNING rows lost when joining field groups (row_limit gives different rows per group)")
436
+ frames.append(merged[fields])
437
+
438
+ pdf = pd.concat(frames, ignore_index=True)
439
+ if pdf.empty:
440
+ return None
441
+
442
+ schema = T.StructType([T.StructField(c, T.StringType(), True) for c in pdf.columns])
443
+ df = spark.createDataFrame(pdf, schema=schema)
444
+ if clean:
445
+ for c in df.columns:
446
+ df = df.withColumn(c, F.when(F.trim(F.col(c)) == "", None).otherwise(F.trim(F.col(c))))
447
+ return df
448
+
449
+
450
+ def sap_tables_to_uc(
451
+ conn,
452
+ tables: dict[str, dict],
453
+ target_schema: str,
454
+ *,
455
+ table_prefix: str = "",
456
+ mode: str = "auto",
457
+ only: list[str] | None = None,
458
+ row_limit: int | None = None,
459
+ lookback_override: int | None = None,
460
+ raise_on_error: bool = True,
461
+ vacuum: bool = True,
462
+ ) -> list[dict]:
463
+ """
464
+ Extracts several SAP tables and writes each one to <target_schema>.<table_prefix><table> in Unity Catalog.
465
+
466
+ Parameters:
467
+ conn: pyrfc Connection from sap_connect.
468
+ tables (dict): table name -> spec, same format as tables.yml:
469
+ keys, fields, load (full|delta), where, lookback_days, change_dates, delta_period, outputs.
470
+ target_schema (str): catalog.schema to write to.
471
+ table_prefix (str, optional): Prefix for the target table names. Defaults to ''.
472
+ mode (str, optional): 'auto' = upsert delta tables and overwrite full tables, 'write' = overwrite everything.
473
+ only (list, optional): Only run these tables from `tables`.
474
+ row_limit (int, optional): Max rows per query, for testing.
475
+ lookback_override (int, optional): Replace lookback_days for all tables, e.g. 2 for a quick test.
476
+ raise_on_error (bool, optional): Run all tables, then raise if any failed. Defaults to True.
477
+ vacuum (bool, optional): Whether to vacuum after writing. Defaults to True.
478
+
479
+ Returns:
480
+ A list with one dict per table: table, status (ok | no rows | failed), rows, target, error.
481
+ """
482
+ if mode not in ("auto", "write"):
483
+ raise ValueError("mode must be 'auto' or 'write'")
484
+ unknown = [t for t in (only or []) if t not in tables]
485
+ if unknown:
486
+ raise ValueError(f"Not in tables: {unknown}")
487
+
488
+ results = []
489
+ for name, spec in tables.items():
490
+ if only and name not in only:
491
+ continue
492
+ target = f"{target_schema}.{table_prefix}{name}"
493
+ lp(f"== {name} ({spec.get('load', 'full')})")
494
+ try:
495
+ df = sap_table(
496
+ conn,
497
+ name,
498
+ fields=spec["fields"],
499
+ keys=spec["keys"],
500
+ where=spec.get("where"),
501
+ lookback_days=lookback_override or spec.get("lookback_days"),
502
+ change_dates=spec.get("change_dates"),
503
+ delta_period=spec.get("delta_period"),
504
+ outputs=spec.get("outputs"),
505
+ row_limit=row_limit,
506
+ )
507
+ if df is None:
508
+ lp(f"{name}: no rows, nothing written")
509
+ results.append({"table": name, "status": "no rows", "rows": 0, "target": None, "error": None})
510
+ continue
511
+
512
+ rows = df.count()
513
+ description = f"SAP MRO {name.upper()} via RFC_READ_TABLE"
514
+ if mode == "auto" and spec.get("load") == "delta":
515
+ upsert_to_uc(df, target=target, upsert_keys=spec["keys"], table_description=description, vacuum=vacuum)
516
+ else:
517
+ write_to_uc(df, target=target, overwrite_schema=True, table_description=description, vacuum=vacuum)
518
+
519
+ results.append({"table": name, "status": "ok", "rows": rows, "target": target, "error": None})
520
+ except Exception as e:
521
+ lp(f"{name}: FAILED {e}")
522
+ results.append({"table": name, "status": "failed", "rows": 0, "target": None, "error": str(e)})
523
+
524
+ lp("Summary")
525
+ for r in results:
526
+ lp(f" {r['table']:10} {r['status']:8} {r['rows']:>9} {r['target'] or ''}")
527
+
528
+ failed = [r["table"] for r in results if r["status"] == "failed"]
529
+ if failed and raise_on_error:
530
+ raise Exception(f"SAP extract failed for: {failed}")
531
+ return results
@@ -70,13 +70,16 @@ def _execute_query_with_backoff(
70
70
 
71
71
 
72
72
  def df_from_sharepoint_masterdata(
73
- list_name, row_limit=5000, site_name="VyMasterdata", internal_name=True
73
+ list_name, row_limit=5000, site_name="VyMasterdata", internal_name=True, get_by_id=False
74
74
  ) -> DataFrame:
75
75
  # Connect and fetch data
76
76
  ctx = _get_sharepoint_ctx(site_name)
77
77
 
78
78
  # Get the list and metadata
79
- s_list = ctx.web.lists.get_by_title(list_name)
79
+ if get_by_id:
80
+ s_list = ctx.web.lists.get_by_id(list_name)
81
+ else:
82
+ s_list = ctx.web.lists.get_by_title(list_name)
80
83
  s_list_fields = s_list.fields
81
84
  ctx.load(s_list_fields)
82
85
  _execute_query_with_backoff(ctx)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: databricks_modules_vy
3
- Version: 1.3.0
3
+ Version: 1.4.0
4
4
  Summary: Databricks modules for Vy
5
5
  Author-email: Sindre Kjetså <sindre.kjetsa@vy.no>
6
6
  Project-URL: Homepage, https://github.com/nsbno/
@@ -10,6 +10,7 @@ databricks_modules_vy/logging.py
10
10
  databricks_modules_vy/oracle.py
11
11
  databricks_modules_vy/qlik.py
12
12
  databricks_modules_vy/s3.py
13
+ databricks_modules_vy/sap_rfc.py
13
14
  databricks_modules_vy/service_platform.py
14
15
  databricks_modules_vy/sharepoint.py
15
16
  databricks_modules_vy/snaplogic.py
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "databricks_modules_vy"
3
- version = "1.3.0"
3
+ version = "1.4.0"
4
4
  authors = [
5
5
  { name="Sindre Kjetså", email="sindre.kjetsa@vy.no" },
6
6
  ]