databricks-modules-vy 1.3.1__tar.gz → 1.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/PKG-INFO +1 -1
- databricks_modules_vy-1.4.0/databricks_modules_vy/sap_rfc.py +531 -0
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy.egg-info/PKG-INFO +1 -1
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy.egg-info/SOURCES.txt +1 -0
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/pyproject.toml +1 -1
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/LICENSE +0 -0
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/__init__.py +0 -0
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/data_product.py +0 -0
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/data_quality_tests.py +0 -0
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/dataframe.py +0 -0
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/extend_class.py +0 -0
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/glue.py +0 -0
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/logging.py +0 -0
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/oracle.py +0 -0
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/qlik.py +0 -0
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/s3.py +0 -0
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/service_platform.py +0 -0
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/sharepoint.py +0 -0
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/snaplogic.py +0 -0
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/styringsinfo_postgres.py +0 -0
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/unity_catalog.py +0 -0
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy.egg-info/dependency_links.txt +0 -0
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy.egg-info/top_level.txt +0 -0
- {databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/setup.cfg +0 -0
|
@@ -0,0 +1,531 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import ctypes
|
|
4
|
+
import glob
|
|
5
|
+
import importlib
|
|
6
|
+
import importlib.util
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import shutil
|
|
10
|
+
import subprocess
|
|
11
|
+
import sys
|
|
12
|
+
import tarfile
|
|
13
|
+
import tempfile
|
|
14
|
+
import zipfile
|
|
15
|
+
from datetime import datetime, timedelta
|
|
16
|
+
|
|
17
|
+
import pandas as pd
|
|
18
|
+
import pyspark.sql.functions as F
|
|
19
|
+
import pyspark.sql.types as T
|
|
20
|
+
from pyspark.dbutils import DBUtils
|
|
21
|
+
from pyspark.sql import DataFrame, SparkSession
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
from .logging import lp, utc_to_oslo
|
|
25
|
+
from .unity_catalog import upsert_to_uc, write_to_uc
|
|
26
|
+
|
|
27
|
+
spark = SparkSession.builder.appName("module").getOrCreate()
|
|
28
|
+
dbutils = DBUtils(spark)
|
|
29
|
+
|
|
30
|
+
# RFC_READ_TABLE limits
|
|
31
|
+
_DELIMITER = "^"
|
|
32
|
+
_MAX_ROW_WIDTH = 500 # the limit is 512 characters including delimiters
|
|
33
|
+
_OPTIONS_WIDTH = 72 # max length of one OPTIONS (WHERE) line
|
|
34
|
+
|
|
35
|
+
_SDK_VOLUME_SCHEMA_AND_NAME = "shared/sap_sdk"
|
|
36
|
+
_WORKSPACE_CATALOGS = {
|
|
37
|
+
"3034386312914956": "dataplattform_test",
|
|
38
|
+
"3366033470006123": "dataplattform",
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _sdk_volume() -> str:
|
|
43
|
+
from dbruntime.databricks_repl_context import get_context
|
|
44
|
+
|
|
45
|
+
workspace_id = str(get_context().workspaceId)
|
|
46
|
+
catalog = _WORKSPACE_CATALOGS.get(workspace_id)
|
|
47
|
+
if not catalog:
|
|
48
|
+
raise RuntimeError(f"No SAP SDK volume configured for workspace {workspace_id}. Pass sdk_source and wheel_dir.")
|
|
49
|
+
return f"/Volumes/{catalog}/{_SDK_VOLUME_SCHEMA_AND_NAME}"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# ----------------------------------------------------------------------------------------------------------------------
|
|
53
|
+
# SAP NW RFC SDK + pyrfc
|
|
54
|
+
# ----------------------------------------------------------------------------------------------------------------------
|
|
55
|
+
def _find_sdk_lib(root):
|
|
56
|
+
hits = glob.glob(os.path.join(root, "**", "libsapnwrfc.so"), recursive=True)
|
|
57
|
+
return os.path.dirname(hits[0]) if hits else None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _copy_tree(src, dst):
|
|
61
|
+
"""
|
|
62
|
+
Copies a folder with file contents only. shutil.copytree also copies permissions and timestamps, which fails
|
|
63
|
+
on volume paths (chmod/utime are not allowed there).
|
|
64
|
+
"""
|
|
65
|
+
for root, _dirs, files in os.walk(src):
|
|
66
|
+
target = os.path.join(dst, os.path.relpath(root, src))
|
|
67
|
+
os.makedirs(target, exist_ok=True)
|
|
68
|
+
for f in files:
|
|
69
|
+
shutil.copyfile(os.path.join(root, f), os.path.join(target, f))
|
|
70
|
+
for root, _dirs, files in os.walk(dst):
|
|
71
|
+
os.chmod(root, 0o755)
|
|
72
|
+
for f in files:
|
|
73
|
+
os.chmod(os.path.join(root, f), 0o755)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _copy_sdk(sdk_source, sdk_dir):
|
|
77
|
+
"""Copies (or unpacks) the NW RFC SDK to local disk and returns the lib folder."""
|
|
78
|
+
lib_dir = _find_sdk_lib(sdk_dir)
|
|
79
|
+
if lib_dir:
|
|
80
|
+
return lib_dir
|
|
81
|
+
|
|
82
|
+
if os.path.exists(sdk_dir): # left over from a failed copy
|
|
83
|
+
shutil.rmtree(sdk_dir, ignore_errors=True)
|
|
84
|
+
try:
|
|
85
|
+
os.makedirs(sdk_dir, exist_ok=True)
|
|
86
|
+
if os.path.exists(sdk_dir) and not os.access(sdk_dir, os.W_OK):
|
|
87
|
+
raise PermissionError(sdk_dir)
|
|
88
|
+
except PermissionError:
|
|
89
|
+
sdk_dir = tempfile.mkdtemp(prefix="sapnwrfc_")
|
|
90
|
+
lp(f"Cannot write to the SDK folder, using {sdk_dir} instead")
|
|
91
|
+
|
|
92
|
+
if os.path.isdir(sdk_source):
|
|
93
|
+
lp(f"Copying SAP NW RFC SDK from {sdk_source} to {sdk_dir} ...")
|
|
94
|
+
_copy_tree(sdk_source, sdk_dir)
|
|
95
|
+
else:
|
|
96
|
+
if sdk_source.lower().endswith(".zip"):
|
|
97
|
+
with zipfile.ZipFile(sdk_source) as z:
|
|
98
|
+
z.extractall(sdk_dir)
|
|
99
|
+
elif sdk_source.lower().endswith((".tar.gz", ".tgz")):
|
|
100
|
+
with tarfile.open(sdk_source) as t:
|
|
101
|
+
t.extractall(sdk_dir)
|
|
102
|
+
else:
|
|
103
|
+
raise ValueError("sdk_source must be a folder, .zip or .tar.gz (a .SAR needs SAPCAR to unpack first)")
|
|
104
|
+
|
|
105
|
+
lib_dir = _find_sdk_lib(sdk_dir)
|
|
106
|
+
if not lib_dir:
|
|
107
|
+
raise FileNotFoundError(f"libsapnwrfc.so not found under {sdk_source}. Is it the Linux x86_64 SDK?")
|
|
108
|
+
return lib_dir
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _install_pyrfc(home, pyrfc_spec, wheel_dir):
|
|
112
|
+
"""Installs pyrfc from a prebuilt wheel in wheel_dir, builds (and saves) the wheel if there is none."""
|
|
113
|
+
wheels = sorted(glob.glob(os.path.join(wheel_dir, "pyrfc-*.whl"))) if wheel_dir else []
|
|
114
|
+
if not wheels:
|
|
115
|
+
lp(f"No pyrfc wheel found, building {pyrfc_spec} ...")
|
|
116
|
+
build_dir = "/tmp/pyrfc_wheels"
|
|
117
|
+
os.makedirs(build_dir, exist_ok=True)
|
|
118
|
+
r = subprocess.run(
|
|
119
|
+
[sys.executable, "-m", "pip", "wheel", pyrfc_spec, "--no-deps", "-w", build_dir],
|
|
120
|
+
env=dict(os.environ, SAPNWRFC_HOME=home),
|
|
121
|
+
capture_output=True,
|
|
122
|
+
text=True,
|
|
123
|
+
)
|
|
124
|
+
if r.returncode != 0:
|
|
125
|
+
raise RuntimeError(f"pyrfc wheel build failed:\n{r.stderr[-1500:]}")
|
|
126
|
+
built = sorted(glob.glob(os.path.join(build_dir, "pyrfc-*.whl")))
|
|
127
|
+
if wheel_dir:
|
|
128
|
+
os.makedirs(wheel_dir, exist_ok=True)
|
|
129
|
+
for f in built:
|
|
130
|
+
shutil.copyfile(f, os.path.join(wheel_dir, os.path.basename(f))) # copy() also chmods, not allowed on volumes
|
|
131
|
+
wheels = sorted(glob.glob(os.path.join(wheel_dir, "pyrfc-*.whl")))
|
|
132
|
+
else:
|
|
133
|
+
wheels = built
|
|
134
|
+
lp(f"Installing {wheels[-1]}")
|
|
135
|
+
subprocess.check_call([sys.executable, "-m", "pip", "install", "-q", "--no-deps", wheels[-1]])
|
|
136
|
+
importlib.invalidate_caches()
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _patch_rpath(lib_dir):
|
|
140
|
+
"""
|
|
141
|
+
LD_LIBRARY_PATH is only read when the Python process starts, so it cannot be set from a notebook.
|
|
142
|
+
Instead the compiled pyrfc module gets an RPATH to the SDK lib folder.
|
|
143
|
+
"""
|
|
144
|
+
pkg_dir = list(importlib.util.find_spec("pyrfc").submodule_search_locations)[0]
|
|
145
|
+
so_files = glob.glob(os.path.join(pkg_dir, "_cyrfc*.so"))
|
|
146
|
+
if not so_files:
|
|
147
|
+
raise FileNotFoundError(f"No compiled _cyrfc*.so in {pkg_dir}")
|
|
148
|
+
subprocess.check_call([sys.executable, "-m", "pip", "install", "-q", "patchelf"])
|
|
149
|
+
patchelf = shutil.which("patchelf") or os.path.join(os.path.dirname(sys.executable), "patchelf")
|
|
150
|
+
for so in so_files:
|
|
151
|
+
# --force-rpath: DT_RPATH (not RUNPATH) also applies to the dependencies of libsapnwrfc.so
|
|
152
|
+
subprocess.check_call([patchelf, "--force-rpath", "--set-rpath", lib_dir, so])
|
|
153
|
+
lp(f"Patched RPATH of {so}")
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _import_connection():
|
|
157
|
+
for m in [m for m in sys.modules if m == "pyrfc" or m.startswith("pyrfc.")]:
|
|
158
|
+
del sys.modules[m] # drop a half-imported pyrfc from an earlier failed attempt
|
|
159
|
+
importlib.invalidate_caches()
|
|
160
|
+
# pyrfc/__init__.py swallows the real error of its compiled module, so import _cyrfc directly
|
|
161
|
+
from pyrfc._cyrfc import Connection
|
|
162
|
+
|
|
163
|
+
return Connection
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def sap_setup_rfc(
|
|
167
|
+
sdk_source: str,
|
|
168
|
+
sdk_dir: str = "/tmp/sapnwrfc",
|
|
169
|
+
wheel_dir: str | None = None,
|
|
170
|
+
pyrfc_spec: str = "pyrfc==3.3.1",
|
|
171
|
+
):
|
|
172
|
+
"""
|
|
173
|
+
Makes the SAP NW RFC SDK and pyrfc usable on the cluster and returns the pyrfc Connection class.
|
|
174
|
+
|
|
175
|
+
The SDK is copied to local disk (/tmp), pyrfc is installed from a prebuilt wheel (built and saved the first time),
|
|
176
|
+
and the compiled pyrfc module gets an RPATH to the SDK libraries. Dedicated (single user) clusters are recommended.
|
|
177
|
+
|
|
178
|
+
Parameters:
|
|
179
|
+
sdk_source (str): The extracted nwrfcsdk folder (with lib/ and include/), or a .zip/.tar.gz of it, e.g. in a volume.
|
|
180
|
+
sdk_dir (str): Local copy of the SDK on the driver. Defaults to /tmp/sapnwrfc.
|
|
181
|
+
wheel_dir (str, optional): Folder with a prebuilt pyrfc wheel, e.g. in a volume. The wheel is saved here if missing.
|
|
182
|
+
pyrfc_spec (str, optional): What to build if there is no wheel. Defaults to pyrfc==3.3.1.
|
|
183
|
+
|
|
184
|
+
Returns:
|
|
185
|
+
The pyrfc Connection class.
|
|
186
|
+
"""
|
|
187
|
+
lib_dir = _copy_sdk(sdk_source, sdk_dir)
|
|
188
|
+
home = os.path.dirname(lib_dir)
|
|
189
|
+
lp(f"SAP NW RFC SDK libs: {lib_dir}")
|
|
190
|
+
|
|
191
|
+
os.environ["SAPNWRFC_HOME"] = home
|
|
192
|
+
os.environ["LD_LIBRARY_PATH"] = lib_dir + ":" + os.environ.get("LD_LIBRARY_PATH", "")
|
|
193
|
+
for name in ["libicudata", "libicuuc", "libicui18n", "libsapucum", "libsapnwrfc"]:
|
|
194
|
+
found = sorted(glob.glob(os.path.join(lib_dir, name + ".so*")))
|
|
195
|
+
if found:
|
|
196
|
+
ctypes.CDLL(found[0], mode=ctypes.RTLD_GLOBAL)
|
|
197
|
+
|
|
198
|
+
if importlib.util.find_spec("pyrfc") is None:
|
|
199
|
+
_install_pyrfc(home, pyrfc_spec, wheel_dir)
|
|
200
|
+
|
|
201
|
+
try:
|
|
202
|
+
return _import_connection()
|
|
203
|
+
except ImportError as e:
|
|
204
|
+
lp(f"pyrfc failed to load ({e}), patching RPATH ...")
|
|
205
|
+
_patch_rpath(lib_dir)
|
|
206
|
+
return _import_connection()
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def sap_connect(
|
|
210
|
+
scope: str = "VL_SAPMRO",
|
|
211
|
+
*,
|
|
212
|
+
host: str | None = None,
|
|
213
|
+
sysnr: str | None = None,
|
|
214
|
+
client: str | None = None,
|
|
215
|
+
user: str | None = None,
|
|
216
|
+
password: str | None = None,
|
|
217
|
+
sdk_source: str | None = None,
|
|
218
|
+
sdk_dir: str = "/tmp/sapnwrfc",
|
|
219
|
+
wheel_dir: str | None = None,
|
|
220
|
+
):
|
|
221
|
+
"""
|
|
222
|
+
Opens an RFC connection to an SAP system.
|
|
223
|
+
|
|
224
|
+
Values that are not given are read from the Databricks secret scope VL_SAPMRO
|
|
225
|
+
with the keys: hostname, sysnr, client, username, password.
|
|
226
|
+
|
|
227
|
+
Parameters:
|
|
228
|
+
scope (str): Databricks secret scope with the SAP credentials.
|
|
229
|
+
host, sysnr, client, user, password (str, optional): Override the values from the secret scope.
|
|
230
|
+
sdk_source (str, optional): Override the SDK location. See sap_setup_rfc.
|
|
231
|
+
sdk_dir (str, optional): See sap_setup_rfc.
|
|
232
|
+
wheel_dir (str, optional): Override the wheel location. See sap_setup_rfc.
|
|
233
|
+
|
|
234
|
+
Returns:
|
|
235
|
+
A pyrfc Connection. Close it with .close() when done.
|
|
236
|
+
"""
|
|
237
|
+
host = host or dbutils.secrets.get(scope, "hostname")
|
|
238
|
+
sysnr = sysnr or dbutils.secrets.get(scope, "sysnr")
|
|
239
|
+
client = client or dbutils.secrets.get(scope, "client")
|
|
240
|
+
user = user or dbutils.secrets.get(scope, "username")
|
|
241
|
+
password = password or dbutils.secrets.get(scope, "password")
|
|
242
|
+
|
|
243
|
+
if not (sdk_source and wheel_dir):
|
|
244
|
+
volume = _sdk_volume()
|
|
245
|
+
sdk_source = sdk_source or f"{volume}/nwrfcsdk"
|
|
246
|
+
wheel_dir = wheel_dir or f"{volume}/wheels"
|
|
247
|
+
Connection = sap_setup_rfc(sdk_source, sdk_dir=sdk_dir, wheel_dir=wheel_dir)
|
|
248
|
+
|
|
249
|
+
lp(f"Connecting to SAP ({host}, sysnr {sysnr}, client {client}) as {user}")
|
|
250
|
+
return Connection(ashost=host, sysnr=sysnr, client=client, user=user, passwd=password, lang="EN")
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
# ----------------------------------------------------------------------------------------------------------------------
|
|
254
|
+
# RFC_READ_TABLE helpers
|
|
255
|
+
# ----------------------------------------------------------------------------------------------------------------------
|
|
256
|
+
def _window_condition(
|
|
257
|
+
lookback_days: int | None,
|
|
258
|
+
change_dates: list[str] | None,
|
|
259
|
+
delta_period: dict | None,
|
|
260
|
+
) -> str | None:
|
|
261
|
+
"""
|
|
262
|
+
The delta window as a WHERE condition, or None for full reads.
|
|
263
|
+
|
|
264
|
+
delta_period: {year: LFGJA, month: LFMON, mode: current_month} this year and month
|
|
265
|
+
{year: LFGJA, month: LFMON, mode: lookback, days: 70} from that month and later
|
|
266
|
+
otherwise: change_dates >= today - lookback_days
|
|
267
|
+
"""
|
|
268
|
+
today = utc_to_oslo(datetime.now()).date()
|
|
269
|
+
|
|
270
|
+
if delta_period:
|
|
271
|
+
y, m = delta_period["year"], delta_period["month"]
|
|
272
|
+
if delta_period.get("mode", "current_month") == "current_month":
|
|
273
|
+
return f"{y} = '{today:%Y}' AND {m} = '{today:%m}'"
|
|
274
|
+
cutoff = today - timedelta(days=int(delta_period["days"]))
|
|
275
|
+
return f"{y} > '{cutoff:%Y}' OR ({y} = '{cutoff:%Y}' AND {m} >= '{cutoff:%m}')"
|
|
276
|
+
|
|
277
|
+
if lookback_days and change_dates:
|
|
278
|
+
cutoff = today - timedelta(days=int(lookback_days))
|
|
279
|
+
return " OR ".join(f"{c} >= '{cutoff:%Y%m%d}'" for c in change_dates)
|
|
280
|
+
|
|
281
|
+
return None
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _expand_in(condition: str) -> str:
|
|
285
|
+
"""The dynamic WHERE parser of this SAP rejects IN ('a', 'b'). Rewrites it to (f = 'a' OR f = 'b')."""
|
|
286
|
+
|
|
287
|
+
def repl(m):
|
|
288
|
+
values = re.findall(r"'[^']*'", m.group(2))
|
|
289
|
+
return "(" + " OR ".join(f"{m.group(1)} = {v}" for v in values) + ")"
|
|
290
|
+
|
|
291
|
+
return re.sub(r"\b(\w+)\s+IN\s*\(([^)]*)\)", repl, condition)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def _build_options(conditions: list[str | None]) -> list[dict]:
|
|
295
|
+
"""
|
|
296
|
+
ANDs the conditions and splits them into RFC_READ_TABLE OPTIONS lines of max 72 characters.
|
|
297
|
+
Continuation lines start with a space because trailing spaces are trimmed in transit.
|
|
298
|
+
"""
|
|
299
|
+
conds = [_expand_in(c) for c in conditions if c]
|
|
300
|
+
if len(conds) > 1:
|
|
301
|
+
conds = [f"({c})" for c in conds]
|
|
302
|
+
text = " AND ".join(conds)
|
|
303
|
+
if not text:
|
|
304
|
+
return []
|
|
305
|
+
|
|
306
|
+
lines, cur = [], ""
|
|
307
|
+
for word in text.split(" "):
|
|
308
|
+
candidate = f"{cur} {word}" if cur else word
|
|
309
|
+
if len(candidate) > _OPTIONS_WIDTH:
|
|
310
|
+
lines.append(cur)
|
|
311
|
+
cur = f" {word}"
|
|
312
|
+
else:
|
|
313
|
+
cur = candidate
|
|
314
|
+
lines.append(cur)
|
|
315
|
+
return [{"TEXT": line} for line in lines]
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def _field_lengths(conn, table: str, fields: list[str]) -> dict:
|
|
319
|
+
"""Field lengths from the RFC_READ_TABLE metadata (NO_DATA). DDIF_FIELDS_GET does not exist on this SAP."""
|
|
320
|
+
r = conn.call(
|
|
321
|
+
"RFC_READ_TABLE",
|
|
322
|
+
QUERY_TABLE=table.upper(),
|
|
323
|
+
DELIMITER=_DELIMITER,
|
|
324
|
+
FIELDS=[{"FIELDNAME": f} for f in fields],
|
|
325
|
+
NO_DATA="X",
|
|
326
|
+
)
|
|
327
|
+
meta = {f["FIELDNAME"]: int(f["LENGTH"]) for f in r["FIELDS"]}
|
|
328
|
+
return {f: meta[f] for f in fields}
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def _make_groups(fields: list[str], keys: list[str], lengths: dict) -> list[list[str]]:
|
|
332
|
+
"""Splits the fields into groups that fit in one RFC_READ_TABLE row. The keys are repeated in every group."""
|
|
333
|
+
|
|
334
|
+
def width(fs):
|
|
335
|
+
return sum(lengths[f] + 1 for f in fs)
|
|
336
|
+
|
|
337
|
+
if width(fields) <= _MAX_ROW_WIDTH:
|
|
338
|
+
return [fields]
|
|
339
|
+
|
|
340
|
+
groups, cur = [], list(keys)
|
|
341
|
+
for f in fields:
|
|
342
|
+
if f in keys:
|
|
343
|
+
continue
|
|
344
|
+
if width(cur + [f]) > _MAX_ROW_WIDTH:
|
|
345
|
+
groups.append(cur)
|
|
346
|
+
cur = list(keys)
|
|
347
|
+
cur.append(f)
|
|
348
|
+
groups.append(cur)
|
|
349
|
+
return groups
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def _read_rfc(conn, table: str, fields: list[str], options: list[dict], row_limit: int | None):
|
|
353
|
+
res = conn.call(
|
|
354
|
+
"RFC_READ_TABLE",
|
|
355
|
+
QUERY_TABLE=table.upper(),
|
|
356
|
+
DELIMITER=_DELIMITER,
|
|
357
|
+
FIELDS=[{"FIELDNAME": f} for f in fields],
|
|
358
|
+
OPTIONS=options,
|
|
359
|
+
ROWCOUNT=row_limit or 0,
|
|
360
|
+
)
|
|
361
|
+
rows, bad = [], 0
|
|
362
|
+
for r in res["DATA"]:
|
|
363
|
+
parts = r["WA"].split(_DELIMITER)
|
|
364
|
+
if len(parts) != len(fields): # e.g. a value that contains the delimiter
|
|
365
|
+
bad += 1
|
|
366
|
+
continue
|
|
367
|
+
rows.append(parts)
|
|
368
|
+
return pd.DataFrame(rows, columns=fields, dtype=str), bad
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
# ----------------------------------------------------------------------------------------------------------------------
|
|
372
|
+
# Public functions
|
|
373
|
+
# ----------------------------------------------------------------------------------------------------------------------
|
|
374
|
+
def sap_table(
|
|
375
|
+
conn,
|
|
376
|
+
table: str,
|
|
377
|
+
*,
|
|
378
|
+
fields: list[str],
|
|
379
|
+
keys: list[str],
|
|
380
|
+
where: str | None = None,
|
|
381
|
+
lookback_days: int | None = None,
|
|
382
|
+
change_dates: list[str] | None = None,
|
|
383
|
+
delta_period: dict | None = None,
|
|
384
|
+
outputs: list[dict | str] | None = None,
|
|
385
|
+
row_limit: int | None = None,
|
|
386
|
+
clean: bool = True,
|
|
387
|
+
) -> DataFrame | None:
|
|
388
|
+
"""
|
|
389
|
+
Reads a SAP table with RFC_READ_TABLE and returns it as a Spark DataFrame with all columns as string
|
|
390
|
+
(dates as YYYYMMDD, 00000000 = unset), like the SnapLogic landing files.
|
|
391
|
+
|
|
392
|
+
Wide tables are read in several field groups (max 512 characters per row) and joined on the keys.
|
|
393
|
+
|
|
394
|
+
Parameters:
|
|
395
|
+
conn: pyrfc Connection from sap_connect.
|
|
396
|
+
table (str): SAP table name, e.g. 'caufv'.
|
|
397
|
+
fields (list): Columns to read.
|
|
398
|
+
keys (list): Key columns. Must be in fields. Used to join field groups.
|
|
399
|
+
where (str, optional): Static filter, ANDed with the delta window.
|
|
400
|
+
lookback_days (int, optional): Delta window in days. Without it (and without delta_period) the table is read in full.
|
|
401
|
+
change_dates (list, optional): Date columns the window is applied to, OR'ed.
|
|
402
|
+
delta_period (dict, optional): Posting period window, see _window_condition.
|
|
403
|
+
outputs (list, optional): One query per entry (extra WHERE), all results are appended.
|
|
404
|
+
Entries are strings or dicts with the key 'where'.
|
|
405
|
+
row_limit (int, optional): Max rows per query, for testing. Defaults to no limit.
|
|
406
|
+
clean (bool, optional): Trim strings and convert empty strings to null. Defaults to True.
|
|
407
|
+
|
|
408
|
+
Returns:
|
|
409
|
+
A Spark DataFrame, or None if no rows were returned (nothing should be written then, same as SnapLogic).
|
|
410
|
+
"""
|
|
411
|
+
missing = set(keys) - set(fields)
|
|
412
|
+
if missing:
|
|
413
|
+
raise ValueError(f"{table}: keys not in fields: {sorted(missing)}")
|
|
414
|
+
|
|
415
|
+
groups = _make_groups(fields, keys, _field_lengths(conn, table, fields))
|
|
416
|
+
window = _window_condition(lookback_days, change_dates, delta_period)
|
|
417
|
+
lp(f"{table}: {len(groups)} field group(s), window: {window}")
|
|
418
|
+
|
|
419
|
+
frames = []
|
|
420
|
+
for output in outputs or [None]:
|
|
421
|
+
output_where = output.get("where") if isinstance(output, dict) else output
|
|
422
|
+
options = _build_options([where, window, output_where])
|
|
423
|
+
lp(f"{table}: WHERE {' '.join(o['TEXT'] for o in options) or '(none)'}")
|
|
424
|
+
|
|
425
|
+
parts = []
|
|
426
|
+
for g in groups:
|
|
427
|
+
pdf, bad = _read_rfc(conn, table, g, options, row_limit)
|
|
428
|
+
lp(f"{table}: {len(pdf)} rows, {len(g)} fields" + (f", {bad} bad rows skipped" if bad else ""))
|
|
429
|
+
parts.append(pdf)
|
|
430
|
+
|
|
431
|
+
merged = parts[0]
|
|
432
|
+
for p in parts[1:]:
|
|
433
|
+
merged = merged.merge(p, on=keys, how="inner")
|
|
434
|
+
if len(parts) > 1 and len(merged) < min(len(p) for p in parts):
|
|
435
|
+
lp(f"{table}: WARNING rows lost when joining field groups (row_limit gives different rows per group)")
|
|
436
|
+
frames.append(merged[fields])
|
|
437
|
+
|
|
438
|
+
pdf = pd.concat(frames, ignore_index=True)
|
|
439
|
+
if pdf.empty:
|
|
440
|
+
return None
|
|
441
|
+
|
|
442
|
+
schema = T.StructType([T.StructField(c, T.StringType(), True) for c in pdf.columns])
|
|
443
|
+
df = spark.createDataFrame(pdf, schema=schema)
|
|
444
|
+
if clean:
|
|
445
|
+
for c in df.columns:
|
|
446
|
+
df = df.withColumn(c, F.when(F.trim(F.col(c)) == "", None).otherwise(F.trim(F.col(c))))
|
|
447
|
+
return df
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def sap_tables_to_uc(
|
|
451
|
+
conn,
|
|
452
|
+
tables: dict[str, dict],
|
|
453
|
+
target_schema: str,
|
|
454
|
+
*,
|
|
455
|
+
table_prefix: str = "",
|
|
456
|
+
mode: str = "auto",
|
|
457
|
+
only: list[str] | None = None,
|
|
458
|
+
row_limit: int | None = None,
|
|
459
|
+
lookback_override: int | None = None,
|
|
460
|
+
raise_on_error: bool = True,
|
|
461
|
+
vacuum: bool = True,
|
|
462
|
+
) -> list[dict]:
|
|
463
|
+
"""
|
|
464
|
+
Extracts several SAP tables and writes each one to <target_schema>.<table_prefix><table> in Unity Catalog.
|
|
465
|
+
|
|
466
|
+
Parameters:
|
|
467
|
+
conn: pyrfc Connection from sap_connect.
|
|
468
|
+
tables (dict): table name -> spec, same format as tables.yml:
|
|
469
|
+
keys, fields, load (full|delta), where, lookback_days, change_dates, delta_period, outputs.
|
|
470
|
+
target_schema (str): catalog.schema to write to.
|
|
471
|
+
table_prefix (str, optional): Prefix for the target table names. Defaults to ''.
|
|
472
|
+
mode (str, optional): 'auto' = upsert delta tables and overwrite full tables, 'write' = overwrite everything.
|
|
473
|
+
only (list, optional): Only run these tables from `tables`.
|
|
474
|
+
row_limit (int, optional): Max rows per query, for testing.
|
|
475
|
+
lookback_override (int, optional): Replace lookback_days for all tables, e.g. 2 for a quick test.
|
|
476
|
+
raise_on_error (bool, optional): Run all tables, then raise if any failed. Defaults to True.
|
|
477
|
+
vacuum (bool, optional): Whether to vacuum after writing. Defaults to True.
|
|
478
|
+
|
|
479
|
+
Returns:
|
|
480
|
+
A list with one dict per table: table, status (ok | no rows | failed), rows, target, error.
|
|
481
|
+
"""
|
|
482
|
+
if mode not in ("auto", "write"):
|
|
483
|
+
raise ValueError("mode must be 'auto' or 'write'")
|
|
484
|
+
unknown = [t for t in (only or []) if t not in tables]
|
|
485
|
+
if unknown:
|
|
486
|
+
raise ValueError(f"Not in tables: {unknown}")
|
|
487
|
+
|
|
488
|
+
results = []
|
|
489
|
+
for name, spec in tables.items():
|
|
490
|
+
if only and name not in only:
|
|
491
|
+
continue
|
|
492
|
+
target = f"{target_schema}.{table_prefix}{name}"
|
|
493
|
+
lp(f"== {name} ({spec.get('load', 'full')})")
|
|
494
|
+
try:
|
|
495
|
+
df = sap_table(
|
|
496
|
+
conn,
|
|
497
|
+
name,
|
|
498
|
+
fields=spec["fields"],
|
|
499
|
+
keys=spec["keys"],
|
|
500
|
+
where=spec.get("where"),
|
|
501
|
+
lookback_days=lookback_override or spec.get("lookback_days"),
|
|
502
|
+
change_dates=spec.get("change_dates"),
|
|
503
|
+
delta_period=spec.get("delta_period"),
|
|
504
|
+
outputs=spec.get("outputs"),
|
|
505
|
+
row_limit=row_limit,
|
|
506
|
+
)
|
|
507
|
+
if df is None:
|
|
508
|
+
lp(f"{name}: no rows, nothing written")
|
|
509
|
+
results.append({"table": name, "status": "no rows", "rows": 0, "target": None, "error": None})
|
|
510
|
+
continue
|
|
511
|
+
|
|
512
|
+
rows = df.count()
|
|
513
|
+
description = f"SAP MRO {name.upper()} via RFC_READ_TABLE"
|
|
514
|
+
if mode == "auto" and spec.get("load") == "delta":
|
|
515
|
+
upsert_to_uc(df, target=target, upsert_keys=spec["keys"], table_description=description, vacuum=vacuum)
|
|
516
|
+
else:
|
|
517
|
+
write_to_uc(df, target=target, overwrite_schema=True, table_description=description, vacuum=vacuum)
|
|
518
|
+
|
|
519
|
+
results.append({"table": name, "status": "ok", "rows": rows, "target": target, "error": None})
|
|
520
|
+
except Exception as e:
|
|
521
|
+
lp(f"{name}: FAILED {e}")
|
|
522
|
+
results.append({"table": name, "status": "failed", "rows": 0, "target": None, "error": str(e)})
|
|
523
|
+
|
|
524
|
+
lp("Summary")
|
|
525
|
+
for r in results:
|
|
526
|
+
lp(f" {r['table']:10} {r['status']:8} {r['rows']:>9} {r['target'] or ''}")
|
|
527
|
+
|
|
528
|
+
failed = [r["table"] for r in results if r["status"] == "failed"]
|
|
529
|
+
if failed and raise_on_error:
|
|
530
|
+
raise Exception(f"SAP extract failed for: {failed}")
|
|
531
|
+
return results
|
|
@@ -10,6 +10,7 @@ databricks_modules_vy/logging.py
|
|
|
10
10
|
databricks_modules_vy/oracle.py
|
|
11
11
|
databricks_modules_vy/qlik.py
|
|
12
12
|
databricks_modules_vy/s3.py
|
|
13
|
+
databricks_modules_vy/sap_rfc.py
|
|
13
14
|
databricks_modules_vy/service_platform.py
|
|
14
15
|
databricks_modules_vy/sharepoint.py
|
|
15
16
|
databricks_modules_vy/snaplogic.py
|
|
File without changes
|
{databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/__init__.py
RENAMED
|
File without changes
|
{databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/data_product.py
RENAMED
|
File without changes
|
|
File without changes
|
{databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/dataframe.py
RENAMED
|
File without changes
|
{databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/extend_class.py
RENAMED
|
File without changes
|
|
File without changes
|
{databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/logging.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/sharepoint.py
RENAMED
|
File without changes
|
{databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/snaplogic.py
RENAMED
|
File without changes
|
|
File without changes
|
{databricks_modules_vy-1.3.1 → databricks_modules_vy-1.4.0}/databricks_modules_vy/unity_catalog.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|