sdmf 0.1.8__py3-none-any.whl → 0.1.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -32,9 +32,9 @@ class Orchestrator:
|
|
|
32
32
|
self.file_hunt_path = config["DEFAULT"]["file_hunt_path"]
|
|
33
33
|
self.system_run_report = pd.DataFrame()
|
|
34
34
|
self.logger.info(f"Current Run Id: {self.run_id}")
|
|
35
|
-
self.logger.info(
|
|
36
|
-
|
|
37
|
-
)
|
|
35
|
+
# self.logger.info(
|
|
36
|
+
# f'Is FAIR: {spark.sparkContext.getConf().get("spark.scheduler.mode")}'
|
|
37
|
+
# )
|
|
38
38
|
|
|
39
39
|
def __system_prerequisites(self) -> bool:
|
|
40
40
|
my_SystemLaunchValidator = SystemLaunchValidator(
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sdmf
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.9
|
|
4
4
|
Summary: SDMF - Standard Data Management Framework
|
|
5
5
|
Author: Harsh Hando
|
|
6
6
|
Author-email: Harsh Handoo <handoo.harsh@gmail.com>
|
|
@@ -223,6 +223,7 @@ Requires-Dist: openpyxl
|
|
|
223
223
|
Requires-Dist: matplotlib
|
|
224
224
|
Requires-Dist: delta-spark
|
|
225
225
|
Requires-Dist: numpy
|
|
226
|
+
Requires-Dist: requests
|
|
226
227
|
|
|
227
228
|
# Standard Data Management Framework (SDMF)
|
|
228
229
|
|
|
@@ -244,7 +245,7 @@ SDMF (Standard Data Management Framework) is an open-source Spark-based data eng
|
|
|
244
245
|
- **Scalable** – Works seamlessly with **Spark**, **Delta Lake**, and distributed environments like **Databricks**.
|
|
245
246
|
- **Logging & Monitoring** – Custom logging with retention and rotation policies.
|
|
246
247
|
|
|
247
|
-
## Installation
|
|
248
|
+
## Installation - test
|
|
248
249
|
|
|
249
250
|
```bash
|
|
250
251
|
pip install sdmf
|
|
@@ -40,7 +40,7 @@ sdmf/exception/StorageFetchException.py,sha256=Zp-r2Xx5PnMIU1PMbvJaoRyDKc4bb34kv
|
|
|
40
40
|
sdmf/exception/SystemError.py,sha256=qdSjWkxvwOb8dhmBQ-ek0s_5FgH1va7ILinOlwTNck8,320
|
|
41
41
|
sdmf/exception/ValidationError.py,sha256=28YDbXd_1D-YfSUgekkyo1ze_FGI5kythpZ5CKv8fN0,328
|
|
42
42
|
sdmf/exception/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
43
|
-
sdmf/orchestrator/Orchestrator.py,sha256=
|
|
43
|
+
sdmf/orchestrator/Orchestrator.py,sha256=Q4GNuDtcWpK3_2gzYk4C4L1UhtGfziBMqrSnv6C9CQ4,6260
|
|
44
44
|
sdmf/orchestrator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
45
45
|
sdmf/result_generator/ResultGenerator.py,sha256=PzmET2fJuhl8bjvUuXOFME2VlcNgKt95q0AD414u-qc,7228
|
|
46
46
|
sdmf/result_generator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -63,7 +63,7 @@ sdmf/validation/validation_rules/StandardCheckStructureCheck.py,sha256=_5CrGlLsQ
|
|
|
63
63
|
sdmf/validation/validation_rules/VacuumHoursCheck.py,sha256=FQI3RRpso2eQc_m6tX41KkOCAViNaKRTS42t8X4cfbQ,982
|
|
64
64
|
sdmf/validation/validation_rules/ValidateFeedSpecsJSON.py,sha256=JRyYA1DaXHZT94oDWt1wm7Q5sghKe6OrjJydVII2ico,1046
|
|
65
65
|
sdmf/validation/validation_rules/ValidateMasterSpecs.py,sha256=PJjL_goDrotit3D0bUWkcDqOgoNxPnFvUeGXfjluH54,666
|
|
66
|
-
sdmf-0.1.
|
|
67
|
-
sdmf-0.1.
|
|
68
|
-
sdmf-0.1.
|
|
69
|
-
sdmf-0.1.
|
|
66
|
+
sdmf-0.1.9.dist-info/METADATA,sha256=nwu5yYoSOm2vCWN4znLpWdeOZPX4E30kVgpsoETI-yY,21190
|
|
67
|
+
sdmf-0.1.9.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
68
|
+
sdmf-0.1.9.dist-info/top_level.txt,sha256=a67a3_q-4a9HG0C80uz5kmOlzfO4AFoqiNag1KhTpUs,5
|
|
69
|
+
sdmf-0.1.9.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|