cloe-nessy 0.3.3__py3-none-any.whl → 0.3.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cloe_nessy/__init__.py +0 -0
- cloe_nessy/clients/__init__.py +0 -0
- cloe_nessy/clients/api_client/__init__.py +0 -0
- cloe_nessy/clients/api_client/api_client.py +0 -0
- cloe_nessy/clients/api_client/api_response.py +0 -0
- cloe_nessy/clients/api_client/auth.py +0 -0
- cloe_nessy/clients/api_client/exceptions.py +0 -0
- cloe_nessy/file_utilities/__init__.py +0 -0
- cloe_nessy/file_utilities/exceptions.py +0 -0
- cloe_nessy/file_utilities/factory.py +0 -0
- cloe_nessy/file_utilities/get_file_paths.py +0 -0
- cloe_nessy/file_utilities/location_types.py +0 -0
- cloe_nessy/file_utilities/strategies/__init__.py +0 -0
- cloe_nessy/file_utilities/strategies/base_strategy.py +0 -0
- cloe_nessy/file_utilities/strategies/local_strategy.py +0 -0
- cloe_nessy/file_utilities/strategies/onelake_strategy.py +0 -0
- cloe_nessy/file_utilities/strategies/utils_strategy.py +0 -0
- cloe_nessy/integration/__init__.py +0 -0
- cloe_nessy/integration/reader/__init__.py +0 -0
- cloe_nessy/integration/reader/api_reader.py +0 -0
- cloe_nessy/integration/reader/catalog_reader.py +0 -0
- cloe_nessy/integration/reader/excel_reader.py +0 -0
- cloe_nessy/integration/reader/exceptions.py +0 -0
- cloe_nessy/integration/reader/file_reader.py +7 -1
- cloe_nessy/integration/reader/reader.py +0 -0
- cloe_nessy/integration/writer/__init__.py +0 -0
- cloe_nessy/integration/writer/catalog_writer.py +1 -1
- cloe_nessy/logging/__init__.py +0 -0
- cloe_nessy/logging/logger_mixin.py +0 -0
- cloe_nessy/models/__init__.py +4 -0
- cloe_nessy/models/adapter/__init__.py +3 -0
- cloe_nessy/models/adapter/unity_catalog_adapter.py +292 -0
- cloe_nessy/models/catalog.py +10 -0
- cloe_nessy/models/column.py +0 -0
- cloe_nessy/models/constraint.py +0 -0
- cloe_nessy/models/foreign_key.py +0 -0
- cloe_nessy/models/mixins/__init__.py +0 -0
- cloe_nessy/models/mixins/read_instance_mixin.py +0 -0
- cloe_nessy/models/mixins/template_loader_mixin.py +0 -0
- cloe_nessy/models/schema.py +19 -0
- cloe_nessy/models/table.py +50 -5
- cloe_nessy/models/types.py +0 -0
- cloe_nessy/models/volume.py +67 -0
- cloe_nessy/object_manager/__init__.py +7 -2
- cloe_nessy/object_manager/table_manager.py +183 -7
- cloe_nessy/object_manager/volume_manager.py +70 -0
- cloe_nessy/pipeline/__init__.py +0 -0
- cloe_nessy/pipeline/actions/__init__.py +2 -0
- cloe_nessy/pipeline/actions/read_api.py +69 -45
- cloe_nessy/pipeline/actions/read_catalog_table.py +9 -9
- cloe_nessy/pipeline/actions/read_excel.py +14 -10
- cloe_nessy/pipeline/actions/read_files.py +54 -28
- cloe_nessy/pipeline/actions/read_metadata_yaml.py +9 -9
- cloe_nessy/pipeline/actions/transform_change_datatype.py +13 -8
- cloe_nessy/pipeline/actions/transform_clean_column_names.py +4 -0
- cloe_nessy/pipeline/actions/transform_concat_columns.py +25 -11
- cloe_nessy/pipeline/actions/transform_decode.py +18 -7
- cloe_nessy/pipeline/actions/transform_deduplication.py +9 -9
- cloe_nessy/pipeline/actions/transform_distinct.py +8 -8
- cloe_nessy/pipeline/actions/transform_filter.py +6 -6
- cloe_nessy/pipeline/actions/transform_generic_sql.py +12 -6
- cloe_nessy/pipeline/actions/transform_group_aggregate.py +20 -26
- cloe_nessy/pipeline/actions/transform_hash_columns.py +209 -0
- cloe_nessy/pipeline/actions/transform_join.py +17 -10
- cloe_nessy/pipeline/actions/transform_json_normalize.py +19 -6
- cloe_nessy/pipeline/actions/transform_rename_columns.py +7 -7
- cloe_nessy/pipeline/actions/transform_replace_values.py +8 -8
- cloe_nessy/pipeline/actions/transform_select_columns.py +38 -9
- cloe_nessy/pipeline/actions/transform_union.py +12 -8
- cloe_nessy/pipeline/actions/write_catalog_table.py +11 -10
- cloe_nessy/pipeline/pipeline.py +44 -2
- cloe_nessy/pipeline/pipeline_action.py +0 -0
- cloe_nessy/pipeline/pipeline_config.py +0 -0
- cloe_nessy/pipeline/pipeline_context.py +0 -0
- cloe_nessy/pipeline/pipeline_parsing_service.py +0 -0
- cloe_nessy/pipeline/pipeline_step.py +0 -0
- cloe_nessy/py.typed +0 -0
- cloe_nessy/session/__init__.py +0 -0
- cloe_nessy/session/session_manager.py +27 -0
- cloe_nessy/settings/__init__.py +0 -0
- cloe_nessy/settings/settings.py +0 -0
- cloe_nessy/utils/__init__.py +0 -0
- cloe_nessy/utils/file_and_directory_handler.py +0 -0
- cloe_nessy-0.3.8.dist-info/METADATA +46 -0
- {cloe_nessy-0.3.3.dist-info → cloe_nessy-0.3.8.dist-info}/RECORD +41 -35
- {cloe_nessy-0.3.3.dist-info → cloe_nessy-0.3.8.dist-info}/WHEEL +1 -1
- {cloe_nessy-0.3.3.dist-info → cloe_nessy-0.3.8.dist-info}/top_level.txt +0 -0
- cloe_nessy-0.3.3.dist-info/METADATA +0 -26
cloe_nessy/pipeline/pipeline.py
CHANGED
|
@@ -134,9 +134,51 @@ class Pipeline(LoggerMixin):
|
|
|
134
134
|
self._console_logger.debug("No more steps to run")
|
|
135
135
|
return False
|
|
136
136
|
|
|
137
|
-
def
|
|
138
|
-
"""
|
|
137
|
+
def _trim_graph(self, graph, until):
|
|
138
|
+
"""Trims the pipeline graph to only include steps up to the specified 'until' step (excluding).
|
|
139
|
+
|
|
140
|
+
This method first verifies that the given step exists in the graph. It then finds all ancestors
|
|
141
|
+
(i.e., steps that precede the 'until' step) and creates a subgraph consisting solely of those steps.
|
|
142
|
+
|
|
143
|
+
Args:
|
|
144
|
+
graph: The complete directed acyclic graph representing the pipeline.
|
|
145
|
+
until: The identifier of the step up to which the graph should be trimmed.
|
|
146
|
+
|
|
147
|
+
Returns:
|
|
148
|
+
A subgraph containing only the steps leading to (and including) the 'until' step.
|
|
149
|
+
|
|
150
|
+
Raises:
|
|
151
|
+
ValueError: If the specified 'until' step is not found in the graph.
|
|
152
|
+
"""
|
|
153
|
+
if until not in graph.nodes:
|
|
154
|
+
raise ValueError(f"Step '{until}' not found in the pipeline.")
|
|
155
|
+
|
|
156
|
+
predecessors = set(nx.ancestors(graph, until))
|
|
157
|
+
predecessors.add(until)
|
|
158
|
+
|
|
159
|
+
trimmed_graph = graph.subgraph(predecessors).copy()
|
|
160
|
+
return trimmed_graph
|
|
161
|
+
|
|
162
|
+
def run(self, until: str | None = None) -> None:
|
|
163
|
+
"""Executes the pipeline steps in the correct order based on dependencies.
|
|
164
|
+
|
|
165
|
+
This method creates a directed acyclic graph (DAG) of the pipeline steps and, if specified, trims
|
|
166
|
+
the graph to only include steps up to the given 'until' step (excluding: the step specified as 'until' will not be executed). It then concurrently executes steps
|
|
167
|
+
with no pending dependencies using a ThreadPoolExecutor, ensuring that all steps are run in order.
|
|
168
|
+
If a cyclic dependency is detected, or if any step fails during execution, the method raises an error.
|
|
169
|
+
|
|
170
|
+
Args:
|
|
171
|
+
until: Optional; the identifier of the step up to which the pipeline should be executed.
|
|
172
|
+
|
|
173
|
+
Raises:
|
|
174
|
+
RuntimeError: If a cyclic dependency is detected.
|
|
175
|
+
Exception: Propagates any error raised during the execution of a step.
|
|
176
|
+
"""
|
|
139
177
|
g = self._create_graph()
|
|
178
|
+
|
|
179
|
+
if until is not None:
|
|
180
|
+
g = self._trim_graph(g, until)
|
|
181
|
+
|
|
140
182
|
remaining_steps = list(g.nodes())
|
|
141
183
|
self._console_logger.info(f"Pipeline [' {self.name} '] started with {len(remaining_steps)} steps.")
|
|
142
184
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
cloe_nessy/py.typed
CHANGED
|
File without changes
|
cloe_nessy/session/__init__.py
CHANGED
|
File without changes
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
1
3
|
from enum import Enum
|
|
2
4
|
from typing import Any
|
|
3
5
|
|
|
@@ -16,12 +18,14 @@ class SessionManager:
|
|
|
16
18
|
- FABRIC_UI: Represents the Fabric user interface.
|
|
17
19
|
- DATABRICKS_CONNECT: Represents the Databricks Connect environment.
|
|
18
20
|
- OTHER_REMOTE_SPARK: Represents other remote Spark environments, such as used in tests.
|
|
21
|
+
- STANDALONE_SPARK: Represents a standalone Spark cluster environment.
|
|
19
22
|
"""
|
|
20
23
|
|
|
21
24
|
DATABRICKS_UI = "databricks_ui"
|
|
22
25
|
FABRIC_UI = "fabric_ui"
|
|
23
26
|
DATABRICKS_CONNECT = "databricks_connect"
|
|
24
27
|
OTHER_REMOTE_SPARK = "other_remote_spark"
|
|
28
|
+
STANDALONE_SPARK = "standalone_spark"
|
|
25
29
|
|
|
26
30
|
_spark: SparkSession | None = None
|
|
27
31
|
_utils = None
|
|
@@ -52,6 +56,15 @@ class SessionManager:
|
|
|
52
56
|
|
|
53
57
|
builder = cls.get_spark_builder()
|
|
54
58
|
|
|
59
|
+
# Check if NESSY_SPARK_CONFIG environment variable is set and load it as config
|
|
60
|
+
nessy_spark_config = os.getenv("NESSY_SPARK_CONFIG")
|
|
61
|
+
if nessy_spark_config:
|
|
62
|
+
try:
|
|
63
|
+
# Parse the JSON configuration from the environment variable
|
|
64
|
+
config = json.loads(nessy_spark_config)
|
|
65
|
+
except json.JSONDecodeError as e:
|
|
66
|
+
raise ValueError(f"Invalid JSON in NESSY_SPARK_CONFIG: {e}") from e
|
|
67
|
+
|
|
55
68
|
if config:
|
|
56
69
|
for key, value in config.items():
|
|
57
70
|
builder.config(key, value) # type: ignore
|
|
@@ -84,6 +97,7 @@ class SessionManager:
|
|
|
84
97
|
cls.Environment.DATABRICKS_UI: cls._get_dbutils,
|
|
85
98
|
cls.Environment.DATABRICKS_CONNECT: cls._get_dbutils,
|
|
86
99
|
cls.Environment.OTHER_REMOTE_SPARK: cls._get_dbutils,
|
|
100
|
+
cls.Environment.STANDALONE_SPARK: cls._get_localsparkutils,
|
|
87
101
|
cls.Environment.FABRIC_UI: cls._get_mssparkutils,
|
|
88
102
|
}
|
|
89
103
|
|
|
@@ -112,6 +126,10 @@ class SessionManager:
|
|
|
112
126
|
|
|
113
127
|
cls._utils = mssparkutils
|
|
114
128
|
|
|
129
|
+
@classmethod
|
|
130
|
+
def _get_localsparkutils(cls):
|
|
131
|
+
return None
|
|
132
|
+
|
|
115
133
|
@classmethod
|
|
116
134
|
def _detect_env(cls) -> Environment | None:
|
|
117
135
|
"""Detects the current execution environment for Spark.
|
|
@@ -169,6 +187,14 @@ class SessionManager:
|
|
|
169
187
|
except ImportError:
|
|
170
188
|
pass
|
|
171
189
|
|
|
190
|
+
try:
|
|
191
|
+
from pyspark.sql import SparkSession # noqa: F401
|
|
192
|
+
|
|
193
|
+
cls._env = cls.Environment.STANDALONE_SPARK
|
|
194
|
+
return cls._env
|
|
195
|
+
except ImportError:
|
|
196
|
+
pass
|
|
197
|
+
|
|
172
198
|
raise RuntimeError("Cannot detect environment.")
|
|
173
199
|
|
|
174
200
|
@classmethod
|
|
@@ -180,6 +206,7 @@ class SessionManager:
|
|
|
180
206
|
cls.Environment.FABRIC_UI: SparkSession.builder,
|
|
181
207
|
cls.Environment.DATABRICKS_CONNECT: cls._get_databricks_connect_builder,
|
|
182
208
|
cls.Environment.OTHER_REMOTE_SPARK: cls._get_databricks_connect_builder,
|
|
209
|
+
cls.Environment.STANDALONE_SPARK: SparkSession.builder,
|
|
183
210
|
}
|
|
184
211
|
builder = builders.get(cls._env)
|
|
185
212
|
if builder is None:
|
cloe_nessy/settings/__init__.py
CHANGED
|
File without changes
|
cloe_nessy/settings/settings.py
CHANGED
|
File without changes
|
cloe_nessy/utils/__init__.py
CHANGED
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cloe-nessy
|
|
3
|
+
Version: 0.3.8
|
|
4
|
+
Summary: Your friendly datalake monster.
|
|
5
|
+
Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
8
|
+
Classifier: Environment :: Console
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Topic :: Database
|
|
14
|
+
Requires-Python: <3.12,>=3.11
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
Requires-Dist: pydantic<3.0.0,>=2.7.2
|
|
17
|
+
Requires-Dist: pyyaml<7.0.0,>=6.0.1
|
|
18
|
+
Requires-Dist: types-pyyaml<7.0.0.0,>=6.0.12.20240311
|
|
19
|
+
Requires-Dist: jinja2<4.0.0,>=3.1.4
|
|
20
|
+
Requires-Dist: pydantic-settings<3.0.0,>=2.4.0
|
|
21
|
+
Requires-Dist: openpyxl<4.0.0,>=3.1.5
|
|
22
|
+
Requires-Dist: requests<3.0.0,>=2.32.3
|
|
23
|
+
Requires-Dist: types-requests<3.0.0.0,>=2.32.0.20240712
|
|
24
|
+
Requires-Dist: pandas-stubs<3.0.0.0,>=2.2.2.240807
|
|
25
|
+
Requires-Dist: azure-identity<2.0.0,>=1.19.0
|
|
26
|
+
Requires-Dist: httpx<1.0.0,>=0.27.2
|
|
27
|
+
Requires-Dist: databricks-sdk<1.0.0,>=0.36.0
|
|
28
|
+
Requires-Dist: networkx<4.0,>=3.3
|
|
29
|
+
Requires-Dist: matplotlib<4.0.0,>=3.9.2
|
|
30
|
+
Requires-Dist: types-networkx<4.0.0.0,>=3.2.1.20240820
|
|
31
|
+
Requires-Dist: fsspec<2025.0.0,>=2024.9.0
|
|
32
|
+
Requires-Dist: cloe-logging[databricks,log-analytics]<0.4,>=0.3.7
|
|
33
|
+
|
|
34
|
+
# cloe-nessy
|
|
35
|
+
|
|
36
|
+
[](https://github.com/copier-org/copier)
|
|
37
|
+
[](https://www.python.org)
|
|
38
|
+
[](https://github.com/astral-sh/uv)
|
|
39
|
+
[](https://github.com/charliermarsh/ruff)
|
|
40
|
+
[](https://mypy-lang.org/)
|
|
41
|
+
[](https://github.com/psf/black)
|
|
42
|
+
[](https://github.com/pre-commit/pre-commit)
|
|
43
|
+
|
|
44
|
+
Owner: initions
|
|
45
|
+
|
|
46
|
+
Your friendly datalake monster.
|
|
@@ -22,60 +22,66 @@ cloe_nessy/integration/reader/api_reader.py,sha256=j3Z5O1oH-Zc43TyA_aYtnDNYC9xFM
|
|
|
22
22
|
cloe_nessy/integration/reader/catalog_reader.py,sha256=tGK-Y0jZQGOrF9eZUzSr7ils-L58uex6qH9PZ81ZLy8,1835
|
|
23
23
|
cloe_nessy/integration/reader/excel_reader.py,sha256=4kifpIakHpGmap0-P0SUgjJoQdY-eeiZBIDrQp87wK8,8012
|
|
24
24
|
cloe_nessy/integration/reader/exceptions.py,sha256=_A9jFpe_RIDZCGY76qzjic9bsshxns6yXPSl141dq1c,203
|
|
25
|
-
cloe_nessy/integration/reader/file_reader.py,sha256=
|
|
25
|
+
cloe_nessy/integration/reader/file_reader.py,sha256=1os8pZIXAGTJBZjGREmHOTlZeabbikC7sDv5xn3bIjE,3950
|
|
26
26
|
cloe_nessy/integration/reader/reader.py,sha256=e2KVPePQme8SBQJEbL-3zpGasOgTiEvKFTslow2wGPw,1034
|
|
27
27
|
cloe_nessy/integration/writer/__init__.py,sha256=NIh0t1RYlG3J1Y5_CvnR36N9tISmcElD5Tq06ksmqoA,71
|
|
28
|
-
cloe_nessy/integration/writer/catalog_writer.py,sha256=
|
|
28
|
+
cloe_nessy/integration/writer/catalog_writer.py,sha256=Gb-hMdADgO_uUJ7mZPHBYyNme2qXsdFFnzwo7GcShHM,2192
|
|
29
29
|
cloe_nessy/logging/__init__.py,sha256=ySVCVbdyR3Dno_tl2ZfiER_7EVaDoQMHVkNyfdMZumY,65
|
|
30
30
|
cloe_nessy/logging/logger_mixin.py,sha256=9iy7BF6drYme-f7Rrt_imbVBRgVqQ89xjcP1X5aMtfY,7467
|
|
31
|
-
cloe_nessy/models/__init__.py,sha256
|
|
31
|
+
cloe_nessy/models/__init__.py,sha256=-FmWEJ1Oq1njSopjc0R7GmT64mLSmALkm8PkHNzy9Y8,327
|
|
32
|
+
cloe_nessy/models/catalog.py,sha256=ayC1sMp4cNLAZtu0ICVV3Us6-o4hn8U9tpzzvxC9RAs,177
|
|
32
33
|
cloe_nessy/models/column.py,sha256=53fBwRnino72XKACsHZpN9QfCBqqSXyKLHZlM0huumg,1988
|
|
33
34
|
cloe_nessy/models/constraint.py,sha256=hsFlhn4n928z81O3dl3v5bMetewPWzMjkJK3_4kASSM,178
|
|
34
35
|
cloe_nessy/models/foreign_key.py,sha256=DwRVHs9sShqqPV-NL7ow_3AmPPWX0Od26yZn_I565pU,1001
|
|
35
|
-
cloe_nessy/models/schema.py,sha256=
|
|
36
|
-
cloe_nessy/models/table.py,sha256=
|
|
36
|
+
cloe_nessy/models/schema.py,sha256=DHlJJ4JA8392vVrFatH22GoPOCJN-eqkmHVihIHpPCg,3382
|
|
37
|
+
cloe_nessy/models/table.py,sha256=W9gqMTCQBRo3Z_rLY5cJ8WgPEUEslNLbaRrODuTy15Y,11744
|
|
37
38
|
cloe_nessy/models/types.py,sha256=XRbuJGdTNa6aXyE3IAzs_J9gVjbfkzMDLfGl-k6jI_4,223
|
|
39
|
+
cloe_nessy/models/volume.py,sha256=7_lH4X-Au8jYNRVFQ5F2x-fRy2J8Z5-cLTn3Z3mpLQs,2197
|
|
40
|
+
cloe_nessy/models/adapter/__init__.py,sha256=m36W_mqwB3dCYnCIt0fLOSHS4E1VU8FRGoaum4Gf95o,90
|
|
41
|
+
cloe_nessy/models/adapter/unity_catalog_adapter.py,sha256=T5v8gN57vRj9OLrfj8N4TEsH_Z_5N38wTqu_7rYgXb0,12572
|
|
38
42
|
cloe_nessy/models/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
39
43
|
cloe_nessy/models/mixins/read_instance_mixin.py,sha256=j5Y4aNWOh1jlskEaxNooZFJgPyxRmik00gAVLJnAaRs,4507
|
|
40
44
|
cloe_nessy/models/mixins/template_loader_mixin.py,sha256=5MXhEGBFlq3dwZvINEyBowSlipNnVun2H_TmhI_fsS4,549
|
|
41
|
-
cloe_nessy/object_manager/__init__.py,sha256=
|
|
42
|
-
cloe_nessy/object_manager/table_manager.py,sha256=
|
|
45
|
+
cloe_nessy/object_manager/__init__.py,sha256=3sle0vNpPwBOkycxA3XVS9m4XZf5LD3Qd4NGxdqcHno,186
|
|
46
|
+
cloe_nessy/object_manager/table_manager.py,sha256=B_00UXaEQQHTAsyZGivhW3TQQpejC3obljAi_IwyFNA,9993
|
|
47
|
+
cloe_nessy/object_manager/volume_manager.py,sha256=6epd3KXzcNH04EvaKubAfLsaUm9qBMeT3KNvMK04gGs,2727
|
|
43
48
|
cloe_nessy/pipeline/__init__.py,sha256=sespmJ5JsgyiFyZiedTiL2kg--zGIX7cjTYsD5vemEg,325
|
|
44
|
-
cloe_nessy/pipeline/pipeline.py,sha256
|
|
49
|
+
cloe_nessy/pipeline/pipeline.py,sha256=-1tJVs9rZf8CcwieH4IP7mqJZ6mL7bQUZ56TNKt8eO8,11154
|
|
45
50
|
cloe_nessy/pipeline/pipeline_action.py,sha256=S7IVFdmG12fRBzHuE_DiWn7qlMtApz6IloVd2Fj31Sg,1944
|
|
46
51
|
cloe_nessy/pipeline/pipeline_config.py,sha256=BN3ZSbr6bC-X9edoh-n5vRfPHFMbgtAU7mQ3dBrcWO8,3131
|
|
47
52
|
cloe_nessy/pipeline/pipeline_context.py,sha256=csElDc6BsynDUtRXgQOSCH7ONc_b-ag0YEg0zlQTz58,1874
|
|
48
53
|
cloe_nessy/pipeline/pipeline_parsing_service.py,sha256=c_nAsgw81QYBM9AFiTxGgqRhNXABkDKplbeoCJPtbpE,6434
|
|
49
54
|
cloe_nessy/pipeline/pipeline_step.py,sha256=UlnmpS6gm_dZ7m9dD1mZvye7mvUF_DA7HjOZo0oGYDU,1977
|
|
50
|
-
cloe_nessy/pipeline/actions/__init__.py,sha256=
|
|
51
|
-
cloe_nessy/pipeline/actions/read_api.py,sha256=
|
|
52
|
-
cloe_nessy/pipeline/actions/read_catalog_table.py,sha256
|
|
53
|
-
cloe_nessy/pipeline/actions/read_excel.py,sha256=
|
|
54
|
-
cloe_nessy/pipeline/actions/read_files.py,sha256=
|
|
55
|
-
cloe_nessy/pipeline/actions/read_metadata_yaml.py,sha256=
|
|
56
|
-
cloe_nessy/pipeline/actions/transform_change_datatype.py,sha256=
|
|
57
|
-
cloe_nessy/pipeline/actions/transform_clean_column_names.py,sha256
|
|
58
|
-
cloe_nessy/pipeline/actions/transform_concat_columns.py,sha256=
|
|
59
|
-
cloe_nessy/pipeline/actions/transform_decode.py,sha256=
|
|
60
|
-
cloe_nessy/pipeline/actions/transform_deduplication.py,sha256=
|
|
61
|
-
cloe_nessy/pipeline/actions/transform_distinct.py,sha256=
|
|
62
|
-
cloe_nessy/pipeline/actions/transform_filter.py,sha256=
|
|
63
|
-
cloe_nessy/pipeline/actions/transform_generic_sql.py,sha256=
|
|
64
|
-
cloe_nessy/pipeline/actions/transform_group_aggregate.py,sha256=
|
|
65
|
-
cloe_nessy/pipeline/actions/
|
|
66
|
-
cloe_nessy/pipeline/actions/
|
|
67
|
-
cloe_nessy/pipeline/actions/
|
|
68
|
-
cloe_nessy/pipeline/actions/
|
|
69
|
-
cloe_nessy/pipeline/actions/
|
|
70
|
-
cloe_nessy/pipeline/actions/
|
|
71
|
-
cloe_nessy/pipeline/actions/
|
|
55
|
+
cloe_nessy/pipeline/actions/__init__.py,sha256=dvIaWVR-L3IQkxMI-TIazn2udeFGsKmhUxw9E0VTz0g,2370
|
|
56
|
+
cloe_nessy/pipeline/actions/read_api.py,sha256=RBv5XeHtjTXuCP09Fqo6JNx6iIhQQI-nuAHCuSaGs2s,7778
|
|
57
|
+
cloe_nessy/pipeline/actions/read_catalog_table.py,sha256=-k2wezkv8bE_xwoW7WM1ORhrCXQagKTUuXkhI2ZEROs,2783
|
|
58
|
+
cloe_nessy/pipeline/actions/read_excel.py,sha256=Mhl3r_2Hqk2XN7Fl5WqqAyE4JdnwSiivbhWMglyBtkE,7961
|
|
59
|
+
cloe_nessy/pipeline/actions/read_files.py,sha256=N9bFgtG1tovhp2JayxE5YiN9PiO2lgG2-6h_Y6tD2eU,5220
|
|
60
|
+
cloe_nessy/pipeline/actions/read_metadata_yaml.py,sha256=3ZDy9qiDYtM1oDQzHPC23hLOvHjhdk5zg1wVHE60m9k,2295
|
|
61
|
+
cloe_nessy/pipeline/actions/transform_change_datatype.py,sha256=24Tn6R3TvUkWCh8V6naLdyNbCbqvyPOOoer-hy_Ebq4,2077
|
|
62
|
+
cloe_nessy/pipeline/actions/transform_clean_column_names.py,sha256=-CEdcXb7Fz5DQNitGlJ8EVBE_LzxfsInyCIO-D7b4iY,3042
|
|
63
|
+
cloe_nessy/pipeline/actions/transform_concat_columns.py,sha256=Nk8YbhxDnFZsWzW9Dj5Yl76Uq6VrcMlevQPHGms65L8,3777
|
|
64
|
+
cloe_nessy/pipeline/actions/transform_decode.py,sha256=JajMwHREtxa8u_1Q3RZDBVMjncoSel-WzQFVTO0MREg,4455
|
|
65
|
+
cloe_nessy/pipeline/actions/transform_deduplication.py,sha256=E0ypz9qkHMSatNfnHekP-E6svQVL149M4PV02M03drg,5099
|
|
66
|
+
cloe_nessy/pipeline/actions/transform_distinct.py,sha256=c7aBxANyqT4aKhm0cSELDtD-bP0Se9vxlBF0K4AgQWs,1976
|
|
67
|
+
cloe_nessy/pipeline/actions/transform_filter.py,sha256=Nz_ggRfKIcNzYFfFOsgq1QeatjdEis0up4I7cOWBdyo,1446
|
|
68
|
+
cloe_nessy/pipeline/actions/transform_generic_sql.py,sha256=_naWfmPdYAUKjPNeHu5qJAohOL7DHCSYz_kwoeRv3OI,2741
|
|
69
|
+
cloe_nessy/pipeline/actions/transform_group_aggregate.py,sha256=KUHeeP-RIDi34dpbsPEJkzea5zFJA6MuyjNpOsFud9o,4045
|
|
70
|
+
cloe_nessy/pipeline/actions/transform_hash_columns.py,sha256=heRjBA-Gfu-nmNHOjTYlipEpKY8oNPAHAY40vjJk3aI,8383
|
|
71
|
+
cloe_nessy/pipeline/actions/transform_join.py,sha256=e_tvMk8YJTAWcUK_EmOgNt0s31ICZoMX_MKOTWx4lBY,3645
|
|
72
|
+
cloe_nessy/pipeline/actions/transform_json_normalize.py,sha256=petF7pnNq1EKc8MqVdG0weFALAHNILSe_eAu4Z5XxIo,4833
|
|
73
|
+
cloe_nessy/pipeline/actions/transform_rename_columns.py,sha256=4zJcPCONMU4C67qeuzsrX3AORRRHoq_selUI7FJyeg0,1952
|
|
74
|
+
cloe_nessy/pipeline/actions/transform_replace_values.py,sha256=1OPHTrjcphfyGepcO7ozYfeqfwA18pjlyHpVKUS_AAU,2049
|
|
75
|
+
cloe_nessy/pipeline/actions/transform_select_columns.py,sha256=-GhSEsb7iNnZIsYRm3BG9BX4_qUDJMbpj1DsKPY046w,4574
|
|
76
|
+
cloe_nessy/pipeline/actions/transform_union.py,sha256=s81Vge0AbYPc7VkskCYfOQ_LEjqcmfNFyDkytfjcZyo,2720
|
|
77
|
+
cloe_nessy/pipeline/actions/write_catalog_table.py,sha256=j7gRuG3Fedh8JgevIFBbHKock3laJVq4l6Mx3CGU5eo,2676
|
|
72
78
|
cloe_nessy/session/__init__.py,sha256=t7_YjUhJYW3km_FrucaUdbIl1boQtwkyhw_8yE10qzc,74
|
|
73
|
-
cloe_nessy/session/session_manager.py,sha256=
|
|
79
|
+
cloe_nessy/session/session_manager.py,sha256=f4OeeyGD3becDQGkdDbck3jVH9ulOCBWjW6Jaj_MIrc,7765
|
|
74
80
|
cloe_nessy/settings/__init__.py,sha256=ZbkneO3WaKOxon7qHFHnou7EnBOSnBFyKMDZblIEvzM,101
|
|
75
81
|
cloe_nessy/settings/settings.py,sha256=I4n129lrujriW-d8q4as2Kb4_kI932ModfZ5Ow_UpVM,3653
|
|
76
82
|
cloe_nessy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
77
83
|
cloe_nessy/utils/file_and_directory_handler.py,sha256=r2EVt9xG81p6ScaJCwETC5an6pMT6WseB0jMOR-JlpU,602
|
|
78
|
-
cloe_nessy-0.3.
|
|
79
|
-
cloe_nessy-0.3.
|
|
80
|
-
cloe_nessy-0.3.
|
|
81
|
-
cloe_nessy-0.3.
|
|
84
|
+
cloe_nessy-0.3.8.dist-info/METADATA,sha256=fnnrNC7Vo8REfpUO5hdflH0_zaZAF437CyRrUuCM8dI,2374
|
|
85
|
+
cloe_nessy-0.3.8.dist-info/WHEEL,sha256=wXxTzcEDnjrTwFYjLPcsW_7_XihufBwmpiBeiXNBGEA,91
|
|
86
|
+
cloe_nessy-0.3.8.dist-info/top_level.txt,sha256=Z7izn8HmQpg2wBUb-0jzaKlYKMU7Ypzuc9__9vPtW_I,11
|
|
87
|
+
cloe_nessy-0.3.8.dist-info/RECORD,,
|
|
File without changes
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.2
|
|
2
|
-
Name: cloe-nessy
|
|
3
|
-
Version: 0.3.3
|
|
4
|
-
Summary: Your friendly datalake monster.
|
|
5
|
-
Home-page: https://initions.com/
|
|
6
|
-
Author: initions
|
|
7
|
-
Author-email: ICSMC_EXT_PYPIORG@accenture.com
|
|
8
|
-
License: MIT
|
|
9
|
-
Requires-Python: <3.12,>=3.11
|
|
10
|
-
Requires-Dist: pydantic<3.0.0,>=2.7.2
|
|
11
|
-
Requires-Dist: pyyaml<7.0.0,>=6.0.1
|
|
12
|
-
Requires-Dist: types-pyyaml<7.0.0.0,>=6.0.12.20240311
|
|
13
|
-
Requires-Dist: jinja2<4.0.0,>=3.1.4
|
|
14
|
-
Requires-Dist: pydantic-settings<3.0.0,>=2.4.0
|
|
15
|
-
Requires-Dist: openpyxl<4.0.0,>=3.1.5
|
|
16
|
-
Requires-Dist: requests<3.0.0,>=2.32.3
|
|
17
|
-
Requires-Dist: types-requests<3.0.0.0,>=2.32.0.20240712
|
|
18
|
-
Requires-Dist: pandas-stubs<3.0.0.0,>=2.2.2.240807
|
|
19
|
-
Requires-Dist: azure-identity<2.0.0,>=1.19.0
|
|
20
|
-
Requires-Dist: httpx<1.0.0,>=0.27.2
|
|
21
|
-
Requires-Dist: databricks-sdk<1.0.0,>=0.36.0
|
|
22
|
-
Requires-Dist: networkx<4.0,>=3.3
|
|
23
|
-
Requires-Dist: matplotlib<4.0.0,>=3.9.2
|
|
24
|
-
Requires-Dist: types-networkx<4.0.0.0,>=3.2.1.20240820
|
|
25
|
-
Requires-Dist: fsspec<2025.0.0,>=2024.9.0
|
|
26
|
-
Requires-Dist: cloe-logging[databricks,log-analytics]<0.4,>=0.3.7
|