cloe-nessy 0.3.18__py3-none-any.whl → 0.3.19__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cloe_nessy/clients/api_client/__init__.py +10 -1
- cloe_nessy/clients/api_client/api_client.py +19 -8
- cloe_nessy/clients/api_client/api_response.py +7 -4
- cloe_nessy/clients/api_client/pagination_config.py +84 -0
- cloe_nessy/clients/api_client/pagination_strategy.py +500 -0
- cloe_nessy/integration/reader/__init__.py +2 -2
- cloe_nessy/integration/reader/api_reader.py +463 -72
- cloe_nessy/integration/reader/catalog_reader.py +6 -4
- cloe_nessy/integration/reader/excel_reader.py +3 -3
- cloe_nessy/integration/reader/file_reader.py +3 -1
- cloe_nessy/integration/reader/reader.py +1 -1
- cloe_nessy/integration/writer/catalog_writer.py +1 -1
- cloe_nessy/pipeline/actions/__init__.py +1 -1
- cloe_nessy/pipeline/actions/read_api.py +272 -75
- cloe_nessy/pipeline/actions/read_excel.py +1 -1
- cloe_nessy/pipeline/actions/transform_decode.py +2 -1
- cloe_nessy/pipeline/pipeline_config.py +2 -0
- cloe_nessy/pipeline/pipeline_context.py +1 -1
- cloe_nessy/pipeline/pipeline_parsing_service.py +104 -39
- cloe_nessy/pipeline/pipeline_step.py +2 -0
- cloe_nessy/session/__init__.py +2 -1
- cloe_nessy/session/pyspark_compat.py +15 -0
- cloe_nessy/session/session_manager.py +1 -1
- {cloe_nessy-0.3.18.dist-info → cloe_nessy-0.3.19.dist-info}/METADATA +3 -3
- {cloe_nessy-0.3.18.dist-info → cloe_nessy-0.3.19.dist-info}/RECORD +26 -23
- {cloe_nessy-0.3.18.dist-info → cloe_nessy-0.3.19.dist-info}/WHEEL +1 -1
|
@@ -3,6 +3,7 @@ import re
|
|
|
3
3
|
from collections import OrderedDict
|
|
4
4
|
from enum import Enum
|
|
5
5
|
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
6
7
|
|
|
7
8
|
import yaml
|
|
8
9
|
|
|
@@ -10,7 +11,7 @@ from ..logging import LoggerMixin
|
|
|
10
11
|
from ..session import SessionManager
|
|
11
12
|
from .actions import PipelineActionType, pipeline_actions
|
|
12
13
|
from .pipeline import Pipeline
|
|
13
|
-
from .pipeline_config import PipelineConfig
|
|
14
|
+
from .pipeline_config import PipelineConfig, PipelineStepConfig
|
|
14
15
|
from .pipeline_step import PipelineStep
|
|
15
16
|
|
|
16
17
|
|
|
@@ -63,49 +64,22 @@ class PipelineParsingService:
|
|
|
63
64
|
if not yaml_str:
|
|
64
65
|
raise ValueError("YAML content is empty.")
|
|
65
66
|
|
|
66
|
-
|
|
67
|
-
|
|
67
|
+
secrets_repl_yaml_str = PipelineParsingService._replace_secret_refs(yaml_str)
|
|
68
|
+
fixed_yaml_str = PipelineParsingService._fix_yaml_str_with_templates(secrets_repl_yaml_str)
|
|
69
|
+
config = yaml.safe_load(fixed_yaml_str)
|
|
68
70
|
pipeline_config = PipelineConfig.metadata_to_instance(config)
|
|
69
|
-
steps = PipelineParsingService._get_steps(pipeline_config.steps)
|
|
71
|
+
steps = PipelineParsingService._get_steps(pipeline_config.steps, pipeline_config.env)
|
|
70
72
|
pipeline = Pipeline(name=pipeline_config.name, steps=steps) # type: ignore
|
|
71
73
|
console_logger.info("Pipeline [ '%s' ] parsed successfully with %d steps.", pipeline.name, len(pipeline.steps))
|
|
72
74
|
return pipeline
|
|
73
75
|
|
|
74
76
|
@staticmethod
|
|
75
|
-
def
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
scope-name is the name of the secret scope and secret-key is the key of
|
|
82
|
-
the secret.
|
|
83
|
-
|
|
84
|
-
Args:
|
|
85
|
-
yaml_str: A string that can be parsed in YAML format.
|
|
86
|
-
|
|
87
|
-
Returns:
|
|
88
|
-
The same YAML string with environment variable placeholders replaced.
|
|
89
|
-
"""
|
|
90
|
-
env_var_pattern = r"\{\{env:([^}]+)\}\}"
|
|
91
|
-
secret_ref_pattern = r"\{\{(?!step|env)([^}]+):([^}]+)\}\}"
|
|
92
|
-
|
|
93
|
-
def replace_with_env_var(match):
|
|
94
|
-
env_var_name = match.group(1)
|
|
95
|
-
env_var_value = os.getenv(env_var_name)
|
|
96
|
-
return env_var_value
|
|
97
|
-
|
|
98
|
-
def replace_with_secret(match):
|
|
99
|
-
secret_scope_name = match.group(1)
|
|
100
|
-
secret_key = match.group(2)
|
|
101
|
-
return SessionManager.get_utils().secrets.get(scope=secret_scope_name, key=secret_key)
|
|
102
|
-
|
|
103
|
-
env_replaced_yaml_string = re.sub(env_var_pattern, replace_with_env_var, yaml_str)
|
|
104
|
-
final_yaml_string = re.sub(secret_ref_pattern, replace_with_secret, env_replaced_yaml_string)
|
|
105
|
-
return final_yaml_string
|
|
106
|
-
|
|
107
|
-
@staticmethod
|
|
108
|
-
def _get_steps(step_configs, last_step_name: str | None = None):
|
|
77
|
+
def _get_steps(
|
|
78
|
+
step_configs: OrderedDict[str, PipelineStepConfig],
|
|
79
|
+
pipeline_env: dict[str, str],
|
|
80
|
+
last_step_name: str | None = None,
|
|
81
|
+
) -> OrderedDict[str, PipelineStep]:
|
|
82
|
+
os_env = dict(os.environ)
|
|
109
83
|
steps = OrderedDict()
|
|
110
84
|
for step_name, step_config in step_configs.items():
|
|
111
85
|
is_successor = step_config.is_successor
|
|
@@ -115,19 +89,99 @@ class PipelineParsingService:
|
|
|
115
89
|
action = PipelineActionType[step_config.action.name].value()
|
|
116
90
|
step = PipelineStep(
|
|
117
91
|
name=step_name,
|
|
92
|
+
env=step_config.env,
|
|
118
93
|
action=action,
|
|
119
94
|
options=step_config.options,
|
|
120
95
|
_context_ref=context_ref,
|
|
121
96
|
_table_metadata_ref=step_config.table_metadata,
|
|
122
97
|
)
|
|
123
|
-
steps[step.name] = step
|
|
98
|
+
steps[step.name] = PipelineParsingService._resolve_env_vars(step, os_env, pipeline_env)
|
|
124
99
|
last_step_name = step_name
|
|
125
100
|
for step in steps.values():
|
|
126
101
|
steps[step.name] = PipelineParsingService._replace_step_refs(steps, step)
|
|
127
102
|
return steps
|
|
128
103
|
|
|
104
|
+
@staticmethod
|
|
105
|
+
def _replace_secret_refs(yaml_str: str) -> str:
|
|
106
|
+
"""Replaces secret reference placeholders in a YAML string.
|
|
107
|
+
|
|
108
|
+
Replaces secret references with the pattern `{{secret-scope-name:secret-key}}`.
|
|
109
|
+
Where scope-name is the name of the secret scope and secret-key is the key of the secret.
|
|
110
|
+
|
|
111
|
+
Args:
|
|
112
|
+
yaml_str: A string that can be parsed in YAML format.
|
|
113
|
+
|
|
114
|
+
Returns:
|
|
115
|
+
The same YAML string with secret reference placeholders replaced.
|
|
116
|
+
"""
|
|
117
|
+
secret_ref_pattern = r"\{\{(?!(?:env|step):)([^}]+):([^}]+)\}\}"
|
|
118
|
+
|
|
119
|
+
def replace_with_secret(match):
|
|
120
|
+
secret_scope_name = match.group(1)
|
|
121
|
+
secret_key = match.group(2)
|
|
122
|
+
return SessionManager.get_utils().secrets.get(scope=secret_scope_name, key=secret_key)
|
|
123
|
+
|
|
124
|
+
return re.sub(secret_ref_pattern, replace_with_secret, yaml_str)
|
|
125
|
+
|
|
126
|
+
@staticmethod
|
|
127
|
+
def _resolve_env_vars(step: PipelineStep, os_env: dict[str, str], pipeline_env: dict[str, str]) -> PipelineStep:
|
|
128
|
+
"""Resolves environment variable placeholders in step definition.
|
|
129
|
+
|
|
130
|
+
Resolves environment variables with the pattern `{{env:var-name}}`,
|
|
131
|
+
where the `var-name` is the name of the environment variable.
|
|
132
|
+
|
|
133
|
+
Args:
|
|
134
|
+
step: Step definition, where replacement is occurred.
|
|
135
|
+
os_env: OS scope environment variable.
|
|
136
|
+
pipeline_env: Pipeline scope environment variables.
|
|
137
|
+
|
|
138
|
+
Returns:
|
|
139
|
+
The same step definition with environment variable placeholders replaced.
|
|
140
|
+
|
|
141
|
+
Raises:
|
|
142
|
+
KeyError: If the specified key is not found in the environment variables.
|
|
143
|
+
"""
|
|
144
|
+
env_var_pattern = re.compile(r"\{\{env:([A-Z_][A-Z0-9_]*)\}\}")
|
|
145
|
+
|
|
146
|
+
def _resolve_object(obj: Any) -> Any:
|
|
147
|
+
if isinstance(obj, str):
|
|
148
|
+
return _resolve_string(obj)
|
|
149
|
+
if isinstance(obj, list):
|
|
150
|
+
return [_resolve_object(i) for i in obj]
|
|
151
|
+
if isinstance(obj, dict):
|
|
152
|
+
return {k: _resolve_object(v) for k, v in obj.items()}
|
|
153
|
+
return obj
|
|
154
|
+
|
|
155
|
+
def _resolve_string(value: str) -> str:
|
|
156
|
+
def repl(match):
|
|
157
|
+
key = match.group(1)
|
|
158
|
+
if key not in effective_env:
|
|
159
|
+
raise KeyError(f"Environment variable '{key}' is not defined")
|
|
160
|
+
return str(effective_env[key])
|
|
161
|
+
|
|
162
|
+
return env_var_pattern.sub(repl, value)
|
|
163
|
+
|
|
164
|
+
if step.options:
|
|
165
|
+
effective_env = {**os_env, **pipeline_env, **step.env}
|
|
166
|
+
for option, value in step.options.items():
|
|
167
|
+
step.options[option] = _resolve_object(value)
|
|
168
|
+
|
|
169
|
+
return step
|
|
170
|
+
|
|
129
171
|
@staticmethod
|
|
130
172
|
def _replace_step_refs(steps: OrderedDict[str, PipelineStep], step: PipelineStep) -> PipelineStep:
|
|
173
|
+
"""Replaces other steps reference placeholders in a step definition.
|
|
174
|
+
|
|
175
|
+
Replaces other steps references with the pattern `((step:step-name))`.
|
|
176
|
+
Where the `step-name` is the name of the referenced step.
|
|
177
|
+
|
|
178
|
+
Args:
|
|
179
|
+
steps: All pipeline steps definitions.
|
|
180
|
+
step: Step definition, where replacement is occurred.
|
|
181
|
+
|
|
182
|
+
Returns:
|
|
183
|
+
The same step definition with referenced step names replaced.
|
|
184
|
+
"""
|
|
131
185
|
step_ref_pattern = r"\(\(step:([^)]+)\)\)"
|
|
132
186
|
|
|
133
187
|
def _handle_string_value(value: str, option: str):
|
|
@@ -154,3 +208,14 @@ class PipelineParsingService:
|
|
|
154
208
|
_handle_list_value(value, option)
|
|
155
209
|
|
|
156
210
|
return step
|
|
211
|
+
|
|
212
|
+
@staticmethod
|
|
213
|
+
def _fix_yaml_str_with_templates(yaml_str: str) -> str:
|
|
214
|
+
"""Fixes unquoted {{env:...}} templates before yaml.safe_load."""
|
|
215
|
+
unquoted_template = re.compile(r"(:)\s*(\{\{env:[^}]+\}\})(?=\s*$|\s+#)", re.MULTILINE)
|
|
216
|
+
|
|
217
|
+
def replacer(match):
|
|
218
|
+
colon, template = match.groups()
|
|
219
|
+
return f'{colon} "{template}"'
|
|
220
|
+
|
|
221
|
+
return unquoted_template.sub(replacer, yaml_str)
|
|
@@ -15,6 +15,7 @@ class PipelineStep:
|
|
|
15
15
|
Attributes:
|
|
16
16
|
name: The name of the step.
|
|
17
17
|
action: The action to be executed.
|
|
18
|
+
env: The step environment variables.
|
|
18
19
|
is_successor: A boolean indicating if the step is a successor and takes
|
|
19
20
|
the previous steps context.
|
|
20
21
|
context: The context of the step.
|
|
@@ -26,6 +27,7 @@ class PipelineStep:
|
|
|
26
27
|
|
|
27
28
|
name: str
|
|
28
29
|
action: PipelineAction
|
|
30
|
+
env: dict[str, str] = field(default_factory=lambda: {})
|
|
29
31
|
context: PipelineContext = field(default_factory=lambda: PipelineContext())
|
|
30
32
|
options: dict[str, Any] = field(default_factory=lambda: {})
|
|
31
33
|
result: PipelineContext = field(default_factory=lambda: PipelineContext())
|
cloe_nessy/session/__init__.py
CHANGED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from typing import TYPE_CHECKING
|
|
2
|
+
|
|
3
|
+
from pyspark.sql.utils import is_remote
|
|
4
|
+
|
|
5
|
+
if TYPE_CHECKING:
|
|
6
|
+
from pyspark.sql import Column, DataFrame, SparkSession
|
|
7
|
+
else:
|
|
8
|
+
# Real runtime imports
|
|
9
|
+
if is_remote():
|
|
10
|
+
from pyspark.sql.connect.dataframe import DataFrame
|
|
11
|
+
from pyspark.sql.connect.session import SparkSession
|
|
12
|
+
else:
|
|
13
|
+
from pyspark.sql import DataFrame, SparkSession
|
|
14
|
+
|
|
15
|
+
__all__ = ["SparkSession", "DataFrame", "Column"]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cloe-nessy
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.19
|
|
4
4
|
Summary: Your friendly datalake monster.
|
|
5
5
|
Project-URL: homepage, https://initions.com/
|
|
6
6
|
Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>
|
|
@@ -58,12 +58,12 @@ Extract-Transform-Load (ETL) Workflow.
|
|
|
58
58
|
|
|
59
59
|
When you are contributing, please refer to our Contribution Guide in the *nessy*
|
|
60
60
|
Docs
|
|
61
|
-
[here](https://
|
|
61
|
+
[here](https://mango-tree-0b8dd3b03.1.azurestaticapps.net/tool_docs/nessy/Developer-Guide/)!
|
|
62
62
|
|
|
63
63
|
## Usage
|
|
64
64
|
|
|
65
65
|
Please find the User Guide
|
|
66
|
-
[here](https://
|
|
66
|
+
[here](https://mango-tree-0b8dd3b03.1.azurestaticapps.net/tool_docs/nessy/User-Guide/)!
|
|
67
67
|
|
|
68
68
|
## Contact
|
|
69
69
|
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
cloe_nessy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
2
|
cloe_nessy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
3
|
cloe_nessy/clients/__init__.py,sha256=GhiRvemNZ4TMS5rrHdmZEF73zozmrhvom2R5Oj6j9FI,71
|
|
4
|
-
cloe_nessy/clients/api_client/__init__.py,sha256=
|
|
5
|
-
cloe_nessy/clients/api_client/api_client.py,sha256=
|
|
6
|
-
cloe_nessy/clients/api_client/api_response.py,sha256=
|
|
4
|
+
cloe_nessy/clients/api_client/__init__.py,sha256=4ucfQnSQScLsvitTOdwTlPME5FX7p7vIt7ZJ-JQ_egc,529
|
|
5
|
+
cloe_nessy/clients/api_client/api_client.py,sha256=DYMxGPOueaBbY2qEA40HkREsf5n5UvB0s8dhApr4-fw,7703
|
|
6
|
+
cloe_nessy/clients/api_client/api_response.py,sha256=51_s6QjfQZLug3gEnojhuF_i3E1Mv-HdWiq6PRiFse0,2958
|
|
7
7
|
cloe_nessy/clients/api_client/auth.py,sha256=TNJQeSfBQ6O6jmqZvoeS-vyyG4PnhpyqbNIwfyElhpM,6737
|
|
8
8
|
cloe_nessy/clients/api_client/exceptions.py,sha256=VR9nYMHWzIRLlMZMrPpOsEX0X_P0jXJCTSBLTEhtN1E,403
|
|
9
|
+
cloe_nessy/clients/api_client/pagination_config.py,sha256=mSv1WjQSifnnpD1Ggl2ExUip_icMAMLpP3pcxske8RE,3722
|
|
10
|
+
cloe_nessy/clients/api_client/pagination_strategy.py,sha256=YcvAee8CrJiOxEvuFQ4KQJlNzHX3Kp0thOJyijnJMcE,17297
|
|
9
11
|
cloe_nessy/file_utilities/__init__.py,sha256=nY8H48jYHvTy0VYSRHVhZaFMlzfch4-T7y3N73tgMpI,73
|
|
10
12
|
cloe_nessy/file_utilities/exceptions.py,sha256=RDeV2S6AQnFhFINRo84HDV_hk2RMrf5oNQ7GhHmAZy0,97
|
|
11
13
|
cloe_nessy/file_utilities/factory.py,sha256=JONYGI8MCkNwG2_ujvjN3iB7BIdl7SqXKgV05YY_i4E,1735
|
|
@@ -25,15 +27,15 @@ cloe_nessy/integration/delta_loader/delta_loader_metadata_table.py,sha256=G_EWUY
|
|
|
25
27
|
cloe_nessy/integration/delta_loader/strategies/__init__.py,sha256=1o5fRWenL5KnUg1hf7kmTuTpG9pbMxchiQTub52Qvwo,255
|
|
26
28
|
cloe_nessy/integration/delta_loader/strategies/delta_cdf_loader.py,sha256=FOOZqtMwp8_LoyG2ab2N19a074CFa2ArCEvNkl7wRWM,16682
|
|
27
29
|
cloe_nessy/integration/delta_loader/strategies/delta_timestamp_loader.py,sha256=YYFH0DkdRPvITUc1JMgkmgIHjwDyZDCjqvEk2qhBMfE,6185
|
|
28
|
-
cloe_nessy/integration/reader/__init__.py,sha256=
|
|
29
|
-
cloe_nessy/integration/reader/api_reader.py,sha256=
|
|
30
|
-
cloe_nessy/integration/reader/catalog_reader.py,sha256=
|
|
31
|
-
cloe_nessy/integration/reader/excel_reader.py,sha256=
|
|
30
|
+
cloe_nessy/integration/reader/__init__.py,sha256=NWQx-v6aKE8YOHhsxfeaZnMVq4KLKyRWXzUduf5aVsk,265
|
|
31
|
+
cloe_nessy/integration/reader/api_reader.py,sha256=FbOyfLVG1ryL2GC-MgE1uClHICsQKBj9yZbY4TG5qrk,19637
|
|
32
|
+
cloe_nessy/integration/reader/catalog_reader.py,sha256=DlnykmFjV_v8SCBh3qaCvf24QM-6TdMFVHx5Mqv7Nvs,4850
|
|
33
|
+
cloe_nessy/integration/reader/excel_reader.py,sha256=JGmxQ16ux0HT-MLvAUp-9XMdKUToMb7cdObciZNsYSs,8027
|
|
32
34
|
cloe_nessy/integration/reader/exceptions.py,sha256=_A9jFpe_RIDZCGY76qzjic9bsshxns6yXPSl141dq1c,203
|
|
33
|
-
cloe_nessy/integration/reader/file_reader.py,sha256=
|
|
34
|
-
cloe_nessy/integration/reader/reader.py,sha256=
|
|
35
|
+
cloe_nessy/integration/reader/file_reader.py,sha256=t5zF-cmZo1X0a1rki6ry1rSiFEu5uXRP2rNGd90fwoY,8163
|
|
36
|
+
cloe_nessy/integration/reader/reader.py,sha256=YHriYkzsBduBjfI2FnP03VEo15a8UCRZ_sXtre8eaEs,1041
|
|
35
37
|
cloe_nessy/integration/writer/__init__.py,sha256=3yzCAGiWZdQWtsbzlTih01sxVTJV2DDYwvl34lEAUlE,243
|
|
36
|
-
cloe_nessy/integration/writer/catalog_writer.py,sha256=
|
|
38
|
+
cloe_nessy/integration/writer/catalog_writer.py,sha256=dQeXmtfs7J6rP6Ye3OCvxBraFScFX_3SHs7Md58hEeM,5296
|
|
37
39
|
cloe_nessy/integration/writer/file_writer.py,sha256=SUDbN13ZzDhbM8DpOGFgM_Gkg70To4L6Q182pXx2HRM,5454
|
|
38
40
|
cloe_nessy/integration/writer/writer.py,sha256=elFPLFrWR-qVE9qnBtzzzhyRALLQcRVuOsPS0rNmRt4,1741
|
|
39
41
|
cloe_nessy/integration/writer/delta_writer/__init__.py,sha256=h2CT6Hllmk0nodlek27uqwniCzVZKMkYcPGyG9K2Z24,164
|
|
@@ -66,22 +68,22 @@ cloe_nessy/object_manager/volume_manager.py,sha256=6epd3KXzcNH04EvaKubAfLsaUm9qB
|
|
|
66
68
|
cloe_nessy/pipeline/__init__.py,sha256=sespmJ5JsgyiFyZiedTiL2kg--zGIX7cjTYsD5vemEg,325
|
|
67
69
|
cloe_nessy/pipeline/pipeline.py,sha256=L4wk3b06LNWRj01nnAkuQpeRrwFTyaV1xTpgYAg4sak,10819
|
|
68
70
|
cloe_nessy/pipeline/pipeline_action.py,sha256=S7IVFdmG12fRBzHuE_DiWn7qlMtApz6IloVd2Fj31Sg,1944
|
|
69
|
-
cloe_nessy/pipeline/pipeline_config.py,sha256=
|
|
70
|
-
cloe_nessy/pipeline/pipeline_context.py,sha256=
|
|
71
|
-
cloe_nessy/pipeline/pipeline_parsing_service.py,sha256=
|
|
71
|
+
cloe_nessy/pipeline/pipeline_config.py,sha256=oVQ-IH4etTGZVVEnE-5iDPLYOtWpvDlltWFv1nevnqQ,3229
|
|
72
|
+
cloe_nessy/pipeline/pipeline_context.py,sha256=eCOcjyE16rGRom3L85Gy_BbncfQD6i1x31yrWqZws-4,1881
|
|
73
|
+
cloe_nessy/pipeline/pipeline_parsing_service.py,sha256=eeC4RbGBILGN6zkbUyjH-qGgEMtOWV4Kv_VxrHbHMY0,9021
|
|
72
74
|
cloe_nessy/pipeline/pipeline_plotting_service.py,sha256=goMQj73FzUVchKn5c2SsPcWR6fr7DtVkVrcQfJsKCq4,13111
|
|
73
|
-
cloe_nessy/pipeline/pipeline_step.py,sha256=
|
|
74
|
-
cloe_nessy/pipeline/actions/__init__.py,sha256=
|
|
75
|
-
cloe_nessy/pipeline/actions/read_api.py,sha256=
|
|
75
|
+
cloe_nessy/pipeline/pipeline_step.py,sha256=oTnlvRpB0fbOBQXbPe1URstA5fv-97igCHt_41fKCAk,2082
|
|
76
|
+
cloe_nessy/pipeline/actions/__init__.py,sha256=Qad9kxOQHoMQ1sj-4AxABNNIdaN5QkZAB14DUFKAtUA,2808
|
|
77
|
+
cloe_nessy/pipeline/actions/read_api.py,sha256=MAc7QfmhnaRUMdE09Ywt41RSAsuW4co8zF0zXHwbM8U,16193
|
|
76
78
|
cloe_nessy/pipeline/actions/read_catalog_table.py,sha256=EkP3JSI7VQMkvUsb97ieUeGnnfvyyUI7egvqNWMqK0I,6894
|
|
77
|
-
cloe_nessy/pipeline/actions/read_excel.py,sha256=
|
|
79
|
+
cloe_nessy/pipeline/actions/read_excel.py,sha256=IG_VmDEt1TvGVEO0SY9Fm3awHNjfisR1_7DUmhC3NEE,7968
|
|
78
80
|
cloe_nessy/pipeline/actions/read_files.py,sha256=hRcM7wG35vxxLVajW3SK5euHW02qxiXCYSkIl11xiQ0,7308
|
|
79
81
|
cloe_nessy/pipeline/actions/read_metadata_yaml.py,sha256=i8fQceV63eAqx_x0ANisCkXWfMHyhqsfFHVFH5yP2po,3544
|
|
80
82
|
cloe_nessy/pipeline/actions/transform_change_datatype.py,sha256=24Tn6R3TvUkWCh8V6naLdyNbCbqvyPOOoer-hy_Ebq4,2077
|
|
81
83
|
cloe_nessy/pipeline/actions/transform_clean_column_names.py,sha256=VxvWqENW63c50L96JA1V_ioe4By6gGzx_iY86njOXEM,3044
|
|
82
84
|
cloe_nessy/pipeline/actions/transform_concat_columns.py,sha256=Nk8YbhxDnFZsWzW9Dj5Yl76Uq6VrcMlevQPHGms65L8,3777
|
|
83
85
|
cloe_nessy/pipeline/actions/transform_convert_timestamp.py,sha256=2SL078tBcOmytDbt-cR81jZbclwqELsUB4XDLjaCnNo,3579
|
|
84
|
-
cloe_nessy/pipeline/actions/transform_decode.py,sha256=
|
|
86
|
+
cloe_nessy/pipeline/actions/transform_decode.py,sha256=_TQc2GFcgdJvtt6BVrCe1xVnJiSHB_J6mEHH01xIKMY,4464
|
|
85
87
|
cloe_nessy/pipeline/actions/transform_deduplication.py,sha256=SfTDrOL0TNSC4wITbozabC0jYvceTLnqU4urnEjYk9g,4910
|
|
86
88
|
cloe_nessy/pipeline/actions/transform_distinct.py,sha256=c7aBxANyqT4aKhm0cSELDtD-bP0Se9vxlBF0K4AgQWs,1976
|
|
87
89
|
cloe_nessy/pipeline/actions/transform_filter.py,sha256=Nz_ggRfKIcNzYFfFOsgq1QeatjdEis0up4I7cOWBdyo,1446
|
|
@@ -100,13 +102,14 @@ cloe_nessy/pipeline/actions/write_delta_merge.py,sha256=zcOk4ytZFUxyGY8U2fdFPLFn
|
|
|
100
102
|
cloe_nessy/pipeline/actions/write_file.py,sha256=JZ8UZslxUn_ttYt5wDyvtHFq2FqYk3vOR8kvExJI8pk,3212
|
|
101
103
|
cloe_nessy/pipeline/utils/__init__.py,sha256=xi02UjBMiXWD7b9gDvww4gyRyowb0eRd_6Wbu0F_cro,118
|
|
102
104
|
cloe_nessy/pipeline/utils/delta_load_utils.py,sha256=KitMNruxePEkecI0h4Jint1JwJpaEog5mCOchMkgan8,1495
|
|
103
|
-
cloe_nessy/session/__init__.py,sha256
|
|
104
|
-
cloe_nessy/session/
|
|
105
|
+
cloe_nessy/session/__init__.py,sha256=-MifkekjFu_3A9rWm30CGFQZ4yjruGaydNpbu3uq7Ww,155
|
|
106
|
+
cloe_nessy/session/pyspark_compat.py,sha256=NrgSWAaWz3GgMNLmzpY4cPgneQytNQlOq_dWrD1MveE,444
|
|
107
|
+
cloe_nessy/session/session_manager.py,sha256=d6qMTkaWJcN4QG261IoxIfQlNlB0ELtVNjTCNx2Elas,9717
|
|
105
108
|
cloe_nessy/settings/__init__.py,sha256=ZbkneO3WaKOxon7qHFHnou7EnBOSnBFyKMDZblIEvzM,101
|
|
106
109
|
cloe_nessy/settings/settings.py,sha256=I4n129lrujriW-d8q4as2Kb4_kI932ModfZ5Ow_UpVM,3653
|
|
107
110
|
cloe_nessy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
108
111
|
cloe_nessy/utils/column_names.py,sha256=dCNtm61mc5aLkY2oE4rlfN3VLCrpot6fOESjAZmCmhA,361
|
|
109
112
|
cloe_nessy/utils/file_and_directory_handler.py,sha256=r2EVt9xG81p6ScaJCwETC5an6pMT6WseB0jMOR-JlpU,602
|
|
110
|
-
cloe_nessy-0.3.
|
|
111
|
-
cloe_nessy-0.3.
|
|
112
|
-
cloe_nessy-0.3.
|
|
113
|
+
cloe_nessy-0.3.19.dist-info/METADATA,sha256=O3LES1mWSPONQE6q47c_j9s_sQcvU2a5RfL1WQW3JPk,3290
|
|
114
|
+
cloe_nessy-0.3.19.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
115
|
+
cloe_nessy-0.3.19.dist-info/RECORD,,
|