cloe-nessy 0.3.13.3b0__py3-none-any.whl → 0.3.13.4b0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -197,7 +197,9 @@ class DeltaMergeWriter(BaseDeltaWriter):
197
197
  config = DeltaMergeConfig(dataframe_columns=data_frame.columns, **kwargs)
198
198
 
199
199
  delta_table = self.table_manager.get_delta_table(
200
- table=table, location=storage_path, spark=data_frame.sparkSession
200
+ table=table,
201
+ location=storage_path,
202
+ spark=data_frame.sparkSession,
201
203
  )
202
204
 
203
205
  match_conditions = self._build_match_conditions(data_frame, config)
@@ -1,66 +1,94 @@
1
- import pathlib
1
+ from pathlib import Path
2
2
  from typing import Any
3
3
 
4
- from ...models import Schema
4
+ from ...models import Table
5
5
  from ..pipeline_action import PipelineAction
6
6
  from ..pipeline_context import PipelineContext
7
7
 
8
8
 
9
9
  class ReadMetadataYAMLAction(PipelineAction):
10
- """Reads schema metadata from a yaml file using the [`Schema`][cloe_nessy.models.schema] model.
10
+ """Reads table metadata from a yaml file using the [`Table`][cloe_nessy.models.table] model.
11
11
 
12
12
  Example:
13
- ```yaml
14
- Read Schema Metadata:
15
- action: READ_METADATA_YAML_ACTION
16
- options:
17
- path: excel_file_folder/excel_files_june/
18
- file_name: sales_schema.yml
19
- table_name: sales
20
- ```
13
+ === "Managed Table"
14
+ ```yaml
15
+ Read Table Metadata:
16
+ action: READ_METADATA_YAML_ACTION
17
+ options:
18
+ file_path: metadata/schemas/bronze/sales_table.yml
19
+ catalog_name: production
20
+ schema_name: sales_data
21
+ ```
22
+ === "External Table"
23
+ ```yaml
24
+ Read Table Metadata:
25
+ action: READ_METADATA_YAML_ACTION
26
+ options:
27
+ file_path: metadata/schemas/bronze/sales_table.yml
28
+ catalog_name: production
29
+ schema_name: sales_data
30
+ storage_path: abfs://external_storage/sales_data/sales_table
31
+ ```
21
32
  """
22
33
 
23
34
  name: str = "READ_METADATA_YAML_ACTION"
24
35
 
25
- @staticmethod
26
36
  def run(
37
+ self,
27
38
  context: PipelineContext,
28
39
  *,
29
- path: str | None = None,
30
- file_name: str | None = None,
31
- table_name: str | None = None,
40
+ file_path: str | None = None,
41
+ catalog_name: str | None = None,
42
+ schema_name: str | None = None,
43
+ storage_path: str | None = None,
32
44
  **_: Any,
33
45
  ) -> PipelineContext:
34
- """Reads schema metadata from a yaml file using the [`Schema`][cloe_nessy.models.schema] model.
46
+ """Reads table metadata from a yaml file using the [`Table`][cloe_nessy.models.table] model.
35
47
 
36
48
  Args:
37
49
  context: The context in which this Action is executed.
38
- path: The path to the data contract directory.
39
- file_name: The name of the file that defines the schema.
40
- table_name: The name of the table for which to retrieve metadata.
50
+ file_path: The path to the file that defines the table.
51
+ catalog_name: The name of the catalog for the table.
52
+ schema_name: The name of the schema for the table.
53
+ storage_path: The storage path for the table, if applicable. If not
54
+ provided, the table will be considered a managed table.
41
55
 
42
56
  Raises:
43
- ValueError: If any issues occur while reading the schema, such as an invalid schema,
44
- missing file, or missing path.
57
+ ValueError: If any issues occur while reading the table metadata, such as an invalid table,
58
+ missing file, missing path, or missing catalog/schema names.
45
59
 
46
60
  Returns:
47
61
  The context after the execution of this Action, containing the table metadata.
48
62
  """
49
- if not path:
50
- raise ValueError("No path provided. Please specify path to schema metadata.")
51
- if not file_name:
52
- raise ValueError("No file_name provided. Please specify file name.")
53
- if not table_name:
54
- raise ValueError("No table_name provided. Please specify table name.")
63
+ missing_params = []
64
+ if not file_path:
65
+ missing_params.append("file_path")
66
+ if not catalog_name:
67
+ missing_params.append("catalog_name")
68
+ if not schema_name:
69
+ missing_params.append("schema_name")
55
70
 
56
- path_obj = pathlib.Path(path)
71
+ if missing_params:
72
+ raise ValueError(
73
+ f"Missing required parameters: {', '.join(missing_params)}. Please specify all required parameters."
74
+ )
57
75
 
58
- schema, errors = Schema.read_instance_from_file(path_obj / file_name)
76
+ final_file_path = Path(file_path) if file_path else Path()
77
+
78
+ table, errors = Table.read_instance_from_file(
79
+ final_file_path,
80
+ catalog_name=catalog_name,
81
+ schema_name=schema_name,
82
+ )
59
83
  if errors:
60
- raise ValueError(f"Errors while reading schema metadata: {errors}")
61
- if not schema:
62
- raise ValueError("No schema found in metadata.")
84
+ raise ValueError(f"Errors while reading table metadata: {errors}")
85
+ if not table:
86
+ raise ValueError("No table found in metadata.")
63
87
 
64
- table = schema.get_table_by_name(table_name=table_name)
88
+ if not table.storage_path and storage_path:
89
+ self._console_logger.info(f"Setting storage path for table [ '{table.name}' ] to [ '{storage_path}' ]")
90
+ table.storage_path = storage_path
91
+ table.is_external = True
65
92
 
93
+ self._console_logger.info(f"Table [ '{table.name}' ] metadata read successfully from [ '{file_path}' ]")
66
94
  return context.from_existing(table_metadata=table)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cloe-nessy
3
- Version: 0.3.13.3b0
3
+ Version: 0.3.13.4b0
4
4
  Summary: Your friendly datalake monster.
5
5
  Project-URL: homepage, https://initions.com/
6
6
  Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>
@@ -30,7 +30,7 @@ cloe_nessy/integration/writer/file_writer.py,sha256=SUDbN13ZzDhbM8DpOGFgM_Gkg70T
30
30
  cloe_nessy/integration/writer/writer.py,sha256=elFPLFrWR-qVE9qnBtzzzhyRALLQcRVuOsPS0rNmRt4,1741
31
31
  cloe_nessy/integration/writer/delta_writer/__init__.py,sha256=h2CT6Hllmk0nodlek27uqwniCzVZKMkYcPGyG9K2Z24,164
32
32
  cloe_nessy/integration/writer/delta_writer/delta_append_writer.py,sha256=TbpW-j87_H9dcUza34uR6VWslJez406y3_5N1ip0SnM,4740
33
- cloe_nessy/integration/writer/delta_writer/delta_merge_writer.py,sha256=3LQsrPNq9Xi71NLFqbB1Qk6tcheNPoQX3ngvCuIKwaw,10147
33
+ cloe_nessy/integration/writer/delta_writer/delta_merge_writer.py,sha256=zhqPIPfAJTzSLFgBUCwFesUW7CcF1zCPRU-N_8yYjok,10172
34
34
  cloe_nessy/integration/writer/delta_writer/delta_table_operation_type.py,sha256=kiacqQ2FYQSzakJqZ9-ZHH3os4X7--QuER_2xx9y21k,971
35
35
  cloe_nessy/integration/writer/delta_writer/delta_writer_base.py,sha256=upUtDZMzwYFU0kzmkelVgkpFToXkrypcR3h_jvGjz14,8596
36
36
  cloe_nessy/integration/writer/delta_writer/exceptions.py,sha256=xPmGiYV0xQXauln5Oh34E5vbm0rVcs6xCh-SJSb2bw0,107
@@ -67,7 +67,7 @@ cloe_nessy/pipeline/actions/read_api.py,sha256=RBv5XeHtjTXuCP09Fqo6JNx6iIhQQI-nu
67
67
  cloe_nessy/pipeline/actions/read_catalog_table.py,sha256=oXbqbc6BfR82dSIGclwzWiTN8EVmpFjNIYLKm4qOU50,2754
68
68
  cloe_nessy/pipeline/actions/read_excel.py,sha256=Mhl3r_2Hqk2XN7Fl5WqqAyE4JdnwSiivbhWMglyBtkE,7961
69
69
  cloe_nessy/pipeline/actions/read_files.py,sha256=N9bFgtG1tovhp2JayxE5YiN9PiO2lgG2-6h_Y6tD2eU,5220
70
- cloe_nessy/pipeline/actions/read_metadata_yaml.py,sha256=3ZDy9qiDYtM1oDQzHPC23hLOvHjhdk5zg1wVHE60m9k,2295
70
+ cloe_nessy/pipeline/actions/read_metadata_yaml.py,sha256=i8fQceV63eAqx_x0ANisCkXWfMHyhqsfFHVFH5yP2po,3544
71
71
  cloe_nessy/pipeline/actions/transform_change_datatype.py,sha256=24Tn6R3TvUkWCh8V6naLdyNbCbqvyPOOoer-hy_Ebq4,2077
72
72
  cloe_nessy/pipeline/actions/transform_clean_column_names.py,sha256=-CEdcXb7Fz5DQNitGlJ8EVBE_LzxfsInyCIO-D7b4iY,3042
73
73
  cloe_nessy/pipeline/actions/transform_concat_columns.py,sha256=Nk8YbhxDnFZsWzW9Dj5Yl76Uq6VrcMlevQPHGms65L8,3777
@@ -94,6 +94,6 @@ cloe_nessy/settings/__init__.py,sha256=ZbkneO3WaKOxon7qHFHnou7EnBOSnBFyKMDZblIEv
94
94
  cloe_nessy/settings/settings.py,sha256=I4n129lrujriW-d8q4as2Kb4_kI932ModfZ5Ow_UpVM,3653
95
95
  cloe_nessy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
96
96
  cloe_nessy/utils/file_and_directory_handler.py,sha256=r2EVt9xG81p6ScaJCwETC5an6pMT6WseB0jMOR-JlpU,602
97
- cloe_nessy-0.3.13.3b0.dist-info/METADATA,sha256=D5hiAAFKj9BZ6RA5SR9dJ7E4BFQhuNH7AXIT-ELLpP0,3294
98
- cloe_nessy-0.3.13.3b0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
99
- cloe_nessy-0.3.13.3b0.dist-info/RECORD,,
97
+ cloe_nessy-0.3.13.4b0.dist-info/METADATA,sha256=Z9Bby2T96l0nm453ZwbiAq3YDP0uau5GfPORfD9KE7E,3294
98
+ cloe_nessy-0.3.13.4b0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
99
+ cloe_nessy-0.3.13.4b0.dist-info/RECORD,,