cloe-nessy 0.3.16.3b0__py3-none-any.whl → 0.3.16.5b0__py3-none-any.whl

cloe_nessy/models/templates/create_table.sql.j2

@@ -42,17 +42,17 @@ ALTER TABLE {{ table.escaped_identifier }} ADD CONSTRAINT {{constraint.name}} CH
 {%- if table.comment %}
 COMMENT ON TABLE {{ table.escaped_identifier }} IS '{{ table.comment }}';
 {%- endif %}
-
+{# Tags do not yet work in Databricks
 {%- if table.business_properties %}
 {%- for tag_key, tag_value in table.business_properties.items() %}
-SET TAG ON TABLE {{ table.escaped_identifier }} "{{ tag_key }}"{% if tag_value %}="{{ tag_value }}"{% endif %};
+SET TAG ON TABLE {{ table.escaped_identifier }} `{{ tag_key }}`{% if tag_value %} = `{{ tag_value }}`{% endif %};
 {%- endfor %}
 {%- endif %}
 
 {%- for column in table.columns %}
 {%- if column.business_properties %}
 {%- for tag_key, tag_value in column.business_properties.items() %}
-SET TAG ON COLUMN {{ table.escaped_identifier }}.`{{ column.name }}` "{{ tag_key }}"{% if tag_value %}="{{ tag_value }}"{% endif %};
+SET TAG ON COLUMN {{ table.escaped_identifier }}.`{{ column.name }}` `{{ tag_key }}`{% if tag_value %} = `{{ tag_value }}`{% endif %};
 {%- endfor %}
 {%- endif %}
-{%- endfor %}
+{%- endfor %} #}

cloe_nessy/pipeline/pipeline.py

@@ -3,10 +3,10 @@ from collections import OrderedDict
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from threading import Lock
 
-import matplotlib.pyplot as plt
 import networkx as nx
 
 from ..logging.logger_mixin import LoggerMixin
+from .pipeline_plotting_service import PipelinePlottingService
 from .pipeline_step import PipelineStep
 
 
@@ -214,30 +214,13 @@ class Pipeline(LoggerMixin):
         self._console_logger.info(f"Pipeline [' {self.name} '] completed successfully.")
 
     def plot_graph(self, save_path: str | None = None) -> None:
-        """Visualizes the graph of the pipeline using matplotlib.
+        """Generates a visual representation of the pipeline steps and their dependencies.
+
+        This method uses the PipelinePlottingService to create a plot of the pipeline graph. If a save path
+        is specified, the plot will be saved to that location; otherwise, it will be displayed interactively.
 
         Args:
-            save_path: If provided, the graph will be saved to this path. Otherwise, it will be shown.
+            save_path: Optional; the file path where the plot should be saved. If None, the plot is displayed interactively.
         """
-        pos = nx.spring_layout(self._graph)  # Position steps (nodes) using the spring layout
-        plt.figure(figsize=(12, 8))
-        nx.draw(
-            self._graph,
-            pos,
-            with_labels=True,
-            node_color="lightblue",
-            font_weight="bold",
-            node_size=3000,
-            font_size=10,
-            edge_color="gray",
-        )
-
-        # Draw edge labels if needed
-        edge_labels = nx.get_edge_attributes(self._graph, "label")
-        nx.draw_networkx_edge_labels(self._graph, pos, edge_labels=edge_labels)
-
-        if save_path:
-            plt.savefig(save_path)
-            self._console_logger.info(f"Graph visual saved to {save_path}")
-        else:
-            plt.show()
+        plotting_service = PipelinePlottingService()
+        plotting_service.plot_graph(self, save_path)

cloe_nessy/pipeline/pipeline_plotting_service.py

@@ -0,0 +1,340 @@
+"""Pipeline plotting service for visualizing pipeline graphs using matplotlib."""
+
+import textwrap
+
+import matplotlib.patches as patches
+import matplotlib.pyplot as plt
+import networkx as nx
+
+from ..logging import LoggerMixin
+
+
+class PipelinePlottingService(LoggerMixin):
+    """Service for plotting pipeline graphs using matplotlib.
+
+    This service handles the visualization of pipeline DAGs, including node positioning,
+    edge drawing, and proper layout management.
+    """
+
+    def __init__(self):
+        """Initialize the plotting service."""
+        self.context_colors = {
+            "initialized": "lightgrey",
+            "successful": "lightgreen",
+            "failed": "red",
+        }
+        self._console_logger = self.get_console_logger()
+
+    def plot_graph(self, pipeline, save_path: str | None = None):
+        """Plot and save the pipeline graph as an image.
+
+        Args:
+            pipeline: The Pipeline object to visualize.
+            save_path: Path where the graph image should be saved.
+        """
+        g: nx.DiGraph = nx.DiGraph()
+        g.add_edges_from(self._generate_edges(pipeline))
+        pos = self._determine_number_of_rows(g.nodes, 75, 5, pipeline)
+
+        _, ax = plt.subplots(figsize=(16, 12))
+
+        adjusted_pos = {}
+        scaling_factor = 0.7
+
+        for node, (x, y) in pos.items():
+            text_length = len(node)
+            node_width = max(7.5, text_length * 0.2)
+            node_height = 2.0
+            x_spacing = max(9.0, node_width * scaling_factor)
+            y_spacing = max(4.5, node_height * scaling_factor)
+            adjusted_pos[node] = (x * x_spacing, y * y_spacing)
+
+        # Calculate bounds to set proper axis limits
+        if adjusted_pos:
+            x_coords = [pos[0] for pos in adjusted_pos.values()]
+            y_coords = [pos[1] for pos in adjusted_pos.values()]
+            x_min, x_max = min(x_coords), max(x_coords)
+            y_min, y_max = min(y_coords), max(y_coords)
+
+            # Add padding
+            x_padding = (x_max - x_min) * 0.1 if x_max != x_min else 5
+            y_padding = (y_max - y_min) * 0.1 if y_max != y_min else 5
+
+            ax.set_xlim(x_min - x_padding, x_max + x_padding)
+            ax.set_ylim(y_min - y_padding, y_max + y_padding)
+
+        self._draw_edges(ax, g, adjusted_pos)
+        self._draw_nodes(ax, pipeline, adjusted_pos)
+        self._add_legend(ax, adjusted_pos)
+
+        ax.set_title(pipeline.name, fontsize=18, weight="bold", pad=20)
+        ax.set_aspect("equal", adjustable="box")
+        ax.axis("off")
+
+        plt.tight_layout()
+        if save_path:
+            plt.savefig(save_path, dpi=300, bbox_inches="tight", facecolor="white", edgecolor="none", format="png")
+            self._console_logger.info(f"Graph saved to {save_path}")
+        else:
+            plt.show()
+        plt.close()
+
+    def _generate_edges(self, pipeline):
+        """Generate edges from pipeline steps."""
+        input_collection = []
+        for _, value in pipeline.steps.items():
+            for predecessor in value._predecessors:
+                input_collection.append((predecessor, value.name))
+        return input_collection
+
+    def _draw_edges(self, ax, graph, adjusted_pos):
+        """Draw edges between nodes with proper arrow positioning."""
+        for src, tgt in graph.edges():
+            x0, y0 = adjusted_pos[src]
+            x1, y1 = adjusted_pos[tgt]
+
+            # Get the dimensions of the source and target nodes
+            src_width = max(7.5, len(src) * 0.2) / 2  # Half width
+            src_height = 1.7 / 2  # Half height
+            tgt_width = max(7.5, len(tgt) * 0.2) / 2  # Half width
+            tgt_height = 1.7 / 2  # Half height
+
+            dx = x1 - x0
+            dy = y1 - y0
+
+            # Calculation of scaling factors so that the arrow starts/ends at rectangle boundary
+            if abs(dx) < 0.001 and abs(dy) < 0.001:
+                # Skip if nodes are at the same position
+                continue
+            if abs(dx) < 0.001:
+                # Vertical line
+                scale_src = src_height / abs(dy) if dy != 0 else 0
+                scale_tgt = tgt_height / abs(dy) if dy != 0 else 0
+            elif abs(dy) < 0.001:
+                # Horizontal line
+                scale_src = src_width / abs(dx)
+                scale_tgt = tgt_width / abs(dx)
+            else:
+                # Diagonal line
+                scale_x_src = src_width / abs(dx)
+                scale_y_src = src_height / abs(dy)
+                scale_src = min(scale_x_src, scale_y_src)
+
+                scale_x_tgt = tgt_width / abs(dx)
+                scale_y_tgt = tgt_height / abs(dy)
+                scale_tgt = min(scale_x_tgt, scale_y_tgt)
+
+            start_x = x0 + dx * scale_src
+            start_y = y0 + dy * scale_src
+            end_x = x1 - dx * scale_tgt
+            end_y = y1 - dy * scale_tgt
+
+            ax.annotate(
+                "",
+                xy=(end_x, end_y),
+                xytext=(start_x, start_y),
+                arrowprops={"arrowstyle": "->", "color": "gray", "lw": 2, "shrinkA": 0, "shrinkB": 0},
+            )
+
+    def _draw_nodes(self, ax, pipeline, adjusted_pos):
+        """Draw nodes as rectangles with text labels."""
+        for node, (x, y) in adjusted_pos.items():
+            context = self._get_context_by_step_info(pipeline, node)
+            fillcolor = self.context_colors.get(context, "lightgrey")
+
+            text_length = len(node)
+            node_width = max(7.5, text_length * 0.2)
+            node_height = 2.0
+
+            rect = patches.Rectangle(
+                (x - node_width / 2, y - node_height / 2),
+                node_width,
+                node_height,
+                linewidth=1,
+                edgecolor="black",
+                facecolor=fillcolor,
+                zorder=2,
+            )
+            ax.add_patch(rect)
+
+            wrapped_label = self._wrap_text(node).replace("<br>", "\n")
+            ax.text(x, y, wrapped_label, ha="center", va="center", fontsize=10, weight="bold", zorder=3)
+
+    def _add_legend(self, ax, adjusted_pos):
+        """Add legend at the bottom of the graph."""
+        legend_text = "Success = Light Green; Waiting = Light Grey; Failed = Bright Red"
+
+        if adjusted_pos:
+            y_coords = [pos[1] for pos in adjusted_pos.values()]
+            legend_y = min(y_coords) - abs(max(y_coords) - min(y_coords)) * 0.2 - 3
+            x_coords = [pos[0] for pos in adjusted_pos.values()]
+            legend_x = (min(x_coords) + max(x_coords)) / 2
+        else:
+            legend_x, legend_y = 0, -5
+
+        ax.text(
+            legend_x,
+            legend_y,
+            legend_text,
+            ha="center",
+            va="center",
+            fontsize=12,
+            bbox={"boxstyle": "round,pad=0.3", "facecolor": "white", "alpha": 0.8},
+        )
+
+    def _determine_number_of_rows(self, nodes, max_row_length, max_nodes_in_row, pipeline):
+        """Structure the node's position in a DAG-aware layout that better shows dependencies."""
+        # Build the graph to understand dependencies
+        graph = nx.DiGraph()
+        graph.add_edges_from(self._generate_edges(pipeline))
+
+        # Try to use a topological layout that respects dependencies
+        try:
+            # Use a hierarchical layout approach
+            pos = self._create_hierarchical_layout(graph, pipeline)
+        except Exception:
+            # Fallback to the original layout if there are issues
+            pos = self._create_simple_layout(nodes, max_row_length, max_nodes_in_row)
+
+        pos = self._post_process_grouping(nodes, pos, pipeline)
+        return pos
+
+    def _create_hierarchical_layout(self, graph, pipeline):
+        """Create a layout based on dependency levels to show parallel branches clearly."""
+        levels = self._calculate_dependency_levels(graph)
+        level_groups = self._group_nodes_by_level(levels)
+        return self._position_nodes_in_levels(level_groups)
+
+    def _calculate_dependency_levels(self, graph):
+        """Calculate the dependency level for each node."""
+        levels = {}
+        remaining_nodes = set(graph.nodes())
+        current_level = 0
+
+        while remaining_nodes:
+            current_level_nodes = self._find_nodes_without_dependencies(graph, remaining_nodes)
+
+            if not current_level_nodes:
+                # Circular dependency or other issue, place remaining nodes at current level
+                current_level_nodes = list(remaining_nodes)
+
+            # Assign level to these nodes
+            for node in current_level_nodes:
+                levels[node] = current_level
+                remaining_nodes.remove(node)
+
+            current_level += 1
+
+        return levels
+
+    def _find_nodes_without_dependencies(self, graph, remaining_nodes):
+        """Find nodes that have no dependencies in the remaining nodes."""
+        nodes_without_deps = []
+        for node in remaining_nodes:
+            predecessors = set(graph.predecessors(node))
+            if not predecessors or predecessors.isdisjoint(remaining_nodes):
+                nodes_without_deps.append(node)
+        return nodes_without_deps
+
+    def _group_nodes_by_level(self, levels):
+        """Group nodes by their dependency level."""
+        level_groups = {}
+        for node, level in levels.items():
+            if level not in level_groups:
+                level_groups[level] = []
+            level_groups[level].append(node)
+        return level_groups
+
+    def _position_nodes_in_levels(self, level_groups):
+        """Position nodes within their levels."""
+        pos = {}
+        for level, nodes_in_level in level_groups.items():
+            for i, node in enumerate(nodes_in_level):
+                # Spread parallel nodes vertically to show they can run in parallel
+                y_pos = i - (len(nodes_in_level) - 1) / 2  # Center around 0
+                pos[node] = (level, -y_pos)  # Negative y to match the original coordinate system
+        return pos
+
+    def _create_simple_layout(self, nodes, max_row_length, max_nodes_in_row):
+        """Fallback to simple row-based layout."""
+        pos = {}
+        current_row = []
+        current_row_length = 0
+        row_index = 0
+        left_to_right = True
+
+        for node in nodes:
+            node_length = len(node)
+            if (current_row_length + node_length > max_row_length) or (len(current_row) >= max_nodes_in_row):
+                self._position_row_nodes(pos, current_row, row_index, left_to_right)
+                current_row = []
+                current_row_length = 0
+                row_index += 1
+                left_to_right = not left_to_right
+
+            current_row.append(node)
+            current_row_length += node_length
+
+        # Handle the last row
+        self._position_row_nodes(pos, current_row, row_index, left_to_right)
+        return pos
+
+    def _position_row_nodes(self, pos, row_nodes, row_index, left_to_right):
+        """Position nodes in a row, either left-to-right or right-to-left."""
+        nodes_to_position = row_nodes if left_to_right else reversed(row_nodes)
+        for i, row_node in enumerate(nodes_to_position):
+            pos[row_node] = (i, -row_index)
+
+    def _post_process_grouping(self, nodes, pos, pipeline):
+        """Re-arrange the node's positions to fit more complex connections."""
+        for node_name in nodes:
+            predecessors = self._get_predecessors_by_step_info(pipeline, node_name)
+            if predecessors is not None and len(predecessors) > 1:
+                pos = self._shift_row_down(pos, node_name)
+        return pos
+
+    def _shift_row_down(self, pos, split_key):
+        """Shift a node and subsequent nodes down by one row."""
+        part1 = {}
+        part2 = {}
+        found_split_key = False
+
+        for key, value in pos.items():
+            if found_split_key:
+                part2[key] = value
+            else:
+                if key == split_key:
+                    found_split_key = True
+                    part2[key] = value
+                else:
+                    part1[key] = value
+
+        for key in part2:
+            if key != split_key:
+                new_tuple = (part2[key][0], part2[key][1] - 1)
+                part2[key] = new_tuple
+
+        part1.update(part2)
+        return part1
+
+    def _wrap_text(self, text, max_length=20):
+        """Add line breaks to text if it exceeds max_length."""
+        if len(text) <= max_length:
+            return text
+
+        wrapped_lines = textwrap.wrap(text, width=max_length)
+        return "<br>".join(wrapped_lines)
+
+    def _get_context_by_step_info(self, pipeline, step_name):
+        """Get the context status of a step."""
+        for _, step in pipeline.steps.items():
+            if step.name == step_name:
+                return step.context.status
+        return None
+
+    def _get_predecessors_by_step_info(self, pipeline, step_name):
+        """Get predecessors of a step."""
+        for _, step in pipeline.steps.items():
+            if step.name == step_name:
+                return step._predecessors
+        return None

{cloe_nessy-0.3.16.3b0.dist-info → cloe_nessy-0.3.16.5b0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloe-nessy
-Version: 0.3.16.3b0
+Version: 0.3.16.5b0
 Summary: Your friendly datalake monster.
 Project-URL: homepage, https://initions.com/
 Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>

{cloe_nessy-0.3.16.3b0.dist-info → cloe_nessy-0.3.16.5b0.dist-info}/RECORD

@@ -50,17 +50,18 @@ cloe_nessy/models/adapter/unity_catalog_adapter.py,sha256=a-14Ys-AevVYQd0xeJU1sy
 cloe_nessy/models/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/models/mixins/read_instance_mixin.py,sha256=j5Y4aNWOh1jlskEaxNooZFJgPyxRmik00gAVLJnAaRs,4507
 cloe_nessy/models/mixins/template_loader_mixin.py,sha256=5MXhEGBFlq3dwZvINEyBowSlipNnVun2H_TmhI_fsS4,549
-cloe_nessy/models/templates/create_table.sql.j2,sha256=_nENxBHT3drRvZYttFgy5o7tWvN86aENgGy1yrSISj0,2435
+cloe_nessy/models/templates/create_table.sql.j2,sha256=71JpUyUZ_ZYO2M0tfIrTXHR7JycypAGsELt2-2d3oO0,2479
 cloe_nessy/models/templates/create_volume.sql.j2,sha256=XIUf1cHcvAxcGTyhzUiv4xpQ1cfDw_ra3_FKmOuLoBs,289
 cloe_nessy/object_manager/__init__.py,sha256=3sle0vNpPwBOkycxA3XVS9m4XZf5LD3Qd4NGxdqcHno,186
 cloe_nessy/object_manager/table_manager.py,sha256=suHx56TYXagaJ2dVkvTP7vwSI4xgTqXNkHYBbYh2pd4,13913
 cloe_nessy/object_manager/volume_manager.py,sha256=6epd3KXzcNH04EvaKubAfLsaUm9qBMeT3KNvMK04gGs,2727
 cloe_nessy/pipeline/__init__.py,sha256=sespmJ5JsgyiFyZiedTiL2kg--zGIX7cjTYsD5vemEg,325
-cloe_nessy/pipeline/pipeline.py,sha256=-1tJVs9rZf8CcwieH4IP7mqJZ6mL7bQUZ56TNKt8eO8,11154
+cloe_nessy/pipeline/pipeline.py,sha256=L4wk3b06LNWRj01nnAkuQpeRrwFTyaV1xTpgYAg4sak,10819
 cloe_nessy/pipeline/pipeline_action.py,sha256=S7IVFdmG12fRBzHuE_DiWn7qlMtApz6IloVd2Fj31Sg,1944
 cloe_nessy/pipeline/pipeline_config.py,sha256=BN3ZSbr6bC-X9edoh-n5vRfPHFMbgtAU7mQ3dBrcWO8,3131
 cloe_nessy/pipeline/pipeline_context.py,sha256=csElDc6BsynDUtRXgQOSCH7ONc_b-ag0YEg0zlQTz58,1874
 cloe_nessy/pipeline/pipeline_parsing_service.py,sha256=c_nAsgw81QYBM9AFiTxGgqRhNXABkDKplbeoCJPtbpE,6434
+cloe_nessy/pipeline/pipeline_plotting_service.py,sha256=goMQj73FzUVchKn5c2SsPcWR6fr7DtVkVrcQfJsKCq4,13111
 cloe_nessy/pipeline/pipeline_step.py,sha256=UlnmpS6gm_dZ7m9dD1mZvye7mvUF_DA7HjOZo0oGYDU,1977
 cloe_nessy/pipeline/actions/__init__.py,sha256=9gjSQKLGrPcaYaJrTYZde8d4yNrN1SoXN_DDHq5KrvY,2600
 cloe_nessy/pipeline/actions/read_api.py,sha256=RBv5XeHtjTXuCP09Fqo6JNx6iIhQQI-nuAHCuSaGs2s,7778
@@ -95,6 +96,6 @@ cloe_nessy/settings/__init__.py,sha256=ZbkneO3WaKOxon7qHFHnou7EnBOSnBFyKMDZblIEv
 cloe_nessy/settings/settings.py,sha256=I4n129lrujriW-d8q4as2Kb4_kI932ModfZ5Ow_UpVM,3653
 cloe_nessy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/utils/file_and_directory_handler.py,sha256=r2EVt9xG81p6ScaJCwETC5an6pMT6WseB0jMOR-JlpU,602
-cloe_nessy-0.3.16.3b0.dist-info/METADATA,sha256=xkQ6JykVWTv9VwFW9vwuY2jsjen3bpDVXBkZeSKGB40,3328
-cloe_nessy-0.3.16.3b0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cloe_nessy-0.3.16.3b0.dist-info/RECORD,,
+cloe_nessy-0.3.16.5b0.dist-info/METADATA,sha256=k06Hu0tT4c7JkwmN7WmDVTBvZi8ypdB5Agxfty42CtQ,3328
+cloe_nessy-0.3.16.5b0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cloe_nessy-0.3.16.5b0.dist-info/RECORD,,