sqlmesh 0.200.1.dev10__tar.gz → 0.200.1.dev12__tar.gz

{sqlmesh-0.200.1.dev10 → sqlmesh-0.200.1.dev12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.200.1.dev10
+Version: 0.200.1.dev12
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.200.1.dev10 → sqlmesh-0.200.1.dev12}/sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.200.1.dev10'
-__version_tuple__ = version_tuple = (0, 200, 1, 'dev10')
+__version__ = version = '0.200.1.dev12'
+__version_tuple__ = version_tuple = (0, 200, 1, 'dev12')

{sqlmesh-0.200.1.dev10 → sqlmesh-0.200.1.dev12}/sqlmesh/core/config/root.py

@@ -260,6 +260,18 @@ class Config(BaseConfig):
 
         return self
 
+    @model_validator(mode="after")
+    def _inherit_project_config_in_cicd_bot(self) -> Self:
+        if self.cicd_bot:
+            # inherit the project-level settings into the CICD bot if they have not been explicitly overridden
+            if self.cicd_bot.auto_categorize_changes_ is None:
+                self.cicd_bot.auto_categorize_changes_ = self.plan.auto_categorize_changes
+
+            if self.cicd_bot.pr_include_unmodified_ is None:
+                self.cicd_bot.pr_include_unmodified_ = self.plan.include_unmodified
+
+        return self
+
     def get_default_test_connection(
         self,
         default_catalog: t.Optional[str] = None,

{sqlmesh-0.200.1.dev10 → sqlmesh-0.200.1.dev12}/sqlmesh/core/console.py

@@ -3049,34 +3049,47 @@ class CaptureTerminalConsole(TerminalConsole):
     def __init__(self, console: t.Optional[RichConsole] = None, **kwargs: t.Any) -> None:
         super().__init__(console=console, **kwargs)
         self._captured_outputs: t.List[str] = []
+        self._warnings: t.List[str] = []
         self._errors: t.List[str] = []
 
     @property
     def captured_output(self) -> str:
         return "".join(self._captured_outputs)
 
+    @property
+    def captured_warnings(self) -> str:
+        return "".join(self._warnings)
+
     @property
     def captured_errors(self) -> str:
         return "".join(self._errors)
 
     def consume_captured_output(self) -> str:
-        output = self.captured_output
-        self.clear_captured_outputs()
-        return output
+        try:
+            return self.captured_output
+        finally:
+            self._captured_outputs = []
 
-    def consume_captured_errors(self) -> str:
-        errors = self.captured_errors
-        self.clear_captured_errors()
-        return errors
+    def consume_captured_warnings(self) -> str:
+        try:
+            return self.captured_warnings
+        finally:
+            self._warnings = []
 
-    def clear_captured_outputs(self) -> None:
-        self._captured_outputs = []
+    def consume_captured_errors(self) -> str:
+        try:
+            return self.captured_errors
+        finally:
+            self._errors = []
 
-    def clear_captured_errors(self) -> None:
-        self._errors = []
+    def log_warning(self, short_message: str, long_message: t.Optional[str] = None) -> None:
+        if short_message not in self._warnings:
+            self._warnings.append(short_message)
+        super().log_warning(short_message, long_message)
 
     def log_error(self, message: str) -> None:
-        self._errors.append(message)
+        if message not in self._errors:
+            self._errors.append(message)
         super().log_error(message)
 
     def log_skipped_models(self, snapshot_names: t.Set[str]) -> None:
@@ -3087,9 +3100,8 @@ class CaptureTerminalConsole(TerminalConsole):
             super().log_skipped_models(snapshot_names)
 
     def log_failed_models(self, errors: t.List[NodeExecutionFailedError]) -> None:
-        if errors:
-            self._errors.append("\n".join(str(ex) for ex in errors))
-            super().log_failed_models(errors)
+        self._errors.extend([str(ex) for ex in errors if str(ex) not in self._errors])
+        super().log_failed_models(errors)
 
     def _print(self, value: t.Any, **kwargs: t.Any) -> None:
         with self.console.capture() as capture:
@@ -3110,6 +3122,11 @@ class MarkdownConsole(CaptureTerminalConsole):
     AUDIT_PADDING = 7
 
     def __init__(self, **kwargs: t.Any) -> None:
+        self.alert_block_max_content_length = int(kwargs.pop("alert_block_max_content_length", 500))
+        self.alert_block_collapsible_threshold = int(
+            kwargs.pop("alert_block_collapsible_threshold", 200)
+        )
+
         super().__init__(
             **{**kwargs, "console": RichConsole(no_color=True, width=kwargs.pop("width", None))}
         )
@@ -3434,18 +3451,40 @@ class MarkdownConsole(CaptureTerminalConsole):
         self._print(msg)
         self._errors.append(msg)
 
-    def log_error(self, message: str) -> None:
-        super().log_error(f"```\n\\[ERROR] {message}```\n\n")
+    @property
+    def captured_warnings(self) -> str:
+        return self._render_alert_block("WARNING", self._warnings)
 
-    def log_warning(self, short_message: str, long_message: t.Optional[str] = None) -> None:
-        logger.warning(long_message or short_message)
+    @property
+    def captured_errors(self) -> str:
+        return self._render_alert_block("CAUTION", self._errors)
 
-        if not short_message.endswith("\n"):
-            short_message += (
-                "\n"  # so that the closing ``` ends up on a newline which is important for GitHub
-            )
+    def _render_alert_block(self, block_type: str, items: t.List[str]) -> str:
+        # GitHub Markdown alert syntax, https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#alerts
+        if items:
+            item_contents = ""
+            list_indicator = "- " if len(items) > 1 else ""
+
+            for item in items:
+                item = item.replace("\n", "\n> ")
+                item_contents += f">\n> {list_indicator}{item}\n"
+
+                if len(item_contents) > self.alert_block_max_content_length:
+                    truncation_msg = (
+                        "...\n>\n> Truncated. Please check the console for full information.\n"
+                    )
+                    item_contents = item_contents[
+                        0 : self.alert_block_max_content_length - len(truncation_msg)
+                    ]
+                    item_contents += truncation_msg
+                    break
+
+            if len(item_contents) > self.alert_block_collapsible_threshold:
+                item_contents = f"> <details>\n{item_contents}> </details>"
+
+            return f"> [!{block_type}]\n{item_contents}\n"
 
-        self._print(f"```\n\\[WARNING] {short_message}```\n\n")
+        return ""
 
     def _print(self, value: t.Any, **kwargs: t.Any) -> None:
         self.console.print(value, **kwargs)
@@ -3472,7 +3511,7 @@ class DatabricksMagicConsole(CaptureTerminalConsole):
         super()._print(value, **kwargs)
         for captured_output in self._captured_outputs:
             print(captured_output)
-        self.clear_captured_outputs()
+        self.consume_captured_output()
 
     def _prompt(self, message: str, **kwargs: t.Any) -> t.Any:
         self._print(message)

{sqlmesh-0.200.1.dev10 → sqlmesh-0.200.1.dev12}/sqlmesh/integrations/github/cicd/command.py

@@ -119,10 +119,7 @@ def _update_pr_environment(controller: GithubController) -> bool:
     except Exception as e:
         logger.exception("Error occurred when updating PR environment")
         conclusion = controller.update_pr_environment_check(
-            status=GithubCheckStatus.COMPLETED,
-            exception=e,
-            plan=controller.pr_plan_or_none,
-            plan_flags=controller.pr_plan_flags,
+            status=GithubCheckStatus.COMPLETED, exception=e
         )
         return (
             conclusion is not None

{sqlmesh-0.200.1.dev10 → sqlmesh-0.200.1.dev12}/sqlmesh/integrations/github/cicd/config.py

@@ -7,6 +7,7 @@ from sqlmesh.core.config import CategorizerConfig
 from sqlmesh.core.config.base import BaseConfig
 from sqlmesh.utils.date import TimeLike
 from sqlmesh.utils.pydantic import model_validator
+from sqlmesh.core.console import get_console
 
 
 class MergeMethod(str, Enum):
@@ -22,10 +23,12 @@ class GithubCICDBotConfig(BaseConfig):
     enable_deploy_command: bool = False
     merge_method: t.Optional[MergeMethod] = None
     command_namespace: t.Optional[str] = None
-    auto_categorize_changes: CategorizerConfig = CategorizerConfig.all_off()
+    auto_categorize_changes_: t.Optional[CategorizerConfig] = Field(
+        default=None, alias="auto_categorize_changes"
+    )
     default_pr_start: t.Optional[TimeLike] = None
-    skip_pr_backfill: bool = True
-    pr_include_unmodified: t.Optional[bool] = None
+    skip_pr_backfill_: t.Optional[bool] = Field(default=None, alias="skip_pr_backfill")
+    pr_include_unmodified_: t.Optional[bool] = Field(default=None, alias="pr_include_unmodified")
     run_on_deploy_to_prod: bool = False
     pr_environment_name: t.Optional[str] = None
     pr_min_intervals: t.Optional[int] = None
@@ -50,6 +53,26 @@ class GithubCICDBotConfig(BaseConfig):
             return [self.prod_branch_names_]
         return ["main", "master"]
 
+    @property
+    def auto_categorize_changes(self) -> CategorizerConfig:
+        return self.auto_categorize_changes_ or CategorizerConfig.all_off()
+
+    @property
+    def pr_include_unmodified(self) -> bool:
+        return self.pr_include_unmodified_ or False
+
+    @property
+    def skip_pr_backfill(self) -> bool:
+        if self.skip_pr_backfill_ is None:
+            get_console().log_warning(
+                "`skip_pr_backfill` is unset, defaulting it to `true` (no data will be backfilled).\n"
+                "Future versions of SQLMesh will default to `skip_pr_backfill: false` to align with the CLI default behaviour.\n"
+                "If you would like to preserve the current behaviour and remove this warning, please explicitly set `skip_pr_backfill: true` in the bot config.\n\n"
+                "For more information on configuring the bot, see: https://sqlmesh.readthedocs.io/en/stable/integrations/github/"
+            )
+            return True
+        return self.skip_pr_backfill_
+
     FIELDS_FOR_ANALYTICS: t.ClassVar[t.Set[str]] = {
         "invalidate_environment_after_deploy",
         "enable_deploy_command",

{sqlmesh-0.200.1.dev10 → sqlmesh-0.200.1.dev12}/sqlmesh/integrations/github/cicd/controller.py

@@ -494,7 +494,7 @@ class GithubController:
 
         try:
             # Clear out any output that might exist from prior steps
-            self._console.clear_captured_outputs()
+            self._console.consume_captured_output()
             if plan.restatements:
                 self._console._print("\n**Restating models**\n")
             else:
@@ -522,13 +522,115 @@ class GithubController:
             if not difference_summary and not missing_dates:
                 return f"No changes to apply.{plan_flags_section}"
 
-            return f"{difference_summary}\n{missing_dates}{plan_flags_section}"
+            warnings_block = self._console.consume_captured_warnings()
+            errors_block = self._console.consume_captured_errors()
+
+            return f"{warnings_block}{errors_block}{difference_summary}\n{missing_dates}{plan_flags_section}"
         except PlanError as e:
             logger.exception("Plan failed to generate")
             return f"Plan failed to generate. Check for pending or unresolved changes. Error: {e}"
         finally:
             self._console.verbosity = orig_verbosity
 
+    def get_pr_environment_summary(
+        self, conclusion: GithubCheckConclusion, exception: t.Optional[Exception] = None
+    ) -> str:
+        heading = ""
+        summary = ""
+
+        if conclusion.is_success:
+            summary = self._get_pr_environment_summary_success()
+        elif conclusion.is_action_required:
+            heading = f":warning: Action Required to create or update PR Environment `{self.pr_environment_name}` :warning:"
+            summary = self._get_pr_environment_summary_action_required(exception)
+        elif conclusion.is_failure:
+            heading = (
+                f":x: Failed to create or update PR Environment `{self.pr_environment_name}` :x:"
+            )
+            summary = self._get_pr_environment_summary_failure(exception)
+        elif conclusion.is_skipped:
+            heading = f":next_track_button: Skipped creating or updating PR Environment `{self.pr_environment_name}` :next_track_button:"
+            summary = self._get_pr_environment_summary_skipped(exception)
+        else:
+            heading = f":interrobang: Got an unexpected conclusion: {conclusion.value}"
+
+        # note: we just add warnings here, errors will be covered by the "failure" conclusion
+        if warnings := self._console.consume_captured_warnings():
+            summary = f"{warnings}\n{summary}"
+
+        return f"{heading}\n\n{summary}".strip()
+
+    def _get_pr_environment_summary_success(self) -> str:
+        prod_plan = self.prod_plan_with_gaps
+
+        if not prod_plan.has_changes:
+            summary = "No models were modified in this PR.\n"
+        else:
+            intro = self._generate_pr_environment_summary_intro()
+            summary = intro + self._generate_pr_environment_summary_list(prod_plan)
+
+        if prod_plan.user_provided_flags:
+            summary += self._generate_plan_flags_section(prod_plan.user_provided_flags)
+
+        return summary
+
+    def _get_pr_environment_summary_skipped(self, exception: t.Optional[Exception] = None) -> str:
+        if isinstance(exception, NoChangesPlanError):
+            skip_reason = "No changes were detected compared to the prod environment."
+        elif isinstance(exception, TestFailure):
+            skip_reason = "Unit Test(s) Failed so skipping PR creation"
+        else:
+            skip_reason = "A prior stage failed resulting in skipping PR creation."
+
+        return skip_reason
+
+    def _get_pr_environment_summary_action_required(
+        self, exception: t.Optional[Exception] = None
+    ) -> str:
+        plan = self.pr_plan_or_none
+        if isinstance(exception, UncategorizedPlanError) and plan:
+            failure_msg = f"The following models could not be categorized automatically:\n"
+            for snapshot in plan.uncategorized:
+                failure_msg += f"- {snapshot.name}\n"
+            failure_msg += (
+                f"\nRun `sqlmesh plan {self.pr_environment_name}` locally to apply these changes.\n\n"
+                "If you would like the bot to automatically categorize changes, check the [documentation](https://sqlmesh.readthedocs.io/en/stable/integrations/github/) for more information."
+            )
+        else:
+            failure_msg = "Please check the Actions Workflow logs for more information."
+
+        return failure_msg
+
+    def _get_pr_environment_summary_failure(self, exception: t.Optional[Exception] = None) -> str:
+        console_output = self._console.consume_captured_output()
+
+        if isinstance(exception, PlanError):
+            failure_msg = f"Plan application failed.\n"
+            if exception.args and (msg := exception.args[0]) and isinstance(msg, str):
+                failure_msg += f"\n{msg}\n"
+            if console_output:
+                failure_msg += f"\n{console_output}"
+        elif isinstance(exception, (SQLMeshError, SqlglotError, ValueError)):
+            # this logic is taken from the global error handler attached to the CLI, which uses `click.echo()` to output the message
+            # so cant be re-used here because it bypasses the Console
+            failure_msg = f"**Error:** {str(exception)}"
+        elif exception:
+            logger.debug(
+                "Got unexpected error. Error Type: "
+                + str(type(exception))
+                + " Stack trace: "
+                + traceback.format_exc()
+            )
+            failure_msg = f"This is an unexpected error.\n\n**Exception:**\n```\n{traceback.format_exc()}\n```"
+
+        if captured_errors := self._console.consume_captured_errors():
+            failure_msg = f"{captured_errors}\n{failure_msg}"
+
+        if plan_flags := self.pr_plan_flags:
+            failure_msg += f"\n\n{self._generate_plan_flags_section(plan_flags)}"
+
+        return failure_msg
+
     def run_tests(self) -> t.Tuple[ModelTextTestResult, str]:
         """
         Run tests for the PR
@@ -539,7 +641,7 @@ class GithubController:
         """
         Run linter for the PR
         """
-        self._console.clear_captured_outputs()
+        self._console.consume_captured_output()
         self._context.lint_models()
 
     def _get_or_create_comment(self, header: str = BOT_HEADER_MSG) -> IssueComment:
@@ -611,7 +713,22 @@ class GithubController:
         Creates a PR environment from the logic present in the PR. If the PR contains changes that are
         uncategorized, then an error will be raised.
         """
-        self._context.apply(self.pr_plan)
+        self._context.apply(self.pr_plan)  # will raise if PR environment creation fails
+
+        # update PR info comment
+        vde_title = "- :eyes: To **review** this PR's changes, use virtual data environment:"
+        comment_value = f"{vde_title}\n  - `{self.pr_environment_name}`"
+        if self.bot_config.enable_deploy_command:
+            comment_value += (
+                "\n- :arrow_forward: To **apply** this PR's plan to prod, comment:\n  - `/deploy`"
+            )
+        dedup_regex = vde_title.replace("*", r"\*") + r".*"
+        updated_comment, _ = self.update_sqlmesh_comment_info(
+            value=comment_value,
+            dedup_regex=dedup_regex,
+        )
+        if updated_comment:
+            self._append_output("created_pr_environment", "true")
 
     def deploy_to_prod(self) -> None:
         """
@@ -855,13 +972,7 @@ class GithubController:
         )
 
     def update_pr_environment_check(
-        self,
-        status: GithubCheckStatus,
-        exception: t.Optional[Exception] = None,
-        plan: t.Optional[Plan] = None,
-        plan_flags: t.Optional[
-            t.Dict[str, UserProvidedFlags]
-        ] = None,  # note: the plan flags are passed separately in case the plan fails to build
+        self, status: GithubCheckStatus, exception: t.Optional[Exception] = None
     ) -> t.Optional[GithubCheckConclusion]:
         """
         Updates the status of the merge commit for the PR environment.
@@ -882,87 +993,7 @@ class GithubController:
         def conclusion_handler(
             conclusion: GithubCheckConclusion, exception: t.Optional[Exception]
         ) -> t.Tuple[GithubCheckConclusion, str, t.Optional[str]]:
-            if conclusion.is_success:
-                prod_plan = self.prod_plan_with_gaps
-
-                if not prod_plan.has_changes:
-                    summary = "No models were modified in this PR.\n"
-                else:
-                    intro = self._generate_pr_environment_summary_intro()
-                    summary = intro + self._generate_pr_environment_summary_list(prod_plan)
-                    if prod_plan.user_provided_flags:
-                        summary += self._generate_plan_flags_section(prod_plan.user_provided_flags)
-
-                vde_title = (
-                    "- :eyes: To **review** this PR's changes, use virtual data environment:"
-                )
-                comment_value = f"{vde_title}\n  - `{self.pr_environment_name}`"
-                if self.bot_config.enable_deploy_command:
-                    comment_value += "\n- :arrow_forward: To **apply** this PR's plan to prod, comment:\n  - `/deploy`"
-                dedup_regex = vde_title.replace("*", r"\*") + r".*"
-                updated_comment, _ = self.update_sqlmesh_comment_info(
-                    value=comment_value,
-                    dedup_regex=dedup_regex,
-                )
-                if updated_comment:
-                    self._append_output("created_pr_environment", "true")
-            else:
-                if isinstance(exception, NoChangesPlanError):
-                    skip_reason = "No changes were detected compared to the prod environment."
-                elif isinstance(exception, TestFailure):
-                    skip_reason = "Unit Test(s) Failed so skipping PR creation"
-                else:
-                    skip_reason = "A prior stage failed resulting in skipping PR creation."
-
-                if not skip_reason and exception:
-                    logger.debug(
-                        f"Got {type(exception).__name__}. Stack trace: " + traceback.format_exc()
-                    )
-
-                captured_errors = self._console.consume_captured_errors()
-                if captured_errors:
-                    logger.debug(f"Captured errors: {captured_errors}")
-                    failure_msg = f"**Errors:**\n{captured_errors}\n"
-                elif isinstance(exception, UncategorizedPlanError) and plan:
-                    failure_msg = f"The following models could not be categorized automatically:\n"
-                    for snapshot in plan.uncategorized:
-                        failure_msg += f"- {snapshot.name}\n"
-                    failure_msg += (
-                        f"\nRun `sqlmesh plan {self.pr_environment_name}` locally to apply these changes.\n\n"
-                        "If you would like the bot to automatically categorize changes, check the [documentation](https://sqlmesh.readthedocs.io/en/stable/integrations/github/) for more information."
-                    )
-                elif isinstance(exception, PlanError):
-                    failure_msg = f"Plan application failed.\n"
-                    if exception.args and (msg := exception.args[0]) and isinstance(msg, str):
-                        failure_msg += f"\n{msg}\n"
-                    if self._console.captured_output:
-                        failure_msg += f"\n{self._console.captured_output}"
-                elif isinstance(exception, (SQLMeshError, SqlglotError, ValueError)):
-                    # this logic is taken from the global error handler attached to the CLI, which uses `click.echo()` to output the message
-                    # so cant be re-used here because it bypasses the Console
-                    failure_msg = f"**Error:** {str(exception)}"
-                else:
-                    logger.debug(
-                        "Got unexpected error. Error Type: "
-                        + str(type(exception))
-                        + " Stack trace: "
-                        + traceback.format_exc()
-                    )
-                    failure_msg = f"This is an unexpected error.\n\n**Exception:**\n```\n{traceback.format_exc()}\n```"
-
-                conclusion_to_summary = {
-                    GithubCheckConclusion.SKIPPED: f":next_track_button: Skipped creating or updating PR Environment `{self.pr_environment_name}`. {skip_reason}",
-                    GithubCheckConclusion.FAILURE: f":x: Failed to create or update PR Environment `{self.pr_environment_name}` :x:\n\n{failure_msg}",
-                    GithubCheckConclusion.CANCELLED: f":stop_sign: Cancelled creating or updating PR Environment `{self.pr_environment_name}`",
-                    GithubCheckConclusion.ACTION_REQUIRED: f":warning: Action Required to create or update PR Environment `{self.pr_environment_name}` :warning:\n\n{failure_msg}",
-                }
-                summary = conclusion_to_summary.get(
-                    conclusion, f":interrobang: Got an unexpected conclusion: {conclusion.value}"
-                )
-                if plan_flags:
-                    plan_flags_section = self._generate_plan_flags_section(plan_flags)
-                    summary += f"\n\n{plan_flags_section}"
-
+            summary = self.get_pr_environment_summary(conclusion, exception)
             self._append_output("pr_environment_name", self.pr_environment_name)
             return conclusion, check_title, summary
 

{sqlmesh-0.200.1.dev10 → sqlmesh-0.200.1.dev12}/sqlmesh.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.200.1.dev10
+Version: 0.200.1.dev12
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.200.1.dev10 → sqlmesh-0.200.1.dev12}/sqlmesh.egg-info/SOURCES.txt

@@ -493,6 +493,7 @@ tests/core/__init__.py
 tests/core/test_audit.py
 tests/core/test_config.py
 tests/core/test_connection_config.py
+tests/core/test_console.py
 tests/core/test_context.py
 tests/core/test_dialect.py
 tests/core/test_environment.py

{sqlmesh-0.200.1.dev10 → sqlmesh-0.200.1.dev12}/tests/core/analytics/test_collector.py

@@ -145,7 +145,7 @@ def test_on_cicd_command(collector: AnalyticsCollector, mocker: MockerFixture):
                 {
                     "seq_num": 1,
                     "event_type": "CICD_COMMAND",
-                    "event": '{"command_name": "test_cicd", "command_args": ["arg_1", "arg_2"], "parent_command_names": ["parent_a", "parent_b"], "cicd_bot_config": {"invalidate_environment_after_deploy": true, "enable_deploy_command": false, "auto_categorize_changes": {"external": "off", "python": "off", "sql": "off", "seed": "off"}, "skip_pr_backfill": true, "run_on_deploy_to_prod": false}}',
+                    "event": '{"command_name": "test_cicd", "command_args": ["arg_1", "arg_2"], "parent_command_names": ["parent_a", "parent_b"], "cicd_bot_config": {"invalidate_environment_after_deploy": true, "enable_deploy_command": false, "run_on_deploy_to_prod": false}}',
                     **common_fields,
                 }
             ),