cognite-toolkit 0.6.36__tar.gz → 0.7.42__tar.gz

cognite_toolkit-0.7.42/PKG-INFO

@@ -0,0 +1,84 @@
+Metadata-Version: 2.4
+Name: cognite_toolkit
+Version: 0.7.42
+Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
+Author: Cognite AS
+Author-email: Cognite AS <support@cognite.com>
+License-Expression: Apache-2.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: cognite-sdk>=7.87.0,<8.0.0
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: pandas>=1.5.3,<3.0.0
+Requires-Dist: pyyaml>=6.0.1
+Requires-Dist: typer>=0.12.0,<1.0.0
+Requires-Dist: rich>=13.9.4
+Requires-Dist: questionary>=2.0.1
+Requires-Dist: tomli>=2.0.1,<3.0.0 ; python_full_version < '3.11'
+Requires-Dist: packaging>=25
+Requires-Dist: typing-extensions>=4.0.0
+Requires-Dist: toml>=0.10.2
+Requires-Dist: sentry-sdk>=2.1.0
+Requires-Dist: mixpanel>=4.10.1
+Requires-Dist: pydantic>=2.11.0
+Requires-Dist: python-dateutil>=2.9.0
+Requires-Dist: pip>=25.0.1
+Requires-Dist: filelock>=3.18.0
+Requires-Dist: sqlparse>=0.5.3 ; extra == 'sql'
+Requires-Dist: pyarrow>=20.0.0 ; extra == 'table'
+Requires-Dist: openpyxl>=3.1.5 ; extra == 'table'
+Requires-Python: >=3.10
+Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases
+Project-URL: Documentation, https://docs.cognite.com/cdf/deploy/cdf_toolkit/references/resource_library
+Project-URL: GitHub, https://github.com/cognitedata/toolkit
+Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
+Provides-Extra: sql
+Provides-Extra: table
+Description-Content-Type: text/markdown
+
+# Cognite Data Fusion Toolkit
+
+[![release](https://img.shields.io/github/actions/workflow/status/cognitedata/toolkit/release.yaml?style=for-the-badge)](https://github.com/cognitedata/toolkit/actions/workflows/release.yaml)
+[![Github](https://shields.io/badge/github-cognite/toolkit-green?logo=github&style=for-the-badge)](https://github.com/cognitedata/toolkit)
+[![PyPI](https://img.shields.io/pypi/v/cognite-toolkit?style=for-the-badge)](https://pypi.org/project/cognite-toolkit/)
+[![Downloads](https://img.shields.io/pypi/dm/cognite-toolkit?style=for-the-badge)](https://pypistats.org/packages/cognite-toolkit)
+[![GitHub](https://img.shields.io/github/license/cognitedata/toolkit?style=for-the-badge)](https://github.com/cognitedata/toolkit/blob/master/LICENSE)
+[![Docker Pulls](https://img.shields.io/docker/pulls/cognite/toolkit?style=for-the-badge)](https://hub.docker.com/r/cognite/toolkit)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json&style=for-the-badge)](https://github.com/astral-sh/ruff)
+[![mypy](https://img.shields.io/badge/mypy-checked-000000.svg?style=for-the-badge&color=blue)](http://mypy-lang.org)
+
+The CDF Toolkit is a command-line interface (`cdf`) used for configuring and administrating
+Cognite Data Fusion (CDF) projects. It ships with modularised `templates` that helps you
+configure Cognite Data Fusion according to best practices.
+
+It supports three different modes of operation:
+
+1. As an **interactive command-line tool** used alongside the Cognite Data Fusion web application to retrieve and
+   push configuration of the different Cognite Data Fusion services like data sets, data models, transformations,
+   and more. This mode also supports configuration of new Cognite Data Fusion projects to quickly get started.
+2. As tool to support the **project life-cycle by scripting and automating** configuration and management of Cognite Data
+   Fusion projects where CDF configurations are kept as yaml-files that can be checked into version
+   control. This mode also supports DevOps workflows with development, staging, and production projects.
+3. As a **tool to deploy official Cognite project templates** to your Cognite Data Fusion project. The tool comes
+   bundled with templates useful for getting started with Cognite Data Fusion, as well as for specific use cases
+   delivered by Cognite or its partners. You can also create your own templates and share them.
+
+## Usage
+
+Install the Toolkit by running:
+
+```bash
+pip install cognite-toolkit
+```
+
+Then run `cdf --help` to get started with the interactive command-line tool.
+
+## For more information
+
+More details about the tool can be found at
+[docs.cognite.com](https://docs.cognite.com/cdf/deploy/cdf_toolkit/).
+
+You can find an overview of the modules and packages in the
+[module and package documentation](https://docs.cognite.com/cdf/deploy/cdf_toolkit/references/resource_library).
+
+See [./CONTRIBUTING.md](./CONTRIBUTING.md) for information about how to contribute to the `cdf-tk` tool or
+templates.

cognite_toolkit-0.7.42/cognite_toolkit/_cdf.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python
+# The Typer parameters get mixed up if we use the __future__ import annotations in the main file.
+# ruff: noqa: E402
+import re
+import sys
+import traceback
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import NoReturn
+
+import typer
+from cognite.client.config import global_config
+from rich.markup import escape
+from rich.panel import Panel
+
+from cognite_toolkit._cdf_tk.hints import Hint
+
+# Do not warn the user about feature previews from the Cognite-SDK we use in Toolkit
+global_config.disable_pypi_version_check = True
+global_config.silence_feature_preview_warnings = True
+
+from rich import print
+
+from cognite_toolkit._cdf_tk.apps import (
+    AuthApp,
+    CoreApp,
+    DataApp,
+    DevApp,
+    DumpApp,
+    ImportApp,
+    LandingApp,
+    MigrateApp,
+    ModulesApp,
+    ProfileApp,
+    RepoApp,
+    RunApp,
+)
+from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
+from cognite_toolkit._cdf_tk.commands import (
+    AboutCommand,
+)
+from cognite_toolkit._cdf_tk.constants import HINT_LEAD_TEXT, URL, USE_SENTRY
+from cognite_toolkit._cdf_tk.exceptions import (
+    ToolkitError,
+)
+from cognite_toolkit._cdf_tk.feature_flags import Flags
+from cognite_toolkit._cdf_tk.plugins import Plugins
+from cognite_toolkit._cdf_tk.tracker import Tracker
+from cognite_toolkit._cdf_tk.utils import (
+    sentry_exception_filter,
+)
+from cognite_toolkit._version import __version__ as current_version
+
+if USE_SENTRY:
+    import sentry_sdk
+
+    sentry_sdk.init(
+        dsn="https://20552f92b525fe551e9adc939024d526@o4508040730968064.ingest.de.sentry.io/4508160801374288",
+        release=current_version,
+        before_send=sentry_exception_filter,
+        # Set traces_sample_rate to 1.0 to capture 100%
+        # of transactions for performance monitoring.
+        traces_sample_rate=1.0,
+    )
+
+
+CDF_TOML = CDFToml.load(Path.cwd())
+
+default_typer_kws = dict(
+    pretty_exceptions_short=False,
+    pretty_exceptions_show_locals=False,
+    pretty_exceptions_enable=False,
+)
+try:
+    typer.Typer(**default_typer_kws)  # type: ignore [arg-type]
+except AttributeError as e:
+    # From Typer version 0.11 -> 0.12, breaks if you have an existing installation.
+    raise ToolkitError(
+        "'cognite-toolkit' uses a dependency named 'typer'. From 'typer' version 0.11 -> 0.12 there was a "
+        "breaking change if you have an existing installation of 'typer'. The workaround is to uninstall "
+        "'typer-slim', and then, reinstall 'typer':\n"
+        "pip uninstall typer-slim\n"
+        "pip install typer\n\n"
+        f"This was triggered by the error: {e!r}"
+    )
+
+_app = CoreApp(**default_typer_kws)
+
+user_app = typer.Typer(**default_typer_kws, hidden=True)  # type: ignore [arg-type]
+landing_app = LandingApp(**default_typer_kws)
+
+_app.add_typer(AuthApp(**default_typer_kws), name="auth")
+_app.add_typer(RepoApp(**default_typer_kws), name="repo")
+
+
+if Plugins.run.value.is_enabled():
+    _app.add_typer(RunApp(**default_typer_kws), name="run")
+
+if Plugins.dump.value.is_enabled():
+    _app.add_typer(DumpApp(**default_typer_kws), name="dump")
+
+
+if Plugins.dev.value.is_enabled():
+    _app.add_typer(DevApp(**default_typer_kws), name="dev")
+
+if Flags.PROFILE.is_enabled():
+    _app.add_typer(ProfileApp(**default_typer_kws), name="profile")
+
+if Flags.MIGRATE.is_enabled():
+    _app.add_typer(MigrateApp(**default_typer_kws), name="migrate")
+
+if Flags.IMPORT_CMD.is_enabled():
+    _app.add_typer(ImportApp(**default_typer_kws), name="import")
+
+if Plugins.data.value.is_enabled():
+    _app.add_typer(DataApp(**default_typer_kws), name="data")
+
+_app.add_typer(ModulesApp(**default_typer_kws), name="modules")
+_app.command("init")(landing_app.main_init)
+
+
+@_app.command("about")
+def about() -> None:
+    """Display information about the Toolkit installation and configuration."""
+    cmd = AboutCommand()
+    cmd.run(lambda: cmd.execute(Path.cwd()))
+
+
+def app() -> NoReturn:
+    # --- Main entry point ---
+    # Users run 'app()' directly, but that doesn't allow us to control excepton handling:
+    try:
+        _app()
+    except ToolkitError as err:
+        if "--verbose" in sys.argv:
+            print(Panel(traceback.format_exc(), title="Traceback", expand=False))
+
+        print(f"  [bold red]ERROR ([/][red]{type(err).__name__}[/][bold red]):[/] {err}")
+        raise SystemExit(1)
+    except SystemExit:
+        if result := re.search(r"click.exceptions.UsageError: No such command '(\w+)'.", traceback.format_exc()):
+            cmd = result.group(1)
+            if cmd in Plugins.list():
+                plugin = r"[plugins]"
+                print(
+                    f"{HINT_LEAD_TEXT} The plugin [bold]{cmd}[/bold] is not enabled."
+                    f"\nEnable it in the [bold]cdf.toml[/bold] file by setting '{cmd} = true' in the "
+                    f"[bold]{escape(plugin)}[/bold] section."
+                    f"\nDocs to learn more: {Hint.link(URL.plugins, URL.plugins)}"
+                )
+        raise
+
+    raise SystemExit(0)
+
+
+@user_app.callback(invoke_without_command=True)
+def user_main(ctx: typer.Context) -> None:
+    """Commands to give information about the toolkit."""
+    if ctx.invoked_subcommand is None:
+        print("Use [bold yellow]cdf user --help[/] to see available commands.")
+    return None
+
+
+@user_app.command("info")
+def user_info() -> None:
+    """Print information about user"""
+    tracker = Tracker()
+    print(f"ID={tracker.get_distinct_id()!r}\nnow={datetime.now(timezone.utc).isoformat(timespec='seconds')!r}")
+
+
+if __name__ == "__main__":
+    app()

cognite_toolkit-0.7.42/cognite_toolkit/_cdf_tk/apps/__init__.py

@@ -0,0 +1,33 @@
+from ._auth_app import AuthApp
+from ._core_app import CoreApp
+from ._data_app import DataApp
+from ._dev_app import DevApp
+from ._download_app import DownloadApp
+from ._dump_app import DumpApp
+from ._import_app import ImportApp
+from ._landing_app import LandingApp
+from ._migrate_app import MigrateApp
+from ._modules_app import ModulesApp
+from ._profile_app import ProfileApp
+from ._purge import PurgeApp
+from ._repo_app import RepoApp
+from ._run import RunApp
+from ._upload_app import UploadApp
+
+__all__ = [
+    "AuthApp",
+    "CoreApp",
+    "DataApp",
+    "DevApp",
+    "DownloadApp",
+    "DumpApp",
+    "ImportApp",
+    "LandingApp",
+    "MigrateApp",
+    "ModulesApp",
+    "ProfileApp",
+    "PurgeApp",
+    "RepoApp",
+    "RunApp",
+    "UploadApp",
+]

cognite_toolkit-0.7.42/cognite_toolkit/_cdf_tk/apps/_core_app.py

@@ -0,0 +1,387 @@
+"""This is the core functionality of the Cognite Data Fusion Toolkit."""
+
+import contextlib
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Annotated, Union
+
+import typer
+from dotenv import load_dotenv
+from rich import print
+from rich.panel import Panel
+
+from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
+from cognite_toolkit._cdf_tk.client import ToolkitClient
+from cognite_toolkit._cdf_tk.commands import BuildCommand, CleanCommand, DeployCommand
+from cognite_toolkit._cdf_tk.commands.build_v2.build_cmd import BuildCommand as BuildCommandV2
+from cognite_toolkit._cdf_tk.commands.clean import AVAILABLE_DATA_TYPES
+from cognite_toolkit._cdf_tk.exceptions import ToolkitFileNotFoundError
+from cognite_toolkit._cdf_tk.feature_flags import Flags
+from cognite_toolkit._cdf_tk.utils import get_cicd_environment
+from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
+from cognite_toolkit._version import __version__ as current_version
+
+
+# Common parameters handled in common callback
+@dataclass
+class Common:
+    override_env: bool
+
+
+CDF_TOML = CDFToml.load(Path.cwd())
+
+
+def _version_callback(value: bool) -> None:
+    if value:
+        typer.echo(f"CDF-Toolkit version: {current_version}.")
+        raise typer.Exit()
+
+
+class CoreApp(typer.Typer):
+    def __init__(self, *args, **kwargs) -> None:  # type: ignore
+        super().__init__(*args, **kwargs)
+        self.callback(invoke_without_command=True)(self.common)
+        self.command()(self.build)
+        self.command()(self.deploy)
+        self.command()(self.clean)
+
+    def common(
+        self,
+        ctx: typer.Context,
+        override_env: Annotated[
+            bool,
+            typer.Option(
+                help="Load the .env file in this or the parent directory, but also override currently set environment variables",
+            ),
+        ] = True,
+        env_path: Annotated[
+            str | None,
+            typer.Option(
+                help="Path to .env file to load. Defaults to .env in current or parent directory.",
+            ),
+        ] = None,
+        version: Annotated[
+            bool,
+            typer.Option(
+                "--version",
+                help="See which version of the Toolkit is installed.",
+                callback=_version_callback,
+            ),
+        ] = False,
+    ) -> None:
+        """
+        Docs: https://docs.cognite.com/cdf/deploy/cdf_toolkit/\n
+        Template reference documentation: https://developer.cognite.com/sdks/toolkit/references/configs
+        """
+        if ctx.invoked_subcommand is None:
+            print(
+                Panel(
+                    "\n".join(
+                        [
+                            "The Cognite Toolkit supports configuration of CDF projects from the command line or in CI/CD pipelines.",
+                            "",
+                            "[bold]Setup:[/]",
+                            "1. Run [underline]cdf repo init[/] [italic]<directory name>[/] to set up a work directory.",
+                            "2. Run [underline]cdf modules init[/] [italic]<directory name>[/] to initialise configuration modules.",
+                            "",
+                            "[bold]Configuration steps:[/]",
+                            "3. Run [underline]cdf build[/] [italic]<directory name>[/] to verify the configuration for your project. Repeat for as many times as needed.",
+                            "   Tip:[underline]cdf modules list[/] [italic]<directory name>[/] gives an overview of all your modules and their status.",
+                            "",
+                            "[bold]Deployment steps:[/]",
+                            "4. Commit the [italic]<directory name>[/] to version control",
+                            "5. Run [underline]cdf auth verify[/] to check that you have access to the relevant CDF project. ",
+                            "    or [underline]cdf auth verify[/] if you have a .env file",
+                            "6. Run [underline]cdf deploy --dry-run[/] to simulate the deployment of the configuration to the CDF project. Review the report provided.",
+                            "7. Run [underline]cdf deploy[/] to deploy the configuration to the CDF project.",
+                        ]
+                    ),
+                    title="Getting started",
+                    style="green",
+                    padding=(1, 2),
+                )
+            )
+            return
+        if override_env:
+            print("  [bold yellow]WARNING:[/] Overriding environment variables with values from .env file...")
+
+        if env_path is not None:
+            if not (dotenv_file := Path(env_path)).is_file():
+                raise ToolkitFileNotFoundError(env_path)
+
+        else:
+            if not (dotenv_file := Path.cwd() / ".env").is_file():
+                if not (dotenv_file := Path.cwd().parent / ".env").is_file():
+                    if get_cicd_environment() == "local":
+                        warn = False
+                        try:
+                            env_vars = EnvironmentVariables.create_from_environment()
+                        except Exception:
+                            warn = True
+                        else:
+                            warn = bool(env_vars.get_missing_vars())
+                        if warn:
+                            print("[bold yellow]WARNING:[/] No .env file found in current or parent directory.")
+
+        if dotenv_file.is_file():
+            has_loaded = load_dotenv(dotenv_file, override=override_env)
+            if not has_loaded:
+                print("  [bold yellow]WARNING:[/] No environment variables found in .env file.")
+
+        ctx.obj = Common(override_env=override_env)
+
+    def build(
+        self,
+        ctx: typer.Context,
+        organization_dir: Annotated[
+            Path,
+            typer.Option(
+                "--organization-dir",
+                "-o",
+                help="Where to find the module templates to build from",
+            ),
+        ] = CDF_TOML.cdf.default_organization_dir,
+        build_dir: Annotated[
+            Path,
+            typer.Option(
+                "--build-dir",
+                "-b",
+                help="Where to save the built module files",
+            ),
+        ] = Path("./build"),
+        selected: Annotated[
+            list[str] | None,
+            typer.Option(
+                "--modules",
+                "-m",
+                help="Specify paths or names to the modules to build",
+            ),
+        ] = None,
+        build_env_name: Annotated[
+            str | None,
+            typer.Option(
+                "--env",
+                "-e",
+                help="The name of the environment to build",
+            ),
+        ] = CDF_TOML.cdf.default_env,
+        no_clean: Annotated[
+            bool,
+            typer.Option(
+                "--no-clean",
+                "-c",
+                help="Whether not to delete the build directory before building the configurations",
+            ),
+        ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+        offline: Annotated[
+            bool,
+            typer.Option(
+                "--offline",
+                help="Do not check CDF for missing dependencies.",
+            ),
+        ] = False,
+        exit_on_warning: Annotated[
+            bool,
+            typer.Option(
+                "--exit-non-zero-on-warning",
+                "-w",
+                help="Exit with non-zero code on warning.",
+            ),
+        ] = False,
+    ) -> None:
+        """Build configuration files from the modules to the build directory."""
+        client: Union[ToolkitClient, None] = None
+        if not offline:
+            with contextlib.redirect_stdout(None), contextlib.suppress(Exception):
+                # Remove the Error message from failing to load the config
+                # This is verified in check_auth
+                client = EnvironmentVariables.create_from_environment().get_client()
+
+        print_warning = True
+        if exit_on_warning:
+            print_warning = False
+
+        cmd = (
+            BuildCommandV2(print_warning=print_warning)
+            if Flags.v08.is_enabled()
+            else BuildCommand(print_warning=print_warning)
+        )
+        cmd.run(
+            lambda: cmd.execute(
+                verbose,
+                organization_dir,
+                build_dir,
+                selected,  # type: ignore[arg-type]
+                build_env_name,
+                no_clean,
+                client,
+                on_error="raise",
+            )
+        )
+
+        if exit_on_warning and cmd.warning_list:
+            print("\n[bold red]Warnings raised during the build process:[/]\n")
+
+            for warning in cmd.warning_list:
+                warning.print_warning(include_timestamp=False)
+                print(end="\n")
+
+            raise typer.Exit(code=1)
+
+    def deploy(
+        self,
+        ctx: typer.Context,
+        build_dir: Annotated[
+            Path,
+            typer.Argument(
+                help="Where to find the module templates to deploy from. Defaults to current directory.",
+                allow_dash=True,
+            ),
+        ] = Path("./build"),
+        build_env_name: Annotated[
+            str | None,
+            typer.Option(
+                "--env",
+                "-e",
+                help="CDF project environment to use for deployment. This is optional and "
+                "if passed it is used to verify against the build environment",
+            ),
+        ] = None,
+        drop: Annotated[
+            bool,
+            typer.Option(
+                "--drop",
+                "-d",
+                help="Whether to drop existing configurations, drop per resource if present.",
+            ),
+        ] = False,
+        drop_data: Annotated[
+            bool,
+            typer.Option(
+                "--drop-data",
+                help="Whether to drop existing data in data model containers and spaces.",
+            ),
+        ] = False,
+        dry_run: Annotated[
+            bool,
+            typer.Option(
+                "--dry-run",
+                "-r",
+                help="Whether to do a dry-run, do dry-run if present.",
+            ),
+        ] = False,
+        include: Annotated[
+            list[str] | None,
+            typer.Option(
+                "--include",
+                help=f"Specify which resources to deploy, available options: {AVAILABLE_DATA_TYPES}.",
+            ),
+        ] = None,
+        force_update: Annotated[
+            bool,
+            typer.Option(
+                "--force-update",
+                help="Whether to force update the resources in the CDF project even if they are considered unchanged.",
+            ),
+        ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+    ) -> None:
+        """Deploys the configuration files in the build directory to the CDF project."""
+        cmd = DeployCommand(print_warning=True)
+        env_vars = EnvironmentVariables.create_from_environment()
+        cmd.run(
+            lambda: cmd.deploy_build_directory(
+                env_vars=env_vars,
+                build_dir=build_dir,
+                build_env_name=build_env_name,
+                dry_run=dry_run,
+                drop=drop,
+                drop_data=drop_data,
+                force_update=force_update,
+                include=include,
+                verbose=verbose,
+            )
+        )
+
+    def clean(
+        self,
+        ctx: typer.Context,
+        build_dir: Annotated[
+            Path,
+            typer.Argument(
+                help="Where to find the module templates to clean from. Defaults to ./build directory.",
+                allow_dash=True,
+            ),
+        ] = Path("./build"),
+        build_env_name: Annotated[
+            str | None,
+            typer.Option(
+                "--env",
+                "-e",
+                help="CDF project environment to use for cleaning. This is optional and "
+                "if passed it is used to verify against the build environment",
+            ),
+        ] = None,
+        dry_run: Annotated[
+            bool,
+            typer.Option(
+                "--dry-run",
+                "-r",
+                help="Whether to do a dry-run, do dry-run if present",
+            ),
+        ] = False,
+        include: Annotated[
+            list[str] | None,
+            typer.Option(
+                "--include",
+                help=f"Specify which resource types to clean, supported types: {AVAILABLE_DATA_TYPES}",
+            ),
+        ] = None,
+        module: Annotated[
+            str | None,
+            typer.Option(
+                "--module",
+                "-m",
+                help="Specify name or path of the module to clean",
+            ),
+        ] = None,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+    ) -> None:
+        """Cleans the resources in the build directory from the CDF project."""
+        # Override cluster and project from the options/env variables
+        cmd = CleanCommand(print_warning=True)
+        env = EnvironmentVariables.create_from_environment()
+        cmd.run(
+            lambda: cmd.execute(
+                env,
+                build_dir,
+                build_env_name,
+                dry_run,
+                include,
+                module,
+                verbose,
+                all_modules=False,
+            )
+        )