cognite-toolkit 0.6.73__tar.gz → 0.7.70__tar.gz

cognite_toolkit-0.7.70/PKG-INFO

@@ -0,0 +1,94 @@
+Metadata-Version: 2.4
+Name: cognite_toolkit
+Version: 0.7.70
+Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
+Author: Cognite AS
+Author-email: Cognite AS <support@cognite.com>
+License-Expression: Apache-2.0
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Build Tools
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Operating System :: OS Independent
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: cognite-sdk>=7.87.0,<8.0.0
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: pandas>=1.5.3,<3.0.0
+Requires-Dist: pyyaml>=6.0.1
+Requires-Dist: typer>=0.12.0,<1.0.0
+Requires-Dist: rich>=13.9.4
+Requires-Dist: questionary>=2.0.1
+Requires-Dist: tomli>=2.0.1,<3.0.0 ; python_full_version < '3.11'
+Requires-Dist: packaging>=25
+Requires-Dist: typing-extensions>=4.0.0
+Requires-Dist: toml>=0.10.2
+Requires-Dist: sentry-sdk>=2.1.0
+Requires-Dist: mixpanel>=4.10.1
+Requires-Dist: pydantic>=2.11.0
+Requires-Dist: python-dateutil>=2.9.0
+Requires-Dist: pip>=25.0.1
+Requires-Dist: filelock>=3.18.0
+Requires-Dist: sqlparse>=0.5.3 ; extra == 'sql'
+Requires-Dist: pyarrow>=20.0.0 ; extra == 'table'
+Requires-Dist: openpyxl>=3.1.5 ; extra == 'table'
+Requires-Python: >=3.10
+Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
+Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases
+Project-URL: GitHub, https://github.com/cognitedata/toolkit
+Project-URL: Documentation, https://docs.cognite.com/cdf/deploy/cdf_toolkit/references/resource_library
+Provides-Extra: sql
+Provides-Extra: table
+Description-Content-Type: text/markdown
+
+# Cognite Data Fusion Toolkit
+
+[![release](https://img.shields.io/github/actions/workflow/status/cognitedata/toolkit/release.yaml?style=for-the-badge)](https://github.com/cognitedata/toolkit/actions/workflows/release.yaml)
+[![Github](https://shields.io/badge/github-cognite/toolkit-green?logo=github&style=for-the-badge)](https://github.com/cognitedata/toolkit)
+[![PyPI](https://img.shields.io/pypi/v/cognite-toolkit?style=for-the-badge)](https://pypi.org/project/cognite-toolkit/)
+[![Downloads](https://img.shields.io/pypi/dm/cognite-toolkit?style=for-the-badge)](https://pypistats.org/packages/cognite-toolkit)
+[![GitHub](https://img.shields.io/github/license/cognitedata/toolkit?style=for-the-badge)](https://github.com/cognitedata/toolkit/blob/master/LICENSE)
+[![Docker Pulls](https://img.shields.io/docker/pulls/cognite/toolkit?style=for-the-badge)](https://hub.docker.com/r/cognite/toolkit)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json&style=for-the-badge)](https://github.com/astral-sh/ruff)
+[![mypy](https://img.shields.io/badge/mypy-checked-000000.svg?style=for-the-badge&color=blue)](http://mypy-lang.org)
+
+The CDF Toolkit is a command-line interface (`cdf`) used for configuring and administrating
+Cognite Data Fusion (CDF) projects. It ships with modularised `templates` that helps you
+configure Cognite Data Fusion according to best practices.
+
+It supports three different modes of operation:
+
+1. As an **interactive command-line tool** used alongside the Cognite Data Fusion web application to retrieve and
+   push configuration of the different Cognite Data Fusion services like data sets, data models, transformations,
+   and more. This mode also supports configuration of new Cognite Data Fusion projects to quickly get started.
+2. As tool to support the **project life-cycle by scripting and automating** configuration and management of Cognite Data
+   Fusion projects where CDF configurations are kept as yaml-files that can be checked into version
+   control. This mode also supports DevOps workflows with development, staging, and production projects.
+3. As a **tool to deploy official Cognite project templates** to your Cognite Data Fusion project. The tool comes
+   bundled with templates useful for getting started with Cognite Data Fusion, as well as for specific use cases
+   delivered by Cognite or its partners. You can also create your own templates and share them.
+
+## Usage
+
+Install the Toolkit by running:
+
+```bash
+pip install cognite-toolkit
+```
+
+Then run `cdf --help` to get started with the interactive command-line tool.
+
+## For more information
+
+More details about the tool can be found at
+[docs.cognite.com](https://docs.cognite.com/cdf/deploy/cdf_toolkit/).
+
+You can find an overview of the modules and packages in the
+[module and package documentation](https://docs.cognite.com/cdf/deploy/cdf_toolkit/references/resource_library).
+
+See [./CONTRIBUTING.md](./CONTRIBUTING.md) for information about how to contribute to the `cdf-tk` tool or
+templates.

cognite_toolkit-0.7.70/cognite_toolkit/_cdf.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python
+# The Typer parameters get mixed up if we use the __future__ import annotations in the main file.
+# ruff: noqa: E402
+import re
+import sys
+import traceback
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import NoReturn
+
+import typer
+from cognite.client.config import global_config
+from rich.markup import escape
+from rich.panel import Panel
+
+from cognite_toolkit._cdf_tk.hints import Hint
+
+# Do not warn the user about feature previews from the Cognite-SDK we use in Toolkit
+global_config.disable_pypi_version_check = True
+global_config.silence_feature_preview_warnings = True
+
+from rich import print
+
+from cognite_toolkit._cdf_tk.apps import (
+    AuthApp,
+    CoreApp,
+    DataApp,
+    DevApp,
+    DumpApp,
+    ImportApp,
+    LandingApp,
+    MigrateApp,
+    ModulesApp,
+    ProfileApp,
+    RepoApp,
+    RunApp,
+)
+from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
+from cognite_toolkit._cdf_tk.commands import (
+    AboutCommand,
+)
+from cognite_toolkit._cdf_tk.constants import HINT_LEAD_TEXT, URL, USE_SENTRY
+from cognite_toolkit._cdf_tk.exceptions import (
+    ToolkitError,
+)
+from cognite_toolkit._cdf_tk.feature_flags import Flags
+from cognite_toolkit._cdf_tk.plugins import Plugins
+from cognite_toolkit._cdf_tk.tracker import Tracker
+from cognite_toolkit._cdf_tk.utils import (
+    sentry_exception_filter,
+)
+from cognite_toolkit._version import __version__ as current_version
+
+if USE_SENTRY:
+    import sentry_sdk
+
+    sentry_sdk.init(
+        dsn="https://20552f92b525fe551e9adc939024d526@o4508040730968064.ingest.de.sentry.io/4508160801374288",
+        release=current_version,
+        before_send=sentry_exception_filter,
+        # Set traces_sample_rate to 1.0 to capture 100%
+        # of transactions for performance monitoring.
+        traces_sample_rate=1.0,
+    )
+
+
+CDF_TOML = CDFToml.load(Path.cwd())
+
+default_typer_kws = dict(
+    pretty_exceptions_short=False,
+    pretty_exceptions_show_locals=False,
+    pretty_exceptions_enable=False,
+)
+try:
+    typer.Typer(**default_typer_kws)  # type: ignore [arg-type]
+except AttributeError as e:
+    # From Typer version 0.11 -> 0.12, breaks if you have an existing installation.
+    raise ToolkitError(
+        "'cognite-toolkit' uses a dependency named 'typer'. From 'typer' version 0.11 -> 0.12 there was a "
+        "breaking change if you have an existing installation of 'typer'. The workaround is to uninstall "
+        "'typer-slim', and then, reinstall 'typer':\n"
+        "pip uninstall typer-slim\n"
+        "pip install typer\n\n"
+        f"This was triggered by the error: {e!r}"
+    )
+
+_app = CoreApp(**default_typer_kws)
+
+user_app = typer.Typer(**default_typer_kws, hidden=True)  # type: ignore [arg-type]
+landing_app = LandingApp(**default_typer_kws)
+
+_app.add_typer(AuthApp(**default_typer_kws), name="auth")
+_app.add_typer(RepoApp(**default_typer_kws), name="repo")
+
+
+if Plugins.run.value.is_enabled():
+    _app.add_typer(RunApp(**default_typer_kws), name="run")
+
+if Plugins.dump.value.is_enabled():
+    _app.add_typer(DumpApp(**default_typer_kws), name="dump")
+
+
+if Plugins.dev.value.is_enabled():
+    _app.add_typer(DevApp(**default_typer_kws), name="dev")
+
+if Flags.PROFILE.is_enabled():
+    _app.add_typer(ProfileApp(**default_typer_kws), name="profile")
+
+if Flags.MIGRATE.is_enabled():
+    _app.add_typer(MigrateApp(**default_typer_kws), name="migrate")
+
+if Flags.IMPORT_CMD.is_enabled():
+    _app.add_typer(ImportApp(**default_typer_kws), name="import")
+
+if Plugins.data.value.is_enabled():
+    _app.add_typer(DataApp(**default_typer_kws), name="data")
+
+_app.add_typer(ModulesApp(**default_typer_kws), name="modules")
+_app.command("init")(landing_app.main_init)
+
+
+@_app.command("about")
+def about() -> None:
+    """Display information about the Toolkit installation and configuration."""
+    cmd = AboutCommand()
+    cmd.run(lambda: cmd.execute(Path.cwd()))
+
+
+def app() -> NoReturn:
+    # --- Main entry point ---
+    # Users run 'app()' directly, but that doesn't allow us to control excepton handling:
+    try:
+        _app()
+    except ToolkitError as err:
+        if "--verbose" in sys.argv:
+            print(Panel(traceback.format_exc(), title="Traceback", expand=False))
+
+        print(f"  [bold red]ERROR ([/][red]{type(err).__name__}[/][bold red]):[/] {err}")
+        raise SystemExit(1)
+    except SystemExit:
+        if result := re.search(r"click.exceptions.UsageError: No such command '(\w+)'.", traceback.format_exc()):
+            cmd = result.group(1)
+            if cmd in Plugins.list():
+                plugin = r"[plugins]"
+                print(
+                    f"{HINT_LEAD_TEXT} The plugin [bold]{cmd}[/bold] is not enabled."
+                    f"\nEnable it in the [bold]cdf.toml[/bold] file by setting '{cmd} = true' in the "
+                    f"[bold]{escape(plugin)}[/bold] section."
+                    f"\nDocs to learn more: {Hint.link(URL.plugins, URL.plugins)}"
+                )
+        raise
+
+    raise SystemExit(0)
+
+
+@user_app.callback(invoke_without_command=True)
+def user_main(ctx: typer.Context) -> None:
+    """Commands to give information about the toolkit."""
+    if ctx.invoked_subcommand is None:
+        print("Use [bold yellow]cdf user --help[/] to see available commands.")
+    return None
+
+
+@user_app.command("info")
+def user_info() -> None:
+    """Print information about user"""
+    tracker = Tracker()
+    print(f"ID={tracker.get_distinct_id()!r}\nnow={datetime.now(timezone.utc).isoformat(timespec='seconds')!r}")
+
+
+if __name__ == "__main__":
+    app()

cognite_toolkit-0.7.70/cognite_toolkit/_cdf_tk/apps/__init__.py

@@ -0,0 +1,33 @@
+from ._auth_app import AuthApp
+from ._core_app import CoreApp
+from ._data_app import DataApp
+from ._dev_app import DevApp
+from ._download_app import DownloadApp
+from ._dump_app import DumpApp
+from ._import_app import ImportApp
+from ._landing_app import LandingApp
+from ._migrate_app import MigrateApp
+from ._modules_app import ModulesApp
+from ._profile_app import ProfileApp
+from ._purge import PurgeApp
+from ._repo_app import RepoApp
+from ._run import RunApp
+from ._upload_app import UploadApp
+
+__all__ = [
+    "AuthApp",
+    "CoreApp",
+    "DataApp",
+    "DevApp",
+    "DownloadApp",
+    "DumpApp",
+    "ImportApp",
+    "LandingApp",
+    "MigrateApp",
+    "ModulesApp",
+    "ProfileApp",
+    "PurgeApp",
+    "RepoApp",
+    "RunApp",
+    "UploadApp",
+]

cognite_toolkit-0.7.70/cognite_toolkit/_cdf_tk/apps/_auth_app.py

@@ -0,0 +1,104 @@
+from typing import Annotated, Any
+
+import typer
+from rich import print
+
+from cognite_toolkit._cdf_tk.commands import AuthCommand
+from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
+
+
+class AuthApp(typer.Typer):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.callback(invoke_without_command=True)(self.main)
+        self.command()(self.init)
+        self.command()(self.verify)
+
+    def main(self, ctx: typer.Context) -> None:
+        """Commands to auth setup"""
+        if ctx.invoked_subcommand is None:
+            print("Use [bold yellow]cdf auth --help[/] for more information.")
+
+    def init(
+        self,
+        no_verify: Annotated[
+            bool,
+            typer.Option(
+                "--no-verify",
+                "-nv",
+                help="Whether to skip the verification of the capabilities after the initialization.",
+            ),
+        ] = False,
+        dry_run: Annotated[
+            bool,
+            typer.Option(
+                "--dry-run",
+                "-r",
+                help="If you verify, and you pass this flag no changes to CDF will be made.",
+            ),
+        ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+    ) -> None:
+        """Sets the OIDC parameters required to authenticate and authorize the Cognite Toolkit in Cognite Data Fusion.
+
+        This will prompt the user to log in and optionally store the credentials in a .env file.
+
+        Needed capabilities for bootstrapping:
+        "projectsAcl": ["LIST", "READ"],
+        "groupsAcl": ["LIST", "READ", "CREATE", "UPDATE", "DELETE"]
+        """
+        # We do not pass in a client here as this is typically used to create the .env file needed for authentication.
+        cmd = AuthCommand()
+        cmd.run(
+            lambda: cmd.init(
+                no_verify=no_verify,
+                dry_run=dry_run,
+            )
+        )
+
+    def verify(
+        self,
+        ctx: typer.Context,
+        dry_run: Annotated[
+            bool,
+            typer.Option(
+                "--dry-run",
+                "-r",
+                help="Whether to do a dry-run. This means that no changes to CDF will be made.",
+            ),
+        ] = False,
+        no_prompt: Annotated[
+            bool,
+            typer.Option(
+                "--no-prompt",
+                "-np",
+                help="Whether to skip the prompt to continue. This is useful for CI/CD pipelines."
+                "If you include this flag, the execution will stop if the user or service principal does not have the required capabilities.",
+            ),
+        ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+    ) -> None:
+        """Verify that the current user or service principal has the required capabilities to run the CDF Toolkit commands."""
+        client = EnvironmentVariables.create_from_environment().get_client()
+        cmd = AuthCommand(client=client)
+        cmd.run(
+            lambda: cmd.verify(
+                client,
+                dry_run=dry_run,
+                no_prompt=no_prompt,
+            )
+        )