cognite-toolkit 0.7.0__tar.gz → 0.7.2__tar.gz

cognite_toolkit-0.7.2/.gitignore

@@ -0,0 +1,322 @@
+# File created using '.gitignore Generator' for Visual Studio Code: https://bit.ly/vscode-gig
+# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode,python,macos,git,windows
+# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode,python,macos,git,windows
+
+### Git ###
+# Created by git for backups. To disable backups in Git:
+# $ git config --global mergetool.keepBackup false
+*.orig
+
+# Created by git when using merge tools for conflicts
+*.BACKUP.*
+*.BASE.*
+*.LOCAL.*
+*.REMOTE.*
+*_BACKUP_*.txt
+*_BASE_*.txt
+*_LOCAL_*.txt
+*_REMOTE_*.txt
+
+### macOS ###
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### macOS Patch ###
+# iCloud generated files
+*.icloud
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+*.env
+.venv
+.local/
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+
+### Python Patch ###
+# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
+poetry.toml
+
+# ruff
+.ruff_cache/
+
+# LSP config files
+pyrightconfig.json
+
+### VisualStudioCode ###
+.vscode/*
+.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/*.code-snippets
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
+
+### Windows ###
+# Windows thumbnail cache files
+Thumbs.db
+Thumbs.db:encryptable
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,python,macos,git,windows
+
+# Custom rules (everything added below won't be overriden by 'Generate .gitignore File' if you use 'Update' option)
+
+tmp/
+playground/
+new_project/
+# If you need to update the cognite_toolkit template files for local.yaml and config.yaml, comment below
+local.yaml
+config.yaml
+!demo/config.yaml
+demo_project/
+tests/tests_unit/pytest-project/
+my_project/
+my_organization/
+*.bck
+
+# Hide environments used for migration testing
+**/.venv*
+module_upgrade/tmp*
+
+cognite_toolkit/config.local.yaml
+**/.venv*
+.venv.*
+build.*
+cognite_toolkit/.env.*
+cognite_toolkit/modules/*
+module_upgrade/project_inits
+function_local_venvs/
+tests/data/run_data/build_info.dev.yaml
+orgdir*
+module_upgrade/cdf.toml
+# Org used by bootcamp
+ice-cream-dataops
+module_upgrade/*.log
+needle/
+
+tmp.py
+tmp*.py
+my_*/
+function_local_venvs*/
+qa_pending_id/
+tests/**/cdf.toml
+
+*.xlsx
+data/
+migration_logs*/
+!tests/data/
+!tests/data/**
+tests/data/**/__pycache__/
+build_info.dev.yaml
+tests/data/run_data/function_local_venvs*/**

cognite_toolkit-0.7.2/PKG-INFO

@@ -0,0 +1,84 @@
+Metadata-Version: 2.4
+Name: cognite_toolkit
+Version: 0.7.2
+Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
+Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
+Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases
+Project-URL: GitHub, https://github.com/cognitedata/toolkit
+Project-URL: Documentation, https://docs.cognite.com/cdf/deploy/cdf_toolkit/references/resource_library
+Author-email: Cognite AS <support@cognite.com>
+License-Expression: Apache-2.0
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: cognite-sdk<8.0.0,>=7.87.0
+Requires-Dist: filelock>=3.18.0
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: mixpanel>=4.10.1
+Requires-Dist: packaging>=25
+Requires-Dist: pandas<3.0.0,>=1.5.3
+Requires-Dist: pip>=25.0.1
+Requires-Dist: pydantic>=2.11.0
+Requires-Dist: python-dateutil>=2.9.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: pyyaml>=6.0.1
+Requires-Dist: questionary>=2.0.1
+Requires-Dist: rich>=13.9.4
+Requires-Dist: sentry-sdk>=2.1.0
+Requires-Dist: toml>=0.10.2
+Requires-Dist: tomli<3.0.0,>=2.0.1; python_version < '3.11'
+Requires-Dist: typer<1.0.0,>=0.12.0
+Requires-Dist: typing-extensions>=4.0.0
+Provides-Extra: sql
+Requires-Dist: sqlparse>=0.5.3; extra == 'sql'
+Provides-Extra: table
+Requires-Dist: openpyxl>=3.1.5; extra == 'table'
+Requires-Dist: pyarrow>=20.0.0; extra == 'table'
+Description-Content-Type: text/markdown
+
+# Cognite Data Fusion Toolkit
+
+[![release](https://img.shields.io/github/actions/workflow/status/cognitedata/toolkit/release.yaml?style=for-the-badge)](https://github.com/cognitedata/toolkit/actions/workflows/release.yaml)
+[![Github](https://shields.io/badge/github-cognite/toolkit-green?logo=github&style=for-the-badge)](https://github.com/cognitedata/toolkit)
+[![PyPI](https://img.shields.io/pypi/v/cognite-toolkit?style=for-the-badge)](https://pypi.org/project/cognite-toolkit/)
+[![Downloads](https://img.shields.io/pypi/dm/cognite-toolkit?style=for-the-badge)](https://pypistats.org/packages/cognite-toolkit)
+[![GitHub](https://img.shields.io/github/license/cognitedata/toolkit?style=for-the-badge)](https://github.com/cognitedata/toolkit/blob/master/LICENSE)
+[![Docker Pulls](https://img.shields.io/docker/pulls/cognite/toolkit?style=for-the-badge)](https://hub.docker.com/r/cognite/toolkit)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json&style=for-the-badge)](https://github.com/astral-sh/ruff)
+[![mypy](https://img.shields.io/badge/mypy-checked-000000.svg?style=for-the-badge&color=blue)](http://mypy-lang.org)
+
+The CDF Toolkit is a command-line interface (`cdf`) used for configuring and administrating
+Cognite Data Fusion (CDF) projects. It ships with modularised `templates` that helps you
+configure Cognite Data Fusion according to best practices.
+
+It supports three different modes of operation:
+
+1. As an **interactive command-line tool** used alongside the Cognite Data Fusion web application to retrieve and
+   push configuration of the different Cognite Data Fusion services like data sets, data models, transformations,
+   and more. This mode also supports configuration of new Cognite Data Fusion projects to quickly get started.
+2. As tool to support the **project life-cycle by scripting and automating** configuration and management of Cognite Data
+   Fusion projects where CDF configurations are kept as yaml-files that can be checked into version
+   control. This mode also supports DevOps workflows with development, staging, and production projects.
+3. As a **tool to deploy official Cognite project templates** to your Cognite Data Fusion project. The tool comes
+   bundled with templates useful for getting started with Cognite Data Fusion, as well as for specific use cases
+   delivered by Cognite or its partners. You can also create your own templates and share them.
+
+## Usage
+
+Install the Toolkit by running:
+
+```bash
+pip install cognite-toolkit
+```
+
+Then run `cdf --help` to get started with the interactive command-line tool.
+
+## For more information
+
+More details about the tool can be found at
+[docs.cognite.com](https://docs.cognite.com/cdf/deploy/cdf_toolkit/).
+
+You can find an overview of the modules and packages in the
+[module and package documentation](https://docs.cognite.com/cdf/deploy/cdf_toolkit/references/resource_library).
+
+See [./CONTRIBUTING.md](./CONTRIBUTING.md) for information about how to contribute to the `cdf-tk` tool or
+templates.

cognite_toolkit-0.7.2/cdf.toml

@@ -0,0 +1,29 @@
+[cdf]
+default_organization_dir = "cognite_toolkit"
+#default_env = "dev"
+file_encoding = "utf-8"
+
+[alpha_flags]
+graphql = true
+import-cmd = true
+module-repeat = true
+profile = true
+infield = true
+migrate = true
+streams = true
+
+[plugins]
+run = true
+dump = true
+dev = true
+data = true
+
+[modules]
+# This is the version of the modules. It should not be changed manually.
+# It will be updated by the 'cdf modules upgrade' command.
+version = "0.7.2"
+
+
+[library.toolkit-data]
+url = "https://github.com/cognitedata/toolkit-data/releases/download/latest/packages.zip" 
+checksum = "sha256:b70c207a78d6c94e0977a733571420b250ca641ac8802d8310f1ffe05bf7b2eb"

cognite_toolkit-0.7.2/cognite_toolkit/_cdf_tk/apps/_upload_app.py

@@ -0,0 +1,107 @@
+from pathlib import Path
+from typing import Annotated, Any
+
+import questionary
+import typer
+from questionary import Choice
+
+from cognite_toolkit._cdf_tk.commands import UploadCommand
+from cognite_toolkit._cdf_tk.constants import DATA_DEFAULT_DIR, DATA_MANIFEST_SUFFIX, DATA_RESOURCE_DIR
+from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
+
+DEFAULT_INPUT_DIR = Path.cwd() / DATA_DEFAULT_DIR
+
+
+class UploadApp(typer.Typer):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.callback(invoke_without_command=True)(self.upload_main)
+        self.command("dir")(self.upload_dir)
+
+    @staticmethod
+    def upload_main(ctx: typer.Context) -> None:
+        """Commands to upload data to CDF."""
+        if ctx.invoked_subcommand is None:
+            print("Use [bold yellow]cdf upload --help[/] for more information.")
+        return None
+
+    @staticmethod
+    def upload_dir(
+        ctx: typer.Context,
+        input_dir: Annotated[
+            Path | None,
+            typer.Argument(
+                help="The directory containing the data to upload. If not specified, an interactive prompt will ask for the directory.",
+                exists=True,
+                file_okay=False,
+                dir_okay=True,
+                resolve_path=True,
+            ),
+        ] = None,
+        dry_run: Annotated[
+            bool,
+            typer.Option(
+                "--dry-run",
+                "-d",
+                help="If set, the command will not actually upload the data, but will print what would be uploaded.",
+            ),
+        ] = False,
+        deploy_resources: Annotated[
+            bool,
+            typer.Option(
+                "--deploy-resources",
+                "-r",
+                help="If set, the command will look for resource configuration files in adjacent folders and create them if they do not exist.",
+            ),
+        ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+    ) -> None:
+        """Commands to upload data to CDF."""
+        cmd = UploadCommand()
+        if input_dir is None:
+            input_candidate = sorted({p.parent for p in DEFAULT_INPUT_DIR.rglob(f"*/**{DATA_MANIFEST_SUFFIX}")})
+            if not input_candidate:
+                typer.echo(f"No data manifests found in default directory: {DEFAULT_INPUT_DIR}")
+                raise typer.Exit(code=1)
+            input_dir = questionary.select(
+                "Select the input directory containing the data to upload:",
+                choices=[Choice(str(option.name), value=option) for option in input_candidate],
+            ).ask()
+            if input_dir is None:
+                typer.echo("No input directory selected. Exiting.")
+                raise typer.Exit(code=1)
+            dry_run = questionary.confirm("Proceed with dry run?", default=dry_run).ask()
+            if dry_run is None:
+                typer.echo("No selection made for dry run. Exiting.")
+                raise typer.Exit(code=1)
+            resource_dir = Path(input_dir) / DATA_RESOURCE_DIR
+            if resource_dir.exists():
+                if resource_dir.is_relative_to(Path.cwd()):
+                    display_name = resource_dir.relative_to(Path.cwd()).as_posix()
+                else:
+                    display_name = resource_dir.as_posix()
+
+                deploy_resources = questionary.confirm(
+                    f"Deploy resources found in {display_name!r}?", default=deploy_resources
+                ).ask()
+                if deploy_resources is None:
+                    typer.echo("No selection made for deploying resources. Exiting.")
+                    raise typer.Exit(code=1)
+
+        client = EnvironmentVariables.create_from_environment().get_client()
+        cmd.run(
+            lambda: cmd.upload(
+                input_dir=input_dir,
+                dry_run=dry_run,
+                verbose=verbose,
+                deploy_resources=deploy_resources,
+                client=client,
+            )
+        )

cognite_toolkit-0.7.2/cognite_toolkit/_cdf_tk/builders/__init__.py

@@ -0,0 +1,40 @@
+from collections.abc import Callable
+from pathlib import Path
+
+from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning
+
+from ._base import Builder, DefaultBuilder, get_resource_crud
+from ._datamodels import DataModelBuilder
+from ._file import FileBuilder
+from ._function import FunctionBuilder
+from ._location import LocationBuilder
+from ._raw import RawBuilder
+from ._streamlit import StreamlitBuilder
+from ._transformation import TransformationBuilder
+
+
+def create_builder(
+    resource_folder: str,
+    build_dir: Path | None,
+    warn: Callable[[ToolkitWarning], None] | None = None,
+) -> Builder:
+    if builder_cls := _BUILDER_BY_RESOURCE_FOLDER.get(resource_folder):
+        return builder_cls(build_dir, warn=warn)  # type: ignore[abstract]
+
+    return DefaultBuilder(build_dir, resource_folder, warn)
+
+
+_BUILDER_BY_RESOURCE_FOLDER = {_builder._resource_folder: _builder for _builder in Builder.__subclasses__()}
+__all__ = [
+    "Builder",
+    "DataModelBuilder",
+    "DefaultBuilder",
+    "FileBuilder",
+    "FunctionBuilder",
+    "LocationBuilder",
+    "RawBuilder",
+    "StreamlitBuilder",
+    "TransformationBuilder",
+    "create_builder",
+    "get_resource_crud",
+]