cognite-toolkit 0.6.82__tar.gz → 0.6.83__tar.gz

{cognite_toolkit-0.6.82 → cognite_toolkit-0.6.83}/.gitignore

@@ -308,7 +308,6 @@ needle/
 tmp.py
 tmp*.py
 my_*/
-!cognite_toolkit/_builtin_modules/custom/my_module/
 function_local_venvs*/
 qa_pending_id/
 tests/**/cdf.toml

{cognite_toolkit-0.6.82 → cognite_toolkit-0.6.83}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.82
+Version: 0.6.83
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.6.82 → cognite_toolkit-0.6.83}/cdf.toml

@@ -20,6 +20,7 @@ download = true
 exit-on-warning = true
 upload = true
 function-cognite-file = true
+external-libraries = true
 v07 = true
 
 [plugins]
@@ -30,4 +31,9 @@ dump = true
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.82"
+version = "0.6.83"
+
+
+[library.toolkit-data]
+url = "https://github.com/cognitedata/toolkit-data/releases/download/latest/packages.zip" 
+checksum = "sha256:b70c207a78d6c94e0977a733571420b250ca641ac8802d8310f1ffe05bf7b2eb"

{cognite_toolkit-0.6.82 → cognite_toolkit-0.6.83}/cognite_toolkit/_cdf_tk/commands/modules.py

@@ -321,16 +321,22 @@ default_organization_dir = "{organization_dir.name}"''',
 
         modules_root_dir = organization_dir / MODULES
 
+        # Determine which library to use (if any)
+        library: Library | None = None
         if library_url:
             if not library_checksum:
                 raise ToolkitRequiredValueError(
                     "The '--library-checksum' is required when '--library-url' is provided."
                 )
+            library = Library(url=library_url, checksum=library_checksum)
+        elif not (organization_dir / CDFToml.file_name).exists():
+            # Load default library from resources when cdf.toml doesn't exist
+            default_cdf_toml = CDFToml.load(cwd=RESOURCES_PATH, use_singleton=False)
+            library = default_cdf_toml.libraries.get("toolkit-data")
+            if library is None:
+                raise ToolkitError("Default cdf.toml in resources is missing 'toolkit-data' library configuration.")
 
-            user_library = Library(url=library_url, checksum=library_checksum)
-            packages, modules_source_path = self._get_available_packages(user_library)
-        else:
-            packages, modules_source_path = self._get_available_packages()
+        packages, modules_source_path = self._get_available_packages(library)
 
         if select_all:
             print(Panel("Instantiating all available modules"))
@@ -767,7 +773,7 @@ default_organization_dir = "{organization_dir.name}"''',
         """
 
         cdf_toml = CDFToml.load()
-        if Flags.EXTERNAL_LIBRARIES.is_enabled() or user_library:
+        if (Flags.EXTERNAL_LIBRARIES.is_enabled() or user_library) and self._module_source_dir is None:
             libraries = {"userdefined": user_library} if user_library else cdf_toml.libraries
 
             for library_name, library in libraries.items():

{cognite_toolkit-0.6.82 → cognite_toolkit-0.6.83}/cognite_toolkit/_cdf_tk/constants.py

@@ -30,7 +30,6 @@ CONFIG_FILE_SUFFIX = "config.yaml"
 # The global config file
 GLOBAL_CONFIG_FILE = "global.yaml"
 
-BUILTIN_MODULES = "_builtin_modules"
 COGNITE_MODULES = "cognite_modules"
 CUSTOM_MODULES = "custom_modules"
 EXTERNAL_PACKAGE = "external_package"
@@ -61,7 +60,6 @@ TABLE_FORMATS = frozenset([".csv", ".parquet"])
 ROOT_PATH = Path(__file__).parent.parent
 COGNITE_MODULES_PATH = ROOT_PATH / COGNITE_MODULES
 MODULES_PATH = ROOT_PATH / MODULES
-BUILTIN_MODULES_PATH = ROOT_PATH / BUILTIN_MODULES
 RESOURCES_PATH = ROOT_PATH / RESOURCES
 SUPPORT_MODULE_UPGRADE_FROM_VERSION = "0.2.0"
 # This is used in the build directory to keep track of order and flatten the

{cognite_toolkit-0.6.82 → cognite_toolkit-0.6.83}/cognite_toolkit/_cdf_tk/data_classes/_config_yaml.py

@@ -15,7 +15,6 @@ from rich import print
 from cognite_toolkit._cdf_tk.constants import (
     _RUNNING_IN_BROWSER,
     BUILD_ENVIRONMENT_FILE,
-    BUILTIN_MODULES,
     DEFAULT_CONFIG_FILE,
     DEFAULT_ENV,
     MODULES,
@@ -468,7 +467,7 @@ class InitConfigYAML(YAMLWithComments[tuple[str, ...], ConfigEntry], ConfigYAMLC
         """Loads all default.config.yaml files in the cognite root module."""
 
         default_files_iterable: Iterable[Path]
-        if cognite_root_module.name in ROOT_MODULES or cognite_root_module.name == BUILTIN_MODULES:
+        if cognite_root_module.name in ROOT_MODULES:
             default_files_iterable = cognite_root_module.glob(f"**/{DEFAULT_CONFIG_FILE}")
         else:
             default_files_iterable = itertools.chain(

{cognite_toolkit-0.6.82 → cognite_toolkit-0.6.83}/cognite_toolkit/_cdf_tk/utils/modules.py

@@ -4,7 +4,6 @@ from pathlib import Path
 from typing import Literal, overload
 
 from cognite_toolkit._cdf_tk.constants import (
-    BUILTIN_MODULES,
     MODULE_PATH_SEP,
     ROOT_MODULES,
 )
@@ -23,9 +22,6 @@ def iterate_modules(root_dir: Path) -> Iterator[tuple[Path, list[Path]]]:
     if root_dir.name in ROOT_MODULES:
         yield from _iterate_modules(root_dir)
         return
-    elif root_dir.name == BUILTIN_MODULES:
-        yield from _iterate_modules(root_dir)
-        return
     for root_module in ROOT_MODULES:
         module_dir = root_dir / root_module
         if module_dir.exists():

{cognite_toolkit-0.6.82 → cognite_toolkit-0.6.83}/cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.6.82
+      image: cognite/toolkit:0.6.83
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.6.82 → cognite_toolkit-0.6.83}/cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.6.82
+      image: cognite/toolkit:0.6.83
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit-0.6.83/cognite_toolkit/_resources/cdf.toml

@@ -0,0 +1,17 @@
+[cdf]#<PLACEHOLDER>
+default_env = "<DEFAULT_ENV_PLACEHOLDER>"
+
+[modules]
+# This is the version of the modules. It should not be changed manually.
+# It will be updated by the 'cdf modules upgrade' command.
+version = "0.6.83"
+
+[alpha_flags]
+external-libraries = true
+
+[plugins]
+run = true
+
+[library.toolkit-data]
+url = "https://github.com/cognitedata/toolkit-data/releases/download/latest/packages.zip" 
+checksum = "sha256:b70c207a78d6c94e0977a733571420b250ca641ac8802d8310f1ffe05bf7b2eb"

cognite_toolkit-0.6.83/cognite_toolkit/_version.py

@@ -0,0 +1 @@
+__version__ = "0.6.83"

cognite_toolkit-0.6.83/last_changelog_entry.md

@@ -0,0 +1,9 @@
+## cdf 
+
+### Removed
+
+- Built-in modules
+
+## templates
+
+No changes.

cognite_toolkit-0.6.83/last_git_message.txt

@@ -0,0 +1,22 @@
+[CDF-26146] Delete _builtin (#2151)
+
+# Description
+
+Builtin-modules have been moved to toolkit-data and/or tests/data, so
+now we can safely delete all those files.
+
+
+## Changelog
+
+- [x] Patch
+- [ ] Skip
+
+## cdf
+
+### Removed
+
+- Built-in modules
+
+## templates
+
+No changes.

cognite_toolkit-0.6.83/last_version.txt

@@ -0,0 +1 @@
+0.6.82

{cognite_toolkit-0.6.82 → cognite_toolkit-0.6.83}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognite_toolkit"
-version = "0.6.82"
+version = "0.6.83"
 description = "Official Cognite Data Fusion tool for project templates and configuration deployment"
 authors = [
     {name ="Cognite AS", email="support@cognite.com"}
@@ -92,7 +92,6 @@ testpaths = [
     # In the case of doctests.
     "cognite_toolkit",
 ]
-norecursedirs = "cognite_toolkit/_builtin_modules"
 
 markers = [
     "toolkit: big smoke tests",
@@ -106,11 +105,7 @@ follow_imports = "normal"
 ignore_missing_imports = true
 warn_redundant_casts = true
 warn_unused_ignores = true
-exclude = [
-    "cognite_toolkit/_builtin_modules/examples/*",
-    "cognite_toolkit/_builtin_modules/inrobot/*",
-    "cognite_toolkit/_builtin_modules/bootcamp/*",
-]
+exclude = []
 
 [tool.ruff]
 line-length = 120
@@ -119,17 +114,10 @@ line-length = 120
 exclude = [
     # Ruff messes up the docstrings which are expected to be in Markdown format
     "cognite_toolkit/_cdf_tk/commands/_changes.py",
-    "cognite_toolkit/_builtin_modules/inrobot/*",
-    "cognite_toolkit/_builtin_modules/examples/*",
-    "cognite_toolkit/_builtin_modules/bootcamp/*",
 ]
 
 [tool.ruff.lint]
-exclude = [
-    "cognite_toolkit/_builtin_modules/inrobot/*",
-    "cognite_toolkit/_builtin_modules/examples/*",
-    "cognite_toolkit/_builtin_modules/bootcamp/*",
-]
+exclude = []
 
 [build-system]
 requires = ["hatchling"]

cognite_toolkit-0.6.83/tests/conftest.py

@@ -0,0 +1,179 @@
+"""
+Root conftest.py for shared test fixtures.
+
+Provides worker-scoped fixtures for external library caching to avoid race conditions
+and expensive repeated downloads when running tests with pytest-xdist.
+"""
+
+import atexit
+import hashlib
+import os
+import shutil
+import tempfile
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+
+import pytest
+from filelock import FileLock
+
+from cognite_toolkit._cdf_tk.commands.modules import ModulesCommand
+from cognite_toolkit._cdf_tk.utils import sanitize_filename
+
+# Global registry to track temp directories for cleanup
+_TEMP_DIRS_TO_CLEANUP: set[Path] = set()
+
+
+def _cleanup_temp_dirs() -> None:
+    """Clean up all registered temp directories at process exit."""
+    for temp_dir in _TEMP_DIRS_TO_CLEANUP:
+        if temp_dir.exists():
+            try:
+                shutil.rmtree(temp_dir, ignore_errors=True)
+            except Exception as e:
+                print(f"Warning: Failed to clean up temp directory {temp_dir}: {e}")
+
+
+atexit.register(_cleanup_temp_dirs)
+
+
+def get_worker_id() -> str:
+    """Get the current pytest-xdist worker ID, or 'master' if not using xdist."""
+    worker_id = os.environ.get("PYTEST_XDIST_WORKER", "master")
+    return sanitize_filename(worker_id)
+
+
+@contextmanager
+def external_library_cache_context(cache_root: Path) -> Iterator[Path]:
+    """Context manager for accessing the external library cache with file locking."""
+    worker_id = get_worker_id()
+    cache_dir = cache_root / f"modules-{worker_id}"
+    lock_file = cache_root / f"modules-{worker_id}.lock"
+
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    with FileLock(lock_file, timeout=300):
+        yield cache_dir
+
+
+def get_unique_run_id() -> str:
+    """Get a unique identifier for this GitHub Actions job run."""
+    run_id = os.getenv("GITHUB_RUN_ID", "")
+    run_attempt = os.getenv("GITHUB_RUN_ATTEMPT", "")
+    job_id = os.getenv("GITHUB_JOB", "")
+
+    if run_id:
+        parts = [run_id]
+        if run_attempt:
+            parts.append(run_attempt)
+        if job_id:
+            job_id_safe = sanitize_filename(job_id)
+            parts.append(job_id_safe)
+        return "-".join(parts)
+
+    import time
+
+    return f"local-{os.getpid()}-{int(time.time() * 1000)}"
+
+
+@pytest.fixture(scope="session")
+def external_library_cache_root() -> Iterator[Path]:
+    """Session-scoped fixture providing a root directory for external library caching."""
+    run_id = get_unique_run_id()
+    session_id = os.getpid()
+    cache_root = Path(tempfile.gettempdir()) / f"toolkit-test-cache-{run_id}-{session_id}"
+    cache_root.mkdir(parents=True, exist_ok=True)
+
+    _TEMP_DIRS_TO_CLEANUP.add(cache_root)
+
+    yield cache_root
+
+    if cache_root.exists():
+        shutil.rmtree(cache_root, ignore_errors=True)
+
+
+@pytest.fixture(scope="session")
+def external_library_modules_dir(external_library_cache_root: Path) -> Iterator[Path]:
+    """Session-scoped fixture providing a worker-specific directory for external library downloads."""
+    worker_id = get_worker_id()
+    modules_dir = external_library_cache_root / f"modules-{worker_id}"
+    modules_dir.mkdir(parents=True, exist_ok=True)
+
+    yield modules_dir
+
+
+@pytest.fixture(scope="function")
+def isolated_external_library_dir(external_library_cache_root: Path) -> Iterator[Path]:
+    """Function-scoped fixture for tests that need a completely isolated library directory."""
+    test_id = os.environ.get("PYTEST_CURRENT_TEST", "unknown")
+    test_hash = hashlib.sha256(test_id.encode()).hexdigest()[:8]
+    worker_id = get_worker_id()
+
+    isolated_dir = external_library_cache_root / f"isolated-{worker_id}-{test_hash}"
+    isolated_dir.mkdir(parents=True, exist_ok=True)
+
+    yield isolated_dir
+
+    if isolated_dir.exists():
+        shutil.rmtree(isolated_dir, ignore_errors=True)
+
+
+@pytest.fixture(scope="function")
+def modules_command_with_cached_download(
+    external_library_cache_root: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> type:
+    """Function-scoped fixture providing a pre-configured ModulesCommand class with caching."""
+    worker_id = get_worker_id()
+    cache_dir = external_library_cache_root / worker_id / "modules"
+    cache_dir.mkdir(parents=True, exist_ok=True)
+
+    lock_file = external_library_cache_root / worker_id / "modules.lock"
+
+    original_init = ModulesCommand.__init__
+    original_download = ModulesCommand._download
+    original_unpack = ModulesCommand._unpack
+
+    def patched_init(self, *args, **kwargs):
+        original_init(self, *args, **kwargs)
+        self._temp_download_dir = cache_dir
+
+    def patched_download(self, url: str, file_path: Path) -> None:
+        """Download with caching and locking - skip if file already exists."""
+        with FileLock(lock_file, timeout=300):
+            if file_path.exists():
+                if file_path.stat().st_size > 0:
+                    return
+
+            unpacked_dir = file_path.parent / file_path.stem
+            if unpacked_dir.exists() and unpacked_dir.is_dir():
+                try:
+                    if any(unpacked_dir.iterdir()):
+                        return
+                except OSError as e:
+                    print(f"Warning: Could not check unpacked directory {unpacked_dir}: {e}")
+
+            original_download(self, url, file_path)
+
+    def patched_unpack(self, file_path: Path) -> None:
+        """Unpack with locking to prevent race conditions."""
+        with FileLock(lock_file, timeout=300):
+            unpacked_dir = file_path.parent / file_path.stem
+            if unpacked_dir.exists() and unpacked_dir.is_dir():
+                try:
+                    if any(unpacked_dir.iterdir()):
+                        return
+                except OSError as e:
+                    print(f"Warning: Could not check unpacked directory {unpacked_dir}: {e}")
+
+            original_unpack(self, file_path)
+
+    def patched_exit(self, *args, **kwargs):
+        """Prevent cleanup - cache persists until session end."""
+        pass
+
+    monkeypatch.setattr(ModulesCommand, "__init__", patched_init)
+    monkeypatch.setattr(ModulesCommand, "_download", patched_download)
+    monkeypatch.setattr(ModulesCommand, "_unpack", patched_unpack)
+    monkeypatch.setattr(ModulesCommand, "__exit__", patched_exit)
+
+    return ModulesCommand

{cognite_toolkit-0.6.82 → cognite_toolkit-0.6.83}/tests/test_unit/test_cdf_tk/test_commands/test_modules.py

@@ -163,8 +163,12 @@ class TestModulesCommand:
         assert config["variables"]["modules"]["my_file_expand_module"]["var"] is not None
         assert (target_path / "modules" / "my_file_expand_module").is_dir()
 
-    def test_add_without_config_yaml(self, tmp_path: Path, monkeypatch: MonkeyPatch) -> None:
-        cmd = ModulesCommand(print_warning=True, skip_tracking=True, module_source_dir=COMPLETE_ORG_MODULES)
+    def test_add_without_config_yaml(
+        self, tmp_path: Path, monkeypatch: MonkeyPatch, modules_command_with_cached_download
+    ) -> None:
+        cmd = modules_command_with_cached_download(
+            print_warning=True, skip_tracking=True, module_source_dir=COMPLETE_ORG_MODULES
+        )
         dummy_resource = "space: my_space"
         my_org = tmp_path / "my_org"
         modules = my_org / "modules"

{cognite_toolkit-0.6.82 → cognite_toolkit-0.6.83}/tests/test_unit/test_cdf_tk/test_data_classes/test_cdf_toml.py

@@ -5,6 +5,7 @@ import pytest
 
 from cognite_toolkit import _version
 from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
+from cognite_toolkit._cdf_tk.constants import RESOURCES_PATH
 from cognite_toolkit._cdf_tk.exceptions import ToolkitTOMLFormatError
 from tests.constants import REPO_ROOT
 
@@ -115,3 +116,18 @@ class TestCDFToml:
 
         config = CDFToml.load(cwd=tmp_path, use_singleton=False)
         assert config.libraries["valid_url"].url == "https://github.com/cognitedata/package/archive/refs/tags/0.0.1.zip"
+
+    def test_default_resources_cdf_toml_has_valid_library_config(self) -> None:
+        """Test that the default cdf.toml in resources has valid library configuration."""
+        default_cdf_toml = CDFToml.load(cwd=RESOURCES_PATH, use_singleton=False)
+
+        # Verify the toolkit-data library exists
+        assert "toolkit-data" in default_cdf_toml.libraries
+        library = default_cdf_toml.libraries["toolkit-data"]
+
+        # Verify the URL is a valid HTTPS URL pointing to toolkit-data
+        assert library.url.startswith("https://github.com/cognitedata/toolkit-data")
+        assert library.url.endswith(".zip")
+
+        # Verify the checksum has the correct format
+        assert library.checksum.startswith("sha256:")

{cognite_toolkit-0.6.82 → cognite_toolkit-0.6.83}/uv.lock

@@ -260,7 +260,7 @@ wheels = [
 
 [[package]]
 name = "cognite-toolkit"
-version = "0.6.82"
+version = "0.6.83"
 source = { editable = "." }
 dependencies = [
     { name = "cognite-sdk" },

cognite_toolkit-0.6.82/cognite_toolkit/_builtin_modules/README.md

@@ -1,7 +0,0 @@
-# Main Cognite Modules modules folder
-
-Modules in this folder come bundled with the `cdf-tk` tool. They are managed
-from a [public repository](https://github.com/cognitedata/toolkit).
-
-See the [documentation](https://docs.cognite.com/cdf/deploy/cdf_toolkit/references/module_reference) for
-an introduction.

cognite_toolkit-0.6.82/cognite_toolkit/_builtin_modules/bootcamp/README.md

@@ -1,5 +0,0 @@
-# Bootcamp module
-
-These are the initial configs and python functions used for the bootcamp.
-[Docs](https://docs.cdf-bootcamp.cogniteapp.com/)
-[Repo](https://github.com/cognitedata/cognite-data-fusion-bootcamp)

cognite_toolkit-0.6.82/cognite_toolkit/_builtin_modules/bootcamp/default.config.yaml

@@ -1,3 +0,0 @@
-# Variables shared by modules
-icapi_ds_external_id: ds_icapi
-icapi_space_name: icapi_dm_space

cognite_toolkit-0.6.82/cognite_toolkit/_builtin_modules/bootcamp/ice_cream_api/default.config.yaml

@@ -1,9 +0,0 @@
-icapi_extractors_source_id: <OBJECT ID FOR ICAPI EXTRACTORS ENTRA GROUP>
-
-tokenUri: ${IDP_TOKEN_URL}
-cdfProjectName: ${CDF_PROJECT}
-scopes: ${IDP_SCOPES}
-icapi_trigger_client_id: ${ICAPI_EXTRACTORS_CLIENT_ID}
-icapi_trigger_client_secret: ${ICAPI_EXTRACTORS_CLIENT_SECRET}
-icapi_extractors_client_id: ${ICAPI_EXTRACTORS_CLIENT_ID}
-icapi_extractors_client_secret: ${ICAPI_EXTRACTORS_CLIENT_SECRET}