cognite-toolkit 0.5.97__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/init.py

@@ -1,30 +1,231 @@
+import shutil
+import tempfile
+import traceback
+from collections.abc import Callable
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+
+import questionary
+import typer
+from questionary import Choice
 from rich import print
 from rich.panel import Panel
 
-from ._base import ToolkitCommand
+from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
+from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
+from cognite_toolkit._cdf_tk.commands.auth import AuthCommand
+from cognite_toolkit._cdf_tk.commands.collect import CollectCommand
+from cognite_toolkit._cdf_tk.commands.modules import ModulesCommand
+from cognite_toolkit._cdf_tk.commands.repo import RepoCommand
+from cognite_toolkit._cdf_tk.exceptions import ToolkitError
+from cognite_toolkit._cdf_tk.feature_flags import FeatureFlag
+
+
+class InitItemStatus(Enum):
+    """Status of an init checklist item"""
+
+    NONE = "none"
+    SUCCESSFUL = "successful"
+    FAILED = "failed"
+
+
+@dataclass
+class InitChecklistItem:
+    name: str
+    description: str
+    function: Callable[[], None]
+    status: InitItemStatus | None = None
+    mandatory: bool = False
+
+    def get_status_display(self) -> str:
+        if self.status == InitItemStatus.SUCCESSFUL:
+            return "✓"
+        elif self.status == InitItemStatus.FAILED:
+            return "✗"
+        else:
+            return "○"
+
+    def get_choice_title(self) -> str:
+        status_icon = self.get_status_display()
+        return f"{status_icon} {self.description} (required)" if self.mandatory else f"{status_icon} {self.description}"
 
 
 class InitCommand(ToolkitCommand):
-    def execute(self) -> None:
+    organization_dir: Path | None
+
+    def __init__(self, print_warning: bool = True, skip_tracking: bool = False, silent: bool = False) -> None:
+        super().__init__(print_warning, skip_tracking, silent)
+        self.organization_dir = None
+
+    def execute(self, dry_run: bool = False) -> None:
+        print("\n")
         print(
             Panel(
-                "\n".join(
-                    [
-                        "[bold]Follow these steps to get started with the Cognite Data Fusion Toolkit:[/b]",
-                        "",
-                        "1. Run [bold]cdf-tk repo init (folder_name)[/] to prepare the working folder.",  # todo: connect the folder name with the modules command
-                        "2. Run [bold]cd (folder_name)[/] to enter the newly created folder.",
-                        "3. Run [bold]cdf-tk modules init[/] to select the configuration modules you would like to start with.",
-                        "4. Run [bold]cdf-tk auth verify --interactive[/] to check that you have access to the relevant CDF project. ",
-                        "   [italic](if you already have a .env file, you can copy it into the folder and run [bold]cdf-tk auth verify[/] directly)[/]",
-                        "5. Run [bold]cdf-tk build modules[/] to build the configuration and look for variables that need your attention. Repeat for as many times as needed.",
-                        "6. Run [bold]cdf-tk deploy --dry-run[/] to simulate the deployment of the configuration to the CDF project. Review the report provided",
-                        "7. Run [bold]cdf-tk deploy[/] to deploy the configuration to the CDF project.",
-                        "8. Commit the changes to your version control system.",
-                    ]
-                ),
-                title="Getting started",
+                "Go through the following steps to set up the Cognite Toolkit. You can re-run steps if you need to change something.",
+                title="Cognite Toolkit Setup",
                 style="green",
                 padding=(1, 2),
             )
         )
+
+        # Initialize checklist items
+        checklist_items = [
+            InitChecklistItem(
+                name="initToml",
+                description="Create toml file",
+                function=lambda: self._init_toml(dry_run=dry_run),
+                mandatory=True,
+            ),
+            InitChecklistItem(
+                name="initAuth",
+                description="Authentication",
+                function=lambda: self._init_auth(dry_run=dry_run),
+            ),
+            InitChecklistItem(
+                name="initModules",
+                description="Modules",
+                function=lambda: self._init_modules(dry_run=dry_run),
+            ),
+            InitChecklistItem(
+                name="initRepo",
+                description="Git repository",
+                function=lambda: self._init_repo(dry_run=dry_run),
+            ),
+            InitChecklistItem(
+                name="initDataCollection",
+                description="Usage statistics",
+                function=lambda: self._init_data_collection(dry_run=dry_run),
+            ),
+        ]
+
+        if CDFToml.load().is_loaded_from_file:
+            checklist_items[0].status = InitItemStatus.SUCCESSFUL
+            print("cdf.toml configuration file already exists. Skipping creation.")
+
+        # Main loop: keep showing checklist until user is done
+        while True:
+            # Build choices for questionary
+            choices = []
+            for item in checklist_items:
+                choices.append(
+                    Choice(
+                        title=item.get_choice_title(),
+                        value=item.name,
+                    )
+                )
+            # Check if all mandatory items are completed
+            mandatory_items = [item for item in checklist_items if item.mandatory]
+            all_mandatory_complete = all(item.status == InitItemStatus.SUCCESSFUL for item in mandatory_items)
+
+            choices.append(Choice(title="> Quit", value="__exit__"))
+
+            # Find the first item with status None to use as default
+            default_item = next((item for item in checklist_items if item.status is None), None)
+            default_value = default_item.name if default_item else "__exit__"
+
+            # Show checklist and get user selection
+            selected = questionary.select(
+                "Select a task:",
+                choices=choices,
+                default=default_value,
+            ).ask()
+
+            # User cancelled (Ctrl+C or similar)
+            if selected is None:
+                return
+
+            if selected == "__exit__":
+                if all_mandatory_complete:
+                    print("Setup complete!")
+                    break
+                else:
+                    incomplete_mandatory = [
+                        item.description for item in mandatory_items if item.status != InitItemStatus.SUCCESSFUL
+                    ]
+                    print(f"Warning: recommended item not completed: {', '.join(incomplete_mandatory)}")
+                    break
+
+            # Find the selected item
+            selected_item = next((item for item in checklist_items if item.name == selected), None)
+            if selected_item is None:
+                continue
+
+            # If item was already run, ask for confirmation to re-run
+            if selected_item.status is not None:
+                status_text = (
+                    "successfully. Re-run it?"
+                    if selected_item.status == InitItemStatus.SUCCESSFUL
+                    else "with failure. Retry?"
+                )
+                confirm = questionary.confirm(
+                    f"'{selected_item.description}' was already run {status_text}",
+                    default=False,
+                ).ask()
+                if not confirm:
+                    continue
+
+            try:
+                selected_item.function()
+                selected_item.status = InitItemStatus.SUCCESSFUL
+                print(f"✓ {selected_item.description} completed successfully")
+            except typer.Exit:
+                # typer.Exit is used for normal exits, treat as success
+                selected_item.status = InitItemStatus.SUCCESSFUL
+                print(f"✓ {selected_item.description} completed successfully")
+            except ToolkitError as e:
+                # Catch expected toolkit errors
+                selected_item.status = InitItemStatus.FAILED
+                print(f"✗ {selected_item.description} failed: {e}")
+            except Exception as e:
+                # Catch unexpected errors and log full traceback for debugging
+                selected_item.status = InitItemStatus.FAILED
+                print(f"✗ {selected_item.description} failed: {e}")
+                print(f"Unexpected error occurred. Full traceback:\n{traceback.format_exc()}")
+
+    def _init_toml(self, dry_run: bool = False) -> None:
+        if self.organization_dir is None:
+            self.organization_dir = ModulesCommand._prompt_organization_dir()
+        if dry_run:
+            print("Would initialize cdf.toml configuration file")
+            return
+        CDFToml.write(self.organization_dir, "dev")
+        FeatureFlag.flush()
+        print(f"cdf.toml configuration file initialized in {self.organization_dir}")
+
+    def _init_auth(self, dry_run: bool = False) -> None:
+        auth_command = AuthCommand()
+        auth_command.run(lambda: auth_command.init(no_verify=True, dry_run=dry_run))
+
+    def _init_modules(self, dry_run: bool = False) -> None:
+        with ModulesCommand() as modules_command:
+            if self.organization_dir is None:
+                self.organization_dir = ModulesCommand._prompt_organization_dir()
+            if dry_run:
+                organization_dir = Path(tempfile.mkdtemp(prefix="init_modules_", suffix=".tmp", dir=Path.cwd()))
+                modules_command.run(lambda: modules_command.init(organization_dir=organization_dir))
+                shutil.rmtree(organization_dir)
+            else:
+                modules_command.run(lambda: modules_command.init(organization_dir=self.organization_dir))
+
+    def _init_repo(self, dry_run: bool = False) -> None:
+        repo_command = RepoCommand()
+        repo_command.run(lambda: repo_command.init(cwd=Path.cwd(), host=None, verbose=False))
+
+    def _init_data_collection(self, dry_run: bool = False) -> None:
+        """Opt in to collect usage statistics"""
+
+        opt_in = questionary.confirm(
+            "Do you want to opt in to collect usage statistics? This will help us improve the Toolkit.",
+            default=True,
+        ).ask()
+        if dry_run:
+            print("Would opt in to collect data" if opt_in else "Would not opt in to collect data")
+            return
+
+        if opt_in:
+            collect_command = CollectCommand()
+            collect_command.run(lambda: collect_command.execute("opt-in"))
+        else:
+            collect_command = CollectCommand()
+            collect_command.run(lambda: collect_command.execute("opt-out"))

cognite_toolkit/_cdf_tk/commands/modules.py

@@ -4,7 +4,6 @@ import tempfile
 import zipfile
 from collections import Counter
 from hashlib import sha256
-from importlib import resources
 from pathlib import Path
 from types import TracebackType
 from typing import Any, Literal
@@ -23,8 +22,7 @@ from rich.rule import Rule
 from rich.table import Table
 from rich.tree import Tree
 
-import cognite_toolkit
-from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
+from cognite_toolkit._cdf_tk.cdf_toml import CDFToml, Library
 from cognite_toolkit._cdf_tk.commands import _cli_commands as CLICommands
 from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
 from cognite_toolkit._cdf_tk.commands._changes import (
@@ -36,9 +34,10 @@ from cognite_toolkit._cdf_tk.commands._changes import (
     UpdateDockerImageVersion,
     UpdateModuleVersion,
 )
+from cognite_toolkit._cdf_tk.commands._questionary_style import custom_style_fancy
 from cognite_toolkit._cdf_tk.constants import (
-    BUILTIN_MODULES,
     MODULES,
+    RESOURCES_PATH,
     SUPPORT_MODULE_UPGRADE_FROM_VERSION,
     EnvType,
 )
@@ -52,9 +51,9 @@ from cognite_toolkit._cdf_tk.data_classes import (
     Packages,
 )
 from cognite_toolkit._cdf_tk.exceptions import ToolkitError, ToolkitRequiredValueError, ToolkitValueError
-from cognite_toolkit._cdf_tk.feature_flags import Flags
 from cognite_toolkit._cdf_tk.hints import verify_module_directory
 from cognite_toolkit._cdf_tk.tk_warnings import MediumSeverityWarning
+from cognite_toolkit._cdf_tk.tk_warnings.other import HighSeverityWarning
 from cognite_toolkit._cdf_tk.utils import humanize_collection, read_yaml_file
 from cognite_toolkit._cdf_tk.utils.file import safe_read, safe_rmtree, safe_write, yaml_safe_dump
 from cognite_toolkit._cdf_tk.utils.modules import module_directory_from_path
@@ -65,20 +64,6 @@ if sys.version_info >= (3, 11):
     from typing import Self
 else:
     from typing_extensions import Self
-custom_style_fancy = questionary.Style(
-    [
-        ("qmark", "fg:#673ab7"),  # token in front of the question
-        ("question", "bold"),  # question text
-        ("answer", "fg:#f44336 bold"),  # submitted answer text behind the question
-        ("pointer", "fg:#673ab7 bold"),  # pointer used in select and checkbox prompts
-        ("highlighted", "fg:#673ab7 bold"),  # pointed-at choice in select and checkbox prompts
-        ("selected", "fg:#673ab7"),  # style for a selected item of a checkbox
-        ("separator", "fg:#cc5454"),  # separator in lists
-        ("instruction", ""),  # user instructions for select, rawselect, checkbox
-        ("text", ""),  # plain text
-        ("disabled", "fg:#858585 italic"),  # disabled choices for select and checkbox prompts
-    ]
-)
 
 INDENT = "  "
 POINTER = INDENT + "▶"
@@ -91,10 +76,19 @@ _FILE_DOWNLOADERS_BY_TYPE: dict[str, type[FileDownloader]] = {
 
 
 class ModulesCommand(ToolkitCommand):
-    def __init__(self, print_warning: bool = True, skip_tracking: bool = False, silent: bool = False):
+    def __init__(
+        self,
+        print_warning: bool = True,
+        skip_tracking: bool = False,
+        silent: bool = False,
+        temp_dir_suffix: str | None = None,
+        module_source_dir: Path | None = None,
+    ):
         super().__init__(print_warning, skip_tracking, silent)
-        self._builtin_modules_path = Path(resources.files(cognite_toolkit.__name__)) / BUILTIN_MODULES  # type: ignore [arg-type]
-        self._temp_download_dir = Path(tempfile.gettempdir()) / "library_downloads"
+        self._module_source_dir: Path | None = module_source_dir
+        # Use suffix to make temp directory unique (useful for parallel test execution)
+        modules_dir_name = f"{MODULES}.{temp_dir_suffix}" if temp_dir_suffix else MODULES
+        self._temp_download_dir = Path(tempfile.gettempdir()) / modules_dir_name
         if not self._temp_download_dir.exists():
             self._temp_download_dir.mkdir(parents=True, exist_ok=True)
 
@@ -144,15 +138,16 @@ class ModulesCommand(ToolkitCommand):
         organization_dir: Path,
         selected_packages: Packages,
         environments: list[EnvType],
+        modules_source_path: Path,
         mode: Literal["new", "clean", "update"] | None,
         download_data: bool = False,
     ) -> None:
-        modules_root_dir = organization_dir / MODULES
-        if mode == "clean" and modules_root_dir.is_dir():
+        modules_target_root_dir = organization_dir / MODULES
+        if mode == "clean" and modules_target_root_dir.is_dir():
             print(f"{INDENT}[yellow]Clearing directory[/]")
-            safe_rmtree(modules_root_dir)
+            safe_rmtree(modules_target_root_dir)
 
-        modules_root_dir.mkdir(parents=True, exist_ok=True)
+        modules_target_root_dir.mkdir(parents=True, exist_ok=True)
 
         seen_modules: set[Path] = set()
         selected_paths: set[Path] = set()
@@ -164,6 +159,11 @@ class ModulesCommand(ToolkitCommand):
             print(f"{INDENT}[{'yellow' if mode == 'clean' else 'green'}]Creating {package_name}[/]")
 
             for module in package.modules:
+                if module.module_id:
+                    self._additional_tracking_info.installed_module_ids.add(module.module_id)
+                if module.package_id:
+                    self._additional_tracking_info.installed_package_ids.add(module.package_id)
+
                 if module.dir in seen_modules:
                     # A module can be part of multiple packages
                     continue
@@ -176,7 +176,7 @@ class ModulesCommand(ToolkitCommand):
                     extra_resources.update(module.definition.extra_resources)
 
                 print(f"{INDENT * 2}[{'yellow' if mode == 'clean' else 'green'}]Creating module {module.name}[/]")
-                target_dir = modules_root_dir / module.relative_path
+                target_dir = modules_target_root_dir / module.relative_path
                 if Path(target_dir).exists() and mode == "update":
                     if questionary.confirm(
                         f"{INDENT}Module {module.name} already exists in folder {target_dir}. Would you like to overwrite?",
@@ -212,8 +212,8 @@ class ModulesCommand(ToolkitCommand):
             created_by_module: dict[Path, int] = Counter()
             for extra in extra_resources:
                 module_dir = module_directory_from_path(extra)
-                extra_full_path = self._builtin_modules_path / extra
-                target_path = modules_root_dir / extra
+                extra_full_path = modules_source_path / extra
+                target_path = modules_target_root_dir / extra
                 if target_path.exists():
                     # Assume that the user has already created this shared resource
                     continue
@@ -238,7 +238,7 @@ class ModulesCommand(ToolkitCommand):
             if mode == "update":
                 config_init = InitConfigYAML.load_existing(
                     safe_read(Path(organization_dir) / f"config.{environment}.yaml"), environment
-                ).load_defaults(self._builtin_modules_path, selected_paths)
+                ).load_defaults(modules_source_path, selected_paths)
             else:
                 ignore_variable_patterns: list[tuple[str, ...]] | None = None
                 if len(selected_packages) == 1 and "quickstart" in selected_packages:
@@ -261,7 +261,7 @@ class ModulesCommand(ToolkitCommand):
                         validation_type=environment,
                         selected=[f"{MODULES}/"],
                     )
-                ).load_defaults(self._builtin_modules_path, selected_paths, ignore_variable_patterns)
+                ).load_defaults(modules_source_path, selected_paths, ignore_variable_patterns)
 
                 if len(selected_packages) == 1 and "quickstart" in selected_packages:
                     config_init.lift()
@@ -271,26 +271,7 @@ class ModulesCommand(ToolkitCommand):
             )
             safe_write(Path(organization_dir) / f"config.{environment}.yaml", config_init.dump_yaml_with_comments())
 
-        cdf_toml_content = self.create_cdf_toml(organization_dir, environments[0] if environments else "dev")
-
-        destination = Path.cwd() / CDFToml.file_name
-        if destination.exists():
-            print(f"{INDENT}[yellow]cdf.toml file already exists. Skipping creation.")
-        else:
-            destination.write_text(cdf_toml_content, encoding="utf-8")
-
-    def create_cdf_toml(self, organization_dir: Path, env: EnvType = "dev") -> str:
-        cdf_toml_content = safe_read(self._builtin_modules_path / CDFToml.file_name)
-        if organization_dir != Path.cwd():
-            cdf_toml_content = cdf_toml_content.replace(
-                "#<PLACEHOLDER>",
-                f'''
-default_organization_dir = "{organization_dir.name}"''',
-            )
-        else:
-            cdf_toml_content = cdf_toml_content.replace("#<PLACEHOLDER>", "")
-        cdf_toml_content = cdf_toml_content.replace("<DEFAULT_ENV_PLACEHOLDER>", env)
-        return cdf_toml_content
+        CDFToml.write(organization_dir, environments[0] if environments else "dev")
 
     def init(
         self,
@@ -300,26 +281,40 @@ default_organization_dir = "{organization_dir.name}"''',
         user_select: str | None = None,
         user_environments: list[str] | None = None,
         user_download_data: bool | None = None,
+        library_url: str | None = None,
+        library_checksum: str | None = None,
     ) -> None:
         if not organization_dir:
-            new_line = "\n    "
-            message = (
-                f"Which directory would you like to create templates in? (default: current directory){new_line}"
-                f"HINT Use an organization directory if you use the repository for more than Toolkit. "
-                f"If not, use the current (repository root) directory '.':"
-            )
-            organization_dir_raw = questionary.text(message=message, default="").ask()
-            organization_dir = Path(organization_dir_raw.strip())
+            organization_dir = ModulesCommand._prompt_organization_dir()
 
         modules_root_dir = organization_dir / MODULES
 
-        packages = self._get_available_packages()
+        # Determine which library to use (if any)
+        library: Library | None = None
+        if library_url:
+            if not library_checksum:
+                raise ToolkitRequiredValueError(
+                    "The '--library-checksum' is required when '--library-url' is provided."
+                )
+            library = Library(url=library_url, checksum=library_checksum)
+        elif not (organization_dir / CDFToml.file_name).exists():
+            # Load default library from resources when cdf.toml doesn't exist
+            default_cdf_toml = CDFToml.load(cwd=RESOURCES_PATH, use_singleton=False)
+            library = default_cdf_toml.libraries.get("toolkit-data")
+            if library is None:
+                raise ToolkitError("Default cdf.toml in resources is missing 'toolkit-data' library configuration.")
+
+        packages, modules_source_path = self._get_available_packages(library)
 
         if select_all:
             print(Panel("Instantiating all available modules"))
             mode = self._verify_clean(modules_root_dir, clean)
             self._create(
-                organization_dir=organization_dir, selected_packages=packages, environments=["dev", "prod"], mode=mode
+                organization_dir=organization_dir,
+                selected_packages=packages,
+                environments=["dev", "prod"],
+                mode=mode,
+                modules_source_path=modules_source_path,
             )
             return
         is_interactive = user_select is not None
@@ -357,7 +352,13 @@ default_organization_dir = "{organization_dir.name}"''',
                 mode = self._verify_clean(bootcamp_org / MODULES, clean)
             # We only ask to verify if the user has not already selected overwrite in the _verify_clean method
             if mode == "clean" or questionary.confirm("Would you like to continue with creation?", default=True).ask():
-                self._create(bootcamp_org, selected, ["test"], mode)
+                self._create(
+                    organization_dir=bootcamp_org,
+                    selected_packages=selected,
+                    environments=["test"],
+                    mode=mode,
+                    modules_source_path=modules_source_path,
+                )
             raise typer.Exit()
 
         if (
@@ -387,7 +388,14 @@ default_organization_dir = "{organization_dir.name}"''',
             download_data = self._get_download_data(selected)
         else:
             download_data = user_download_data
-        self._create(organization_dir, selected, environments, mode, download_data)
+        self._create(
+            organization_dir=organization_dir,
+            selected_packages=selected,
+            environments=environments,
+            mode=mode,
+            download_data=download_data,
+            modules_source_path=modules_source_path,
+        )
 
         print(
             Panel(
@@ -405,6 +413,21 @@ default_organization_dir = "{organization_dir.name}"''',
         if not is_interactive:
             raise typer.Exit()
 
+    @staticmethod
+    def _prompt_organization_dir() -> Path:
+        print(
+            "\n".join(
+                [
+                    "Enter an optional subdirectory for Toolkit content.",
+                    "    [bold yellow]HINT[/bold yellow] Use an organization directory if you use the repository for more than Toolkit.",
+                    "    [dim]The current directory is the default.[/dim]",
+                ]
+            )
+        )
+
+        organization_dir_raw = questionary.text(message="", default="").ask()
+        return Path(organization_dir_raw.strip())
+
     @staticmethod
     def _get_download_data(selected: Packages) -> bool:
         example_data = {
@@ -427,7 +450,7 @@ default_organization_dir = "{organization_dir.name}"''',
         choices = sorted(
             [
                 questionary.Choice(
-                    title=module.title,
+                    title=module.title or module.name,
                     value=module,
                     checked=module.name in dependencies or module.is_selected_by_default,
                     disabled="required" if module.name in dependencies else None,
@@ -713,30 +736,62 @@ default_organization_dir = "{organization_dir.name}"''',
             build_env = default.environment.validation_type
 
         existing_module_names = [module.name for module in ModuleResources(organization_dir, build_env).list()]
-        available_packages = self._get_available_packages()
+        available_packages, modules_source_path = self._get_available_packages()
         added_packages = self._select_packages(available_packages, existing_module_names)
 
         download_data = self._get_download_data(added_packages)
-        self._create(organization_dir, added_packages, environments, "update", download_data)
+        self._create(
+            organization_dir=organization_dir,
+            selected_packages=added_packages,
+            environments=environments,
+            mode="update",
+            download_data=download_data,
+            modules_source_path=modules_source_path,
+        )
 
-    def _get_available_packages(self) -> Packages:
+    def _get_available_packages(self, user_library: Library | None = None) -> tuple[Packages, Path]:
         """
         Returns a list of available packages, either from the CDF TOML file or from external libraries if the feature flag is enabled.
-        If the feature flag is not enabled and no libraries are specified, it returns the built-in modules.
         """
 
         cdf_toml = CDFToml.load()
-        if Flags.EXTERNAL_LIBRARIES.is_enabled() and cdf_toml.libraries:
-            for library_name, library in cdf_toml.libraries.items():
+
+        if self._module_source_dir is None:
+            libraries = {"userdefined": user_library} if user_library else cdf_toml.libraries
+
+            for library_name, library in libraries.items():
                 try:
-                    print(f"[green]Adding library {library_name}[/]")
-                    file_path = self._temp_download_dir / f"{library_name}.zip"
+                    if library_name:
+                        self._additional_tracking_info.downloaded_library_ids.add(library_name)
+
+                    print(f"[green]Adding library {library_name} from {library.url}[/]")
+                    # Extract filename from URL, fallback to library_name.zip if no filename found
+                    from urllib.parse import urlparse
+
+                    url_path = urlparse(library.url).path
+                    filename = (
+                        url_path.split("/")[-1] if url_path and url_path.split("/")[-1] else f"{library_name}.zip"
+                    )
+                    file_path = self._temp_download_dir / filename
                     self._download(library.url, file_path)
                     self._validate_checksum(library.checksum, file_path)
                     self._unpack(file_path)
                     packages = Packages().load(file_path.parent)
+                    if packages.warnings:
+                        for warning in packages.warnings:
+                            self.warn(warning)
                     self._validate_packages(packages, f"library {library_name}")
-                    return packages
+
+                    # Track deployment pack download for each package and module
+                    for package in packages.values():
+                        if package.id:
+                            self._additional_tracking_info.downloaded_package_ids.add(package.id)
+
+                        for module in package.modules:
+                            if module.module_id:
+                                self._additional_tracking_info.downloaded_module_ids.add(module.module_id)
+
+                    return packages, file_path.parent
                 except Exception as e:
                     if isinstance(e, ToolkitError):
                         raise e
@@ -746,12 +801,24 @@ default_organization_dir = "{organization_dir.name}"''',
                         ) from e
 
                     raise ToolkitError(f"Failed to add library {library_name}, {e}")
-            # If no libraries are specified or the flag is not enabled, load the built-in modules
+            # If no libraries are specified or the flag is not enabled, raise an error
             raise ValueError("No valid libraries found.")
         else:
-            packages = Packages.load(self._builtin_modules_path)
-            self._validate_packages(packages, "built-in modules")
-            return packages
+            if user_library:
+                self.warn(
+                    MediumSeverityWarning(
+                        "External library provided but not enabled in cdf.toml. Please enable the feature flag and try again."
+                    )
+                )
+
+            if self._module_source_dir is None:
+                self.warn(
+                    MediumSeverityWarning("No external libraries and no local module source directory specified.")
+                )
+                return Packages(), Path(".")
+            packages = Packages.load(self._module_source_dir)
+            self._validate_packages(packages, self._module_source_dir.name)
+            return packages, self._module_source_dir
 
     def _validate_packages(self, packages: Packages, source_name: str) -> None:
         """
@@ -834,12 +901,21 @@ default_organization_dir = "{organization_dir.name}"''',
                 for chunk in iter(lambda: f.read(chunk_size), b""):
                     sha256_hash.update(chunk)
             calculated = sha256_hash.hexdigest()
-            if calculated != checksum:
-                raise ToolkitError(f"Checksum mismatch. Expected {checksum}, got {calculated}.")
-            else:
-                print("Checksum verified")
-        except Exception as e:
+        except OSError as e:
             raise ToolkitError(f"Failed to calculate checksum for {file_path}: {e}") from e
+        except Exception as e:
+            raise ToolkitError(f"Unexpected error during checksum calculation for {file_path}: {e}") from e
+
+        if calculated != checksum:
+            self.warn(
+                HighSeverityWarning(
+                    f"The provided checksum sha256:{checksum} does not match downloaded file hash sha256:{calculated}.\n"
+                    "Please verify the checksum with the source and update cdf.toml if needed.\n"
+                    "This may indicate that the package content has changed."
+                )
+            )
+        else:
+            print("[green]✓ Checksum verified[/green]")
 
     def _unpack(self, file_path: Path) -> None:
         """