cognite-toolkit 0.7.25__tar.gz → 0.7.26__tar.gz

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.7.25
+Version: 0.7.26
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Author: Cognite AS
 Author-email: Cognite AS <support@cognite.com>

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.26}/cognite_toolkit/_cdf_tk/cruds/_base_cruds.py

@@ -1,6 +1,6 @@
 import sys
 from abc import ABC, abstractmethod
-from collections.abc import Hashable, Iterable, Sequence, Set, Sized
+from collections.abc import Hashable, Iterable, Sequence, Sized
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generic, TypeVar
 
@@ -9,7 +9,7 @@ from cognite.client.utils.useful_types import SequenceNotStr
 from rich.console import Console
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING, EXCL_FILES
+from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING, YAML_SUFFIX
 from cognite_toolkit._cdf_tk.protocols import T_ResourceRequest, T_ResourceResponse
 from cognite_toolkit._cdf_tk.resource_classes import ToolkitResource
 from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning
@@ -39,11 +39,9 @@ class Loader(ABC):
         exclude_filetypes: A set of filetypes that should be excluded from the supported filetypes.
     """
 
-    filetypes: frozenset[str]
     folder_name: str
     kind: str
     dependencies: "frozenset[type[ResourceCRUD]]" = frozenset()
-    exclude_filetypes: frozenset[str] = frozenset()
     _doc_base_url: str = "https://api-docs.cognite.com/20230101/tag/"
     _doc_url: str = ""
     sub_folder_name: str | None = None
@@ -74,13 +72,12 @@ class Loader(ABC):
     def doc_url(cls) -> str:
         return cls._doc_base_url + cls._doc_url
 
-    def find_files(self, dir_or_file: Path | None = None, include_formats: Set[str] | None = None) -> list[Path]:
+    def find_files(self, dir_or_file: Path | None = None) -> list[Path]:
         """Find all files that are supported by this loader in the given directory or file.
 
         Args:
             dir_or_file (Path): The directory or file to search in. If no path is given,
                 the build directory is used.
-            include_formats (set[str]): A set of file formats to include. If not set, all formats are included.
 
         Returns:
             list[Path]: A sorted list of all files that are supported by this loader.
@@ -94,12 +91,7 @@ class Loader(ABC):
                 raise ValueError("Invalid file type")
             return [dir_or_file]
         elif dir_or_file.is_dir():
-            file_paths = [
-                file
-                for file in dir_or_file.glob("**/*")
-                if self.is_supported_file(file) and (include_formats is None or file.suffix in include_formats)
-            ]
-            return sorted(file_paths)
+            return sorted([file for file in dir_or_file.rglob("*") if self.is_supported_file(file)])
         else:
             return []
 
@@ -118,11 +110,7 @@ class Loader(ABC):
             bool: True if the file is supported, False otherwise.
 
         """
-        if cls.filetypes and file.suffix[1:] not in cls.filetypes:
-            return False
-        if cls.exclude_filetypes and file.suffix[1:] in cls.exclude_filetypes:
-            return False
-        return file.stem.casefold().endswith(cls.kind.casefold())
+        return file.suffix in YAML_SUFFIX and file.stem.casefold().endswith(cls.kind.casefold())
 
 
 T_Loader = TypeVar("T_Loader", bound=Loader)
@@ -144,8 +132,6 @@ class ResourceCRUD(Loader, ABC, Generic[T_ID, T_ResourceRequest, T_ResourceRespo
         yaml_cls: The File format for this resource. This is used to validate the user input.
         support_drop: Whether the resource supports the drop flag.
         support_update: Whether the resource supports the update operation.
-        filetypes: The filetypes that are supported by this crud. This should not be set in the subclass, it
-            should always be yaml and yml.
         dependencies: A set of other resource cruds that must be loaded before this crud.
         parent_resource: A set of other resource cruds that are parent resources to this resource. This is used
             to determine if the iterate method should return any resources when filtering by parent ids.
@@ -158,7 +144,6 @@ class ResourceCRUD(Loader, ABC, Generic[T_ID, T_ResourceRequest, T_ResourceRespo
     # Optional to set in the subclass
     support_drop = True
     support_update = True
-    filetypes = frozenset({"yaml", "yml"})
     dependencies: "frozenset[type[ResourceCRUD]]" = frozenset()
     # For example, TransformationNotification and Schedule has Transformation as the parent resource
     # This is used in the iterate method to ensure that nothing is returned if
@@ -437,10 +422,3 @@ class DataCRUD(Loader, ABC):
     @abstractmethod
     def upload(self, state: "BuildEnvironment", dry_run: bool) -> Iterable[tuple[str, int]]:
         raise NotImplementedError
-
-    def _find_data_files(self, directory: Path) -> list[Path]:
-        return [
-            path
-            for path in directory.rglob("*")
-            if path.is_file() and path.name not in EXCL_FILES and self.is_supported_file(path)
-        ]

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.26}/cognite_toolkit/_cdf_tk/cruds/_data_cruds.py

@@ -28,7 +28,6 @@ class DatapointsCRUD(DataCRUD):
     item_name = "datapoints"
     folder_name = "timeseries"
     kind = "Datapoints"
-    filetypes = frozenset({"csv", "parquet"})
     dependencies = frozenset({TimeSeriesCRUD})
     _doc_url = "Time-series/operation/postMultiTimeSeriesDatapoints"
 
@@ -43,7 +42,9 @@ class DatapointsCRUD(DataCRUD):
         resource_directories = state.built_resources[self.folder_name].get_resource_directories(self.folder_name)
 
         for resource_dir in resource_directories:
-            for datafile in self._find_data_files(resource_dir):
+            for datafile in resource_dir.rglob("*"):
+                if not datafile.stem.casefold().endswith(self.kind.casefold()):
+                    continue
                 if datafile.suffix == ".csv":
                     # The replacement is used to ensure that we read exactly the same file on Windows and Linux
                     file_content = datafile.read_bytes().replace(b"\r\n", b"\n").decode("utf-8")
@@ -52,7 +53,7 @@ class DatapointsCRUD(DataCRUD):
                 elif datafile.suffix == ".parquet":
                     data = pd.read_parquet(datafile, engine="pyarrow")
                 else:
-                    raise ValueError(f"Unsupported file type {datafile.suffix} for {datafile.name}")
+                    continue
                 timeseries_ids = list(data.columns)
                 if len(timeseries_ids) == 1:
                     ts_str = timeseries_ids[0]
@@ -87,8 +88,6 @@ class FileCRUD(DataCRUD):
     item_name = "file contents"
     folder_name = "files"
     kind = "File"
-    filetypes = frozenset()
-    exclude_filetype: frozenset[str] = frozenset({})
     dependencies = frozenset({FileMetadataCRUD, CogniteFileCRUD})
     _doc_url = "Files/operation/initFileUpload"
 
@@ -149,7 +148,6 @@ class FileCRUD(DataCRUD):
 class RawFileCRUD(DataCRUD):
     item_name = "rows"
     folder_name = "raw"
-    filetypes = frozenset({"csv", "parquet"})
     kind = "Raw"
     dependencies = frozenset({RawTableCRUD})
     _doc_url = "Raw/operation/postRows"
@@ -169,7 +167,7 @@ class RawFileCRUD(DataCRUD):
             datafile = next(
                 (
                     resource.source.path.with_suffix(f".{file_type}")
-                    for file_type in self.filetypes
+                    for file_type in ["csv", "parquet"]
                     if (resource.source.path.with_suffix(f".{file_type}").exists())
                 ),
                 None,

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.26}/cognite_toolkit/_cdf_tk/cruds/_resource_cruds/classic.py

@@ -29,20 +29,23 @@ from cognite_toolkit._cdf_tk.client.data_classes.sequences import (
     ToolkitSequenceRowsList,
     ToolkitSequenceRowsWrite,
 )
+from cognite_toolkit._cdf_tk.constants import TABLE_FORMATS, YAML_SUFFIX
 from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
+from cognite_toolkit._cdf_tk.feature_flags import Flags
 from cognite_toolkit._cdf_tk.resource_classes import AssetYAML, EventYAML, SequenceRowYAML, SequenceYAML
-from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning
+from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning, ToolkitDeprecationWarning
 from cognite_toolkit._cdf_tk.utils import load_yaml_inject_variables
 from cognite_toolkit._cdf_tk.utils.diff_list import diff_list_hashable, diff_list_identifiable
 from cognite_toolkit._cdf_tk.utils.file import read_csv
 
 from .data_organization import DataSetsCRUD, LabelCRUD
 
+_DEPRECATION_WARNING_ISSUED = False
+
 
 @final
 class AssetCRUD(ResourceCRUD[str, AssetWrite, Asset]):
     folder_name = "classic"
-    filetypes = frozenset({"yaml", "yml", "csv", "parquet"})
     resource_cls = Asset
     resource_write_cls = AssetWrite
     yaml_cls = AssetYAML
@@ -50,6 +53,26 @@ class AssetCRUD(ResourceCRUD[str, AssetWrite, Asset]):
     dependencies = frozenset({DataSetsCRUD, LabelCRUD})
     _doc_url = "Assets/operation/createAssets"
 
+    @classmethod
+    def is_supported_file(cls, file: Path) -> bool:
+        if Flags.v08.is_enabled():
+            return super().is_supported_file(file)
+        global _DEPRECATION_WARNING_ISSUED
+        if not file.stem.casefold().endswith(cls.kind.casefold()):
+            return False
+        if file.suffix in YAML_SUFFIX:
+            return True
+        if file.suffix in TABLE_FORMATS:
+            if not _DEPRECATION_WARNING_ISSUED:
+                ToolkitDeprecationWarning(
+                    feature="deployment of asset from CSV or Parquet files",
+                    alternative="data plugin and cdf data upload commands",
+                    removal_version="0.8",
+                ).print_warning()
+                _DEPRECATION_WARNING_ISSUED = True
+            return True
+        return False
+
     @property
     def display_name(self) -> str:
         return "assets"
@@ -479,7 +502,6 @@ class SequenceRowCRUD(ResourceCRUD[str, ToolkitSequenceRowsWrite, ToolkitSequenc
 @final
 class EventCRUD(ResourceCRUD[str, EventWrite, Event]):
     folder_name = "classic"
-    filetypes = frozenset({"yaml", "yml"})
     resource_cls = Event
     resource_write_cls = EventWrite
     yaml_cls = EventYAML

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.26}/cognite_toolkit/_cdf_tk/cruds/_resource_cruds/fieldops.py

@@ -40,7 +40,6 @@ from .group_scoped import GroupResourceScopedCRUD
 @final
 class InfieldV1CRUD(ResourceCRUD[str, APMConfigWrite, APMConfig]):
     folder_name = "cdf_applications"
-    filetypes = frozenset({"yaml", "yml"})
     resource_cls = APMConfig
     resource_write_cls = APMConfigWrite
     kind = "InfieldV1"
@@ -249,7 +248,6 @@ class InfieldV1CRUD(ResourceCRUD[str, APMConfigWrite, APMConfig]):
 @final
 class InFieldLocationConfigCRUD(ResourceCRUD[NodeIdentifier, InfieldLocationConfig, InfieldLocationConfig]):
     folder_name = "cdf_applications"
-    filetypes = frozenset({"yaml", "yml"})
     resource_cls = InfieldLocationConfig
     resource_write_cls = InfieldLocationConfig
     kind = "InFieldLocationConfig"
@@ -349,7 +347,6 @@ class InFieldLocationConfigCRUD(ResourceCRUD[NodeIdentifier, InfieldLocationConf
 @final
 class InFieldCDMLocationConfigCRUD(ResourceCRUD[NodeIdentifier, InFieldCDMLocationConfig, InFieldCDMLocationConfig]):
     folder_name = "cdf_applications"
-    filetypes = frozenset({"yaml", "yml"})
     resource_cls = InFieldCDMLocationConfig
     resource_write_cls = InFieldCDMLocationConfig
     kind = "InFieldCDMLocationConfig"

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.26}/cognite_toolkit/_cdf_tk/cruds/_resource_cruds/migration.py

@@ -21,7 +21,6 @@ from .datamodel import SpaceCRUD, ViewCRUD
 @final
 class ResourceViewMappingCRUD(ResourceCRUD[str, ResourceViewMappingApply, ResourceViewMapping]):
     folder_name = "migration"
-    filetypes = frozenset({"yaml", "yml"})
     resource_cls = ResourceViewMapping
     resource_write_cls = ResourceViewMappingApply
     kind = "ResourceViewMapping"

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.26}/cognite_toolkit/_cdf_tk/cruds/_resource_cruds/relationship.py

@@ -23,7 +23,6 @@ from .timeseries import TimeSeriesCRUD
 @final
 class RelationshipCRUD(ResourceCRUD[str, RelationshipWrite, Relationship]):
     folder_name = "classic"
-    filetypes = frozenset({"yaml", "yml"})
     resource_cls = Relationship
     resource_write_cls = RelationshipWrite
     kind = "Relationship"

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.26}/cognite_toolkit/_cdf_tk/cruds/_resource_cruds/streams.py

@@ -19,7 +19,6 @@ from .datamodel import ContainerCRUD
 @final
 class StreamCRUD(ResourceCRUD[str, StreamRequest, StreamResponse]):
     folder_name = "streams"
-    filetypes = frozenset({"yaml", "yml"})
     resource_cls = StreamResponse
     resource_write_cls = StreamRequest
     kind = "Streams"

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.26}/cognite_toolkit/_cdf_tk/cruds/_worker.py

@@ -14,6 +14,7 @@ from yaml import YAMLError
 
 from cognite_toolkit._cdf_tk.constants import TABLE_FORMATS
 from cognite_toolkit._cdf_tk.exceptions import ToolkitWrongResourceError, ToolkitYAMLFormatError
+from cognite_toolkit._cdf_tk.feature_flags import Flags
 from cognite_toolkit._cdf_tk.tk_warnings import EnvironmentVariableMissingWarning, catch_warnings
 from cognite_toolkit._cdf_tk.utils import to_diff
 
@@ -47,12 +48,16 @@ class ResourceWorker(Generic[T_ID, T_ResourceRequest, T_ResourceResponse]):
     ) -> list[Path]:
         filepaths = self.loader.find_files(directory)
 
-        for read_module in read_modules or []:
-            if resource_dir := read_module.resource_dir_path(self.loader.folder_name):
-                # As of 05/11/24, Asset support csv and parquet files in addition to YAML.
-                # These table formats are not built, i.e., no variable replacement is done,
-                # so we load them directly from the source module.
-                filepaths.extend(self.loader.find_files(resource_dir, include_formats=TABLE_FORMATS))
+        if not Flags.v08.is_enabled():
+            for read_module in read_modules or []:
+                if resource_dir := read_module.resource_dir_path(self.loader.folder_name):
+                    # As of 05/11/24, Asset support csv and parquet files in addition to YAML.
+                    # These table formats are not built, i.e., no variable replacement is done,
+                    # so we load them directly from the source module.
+                    table_files = [
+                        file for file in self.loader.find_files(resource_dir) if file.suffix in TABLE_FORMATS
+                    ]
+                    filepaths.extend(table_files)
 
         if not sort:
             return filepaths

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.26}/cognite_toolkit/_cdf_tk/tk_warnings/other.py

@@ -136,15 +136,19 @@ class MissingCapabilityWarning(GeneralWarning):
 
 @dataclass(frozen=True)
 class ToolkitDeprecationWarning(ToolkitWarning, DeprecationWarning):
+    severity = SeverityLevel.HIGH
     message: ClassVar[str] = "The '{feature}' is deprecated and will be removed in a future version."
 
     feature: str
     alternative: str | None = None
+    removal_version: str | None = None
 
     def get_message(self) -> str:
         msg = self.message.format(feature=self.feature)
         if self.alternative:
             msg += f"\nUse {self.alternative!r} instead."
+        if self.removal_version:
+            msg += f"\nIt will be removed in version {self.removal_version}."
 
         return msg
 

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.26}/cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.7.25
+      image: cognite/toolkit:0.7.26
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.26}/cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.7.25
+      image: cognite/toolkit:0.7.26
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.26}/cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.25"
+version = "0.7.26"
 
 
 [plugins]

cognite_toolkit-0.7.26/cognite_toolkit/_version.py

@@ -0,0 +1 @@
+__version__ = "0.7.26"

{cognite_toolkit-0.7.25 → cognite_toolkit-0.7.26}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognite_toolkit"
-version = "0.7.25"
+version = "0.7.26"
 description = "Official Cognite Data Fusion tool for project templates and configuration deployment"
 authors = [
     {name ="Cognite AS", email="support@cognite.com"}

cognite_toolkit-0.7.25/cognite_toolkit/_version.py

@@ -1 +0,0 @@
-__version__ = "0.7.25"