cognite-neat 0.75.4__tar.gz → 0.75.6__tar.gz

{cognite_neat-0.75.4 → cognite_neat-0.75.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.75.4
+Version: 0.75.6
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

cognite_neat-0.75.6/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.75.6"

{cognite_neat-0.75.4 → cognite_neat-0.75.6}/cognite/neat/rules/exporters/_rules2dms.py

@@ -27,7 +27,7 @@ from cognite.neat.utils.cdf_loaders import (
 from ._base import CDFExporter
 from ._models import UploadResult
 
-Component: TypeAlias = Literal["all", "spaces", "data_models", "views", "containers"]
+Component: TypeAlias = Literal["all", "spaces", "data_models", "views", "containers", "node_types"]
 
 
 class DMSExporter(CDFExporter[DMSSchema]):
@@ -99,7 +99,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
         if "all" in self.export_components:
             exclude = set()
         else:
-            exclude = {"spaces", "data_models", "views", "containers"} - self.export_components
+            exclude = {"spaces", "data_models", "views", "containers", "node_types"} - self.export_components
         return exclude
 
     def export(self, rules: Rules) -> DMSSchema:
@@ -118,12 +118,12 @@ class DMSExporter(CDFExporter[DMSSchema]):
             return dms_rules.as_schema(self.export_pipeline, self.instance_space)
 
         # This is an extension of an existing model.
-        reference_rules = cast(DMSRules, dms_rules.reference).copy(deep=True)
+        reference_rules = cast(DMSRules, dms_rules.reference).model_copy(deep=True)
         reference_schema = reference_rules.as_schema(self.export_pipeline)
 
         # Todo Move this to an appropriate location
         # Merging Reference with User Rules
-        combined_rules = dms_rules.copy(deep=True)
+        combined_rules = dms_rules.model_copy(deep=True)
         existing_containers = {container.class_ for container in combined_rules.containers or []}
         if combined_rules.containers is None:
             combined_rules.containers = SheetList[DMSContainer](data=[])

{cognite_neat-0.75.4 → cognite_neat-0.75.6}/cognite/neat/rules/exporters/_rules2excel.py

@@ -29,6 +29,11 @@ class ExcelExporter(BaseExporter[Workbook]):
         new_model_id: The new model ID to use for the exported spreadsheet. This is only applicable if the input
             rules have 'is_reference' set. If provided, the model ID will be used to automatically create the
             new metadata sheet in the Excel file.
+        is_reference: If True, the rules are considered to be a reference model. The exported Excel file will
+            then contain empty sheets for the main rules and this data model will be dumped to the reference sheets.
+            This is useful when you are building a solution model based on an Enterprise model, then the
+            Enterprise model will serve as the reference model. It is also useful when you are extending an existing
+            model, then the existing model will serve as the reference model. Defaults to False.
 
     The following styles are available:
 
@@ -54,6 +59,7 @@ class ExcelExporter(BaseExporter[Workbook]):
         styling: Style = "default",
         output_role: RoleTypes | None = None,
         new_model_id: tuple[str, str, str] | None = None,
+        is_reference: bool = False,
     ):
         if styling not in self.style_options:
             raise ValueError(f"Invalid styling: {styling}. Valid options are {self.style_options}")
@@ -61,6 +67,7 @@ class ExcelExporter(BaseExporter[Workbook]):
         self._styling_level = self.style_options.index(styling)
         self.output_role = output_role
         self.new_model_id = new_model_id
+        self.is_reference = is_reference
 
     def export_to_file(self, rules: Rules, filepath: Path) -> None:
         """Exports transformation rules to excel file."""
@@ -79,7 +86,7 @@ class ExcelExporter(BaseExporter[Workbook]):
 
         dumped_rules: dict[str, Any]
         dumped_reference_rules: dict[str, Any] | None = None
-        if rules.is_reference:
+        if self.is_reference:
             # Writes empty reference sheets
             dumped_rules = {
                 "Metadata": self._create_metadata_sheet_user_rules(rules),

{cognite_neat-0.75.4 → cognite_neat-0.75.6}/cognite/neat/rules/importers/_base.py

@@ -20,21 +20,18 @@ class BaseImporter(ABC):
     """
 
     @overload
-    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None, is_reference: bool = False) -> Rules:
+    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
         ...
 
     @overload
     def to_rules(
-        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None, is_reference: bool = False
+        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList]:
         ...
 
     @abstractmethod
     def to_rules(
-        self,
-        errors: Literal["raise", "continue"] = "continue",
-        role: RoleTypes | None = None,
-        is_reference: bool = False,
+        self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList] | Rules:
         """
         Creates `Rules` object from the data for target role.
@@ -48,7 +45,6 @@ class BaseImporter(ABC):
         issues: IssueList,
         errors: Literal["raise", "continue"] = "continue",
         role: RoleTypes | None = None,
-        is_reference: bool = False,
     ) -> tuple[Rules | None, IssueList] | Rules:
         """Converts the rules to the output format."""
 
@@ -61,9 +57,6 @@ class BaseImporter(ABC):
         else:
             raise NotImplementedError(f"Role {role} is not supported for {type(rules).__name__} rules")
 
-        if is_reference:
-            output.is_reference = True
-
         if errors == "raise":
             return output
         else:

{cognite_neat-0.75.4 → cognite_neat-0.75.6}/cognite/neat/rules/importers/_dms2rules.py

@@ -82,20 +82,17 @@ class DMSImporter(BaseImporter):
         return cls(DMSSchema.from_zip(zip_file))
 
     @overload
-    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None, is_reference: bool = False) -> Rules:
+    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
         ...
 
     @overload
     def to_rules(
-        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None, is_reference: bool = False
+        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList]:
         ...
 
     def to_rules(
-        self,
-        errors: Literal["raise", "continue"] = "continue",
-        role: RoleTypes | None = None,
-        is_reference: bool = False,
+        self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList] | Rules:
         if role is RoleTypes.domain_expert:
             raise ValueError(f"Role {role} is not supported for DMSImporter")
@@ -210,7 +207,6 @@ class DMSImporter(BaseImporter):
                 data=[DMSContainer.from_container(container) for container in self.schema.containers]
             ),
             views=SheetList[DMSView](data=[DMSView.from_view(view, data_model_view_ids) for view in self.schema.views]),
-            is_reference=is_reference,
         )
         output_rules: Rules
         if role is RoleTypes.information_architect:

{cognite_neat-0.75.4 → cognite_neat-0.75.6}/cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py

@@ -119,20 +119,17 @@ class DTDLImporter(BaseImporter):
         return cls(items, zip_file.stem, read_issues=issues)
 
     @overload
-    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None, is_reference: bool = False) -> Rules:
+    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
         ...
 
     @overload
     def to_rules(
-        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None, is_reference: bool = False
+        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList]:
         ...
 
     def to_rules(
-        self,
-        errors: Literal["raise", "continue"] = "continue",
-        role: RoleTypes | None = None,
-        is_reference: bool = False,
+        self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList] | Rules:
         converter = _DTDLConverter(self._read_issues)
 
@@ -161,4 +158,4 @@ class DTDLImporter(BaseImporter):
             else:
                 raise converter.issues.as_errors()
 
-        return self._to_output(rules, converter.issues, errors, role, is_reference)
+        return self._to_output(rules, converter.issues, errors, role)

{cognite_neat-0.75.4 → cognite_neat-0.75.6}/cognite/neat/rules/importers/_owl2rules/_owl2rules.py

@@ -43,20 +43,17 @@ class OWLImporter(BaseImporter):
         self.make_compliant = make_compliant
 
     @overload
-    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None, is_reference: bool = False) -> Rules:
+    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
         ...
 
     @overload
     def to_rules(
-        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None, is_reference: bool = False
+        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList]:
         ...
 
     def to_rules(
-        self,
-        errors: Literal["raise", "continue"] = "continue",
-        role: RoleTypes | None = None,
-        is_reference: bool = False,
+        self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList] | Rules:
         graph = Graph()
         try:
@@ -82,7 +79,7 @@ class OWLImporter(BaseImporter):
             components = make_components_compliant(components)
 
         rules = InformationRules.model_validate(components)
-        return self._to_output(rules, IssueList(), errors, role, is_reference)
+        return self._to_output(rules, IssueList(), errors, role)
 
 
 def make_components_compliant(components: dict) -> dict:

{cognite_neat-0.75.4 → cognite_neat-0.75.6}/cognite/neat/rules/importers/_spreadsheet2rules.py

@@ -152,7 +152,7 @@ class ExcelImporter(BaseImporter):
         self.filepath = filepath
 
     @overload
-    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None, is_reference: bool = False) -> Rules:
+    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
         ...
 
     @overload
@@ -160,29 +160,23 @@ class ExcelImporter(BaseImporter):
         self,
         errors: Literal["continue"] = "continue",
         role: RoleTypes | None = None,
-        is_reference: bool = False,
     ) -> tuple[Rules | None, IssueList]:
         ...
 
     def to_rules(
-        self,
-        errors: Literal["raise", "continue"] = "continue",
-        role: RoleTypes | None = None,
-        is_reference: bool = False,
+        self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList] | Rules:
         issue_list = IssueList(title=f"'{self.filepath.name}'")
         if not self.filepath.exists():
             issue_list.append(issues.spreadsheet_file.SpreadsheetNotFoundError(self.filepath))
             return self._return_or_raise(issue_list, errors)
 
-        user_result: ReadResult | None = None
-        if not is_reference:
-            user_result = SpreadsheetReader(issue_list, is_reference=False).read(self.filepath)
-            if user_result is None or issue_list.has_errors:
-                return self._return_or_raise(issue_list, errors)
+        user_result = SpreadsheetReader(issue_list, is_reference=False).read(self.filepath)
+        if user_result is None or issue_list.has_errors:
+            return self._return_or_raise(issue_list, errors)
 
         reference_result: ReadResult | None = None
-        if is_reference or (
+        if (
             user_result
             and user_result.role != RoleTypes.domain_expert
             and user_result.schema == SchemaCompleteness.extended
@@ -230,7 +224,6 @@ class ExcelImporter(BaseImporter):
             issue_list,
             errors=errors,
             role=role,
-            is_reference=is_reference,
         )
 
     @classmethod
@@ -246,20 +239,17 @@ class GoogleSheetImporter(BaseImporter):
         self.skiprows = skiprows
 
     @overload
-    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None, is_reference: bool = False) -> Rules:
+    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
         ...
 
     @overload
     def to_rules(
-        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None, is_reference: bool = False
+        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList]:
         ...
 
     def to_rules(
-        self,
-        errors: Literal["raise", "continue"] = "continue",
-        role: RoleTypes | None = None,
-        is_reference: bool = False,
+        self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList] | Rules:
         local_import("gspread", "google")
         import gspread  # type: ignore[import]
@@ -284,4 +274,4 @@ class GoogleSheetImporter(BaseImporter):
         else:
             raise ValueError(f"Role {role} is not valid.")
 
-        return self._to_output(output, IssueList(), errors=errors, role=role, is_reference=is_reference)
+        return self._to_output(output, IssueList(), errors=errors, role=role)

{cognite_neat-0.75.4 → cognite_neat-0.75.6}/cognite/neat/rules/importers/_yaml2rules.py

@@ -45,20 +45,17 @@ class YAMLImporter(BaseImporter):
         return cls(yaml.safe_load(filepath.read_text()), filepaths=[filepath])
 
     @overload
-    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None, is_reference: bool = False) -> Rules:
+    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
         ...
 
     @overload
     def to_rules(
-        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None, is_reference: bool = False
+        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList]:
         ...
 
     def to_rules(
-        self,
-        errors: Literal["raise", "continue"] = "continue",
-        role: RoleTypes | None = None,
-        is_reference: bool = False,
+        self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
     ) -> tuple[Rules | None, IssueList] | Rules:
         if any(issue for issue in self._read_issues if isinstance(issue, NeatValidationError)) or not self.raw_data:
             if errors == "raise":
@@ -107,4 +104,4 @@ class YAMLImporter(BaseImporter):
             else:
                 raise issue_list.as_errors()
 
-        return self._to_output(rules, issue_list, errors, role, is_reference)
+        return self._to_output(rules, issue_list, errors, role)

{cognite_neat-0.75.4 → cognite_neat-0.75.6}/cognite/neat/rules/models/rules/_dms_architect_rules.py

@@ -282,7 +282,6 @@ class DMSRules(BaseRules):
     views: SheetList[DMSView] = Field(alias="Views")
     containers: SheetList[DMSContainer] | None = Field(None, alias="Containers")
     reference: "DMSRules | None" = Field(None, alias="Reference")
-    is_reference: bool = False
 
     @model_validator(mode="after")
     def set_default_space_and_version(self) -> "DMSRules":
@@ -583,7 +582,7 @@ class DMSRules(BaseRules):
                     "The schema is set to 'extended', but no reference rules are provided to validate against"
                 )
             # This is an extension of the reference rules, we need to merge the two
-            rules = self.copy(deep=True)
+            rules = self.model_copy(deep=True)
             rules.properties.extend(self.reference.properties.data)
             existing_views = {view.view.as_id(False) for view in rules.views}
             rules.views.extend([view for view in self.reference.views if view.view.as_id(False) not in existing_views])
@@ -666,7 +665,6 @@ class DMSRules(BaseRules):
             "Properties" if info.by_alias else "properties": properties,
             "Views" if info.by_alias else "views": views,
             "Containers" if info.by_alias else "containers": containers,
-            "is_reference": self.is_reference,
         }
         if self.reference is not None:
             output["Reference" if info.by_alias else "reference"] = self.reference.model_dump(**kwargs)
@@ -682,7 +680,7 @@ class DMSRules(BaseRules):
         return _DMSRulesConverter(self).as_domain_rules()
 
     def reference_self(self) -> Self:
-        new_rules = self.copy(deep=True)
+        new_rules = self.model_copy(deep=True)
         for prop in new_rules.properties:
             prop.reference = ReferenceEntity(
                 prefix=prop.view.prefix, suffix=prop.view.suffix, version=prop.view.version, property_=prop.property_
@@ -762,7 +760,8 @@ class _DMSExporter:
             data_model.space = used_spaces.pop()
             spaces = dm.SpaceApplyList([dm.SpaceApply(space=data_model.space)])
         else:
-            spaces = dm.SpaceApplyList([metadata.as_space()] + [dm.SpaceApply(space=space) for space in used_spaces])
+            used_spaces.add(metadata.space)
+            spaces = dm.SpaceApplyList([dm.SpaceApply(space=space) for space in used_spaces])
         if self.instance_space and self.instance_space not in {space.space for space in spaces}:
             spaces.append(dm.SpaceApply(space=self.instance_space, name=self.instance_space))
         return spaces
@@ -1135,7 +1134,6 @@ class _DMSRulesConverter:
             properties=SheetList[InformationProperty](data=properties),
             classes=SheetList[InformationClass](data=classes),
             reference=self.dms.reference and self.dms.reference.as_information_architect_rules(),  # type: ignore[arg-type]
-            is_reference=self.dms.is_reference,
         )
 
     @classmethod

{cognite_neat-0.75.4 → cognite_neat-0.75.6}/cognite/neat/rules/models/rules/_domain_rules.py

@@ -48,7 +48,6 @@ class DomainRules(RuleModel):
     properties: SheetList[DomainProperty] = Field(alias="Properties")
     classes: SheetList[DomainClass] | None = Field(None, alias="Classes")
     reference: "DomainRules | None" = Field(None, alias="Reference")
-    is_reference: bool = False
 
     @model_serializer(mode="plain", when_used="always")
     def domain_rules_serializer(self, info: SerializationInfo) -> dict[str, Any]:
@@ -65,4 +64,4 @@ class DomainRules(RuleModel):
 
     def reference_self(self) -> "DomainRules":
         """DomainRules does not have reference field, so it returns a copy of itself."""
-        return self.copy(deep=True)
+        return self.model_copy(deep=True)

{cognite_neat-0.75.4 → cognite_neat-0.75.6}/cognite/neat/rules/models/rules/_information_rules.py

@@ -247,7 +247,6 @@ class InformationRules(RuleModel):
     classes: SheetList[InformationClass] = Field(alias="Classes")
     prefixes: dict[str, Namespace] = Field(default_factory=lambda: PREFIXES.copy())
     reference: "InformationRules | None" = Field(None, alias="Reference")
-    is_reference: bool = False
 
     @field_validator("prefixes", mode="before")
     def parse_str(cls, values: Any) -> Any:
@@ -355,7 +354,7 @@ class InformationRules(RuleModel):
         return _InformationRulesConverter(self).as_dms_architect_rules()
 
     def reference_self(self) -> "InformationRules":
-        new_self = self.copy(deep=True)
+        new_self = self.model_copy(deep=True)
         for prop in new_self.properties:
             prop.reference = ReferenceEntity(
                 prefix=prop.class_.prefix,
@@ -451,7 +450,6 @@ class _InformationRulesConverter:
             views=SheetList[DMSView](data=views),
             containers=SheetList[DMSContainer](data=containers),
             reference=self.information.reference and self.information.reference.as_dms_architect_rules(),  # type: ignore[arg-type]
-            is_reference=self.information.is_reference,
         )
 
     @classmethod

{cognite_neat-0.75.4 → cognite_neat-0.75.6}/cognite/neat/rules/models/rules/_types/_base.py

@@ -30,7 +30,7 @@ if sys.version_info >= (3, 11):
 else:
     from typing_extensions import Self
 
-
+# mypy: ignore-errors
 VERSION_ID_REGEX = rf"\(version=(?P<version>{VERSION_REGEX})\)"
 
 

{cognite_neat-0.75.4 → cognite_neat-0.75.6}/cognite/neat/workflows/steps/lib/rules_importer.py

@@ -72,7 +72,7 @@ class ExcelToRules(Step):
             role_enum = RoleTypes[role]
 
         excel_importer = importers.ExcelImporter(rules_file_path)
-        rules, issues = excel_importer.to_rules(role=role_enum, errors="continue")
+        rules, issues = excel_importer.to_rules(errors="continue", role=role_enum)
 
         if rules is None:
             output_dir = self.data_store_path / Path("staging")
@@ -151,7 +151,7 @@ class OntologyToRules(Step):
             role_enum = RoleTypes[role]
 
         ontology_importer = importers.OWLImporter(filepath=rules_file_path, make_compliant=make_compliant)
-        rules, issues = ontology_importer.to_rules(role=role_enum, errors="continue")
+        rules, issues = ontology_importer.to_rules(errors="continue", role=role_enum)
 
         if rules is None:
             output_dir = self.data_store_path / Path("staging")
@@ -196,13 +196,6 @@ class DMSToRules(Step):
             label="For what role Rules are intended?",
             options=["infer", *RoleTypes.__members__.keys()],
         ),
-        Configurable(
-            name="Reference",
-            value="false",
-            label="Whether the imported rules are a reference model or not. This is "
-            "used when you want to extend an existing model and this is the model that is being extended from.",
-            options=["true", "false"],
-        ),
     ]
 
     def run(self, cdf_client: CogniteClient) -> (FlowMessage, MultiRuleData):  # type: ignore[syntax, override]
@@ -222,8 +215,6 @@ class DMSToRules(Step):
             )
             return FlowMessage(error_text=error_text, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
 
-        is_reference = self.configs.get("Reference", "false") == "true"
-
         dms_importer = importers.DMSImporter.from_data_model_id(cdf_client, datamodel_entity.as_id())
 
         # if role is None, it will be inferred from the rules file
@@ -232,7 +223,7 @@ class DMSToRules(Step):
         if role != "infer" and role is not None:
             role_enum = RoleTypes[role]
 
-        rules, issues = dms_importer.to_rules(role=role_enum, errors="continue", is_reference=is_reference)
+        rules, issues = dms_importer.to_rules(errors="continue", role=role_enum)
 
         if rules is None:
             output_dir = self.data_store_path / Path("staging")

{cognite_neat-0.75.4 → cognite_neat-0.75.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite-neat"
-version = "0.75.4"
+version = "0.75.6"
 readme = "README.md"
 description = "Knowledge graph transformation"
 authors = [

cognite_neat-0.75.4/cognite/neat/_version.py

@@ -1 +0,0 @@
-__version__ = "0.75.4"