cognite-extractor-utils 7.4.5__tar.gz → 7.4.7__tar.gz

{cognite_extractor_utils-7.4.5 → cognite_extractor_utils-7.4.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-extractor-utils
-Version: 7.4.5
+Version: 7.4.7
 Summary: Utilities for easier development of extractors for CDF
 Home-page: https://github.com/cognitedata/python-extractor-utils
 License: Apache-2.0

{cognite_extractor_utils-7.4.5 → cognite_extractor_utils-7.4.7}/cognite/extractorutils/__init__.py

@@ -16,5 +16,5 @@
 Cognite extractor utils is a Python package that simplifies the development of new extractors.
 """
 
-__version__ = "7.4.5"
+__version__ = "7.4.7"
 from .base import Extractor

{cognite_extractor_utils-7.4.5 → cognite_extractor_utils-7.4.7}/cognite/extractorutils/configtools/elements.py

@@ -696,3 +696,51 @@ class StateStoreConfig:
             return LocalStateStore(file_path="states.json", cancellation_token=cancellation_token)
         else:
             return NoStateStore()
+
+
+class RegExpFlag(Enum):
+    IGNORECASE = "ignore-case"
+    IC = "i"
+    ASCII = "ascii-only"
+    A = "a"
+
+    def get_regex_flag(self) -> int:
+        if self in (RegExpFlag.IGNORECASE, RegExpFlag.IC):
+            return re.IGNORECASE
+        elif self.value in (RegExpFlag.ASCII, RegExpFlag.A):
+            return re.ASCII
+        return 0
+
+
+@dataclass
+class IgnorePattern:
+    """
+    Configuration for regexp for ignore pattern
+    """
+
+    pattern: str
+    options: Optional[list[RegExpFlag]] = None
+    flags: Optional[list[RegExpFlag]] = None
+
+    def compile(self) -> re.Pattern[str]:
+        """
+        Compile RegExp pattern.
+
+        Returns:
+            Compiled pattern.
+        """
+        flag = 0
+        for f in self.options or []:
+            flag |= f.get_regex_flag()
+        return re.compile(self.pattern, flag)
+
+    def __post_init__(self) -> None:
+        if self.options is not None and self.flags is not None:
+            raise ValueError("Only one of either 'options' or 'flags' can be specified.")
+        if self.options is None and self.flags is None:
+            raise ValueError("'options' is required.")
+
+        if self.flags is not None:
+            _logger.warning("'options' is preferred over 'flags' as this may be removed in a future release")
+            self.options = self.flags
+            self.flags = None

{cognite_extractor_utils-7.4.5 → cognite_extractor_utils-7.4.7}/cognite/extractorutils/configtools/loaders.py

@@ -22,7 +22,7 @@ import sys
 from enum import Enum
 from hashlib import sha256
 from pathlib import Path
-from typing import Any, Callable, Dict, Generic, Iterable, Optional, TextIO, Type, TypeVar, Union, cast
+from typing import Any, Callable, Dict, Generic, Iterable, List, Optional, TextIO, Type, TypeVar, Union, cast
 
 import dacite
 import yaml
@@ -37,6 +37,7 @@ from cognite.extractorutils.configtools._util import _to_snake_case
 from cognite.extractorutils.configtools.elements import (
     BaseConfig,
     ConfigType,
+    IgnorePattern,
     TimeIntervalConfig,
     _BaseConfig,
 )
@@ -320,6 +321,25 @@ def load_yaml_dict(
     )
 
 
+def compile_patterns(ignore_patterns: List[Union[str, IgnorePattern]]) -> list[re.Pattern[str]]:
+    """
+    List of patterns to compile
+
+    Args:
+        ignore_patterns: A list of strings or IgnorePattern to be compiled.
+
+    Returns:
+        A list of compiled RegExp patterns.
+    """
+    compiled = []
+    for p in ignore_patterns:
+        if isinstance(p, IgnorePattern):
+            compiled.append(re.compile(p.compile()))
+        else:
+            compiled.append(re.compile(p))
+    return compiled
+
+
 class ConfigResolver(Generic[CustomConfigClass]):
     def __init__(self, config_path: str, config_type: Type[CustomConfigClass]):
         self.config_path = config_path

cognite_extractor_utils-7.4.7/cognite/extractorutils/configtools/validators.py

@@ -0,0 +1,37 @@
+import logging
+import re
+from typing import Union
+
+_logger = logging.getLogger(__name__)
+
+
+def matches_patterns(patterns: list[Union[str, re.Pattern[str]]], string: str) -> bool:
+    """
+    Check string against list of RegExp patterns.
+
+    Args:
+        patterns: A list of (re) patterns to match string against.
+        string: String to which we match the pattern.
+
+    Returns:
+        boolean value indicating whether string matches any of the patterns.
+    """
+    return any([matches_pattern(pattern, string) for pattern in patterns])
+
+
+def matches_pattern(pattern: Union[str, re.Pattern[str]], string: str) -> bool:
+    """
+    Match pattern against a string.
+
+    Args:
+        pattern: (re) Pattern to match against a string.
+        string: String to which we match the pattern.
+
+    Returns:
+        boolean value indicating a match or otherwise.
+    """
+    try:
+        return re.search(pattern, string) is not None
+    except re.error as e:
+        _logger.warning(f"Could not apply RegExp: {pattern}\nReason: {e}")
+        return False

{cognite_extractor_utils-7.4.5 → cognite_extractor_utils-7.4.7}/cognite/extractorutils/uploader/files.py

@@ -255,10 +255,20 @@ class IOFileUploadQueue(AbstractUploadQueue):
     ) -> tuple[FileMetadataOrCogniteExtractorFile, str]:
         if isinstance(file_meta, CogniteExtractorFileApply):
             node_id = self._apply_cognite_file(file_meta)
-            file_meta, url = self._create_cdm(instance_id=node_id)
+            file_meta_response, url = self._create_cdm(instance_id=node_id)
         else:
-            file_meta, url = self.cdf_client.files.create(file_metadata=file_meta, overwrite=self.overwrite_existing)
-        return file_meta, url
+            file_meta_response, url = self.cdf_client.files.create(
+                file_metadata=file_meta, overwrite=self.overwrite_existing
+            )
+            # trigger update after creation (upsert =P)
+            basic_attributes = set(["externalId", "name"])
+            attr = set(file_meta.dump().keys())
+            diff = attr - basic_attributes
+
+            if len(diff) >= 1 and "externalId" in attr:
+                file_meta_response = self.cdf_client.files.update(file_meta)
+
+        return file_meta_response, url
 
     def _upload_bytes(self, size: int, file: BinaryIO, file_meta: FileMetadataOrCogniteExtractorFile) -> None:
         file_meta, url = self._upload_empty(file_meta)

{cognite_extractor_utils-7.4.5 → cognite_extractor_utils-7.4.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite-extractor-utils"
-version = "7.4.5"
+version = "7.4.7"
 description = "Utilities for easier development of extractors for CDF"
 authors = ["Mathias Lohne <mathias.lohne@cognite.com>"]
 license = "Apache-2.0"
@@ -93,7 +93,7 @@ parameterized = "*"
 requests = "^2.31.0"
 types-requests = "^2.31.0.20240125"
 httpx = "^0.27.0"
-faker = "^28.0.0"
+faker = "^29.0.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]