cognite-neat 0.70.1__py3-none-any.whl → 0.127.19__py3-none-any.whl

cognite/neat/v0/core/_utils/spreadsheet.py

@@ -0,0 +1,182 @@
+import warnings
+from dataclasses import dataclass, field
+from typing import Any, Literal, cast, overload
+from warnings import catch_warnings
+
+import pandas as pd
+from openpyxl import load_workbook
+from openpyxl.worksheet.datavalidation import DataValidation
+from openpyxl.worksheet.worksheet import Worksheet
+
+from cognite.neat.v0.core._data_model._constants import get_internal_properties
+
+
+@dataclass
+class SpreadsheetRead:
+    """This class is used to store information about the source spreadsheet.
+
+    It is used to adjust row numbers to account for header rows and empty rows
+    such that the error/warning messages are accurate.
+    """
+
+    header_row: int = 1
+    empty_rows: list[int] = field(default_factory=list)
+    skipped_rows: list[int] = field(default_factory=list)
+    is_one_indexed: bool = True
+
+    def __post_init__(self) -> None:
+        self.empty_rows = sorted(self.empty_rows)
+
+    def adjusted_row_number(self, row_no: int) -> int:
+        output = row_no
+        for empty_row in self.empty_rows:
+            if empty_row <= output:
+                output += 1
+            else:
+                break
+
+        for skipped_rows in self.skipped_rows:
+            if skipped_rows <= output:
+                output += 1
+            else:
+                break
+
+        return output + self.header_row + (1 if self.is_one_indexed else 0)
+
+
+@overload
+def read_individual_sheet(
+    excel_file: pd.ExcelFile,
+    sheet_name: str,
+    return_read_info: Literal[True],
+    expected_headers: list[str] | None = None,
+) -> tuple[list[dict], SpreadsheetRead]: ...
+
+
+@overload
+def read_individual_sheet(
+    excel_file: pd.ExcelFile,
+    sheet_name: str,
+    return_read_info: Literal[False] = False,
+    expected_headers: list[str] | None = None,
+) -> list[dict]: ...
+
+
+def read_individual_sheet(
+    excel_file: pd.ExcelFile,
+    sheet_name: str,
+    return_read_info: bool = False,
+    expected_headers: list[str] | None = None,
+) -> tuple[list[dict], SpreadsheetRead] | list[dict]:
+    if expected_headers:
+        with catch_warnings():
+            # When reading spreadsheets produced by neat, they contain dropdowns. These
+            # are not supported by openpyxl and will raise a warning as openpyxl cannot validate these.
+            # We ignore these warnings as Neat will do the same checks.
+            warnings.simplefilter("ignore")
+            target_row = _get_row_number(cast(Worksheet, load_workbook(excel_file)[sheet_name]), expected_headers)
+        skiprows = target_row - 1 if target_row is not None else 0
+    else:
+        skiprows = 0
+
+    with catch_warnings():
+        # When reading spreadsheets produced by neat, they contain dropdowns. These
+        # are not supported by openpyxl and will raise a warning as openpyxl cannot validate these.
+        # We ignore these warnings as Neat will do the same checks.
+        warnings.simplefilter("ignore")
+        raw = pd.read_excel(excel_file, sheet_name, skiprows=skiprows)
+    is_na = raw.isnull().all(axis=1)
+    skip_rows = _find_rows_to_skip(raw)
+    empty_rows = is_na[is_na].index.tolist()
+
+    if skip_rows:
+        raw = raw.drop(skip_rows)
+
+    raw.dropna(axis=0, how="all", inplace=True)
+
+    if "Value Type" in raw.columns:
+        # Special handling for Value Type column, #N/A is treated specially by NEAT it means Unknown
+        raw["Value Type"] = raw["Value Type"].replace(float("nan"), "#N/A")
+
+    if "Concept" in raw.columns:
+        # Special handling for Concept column, #N/A is treated specially by NEAT it means Unknown
+        raw["Concept"] = raw["Concept"].replace(float("nan"), "#N/A")
+
+    output = raw.replace(float("nan"), None).to_dict(orient="records")
+    if return_read_info:
+        # If no rows are skipped, row 1 is the header row.
+        return output, SpreadsheetRead(
+            header_row=skiprows + 1,
+            empty_rows=empty_rows,
+            is_one_indexed=True,
+            skipped_rows=skip_rows,
+        )
+    return output
+
+
+def _find_rows_to_skip(
+    df: pd.DataFrame,
+) -> list:
+    """Find rows which are having all values as None except for internal properties."""
+    rows_to_skip = []
+
+    internal_cols = {val.lower() for val in get_internal_properties()}
+    for i, row in df.iterrows():
+        user_cols_state = []
+        internal_cols_state = []
+        for col in df.columns:
+            if col.lower() not in internal_cols:
+                user_cols_state.append(row[col] == "#N/A" or row[col].__str__().lower() in ["none", "nan"])
+            else:
+                internal_cols_state.append(row[col] is not None)
+
+        if all(user_cols_state) and any(internal_cols_state):
+            rows_to_skip.append(i)
+
+    return rows_to_skip
+
+
+def _get_row_number(sheet: Worksheet, values_to_find: list[str]) -> int | None:
+    for row_number, row in enumerate(sheet.iter_rows(values_only=True), start=1):
+        if any(value in row for value in values_to_find):
+            return row_number
+    return None
+
+
+@overload
+def find_column_and_row_with_value(
+    sheet: Worksheet, value: Any, column_letter: Literal[True] = True
+) -> tuple[str, int] | tuple[None, None]: ...
+
+
+@overload
+def find_column_and_row_with_value(
+    sheet: Worksheet, value: Any, column_letter: Literal[False]
+) -> tuple[int, int] | tuple[None, None]: ...
+
+
+def find_column_and_row_with_value(
+    sheet: Worksheet, value: Any, column_letter: bool = True
+) -> tuple[int, int] | tuple[str, int] | tuple[None, None]:
+    for row in sheet.iter_rows():
+        for cell in row:
+            if cell.value and isinstance(cell.value, str) and cell.value.lower() == value.lower():
+                return (cell.column_letter, cell.row) if column_letter else (cell.column, cell.row)
+
+    return None, None
+
+
+def generate_data_validation(sheet: str, column: str, total_header_rows: int, validation_range: int) -> DataValidation:
+    """Creates openpyxl data validation object for a cell in a sheet
+
+    Args:
+        sheet: The name of the sheet where the data validation is applied.
+        column: The column letter where the data validation is applied.
+        total_header_rows: The number of header rows in the sheet.
+        validation_range: The total number of validation values in the column.
+    """
+
+    return DataValidation(
+        type="list",
+        formula1=f"={sheet}!{column}${total_header_rows + 1}:{column}${validation_range + total_header_rows + 1}",
+    )

cognite/neat/v0/core/_utils/tarjan.py

@@ -0,0 +1,44 @@
+from typing import TypeVar
+
+T = TypeVar("T")
+
+
+def tarjan(dependencies_by_id: dict[T, set[T]]) -> list[set[T]]:
+    """Returns the strongly connected components of the dependency graph
+     in topological order.
+    Args:
+        dependencies_by_id: A dictionary where the keys are ids and the values are sets of ids that the key depends on.
+    Returns:
+        A list of sets of ids that are strongly connected components in the dependency graph.
+    """
+
+    stack = []
+    stack_set = set()
+    index: dict[T, int] = {}
+    lowlink = {}
+    result = []
+
+    def visit(v: T) -> None:
+        index[v] = len(index)
+        lowlink[v] = index[v]
+        stack.append(v)
+        stack_set.add(v)
+        for w in dependencies_by_id.get(v, []):
+            if w not in index:
+                visit(w)
+                lowlink[v] = min(lowlink[w], lowlink[v])
+            elif w in stack_set:
+                lowlink[v] = min(lowlink[v], index[w])
+        if lowlink[v] == index[v]:
+            scc = set()
+            dependency: T | None = None
+            while v != dependency:
+                dependency = stack.pop()
+                scc.add(dependency)
+                stack_set.remove(dependency)
+            result.append(scc)
+
+    for view_id in dependencies_by_id.keys():
+        if view_id not in index:
+            visit(view_id)
+    return result

cognite/neat/v0/core/_utils/text.py

@@ -0,0 +1,277 @@
+import re
+import urllib.parse
+from collections.abc import Collection, Set
+from re import Pattern
+from typing import Any
+
+from cognite.neat.v0.core._data_model._constants import get_reserved_words
+
+PREPOSITIONS = frozenset(
+    {
+        "in",
+        "on",
+        "at",
+        "by",
+        "for",
+        "with",
+        "about",
+        "against",
+        "between",
+        "into",
+        "through",
+        "during",
+        "before",
+        "after",
+        "above",
+        "below",
+        "to",
+        "from",
+        "up",
+        "down",
+        "out",
+        "off",
+        "over",
+        "under",
+        "again",
+        "further",
+        "then",
+        "once",
+    }
+)
+
+
+def to_camel_case(string: str) -> str:
+    """Convert snake_case_name to camelCaseName.
+
+    Args:
+        string: The string to convert.
+    Returns:
+        camelCase of the input string.
+
+    Examples:
+        >>> to_camel_case("a_b")
+        'aB'
+        >>> to_camel_case("ScenarioInstance_priceForecast")
+        'scenarioInstancePriceForecast'
+    """
+    string = re.sub(r"[^a-zA-Z0-9_]", "_", string)
+    string = re.sub("_+", "_", string)
+    is_all_upper = string.upper() == string
+    is_first_upper = (
+        len(string) >= 2 and string[:2].upper() == string[:2] and "_" not in string[:2] and not is_all_upper
+    )
+    return _to_camel_case(string, is_all_upper, is_first_upper)
+
+
+def _to_camel_case(string: str, is_all_upper: bool, is_first_upper: bool) -> str:
+    if "_" in string:
+        pascal_splits = [
+            _to_pascal_case(part, is_all_upper, is_first_upper and no == 0)
+            for no, part in enumerate(string.split("_"), 0)
+        ]
+    else:
+        # Ensure pascal
+        if string:
+            string = string[0].upper() + string[1:]
+        pascal_splits = [string]
+    cleaned: list[str] = []
+    for part in pascal_splits:
+        if part.upper() == part and is_all_upper:
+            cleaned.append(part.capitalize())
+        else:
+            cleaned.append(part)
+
+    string_split = []
+    for part in cleaned:
+        string_split.extend(re.findall(r"[A-Z][a-z0-9]*", part))
+    if not string_split:
+        string_split = [string]
+    if len(string_split) == 0:
+        return ""
+    # The first word is a single letter, keep the original case
+    if is_first_upper:
+        return "".join(word for word in string_split)
+    else:
+        return string_split[0].casefold() + "".join(word for word in string_split[1:])
+
+
+def to_pascal_case(string: str) -> str:
+    """Convert string to PascalCaseName.
+
+    Args:
+        string: The string to convert.
+    Returns:
+        PascalCase of the input string.
+
+    Examples:
+        >>> to_pascal_case("a_b")
+        'AB'
+        >>> to_pascal_case('camel_case')
+        'CamelCase'
+    """
+    return _to_pascal_case(string, string == string.upper(), True)
+
+
+def _to_pascal_case(string: str, is_all_upper: bool, is_first_upper: bool) -> str:
+    camel = _to_camel_case(string, is_all_upper, is_first_upper)
+    return f"{camel[0].upper()}{camel[1:]}" if camel else ""
+
+
+def to_snake_case(string: str) -> str:
+    """
+    Convert input string to snake_case
+
+    Args:
+        string: The string to convert.
+    Returns:
+        snake_case of the input string.
+
+    Examples:
+        >>> to_snake_case("aB")
+        'a_b'
+        >>> to_snake_case('CamelCase')
+        'camel_case'
+        >>> to_snake_case('camelCamelCase')
+        'camel_camel_case'
+        >>> to_snake_case('Camel2Camel2Case')
+        'camel_2_camel_2_case'
+        >>> to_snake_case('getHTTPResponseCode')
+        'get_http_response_code'
+        >>> to_snake_case('get200HTTPResponseCode')
+        'get_200_http_response_code'
+        >>> to_snake_case('getHTTP200ResponseCode')
+        'get_http_200_response_code'
+        >>> to_snake_case('HTTPResponseCode')
+        'http_response_code'
+        >>> to_snake_case('ResponseHTTP')
+        'response_http'
+        >>> to_snake_case('ResponseHTTP2')
+        'response_http_2'
+        >>> to_snake_case('Fun?!awesome')
+        'fun_awesome'
+        >>> to_snake_case('Fun?!Awesome')
+        'fun_awesome'
+        >>> to_snake_case('10CoolDudes')
+        '10_cool_dudes'
+        >>> to_snake_case('20coolDudes')
+        '20_cool_dudes'
+    """
+    pattern = re.compile(r"[A-Z]?[a-z]+|[A-Z]+(?=[A-Z][a-z]|\d|\W|$)|\d+")
+    if "_" in string:
+        words = [word for section in string.split("_") for word in pattern.findall(section)]
+    else:
+        words = pattern.findall(string)
+    return "_".join(map(str.lower, words))
+
+
+def to_words(string: str) -> str:
+    """Converts snake_case camelCase or PascalCase to words."""
+    return to_snake_case(string).replace("_", " ")
+
+
+def title(text: str, skip_words: Set[str] = PREPOSITIONS) -> str:
+    """Converts text to title case, skipping prepositions."""
+    words = (word.lower() for word in text.split())
+    titled_words = (word.capitalize() if word not in skip_words else word for word in words)
+    return " ".join(titled_words)
+
+
+def sentence_or_string_to_camel(string: str) -> str:
+    # Could be a combination of kebab and pascal/camel case
+    if " " in string:
+        parts = string.split(" ")
+        try:
+            return parts[0].casefold() + "".join(word.capitalize() for word in parts[1:])
+        except IndexError:
+            return ""
+    else:
+        return to_camel_case(string)
+
+
+def replace_non_alphanumeric_with_underscore(text: str) -> str:
+    return re.sub(r"\W+", "_", text)
+
+
+def humanize_collection(collection: Collection[Any], /, *, sort: bool = True, bind_word: str = "and") -> str:
+    """Convert a collection of items to a human-readable string.
+
+    Args:
+        collection: The collection of items to convert.
+        sort: Whether to sort the collection before converting. Default is True.
+        bind_word: The word to use to bind the last two items. Default is "and".
+
+    Returns:
+        A human-readable string of the collection.
+
+    Examples:
+        >>> humanize_collection(["b", "c", "a"])
+        'a, b and c'
+        >>> humanize_collection(["b", "c", "a"], sort=False)
+        'b, c and a'
+        >>> humanize_collection(["a", "b"])
+        'a and b'
+        >>> humanize_collection(["a"])
+        'a'
+        >>> humanize_collection([])
+
+    """
+    if not collection:
+        return ""
+    elif len(collection) == 1:
+        return str(next(iter(collection)))
+
+    strings = (str(item) for item in collection)
+    if sort:
+        sequence = sorted(strings)
+    else:
+        sequence = list(strings)
+
+    return f"{', '.join(sequence[:-1])} {bind_word} {sequence[-1]}"
+
+
+class NamingStandardization:
+    _letter_number_underscore = re.compile(r"[^a-zA-Z0-9_]+")
+    _letter_number_underscore_hyphen = re.compile(r"[^a-zA-Z0-9_-]+")
+    _multi_underscore_pattern = re.compile(r"_+")
+    _start_letter_pattern = re.compile(r"^[a-zA-Z]")
+
+    @classmethod
+    def standardize_concept_str(cls, raw: str) -> str:
+        clean = cls._clean_string(raw)
+        if not cls._start_letter_pattern.match(clean):
+            # Underscore ensure that 'Class' it treated as a separate word
+            # in the to_pascale function
+            clean = f"Class_{clean}"
+        return to_pascal_case(clean)
+
+    @classmethod
+    def standardize_property_str(cls, raw: str) -> str:
+        clean = cls._clean_string(raw)
+        if not cls._start_letter_pattern.match(clean):
+            # Underscore ensure that 'property' it treated as a separate word
+            # in the to_camel function
+            clean = f"property_{clean}"
+        return to_camel_case(clean)
+
+    @classmethod
+    def standardize_space_str(cls, raw: str) -> str:
+        clean = cls._clean_string(raw, cls._letter_number_underscore_hyphen)
+        if not cls._start_letter_pattern.match(clean):
+            clean = f"sp_{clean}"
+        if clean in set(get_reserved_words("space")):
+            clean = f"my_{clean}"
+        if len(clean) > 43:
+            clean = clean[:43]
+        if not (clean[-1].isalnum()) and len(clean) == 43:
+            clean = f"{clean[:-1]}x"
+        elif not clean[-1].isalnum():
+            clean = f"{clean}x"
+        if not clean:
+            raise ValueError("Space name must contain at least one letter.")
+        return to_snake_case(clean)
+
+    @classmethod
+    def _clean_string(cls, raw: str, clean_pattern: Pattern[str] = _letter_number_underscore) -> str:
+        raw = urllib.parse.unquote(raw)
+        raw = clean_pattern.sub("_", raw)
+        return cls._multi_underscore_pattern.sub("_", raw)

cognite/neat/v0/core/_utils/time_.py

@@ -0,0 +1,17 @@
+import sys
+from datetime import datetime
+
+if sys.version_info >= (3, 11):
+    from datetime import UTC
+else:
+    from datetime import timezone
+
+    UTC = timezone.utc
+
+
+def epoch_now_ms() -> int:
+    return int((datetime.now(UTC) - datetime(1970, 1, 1, tzinfo=UTC)).total_seconds() * 1000)
+
+
+def datetime_utc_now() -> datetime:
+    return datetime.now(UTC)

cognite/neat/v0/core/_utils/upload.py

@@ -0,0 +1,134 @@
+from abc import ABC
+from dataclasses import dataclass, field
+from functools import total_ordering
+from typing import Any, Generic
+
+from cognite.neat.v0.core._issues import IssueList
+from cognite.neat.v0.core._issues.errors import NeatValueError
+from cognite.neat.v0.core._shared import T_ID, NeatList, NeatObject
+
+
+@total_ordering
+@dataclass
+class UploadResultCore(NeatObject, ABC):
+    name: str
+    error_messages: list[str] = field(default_factory=list)
+    issues: IssueList = field(default_factory=IssueList)
+
+    def __lt__(self, other: object) -> bool:
+        if isinstance(other, UploadResultCore):
+            return self.name < other.name
+        else:
+            return NotImplemented
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, UploadResultCore):
+            return self.name == other.name
+        else:
+            return NotImplemented
+
+    def dump(self, aggregate: bool = True) -> dict[str, Any]:
+        output: dict[str, Any] = {"name": self.name}
+        if self.error_messages:
+            output["error_messages"] = len(self.error_messages) if aggregate else self.error_messages
+        if self.issues:
+            output["issues"] = len(self.issues) if aggregate else [issue.dump() for issue in self.issues]
+        return output
+
+
+class UploadResultList(NeatList[UploadResultCore]):
+    def _repr_html_(self) -> str:
+        df = self.to_pandas().fillna(0)
+        df = df.style.format({column: "{:,.0f}".format for column in df.select_dtypes(include="number").columns})
+        return df._repr_html_()  # type: ignore[attr-defined]
+
+
+@dataclass
+class UploadResult(UploadResultCore, Generic[T_ID]):
+    created: set[T_ID] = field(default_factory=set)
+    upserted: set[T_ID] = field(default_factory=set)
+    deleted: set[T_ID] = field(default_factory=set)
+    changed: set[T_ID] = field(default_factory=set)
+    unchanged: set[T_ID] = field(default_factory=set)
+    skipped: set[T_ID] = field(default_factory=set)
+    failed_created: set[T_ID] = field(default_factory=set)
+    failed_upserted: set[T_ID] = field(default_factory=set)
+    failed_changed: set[T_ID] = field(default_factory=set)
+    failed_deleted: set[T_ID] = field(default_factory=set)
+    failed_items: list = field(default_factory=list)
+
+    @property
+    def failed(self) -> int:
+        return (
+            len(self.failed_created) + len(self.failed_changed) + len(self.failed_deleted) + len(self.failed_upserted)
+        )
+
+    @property
+    def success(self) -> int:
+        return (
+            len(self.created)
+            + len(self.deleted)
+            + len(self.changed)
+            + len(self.upserted)
+            + len(self.unchanged)
+            + len(self.skipped)
+        )
+
+    def dump(self, aggregate: bool = True) -> dict[str, Any]:
+        output = super().dump(aggregate)
+        if self.created:
+            output["created"] = len(self.created) if aggregate else list(self.created)
+        if self.upserted:
+            output["upserted"] = len(self.upserted) if aggregate else list(self.upserted)
+        if self.deleted:
+            output["deleted"] = len(self.deleted) if aggregate else list(self.deleted)
+        if self.changed:
+            output["changed"] = len(self.changed) if aggregate else list(self.changed)
+        if self.unchanged:
+            output["unchanged"] = len(self.unchanged) if aggregate else list(self.unchanged)
+        if self.skipped:
+            output["skipped"] = len(self.skipped) if aggregate else list(self.skipped)
+        if self.failed_created:
+            output["failed_created"] = len(self.failed_created) if aggregate else list(self.failed_created)
+        if self.failed_upserted:
+            output["failed_upserted"] = len(self.failed_upserted) if aggregate else list(self.failed_upserted)
+        if self.failed_changed:
+            output["failed_changed"] = len(self.failed_changed) if aggregate else list(self.failed_changed)
+        if self.failed_deleted:
+            output["failed_deleted"] = len(self.failed_deleted) if aggregate else list(self.failed_deleted)
+        if "error_messages" in output:
+            # Trick to move error_messages to the end of the dict
+            output["error_messages"] = output.pop("error_messages")
+        if "issues" in output:
+            # Trick to move issues to the end of the dict
+            output["issues"] = output.pop("issues")
+        return output
+
+    def __str__(self) -> str:
+        dumped = self.dump(aggregate=True)
+        lines: list[str] = []
+        for key, value in dumped.items():
+            if key in ["name", "error_messages", "issues"]:
+                continue
+            lines.append(f"{key}: {value}")
+        return f"{self.name.title()}: {', '.join(lines)}"
+
+    def merge(self, other: "UploadResult[T_ID]") -> "UploadResult[T_ID]":
+        if self.name != other.name:
+            raise NeatValueError(f"Cannot merge UploadResults with different names: {self.name} and {other.name}")
+        return UploadResult(
+            name=self.name,
+            error_messages=self.error_messages + other.error_messages,
+            issues=IssueList(self.issues + other.issues),
+            created=self.created.union(other.created),
+            upserted=self.upserted.union(other.upserted),
+            deleted=self.deleted.union(other.deleted),
+            changed=self.changed.union(other.changed),
+            unchanged=self.unchanged.union(other.unchanged),
+            skipped=self.skipped.union(other.skipped),
+            failed_created=self.failed_created.union(other.failed_created),
+            failed_upserted=self.failed_upserted.union(other.failed_upserted),
+            failed_changed=self.failed_changed.union(other.failed_changed),
+            failed_deleted=self.failed_deleted.union(other.failed_deleted),
+            failed_items=self.failed_items + other.failed_items,
+        )