otcfinutils 0.0.24.15__tar.gz

otcfinutils-0.0.24.15/OTCFinUtils/__init__.py

@@ -0,0 +1,69 @@
+from OTCFinUtils.dataverse_handler import (
+    DVHandler,
+    get_dv_handler,
+    set_up_dv_handler,
+)
+from OTCFinUtils.buffer import Buffer
+from OTCFinUtils.data_structs import (
+    ResponseIds,
+    APIParams,
+    HTTPMethod,
+    KeyLookupParams,
+    EntityKeyLookupParams,
+    IDParams,
+    EntityLookupParams,
+    Category,
+    Lookup,
+    EntityLookups,
+    MappingFields,
+    Operator,
+    Condition,
+    DEFAULT_SHEET_CHUNK_SIZE,
+    SHEETS_RULE_TYPE_CHOICE,
+    FileSource,
+)
+from OTCFinUtils.mapper_loading import (
+    _extract_key_api_params,
+    _extract_batch_response_objects,
+    _extract_object_ids,
+    _extract_object_lookups,
+    _extract_curr_object_lookups,
+    get_buffer_lookups,
+    get_buffer_ids,
+)
+from OTCFinUtils.mapper import DataMapper
+from OTCFinUtils.security import (
+    get_client_credential,
+    get_dataverse_token,
+    get_graph_token,
+)
+from OTCFinUtils.sharepoint import (
+    load_document,
+    get_connection_details,
+    create_excel_document,
+    update_excel,
+)
+from OTCFinUtils.utils import (
+    extract_segment_object_line,
+    extract_segment_response_id,
+    extract_segment_status_code,
+    extract_segment_table_name,
+    process_data_type,
+    extract_lookup_column,
+    extract_file_data,
+    data_equals_exclude_value,
+    trim_string_values,
+    is_date,
+    extract_date_string,
+    choice_value,
+    normalize_choice_map,
+    default_value,
+    cast_data,
+    get_skip_flag,
+    data_is_empty,
+    evaluate_condition,
+    is_valid,
+    group_mappings,
+    extract_sorted_categories,
+    extract_sheets,
+)

otcfinutils-0.0.24.15/OTCFinUtils/buffer.py

@@ -0,0 +1,103 @@
+from OTCFinUtils.data_structs import (
+    EntityKeyLookupParams, 
+    EntityLookupParams, 
+    IDParams,
+)
+
+
+class Buffer:
+    def __init__(self) -> None:
+        """
+        Buffers for temporarily storing entity information.
+        Need to use them for the batch reading / writing operations.
+        """
+        self.entity_data:                dict = dict()
+        self.entity_id_params:           IDParams = IDParams()
+        self.entity_lookup_params:       dict[str, IDParams] = dict()
+        self.entity_key_lookup_params:   EntityKeyLookupParams = dict()
+
+        """
+        Buffers for temporarily storing information for all entities in a category.
+        Need to use them for the batch reading / writing operations.
+        """
+        self.category_data:              list[dict] = []
+        self.category_id_params:         list[IDParams] = []
+        self.category_lookup_params:     list[EntityLookupParams] = []
+        self.category_key_lookup_params: list[EntityKeyLookupParams] = []
+
+
+    def _load_id_params_buffer(self) -> None:
+        """
+        Loads the buffer for the entity to the buffer for the category.
+        This buffer is used for a batch query for the ids of the entities.
+        Resets the entity buffer.
+        """
+        self.category_id_params.append(self.entity_id_params)
+        self.entity_id_params = IDParams()
+
+
+    def _load_data_buffer(self) -> None:
+        """
+        Loads the buffer for the entity to the buffer for the category.
+        This is the buffer that will be bulk-upserted in the end.
+        Resets the entity buffer.
+        """
+        self.category_data.append(self.entity_data)
+        self.entity_data = dict()
+
+
+    def _load_lookup_params_buffer(self) -> None:
+        """
+        Loads the lookup buffer for the entity, which is a dictionary,
+        into the lookup buffer for the category. Transforms the dictionary 
+        to a list, because that's how the the DV Handler requires it.
+        Resets the entity buffer.
+        """
+        entity_lookup_params_list = list(self.entity_lookup_params.values())
+        self.category_lookup_params.append(entity_lookup_params_list)
+        self.entity_lookup_params = dict()
+
+
+    def _load_key_lookup_params_buffer(self) -> None:
+        """
+        Loads the key-lookup params for the entity into
+        the key-lookup buffer for the category of data.
+        """
+        self.category_key_lookup_params.append(self.entity_key_lookup_params)
+        self.entity_key_lookup_params = dict()
+
+
+    def reset_entity_buffers(self) -> None:
+        """
+        Resets all buffers for the current entity.
+        Used in the entity cycle.
+        """
+        self.entity_data = dict()
+        self.entity_id_params = IDParams()
+        self.entity_lookup_params = dict()
+        self.entity_key_lookup_params = dict()
+
+
+    def reset_category_buffers(self):
+        """
+        Reset all category-level buffers.
+        """
+        self.category_id_params = []
+        self.category_lookup_params = []
+        self.category_key_lookup_params = []
+        self.category_data = []
+
+
+    def load_buffers(self) -> None:
+        """
+        Because we will query and upsert the data in batches,
+        we need to store everything in our buffers.
+        """
+        try:
+            self._load_lookup_params_buffer()
+            self._load_id_params_buffer()
+            self._load_data_buffer()
+            self._load_key_lookup_params_buffer()
+        
+        except Exception as e:
+            raise RuntimeError(f"Error while loading buffers: {e}")

otcfinutils-0.0.24.15/OTCFinUtils/data_structs.py

@@ -0,0 +1,242 @@
+from http import HTTPStatus as status
+from enum import Enum
+from dataclasses import dataclass, field
+from typing import Optional, Any
+from collections import namedtuple
+
+
+@dataclass
+class ResponseIds:
+    """
+    Used as a return value from the upsert operation on the DV.
+    Contains the table names and ids of the entities that were 
+    updated and created, stored separately.
+    """
+    def __init__(self) -> None:
+        self.updated_entity_ids: dict[str, set[str]] = dict()
+        self.created_entity_ids: dict[str, set[str]] = dict()
+        self.failed_operations: list[str] = list()
+
+    
+    def append(self, status_code: int, id: str, table: str) -> None:
+        """
+        Depending on the status code appends the id and 
+        table name to the create / update category.
+        """
+        if status_code == status.OK.value and table in self.updated_entity_ids:
+            self.updated_entity_ids[table].add(id)
+        
+        elif status_code == status.OK.value and table not in self.updated_entity_ids:
+            self.updated_entity_ids[table] = set()
+            self.updated_entity_ids[table].add(id)
+        
+        elif status_code == status.CREATED.value and table in self.created_entity_ids:
+            self.created_entity_ids[table].add(id)
+        
+        elif status_code == status.CREATED.value and table not in self.created_entity_ids:
+            self.created_entity_ids[table] = set()
+            self.created_entity_ids[table].add(id)
+        
+        return
+    
+
+    def merge(self, delta_ids: "ResponseIds") -> None:
+        """
+        Merges the values from the provided object onto the
+        the values of the calling object.
+        """
+        for table, ids in delta_ids.created_entity_ids.items():
+            if table not in self.created_entity_ids:
+                self.created_entity_ids[table] = set()
+            self.created_entity_ids[table].update(ids)
+
+        for table, ids in delta_ids.updated_entity_ids.items():
+            if table not in self.updated_entity_ids:
+                self.updated_entity_ids[table] = set()
+            self.updated_entity_ids[table].update(ids)
+
+        self.failed_operations.extend(delta_ids.failed_operations)
+
+        return
+    
+
+    def json_updated_entities(self) -> dict[str, list]:
+        result = dict()
+        for table, ids in self.updated_entity_ids.items():
+            result[table] = list(ids)
+        return result
+    
+
+    def json_created_entities(self) -> dict[str, list]:
+        result = dict()
+        for table, ids in self.created_entity_ids.items():
+            result[table] = list(ids)
+        return result
+    
+
+    def num_updated_entities(self) -> int:
+        total = 0
+        for ids in self.updated_entity_ids.values():
+            total += len(ids)
+        return total
+    
+    def num_created_entities(self) -> int:
+        total = 0
+        for ids in self.created_entity_ids.values():
+            total += len(ids)
+        return total
+    
+    def num_failed_operations(self) -> int:
+        return len(self.failed_operations)
+    
+
+class FileSource(Enum):
+    DV = "DV"
+    SP = "SP"
+
+
+@dataclass
+class APIParams:
+    """
+    Basic helper data structure for making API calls.
+    """
+    url: str
+    method: str
+    data: dict | None = None
+
+
+class QueryOperator(Enum):
+    AND = "and"
+    OR = "or"
+
+
+@dataclass
+class QueryParams:
+    table: str
+    columns: Optional[list[str]] = None
+    conditions: Optional[list[str]] = None
+    operator: QueryOperator = QueryOperator.AND
+    top_num: Optional[int] = None
+
+
+class HTTPMethod(Enum):
+    GET = "GET"
+    POST = "POST"
+    PATCH = "PATCH"
+    DELETE = "DELETE"
+
+
+@dataclass
+class KeyLookupParams:
+    lookup_id_column: str
+    lookup_id: Optional[str]
+
+
+EntityKeyLookupParams = dict[str, KeyLookupParams]
+
+
+@dataclass
+class IDParams:
+    """
+    A group of parameters used for querying id info of entities.
+    """
+    table: Optional[str] = None
+    id_column: Optional[str] = None
+    conditions: list[str] = field(default_factory=list)
+    
+    # If we have a lookup
+    bind_column: Optional[str] = None
+    
+
+EntityLookupParams = list[IDParams]
+
+"""
+Used to divide and sort the file loading mappings
+"""
+Category = namedtuple("Category", ["group_num", "table", "sheet"])
+
+"""
+Used for storing the id info of the lookup entity
+"""
+Lookup = namedtuple("Lookup", ["bind_column", "table", "id"])
+
+"""
+Needed because each entity can have several lookups to other entities
+"""
+EntityLookups = list[Lookup]
+
+
+class MappingFields:
+    LOOKUP_TABLE = "new_lookuptable"
+    LOOKUP_RELATIONSHIP = "new_lookuprelationship"
+    LOOKUP_COLUMN = "new_lookupcolumn"
+    DV_COLUMN = "new_dataversecolumnname"
+    DV_TABLE = "new_dataversetablenametext"
+    FILE_COLUMN = "new_filecolumnname"
+    DEFAULT_VALUE = "new_defaultvalue"
+    DATA_TYPE = "new_dataversedatatype"
+    IS_LOOKUP = "new_islookup"
+    IS_KEY = "new_iskey"
+    IS_CHOICE = "new_ischoice"
+    CHOICE_DICTIONARY = "new_choicedictionary"
+    CONDITION = "new_condition"
+    ALTERNATIVE_FILE_COLUMN = "new_alternativefilecolumnname"
+    EXCLUDE_VALUE = "new_excludevalue"
+    MAP_GROUP_ORDER = "new_mapgrouporder"
+    SHEET = "new_sheetname"
+
+
+@dataclass
+class Mapping:
+    dv_column: str
+    dv_table: str
+    file_column: str
+    map_group_order: int
+    sheet: str
+    default_value: Optional[str] = None
+    data_type: Optional[str] = None
+    lookup_table: Optional[str] = None
+    lookup_relationship: Optional[str] = None
+    lookup_column: Optional[str] = None
+    is_lookup: bool = False
+    is_key: bool = False
+    is_choice: bool = False
+    choice_dictionary: Optional[str] = None
+    condition: Optional[str] = None
+    alternative_file_column: Optional[str] = None
+    exclude_value: Optional[str] = None
+
+
+class Operator(Enum):
+    EQUALS = "EQUALS"
+    NOT_EQUALS = "NOT_EQUALS"
+    IS_NULL = "IS_NULL"
+    IS_NOT_NULL = "IS_NOT_NULL"
+    SMALLER = "SMALLER"
+    GREATER = "GREATER"
+    GREATER_OR_EQUAL = "GREATER_OR_EQUAL"
+    SMALLER_OR_EQUAL = "SMALLER_OR_EQUAL"
+
+
+@dataclass
+class Condition:
+    column: str
+    operator: Operator
+    is_static_value: bool
+    compare_value: Any
+    result_value: Any
+
+    @classmethod
+    def create(cls, obj: dict) -> "Condition":
+        return cls(
+            column=obj["column"],
+            operator=Operator(obj["operator"]),
+            is_static_value=obj["is_static_value"],
+            compare_value=obj["compare_value"],
+            result_value=obj["result_value"],
+        )
+    
+
+DEFAULT_SHEET_CHUNK_SIZE = 1000
+SHEETS_RULE_TYPE_CHOICE = 12
+NOT_DEFINED = "NOT DEFINED"