tts-data-utils 0.7.3__py3-none-any.whl

tts_data_utils/core/generic.py

@@ -0,0 +1,78 @@
+#Python Imports
+import pdb
+from abc import ABC, abstractmethod
+from datetime import datetime
+
+#JPL Imports
+from jpl_time import Time
+from jpl_time.jpl_time import TimeConversionError
+
+#This Library Imports
+from tts_data_utils.core.data_container import DataContainer
+from tts_data_utils.core.data_item import DataItem
+
+class GenericItem(DataItem):
+    """
+    DataItem to go with GenericContainer
+
+    No validation requirements on these items.
+    """
+    #these are the fields that will be validated against these
+    #data types. Other fields are allowed
+
+    DICT_VALID_KEYS = []
+
+    TIME_FORMATS = {}
+    NAME = 'Generic Data Item'
+
+
+    @property
+    def default_html_row_style(self):
+        return {}
+    
+    @property
+    def time(self):
+        return
+
+    @property
+    def time_str(self):
+        return
+
+class GenericContainer(DataContainer):
+    """
+    Container provided for hooking into DataContainer infrastructure for which there is
+    no extended class.
+
+    Usually this class shouldn't be used in production, but is nice to have in development,
+    especially when prototyping behavior for data types for which you are not certain you
+    will want to eventually define an extension.
+
+    Some behavior will be missing without data validation defined, and no default styles
+    for html or text repr will be available.
+
+    For parameters, see DataContainer
+    """
+    NAME = 'Generic Items'
+    DATA_ITEM_CLS = GenericItem
+    
+    def __init__(self, raw_data=None, name=None, **kwargs):
+        super().__init__(raw_data=raw_data, **kwargs)
+        self.name = 'Generic Container' if name is None else name
+        
+        # Check if we have records to determine default columns
+        if self.records:
+            # Use the merged values from the first record instead of raw_data
+            self._repr_cols = list(self.records[0].values.keys())
+            self._csv_cols = self._repr_cols
+
+    def _impl_init(self):
+        return
+
+    @property
+    def repr_cols(self):
+        return self._repr_cols
+
+    @property
+    def default_time_label(self):
+        return
+

tts_data_utils/core/lorem_utils.py

@@ -0,0 +1,206 @@
+"""
+Utilities for generating lorem ipsum dummy data for DataContainer instances.
+
+This module provides functions to generate realistic dummy data based on
+the structure defined in a DataItem class.
+"""
+
+import random
+from datetime import datetime, timedelta
+import string
+import uuid
+
+# Lorem ipsum words for generating text
+LOREM_WORDS = [
+    "lorem", "ipsum", "dolor", "sit", "amet", "consectetur", "adipiscing", "elit",
+    "sed", "do", "eiusmod", "tempor", "incididunt", "ut", "labore", "et", "dolore",
+    "magna", "aliqua", "enim", "ad", "minim", "veniam", "quis", "nostrud", "exercitation",
+    "ullamco", "laboris", "nisi", "aliquip", "ex", "ea", "commodo", "consequat",
+    "duis", "aute", "irure", "reprehenderit", "voluptate", "velit", "esse", "cillum",
+    "dolore", "fugiat", "nulla", "pariatur", "excepteur", "sint", "occaecat", "cupidatat",
+    "non", "proident", "sunt", "culpa", "qui", "officia", "deserunt", "mollit", "anim",
+    "id", "est", "laborum"
+]
+
+# Common status values
+STATUS_VALUES = ["pending", "in_progress", "completed", "failed", "success", "unknown", "error"]
+
+# Common severity levels
+SEVERITY_LEVELS = ["info", "debug", "warning", "error", "critical"]
+
+def generate_lorem_text(min_words=3, max_words=10):
+    """Generate lorem ipsum text with a random number of words."""
+
+    # nosec B311: Standard random is fine for placeholder text (not cryptographic)
+    num_words = random.randint(min_words, max_words) #nosec
+    text = " ".join(random.sample(LOREM_WORDS, k=min(num_words, len(LOREM_WORDS)))) #nosec
+    return text.capitalize()
+
+def generate_value_for_type(type_spec):
+    """
+    Generate a random value that matches the given type specification.
+    
+    Args:
+        type_spec: A type or tuple of types (from DICT_VALID_KEYS)
+        
+    Returns:
+        A random value of the appropriate type
+    """
+    # If type_spec is not a tuple, make it one for consistent handling
+    if not isinstance(type_spec, tuple):
+        types = (type_spec,)
+    else:
+        types = type_spec
+    
+    # If None is an acceptable type, occasionally return None
+    if None in types and random.random() < 0.1: #nosec
+        return None
+    
+    # Filter out None from types for actual value generation
+    valid_types = [t for t in types if t is not None]
+    
+    # If no valid types remain, return None
+    if not valid_types:
+        return None
+    
+    # Choose a random type from the valid options
+    chosen_type = random.choice(valid_types) #nosec
+    
+    # Generate a value based on the chosen type
+    if chosen_type == str:
+        # For strings, generate different formats based on field name patterns
+        return generate_lorem_text(2, 8)
+    elif chosen_type == int:
+        return random.randint(0, 10000) #nosec
+    elif chosen_type == float:
+        return round(random.uniform(0, 100), 2) #nosec
+    elif chosen_type == bool:
+        return random.choice([True, False]) #nosec
+    elif chosen_type == datetime:
+        # Generate a random datetime within the last year
+        days_ago = random.randint(0, 365) #nosec
+        return datetime.now().replace(microsecond=0) - timedelta(days=days_ago)
+    elif chosen_type == dict:
+        # Generate a simple dictionary with 1-3 key-value pairs
+        num_items = random.randint(1, 3) #nosec
+        return {f"key{i}": generate_lorem_text(1, 3) for i in range(num_items)}
+    elif chosen_type == list:
+        # Generate a list with 1-5 string items
+        num_items = random.randint(1, 5) #nosec
+        return [generate_lorem_text(1, 2) for _ in range(num_items)]
+    else:
+        # For any other types, return a string representation
+        return str(chosen_type.__name__)
+
+def generate_smart_value(key, type_spec):
+    """
+    Generate a value for a specific key, using naming conventions to create more realistic data.
+    
+    Args:
+        key: The field name
+        type_spec: The type specification
+        
+    Returns:
+        A value appropriate for the field name and type
+    """
+    # Handle common field name patterns
+    key_lower = key.lower()
+    
+    # Handle None in type_spec
+    if isinstance(type_spec, tuple) and None in type_spec and random.random() < 0.1: #nosec
+        return None
+    
+    # ID fields
+    if key_lower in ('id', 'channelid', 'eventid', 'sessionid') or key_lower.endswith('id'):
+        if isinstance(type_spec, tuple):
+            if str in type_spec:
+                return f"ID-{random.randint(1000, 9999)}" #nosec
+            elif int in type_spec:
+                return random.randint(1000, 9999) #nosec
+        elif type_spec == int:
+            return random.randint(1000, 9999) #nosec
+        elif type_spec == str:
+            return f"ID-{random.randint(1000, 9999)}" #nosec
+    
+    # Name fields
+    if key_lower == 'name' or key_lower.endswith('name'):
+        return generate_lorem_text(1, 3)
+    
+    # Message fields
+    if 'message' in key_lower:
+        return generate_lorem_text(5, 15)
+    
+    # Status fields
+    if 'status' in key_lower:
+        return random.choice(STATUS_VALUES) #nosec
+    
+    # Level or severity fields
+    if 'level' in key_lower or 'severity' in key_lower:
+        return random.choice(SEVERITY_LEVELS) #nosec
+    
+    # Time fields
+    if any(time_word in key_lower for time_word in ['time', 'date', 'scet', 'ert', 'lst', 'rct']):
+        if isinstance(type_spec, tuple):
+            if datetime in type_spec:
+                days_ago = random.randint(0, 365) #nosec
+                return datetime.now().replace(microsecond=0) - timedelta(days=days_ago)
+            elif str in type_spec:
+                # Return an ISO format date string
+                days_ago = random.randint(0, 365) #nosec
+                dt = datetime.now() - timedelta(days=days_ago)
+                return dt.isoformat()
+        elif type_spec == datetime:
+            days_ago = random.randint(0, 365) #nosec
+            return datetime.now().replace(microsecond=0) - timedelta(days=days_ago)
+        elif type_spec == str:
+            days_ago = random.randint(0, 365) #nosec
+            dt = datetime.now() - timedelta(days=days_ago)
+            return dt.isoformat()
+    
+    # Clock fields
+    if 'sclk' in key_lower:
+        return random.uniform(1000000, 9999999) #nosec
+    
+    # Boolean fields
+    if any(bool_word in key_lower for bool_word in ['is_', 'has_', 'can_', 'should_', 'fromsse', 'realtime']):
+        return random.choice([True, False]) #nosec
+    
+    # Fallback to generic value generation
+    return generate_value_for_type(type_spec)
+
+def generate_lorem_data_for_item(data_item_cls, num_records=10):
+    """
+    Generate lorem ipsum data for a specific DataItem class.
+    
+    Args:
+        data_item_cls: The DataItem class to generate data for
+        num_records: Number of records to generate
+        
+    Returns:
+        List of dictionaries with dummy data matching the DataItem structure
+    """
+    data = []
+    
+    # If the class has no DICT_VALID_KEYS, generate generic data
+    if not hasattr(data_item_cls, 'DICT_VALID_KEYS') or not data_item_cls.DICT_VALID_KEYS:
+        for i in range(num_records):
+            record = {
+                "id": i + 1,
+                "title": generate_lorem_text(2, 5),
+                "description": generate_lorem_text(10, 20),
+                "timestamp": datetime.now() - timedelta(days=random.randint(0, 365)), #nosec
+                "value": round(random.uniform(0, 100), 2), #nosec
+                "count": random.randint(1, 1000), #nosec
+                "status": random.choice(STATUS_VALUES) #nosec
+            }
+            data.append(record)
+        return data
+    
+    # Generate data based on DICT_VALID_KEYS
+    for i in range(num_records):
+        record = {}
+        for key, type_spec in data_item_cls.DICT_VALID_KEYS:
+            record[key] = generate_smart_value(key, type_spec)
+        data.append(record)
+    
+    return data

tts_data_utils/core/visual_diff.py

@@ -0,0 +1,148 @@
+#Python Imports
+import pdb
+from abc import ABC, abstractmethod
+from datetime import datetime
+
+#JPL Imports
+from jpl_time import Time
+from jpl_time.jpl_time import TimeConversionError
+
+#Teamtool Studio Imports
+# None
+
+#This Library Imports
+from tts_data_utils.core.data_container import DataContainer
+from tts_data_utils.core.data_item import DataItem
+
+class VisualDiffItem(DataItem):
+    """
+    A specialized data record designed for side-by-side comparison.
+
+    **The Concept:**
+    When comparing two versions of a dataset (a "Diff"), we don't just care about the 
+    raw values—we care about the *state* of each row relative to its counterpart. 
+    Is it new? Was it deleted? Was it modified?
+
+    A `VisualDiffItem` carries the data values along with metadata (prefixed with `_visdiff`) 
+    that tells a renderer exactly how to style the row (e.g., green for an insert, 
+    red for a delete) to make differences immediately obvious to a human reviewer.
+
+    **Styling Logic:**
+    * **Row Level:** The entire background color is determined by the `_visdiff_match` status.
+    * **Cell Level:** Individual cells are highlighted if their specific key exists in 
+      the `_mismatched_keys` list.
+    """
+
+    PALETTE = {
+        'green': {'background-color': '#95FB95', 'color': '#333333'},
+        'blue': {'background-color': '#ADC4DF', 'color': '#333333'},
+        'grey': {'background-color': '#BDBEBD', 'color': '#BDBEBD'},
+        'red': {'background-color': '#FF614A', 'color': '#333333'},
+        'white': {'background-color': '#FFFFF', 'color': '#333333'},
+        'error': {'background-color': '#000000', 'color': '#FF614A'}
+        }
+
+
+    MATCH_STATUS_COLORS = {
+        'insert':   PALETTE['green'],
+        'replace':  PALETTE['blue'],
+        'delete':   PALETTE['red'],
+        'equal':    PALETTE['white'],
+        'empty_from_delete':  PALETTE['grey'], 
+        'empty_from_insert': PALETTE['grey'],
+        'unmatched': PALETTE['red'],
+    }
+
+    DICT_VALID_KEYS = [
+        ('_visdiff_index', (int, None)),
+        ('_visdiff_match', (str, None)),
+        ('_mismatched_keys', list)
+        ]
+
+    TIME_FORMATS = {
+        'Time': '%Y-%jT%H:%M:%S.%f'
+    }
+
+    NAME = 'Generic Data Item'
+
+
+    @property
+    def default_html_row_style(self):
+        """
+        Returns the CSS style dictionary for the table row based on the match status.
+        """
+        return self.MATCH_STATUS_COLORS[self['_visdiff_match']]
+
+    @property
+    def default_html_cell_styles(self):
+        """
+        Returns a mapping of column keys to CSS styles, highlighting specific 
+        mismatched cells.
+        """
+        return {k: {'background-color': '#218FFF'} if k in self['_mismatched_keys'] else {} for k in self.values.keys()}
+
+    
+    @property
+    def time(self):
+        """
+        Returns the native time object for the item.
+        """
+        return
+
+    @property
+    def time_str(self):
+        """
+        Returns a formatted string representation of the item's timestamp.
+        """
+        return
+
+class VisualDiffContainer(DataContainer):
+    """
+    A collection of VisualDiffItems, representing a full comparison report.
+
+    **Concept:**
+    The `VisualDiffContainer` acts as the manager for a set of diffed records. It 
+    distinguishes between "Display" columns (what the user needs to see) and 
+    "Metadata" columns (the internal flags used to calculate differences).
+
+    **Usage Note:**
+    By default, any column starting with an underscore (`_`) is treated as internal 
+    metadata and is hidden from the standard `repr_cols` view, though it remains 
+    accessible for CSV exports and logic processing.
+
+    :param raw_data: A list of dictionaries representing the diffed rows.
+    :type raw_data: list[dict]
+    :param name: The name of the container/report. Defaults to 'Generic Container'.
+    :type name: str, optional
+    :param kwargs: Additional arguments passed to the parent `DataContainer`.
+    """
+    NAME = 'Visual Diff Container'
+    DATA_ITEM_CLS = VisualDiffItem
+    
+    def __init__(self, raw_data=None, name=None, **kwargs):
+        #TO DO; Clean this up. Shouldn't be checking for a string like this
+        #and need to clean up at a higher level
+        super().__init__(raw_data=raw_data, **kwargs)
+        self.name = 'Generic Container' if name is None else name
+        self._repr_cols = [x for x, _ in raw_data[0].items() if not x.startswith('_')]
+        self._csv_cols = [x for x, _ in raw_data[0].items()]
+
+    def _impl_init(self):
+        """
+        Internal initialization hook for custom setup logic.
+        """
+        return
+
+    @property
+    def repr_cols(self):
+        """
+        The list of columns intended for visual display (excludes internal metadata).
+        """
+        return self._repr_cols
+
+    @property
+    def default_time_label(self):
+        """
+        The primary time-based column label used for sorting or indexing.
+        """
+        return

tts_data_utils/invulnerable_data_manager/batch.py

@@ -0,0 +1,176 @@
+from abc import ABC, abstractmethod
+
+class Batcher(ABC):
+    """
+    An abstract factory for organizing raw data into logical processing groups.
+
+    **The Concept:**
+    Think of a `Batcher` as a sorting machine at a postal office. It takes a giant 
+    unsorted bin of mail (`source_batch`) and uses specific internal rules to 
+    place that mail into smaller, specialized bags (`batches`) based on 
+    destination or priority (e.g., 'daily_maintenance' or 'system_updates').
+
+    **How it works:**
+    1. The `InvulnerableDataManager` provides the `source_batch`.
+    2. The `Batcher` checks `REQUIRED_DATA` to ensure it has what it needs.
+    3. The internal `_make_batches()` method runs the sorting logic.
+    4. Each resulting `Batch` is stored in the `self.batches` list.
+
+    :param source_batch: The pool of data to be sorted.
+    :type source_batch: AllDataBatch
+    """
+    @classmethod
+    @property
+    @abstractmethod
+    def NAME(cls):
+        """Name of the data batch type, e.g. 'mobility_backbone' or 'daily_maintenance'"""
+        raise NotImplementedError
+        
+    REQUIRED_DATA = []
+        
+    def __init__(self, source_batch):
+        self.source_batch = source_batch
+        
+        self.batches = []
+        self._make_batches()
+        
+    @abstractmethod
+    def _make_batches(self):
+        """
+        Subclasses must implement this to define how the source data 
+        is split into individual Batch objects.
+        """
+        pass
+    
+    def _add_batch(self, new_batch):
+        """
+        Internal helper to append a completed batch to the registry.
+        """
+        self.batches.append(new_batch)
+        
+class Batch:
+    """
+    A labeled container for a subset of data items.
+
+    **The Concept:**
+    A `Batch` is a logical "slice" of data. Unlike a standard `DataContainer`, 
+    a Batch can hold multiple types of data containers simultaneously, grouped 
+    by a common theme (like a specific time window or a shared asset ID).
+
+    **Data Tagging:**
+    If `tag_data` is True, every `DataItem` added to this batch is "stamped" 
+    with a reference back to this batch. This allows individual data rows to 
+    "know" which logical groups they belong to later in the pipeline.
+
+    :param name: Human-readable name for the batch.
+    :type name: str
+    :param tag_data: If True, calls `.tag_with_batch()` on all ingested items.
+    :type tag_data: bool
+    """
+    def __init__(self, name, tag_data=True):
+        # Inputs
+        self.NAME = name
+        self.tag_data = tag_data
+        # Data storage
+        self._data_map = {}
+        
+    @property
+    def data_map(self):
+        """The internal dictionary mapping container names to DataContainers."""
+        return self._data_map
+        
+    def has_data(self, name):
+        """Check if a specific data container exists in this batch."""
+        return name in self._data_map
+    
+    def set_data(self, **data_map):
+        """
+        Bulk-add data containers to the batch via keyword arguments.
+        """
+        for name, data in data_map.items():
+            self.set_data_one(name, data)
+        
+    def set_data_one(self, name, data):
+        """
+        Add a single named DataContainer to the batch.
+
+        :raises ValueError: If a container with the same name is already present.
+        """
+        if self.has_data(name):
+            raise ValueError(f'Attempt to set data {name} in batch {self.NAME} twice')
+        self._data_map[name] = data
+        # Tag the data with this batch if desired
+        if self.tag_data:
+            for data_item in data:
+                data_item.tag_with_batch(self)
+        
+    def get_data(self, name):
+        """Retrieve a DataContainer by name from the internal map."""
+        return self._data_map.get(name)
+    
+    def subdivide_on_condition(self, sub_f):
+        """
+        Creates a new data map by applying a filter function to every 
+        container in the batch.
+
+        :param sub_f: A function that returns a boolean for each DataItem.
+        :type sub_f: function
+        :return: A dictionary of filtered DataContainers.
+        :rtype: dict
+        """
+        new_data_map = {}
+        for name, data in self.data_map.items():
+            new_data_map[name] = data.subdivide_f(sub_f)
+        return new_data_map
+    
+    def subdivide_on_time(self, start_time=None, end_time=None):
+        """
+        A specialized subdivision helper for temporal windowing.
+
+        **Usage Note:**
+        This relies on the `DataItem.time` property being properly implemented 
+        in the underlying data objects.
+
+        :param start_time: The beginning of the window (inclusive).
+        :param end_time: The end of the window (inclusive).
+        :type start_time, end_time: datetime, optional
+        """
+        if start_time is None and end_time is None:
+            raise ValueError(f'Attempting to subdivide batch {self.NAME} on time with no time arguments')
+        if start_time is None:
+            sub_f = lambda x: x.time <= end_time
+        elif end_time is None:
+            sub_f = lambda x: start_time <= x.time
+        else:
+            sub_f = lambda x: start_time <= x.time <= end_time
+        return self.subdivide_on_condition(sub_f)
+            
+        
+class AllDataBatch(Batch):
+    """
+    A specialized Batch that serves as the global "Master List" for all input data.
+
+    **Concept:**
+    This batch is initialized with `tag_data=False` because it represents the 
+    raw, un-categorized state of the system. It is the "Source of Truth" from 
+    which all other Batchers draw their data.
+    """
+    def __init__(self):
+        super().__init__('All Data', tag_data=False)
+
+class UntaggedBatch(Batch):
+    """
+    A "Lost and Found" batch for data that escaped categorization.
+
+    **Concept:**
+    After all Batchers have run, the `UntaggedBatch` scans the `AllDataBatch` 
+    to find any `DataItem` that does not have any batch tags associated with it. 
+    This is a critical auditing tool to ensure no data "fell through the cracks" 
+    of the defined sorting logic.
+
+    :param all_data_batch: The master data batch to scan for untagged items.
+    :type all_data_batch: AllDataBatch
+    """
+    def __init__(self, all_data_batch):
+        super().__init__('Untagged Data', tag_data=False)
+        self.set_data(**all_data_batch.subdivide_on_condition(lambda x: not x.any_batches()))