tts-data-utils 0.7.3__py3-none-any.whl

tts_data_utils/test/core/test_visdiff.py

@@ -0,0 +1,193 @@
+#Standard Library Imports
+from pathlib import Path
+import pdb
+
+#Installed Library Importsimport pytest
+from datetime import datetime
+import pytest
+
+#Teamtool Studio Imports
+from tts_utilities.logger import create_logger
+from tts_utilities.test_utilities import safe_test_paths
+from tts_html_utils.visdiff.visdiff import VisualDiff
+from tts_html_utils.core.compiler import HtmlCompiler
+
+#This Library Imports
+from tts_data_utils.core.generic import GenericContainer
+
+logger = create_logger(f'data_utils.core.filters')
+TEST_INPUT_DIR, TEST_OUTPUT_DIR = safe_test_paths(Path(__file__).parent, 'test_files/visdiff')
+
+def save_html(visdiff_left, visdiff_right, filepath):
+    power_table_left = visdiff_left.power_table(id='visdiff-left', style={'width': '130%', 'float': 'left'})
+    power_table_right = visdiff_right.power_table(id='visdiff-right', style={'width': '130%', 'float': 'right'})
+    compiler = HtmlCompiler('VisDiff Test')
+    compiler.add_body_component(VisualDiff(power_table_left, power_table_right))
+    file = open(filepath, 'w')
+    file.write(compiler.render())
+    file.close()
+
+@pytest.fixture(scope="module")
+def test_container():
+    time = [datetime.fromtimestamp(ii) for ii in range(100)]
+    col1 = [ii for ii in range(50)]*2
+    col2 = [ii for ii in range(100, 200)]
+    col3 = [str(ii) for ii in range(200, 300)]
+    col4 = [ii for ii in range(300, 400)]
+    col5 = [ii for ii in range(400, 500)]
+    raw_data = [{
+        'Time': t,
+        'Column 1': c1,
+        'Column 2': c2,
+        'Column 3': c3,
+        'Column 4': c4,
+        'Column 5': c5,
+    } for t, c1, c2, c3, c4, c5 in zip(time, col1, col2, col3, col4, col5)]
+    test_container = GenericContainer(raw_data=raw_data)
+    return test_container
+
+
+class TestVisDiff:
+    def test_no_diff(self, test_container):
+        visdiff_left, visdiff_right = test_container.visual_diff(test_container)
+        save_html(visdiff_left, visdiff_right, TEST_OUTPUT_DIR.joinpath('no_diff.html'))
+
+        assert visdiff_left['_visdiff_match'] == ['equal']*100
+        assert visdiff_left['_visdiff_index'] == [ii for ii in range(100)]
+        assert visdiff_left['_mismatched_keys'] == [[]]*100
+
+        assert visdiff_right['_visdiff_match'] == ['equal']*100
+        assert visdiff_right['_visdiff_index'] == [ii for ii in range(100)]
+        assert visdiff_right['_mismatched_keys'] == [[]]*100
+
+    def test_simple_delete(self, test_container):
+        left = test_container
+        right = test_container[:10] + test_container[20:]
+        visdiff_left, visdiff_right = left.visual_diff(right)
+
+        save_html(visdiff_left, visdiff_right, TEST_OUTPUT_DIR.joinpath('simple_delete.html'))
+
+        assert visdiff_left['_visdiff_match'] == ['equal']*10 + ['delete']*10 + ['equal']*80
+        assert visdiff_left['_visdiff_index'] == [ii for ii in range(10)] + [None]*10 + [ii for ii in range(10, 90)]
+        assert visdiff_left['_mismatched_keys'] == [[]]*100
+
+        assert visdiff_right['_visdiff_match'] == ['equal']*10 + ['empty_from_delete']*10 + ['equal']*80
+        assert visdiff_right['_visdiff_index'] == [ii for ii in range(10)] + [None]*10 + [ii for ii in range(20, 100)]
+        assert visdiff_right['_mismatched_keys'] == [[]]*100
+
+    def test_simple_insert(self, test_container):
+        left = test_container[:10] + test_container[20:]
+        right = test_container
+        visdiff_left, visdiff_right = left.visual_diff(right)
+        save_html(visdiff_left, visdiff_right, TEST_OUTPUT_DIR.joinpath('simple_insert.html'))
+
+        assert visdiff_left['_visdiff_match'] == ['equal']*10 + ['empty_from_insert']*10 + ['equal']*80
+        assert visdiff_left['_visdiff_index'] == [ii for ii in range(10)] + [None]*10 + [ii for ii in range(20,100)]
+        assert visdiff_left['_mismatched_keys'] == [[]]*100
+
+        assert visdiff_right['_visdiff_match'] == ['equal']*10 + ['insert']*10 + ['equal']*80
+        assert visdiff_right['_visdiff_index'] == [ii for ii in range(10)] + [None]*10 + [ii for ii in range(10, 90)]
+        assert visdiff_right['_mismatched_keys'] == [[]]*100
+
+    def test_insert_and_delete(self, test_container):
+
+        left = test_container[:10] + test_container[20:]
+        right = test_container[:50] + test_container[60:]
+        visdiff_left, visdiff_right = left.visual_diff(right)
+        save_html(visdiff_left, visdiff_right, TEST_OUTPUT_DIR.joinpath('insert_and_delete.html'))
+
+        assert visdiff_left['_visdiff_match'] == ['equal']*10 + ['empty_from_insert']*10 + ['equal']*30 + ['delete']*10 + ['equal']*40
+        assert visdiff_left['_visdiff_index'] == [ii for ii in range(10)] + [None]*10 + [ii for ii in range(20,50)] + [None]*10 + [ii for ii in range(50, 90)]
+        assert visdiff_left['_mismatched_keys'] == [[]]*100
+
+        assert visdiff_right['_visdiff_match'] == ['equal']*10 + ['insert']*10 + ['equal']*30 + ['empty_from_delete']*10 + ['equal']*40
+        assert visdiff_right['_visdiff_index'] == [ii for ii in range(10)] + [None]*10 + [ii for ii in range(10,40)] + [None]*10 + [ii for ii in range(50, 90)]
+        assert visdiff_right['_mismatched_keys'] == [[]]*100
+
+
+    def test_adjacent_insert_delete(self, test_container):
+
+        left = test_container[:10] + test_container[20:]
+        right = test_container[:20] + test_container[30:]
+        visdiff_left, visdiff_right = left.visual_diff(right)
+        save_html(visdiff_left, visdiff_right, TEST_OUTPUT_DIR.joinpath('adjacent_insert_delete.html'))
+
+        assert visdiff_left['_visdiff_match'] == ['equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal']
+        assert visdiff_left['_visdiff_index'] == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89]
+        assert visdiff_left['_mismatched_keys'] == [[]]*100
+
+        assert visdiff_right['_visdiff_match'] == ['equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal']
+        assert visdiff_right['_visdiff_index'] == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89]
+        assert visdiff_right['_mismatched_keys'] == [[]]*100
+
+    def test_overlapping_insert_delete(self, test_container):
+        left = test_container[:10] + test_container[20:]
+        right = test_container[:15] + test_container[25:]
+        visdiff_left, visdiff_right = left.visual_diff(right)
+        save_html(visdiff_left, visdiff_right, TEST_OUTPUT_DIR.joinpath('overlapping_insert_delete.html'))
+
+        assert visdiff_left['_visdiff_match'] == ['equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'delete', 'delete', 'delete', 'delete', 'delete', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal']
+        assert visdiff_left['_visdiff_index'] == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, None, None, None, None, None, None, None, None, None, None, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89]
+        assert visdiff_left['_mismatched_keys'] == [[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]
+
+        assert visdiff_right['_visdiff_match'] == ['equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'insert', 'insert', 'insert', 'insert', 'insert', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal', 'equal']
+        assert visdiff_right['_visdiff_index'] == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, None, None, None, None, None, None, None, None, None, None, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89]
+        assert visdiff_right['_mismatched_keys'] == [[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]
+
+    def test_replace(self, test_container):
+        left = test_container._copy()
+        right = left._copy()
+        for ii in range(len(right)):
+            if ii%3 == 0: right[ii]['Column 1'] += 3
+            if ii%5 == 0: right[ii]['Column 2'] += 2
+
+        visdiff_left, visdiff_right = left.visual_diff(right)
+        save_html(visdiff_left, visdiff_right, TEST_OUTPUT_DIR.joinpath('replace.html'))
+
+        assert visdiff_left['_visdiff_match'] == ['replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace', 'replace', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace', 'replace', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace', 'replace', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace', 'replace', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace', 'replace', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace', 'replace', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace']
+        assert visdiff_left['_visdiff_index'] == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99]
+        assert visdiff_left['_mismatched_keys'] == [['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1'], ['Column 2'], [], ['Column 1'], [], [], ['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1'], ['Column 2'], [], ['Column 1'], [], [], ['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1'], ['Column 2'], [], ['Column 1'], [], [], ['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1'], ['Column 2'], [], ['Column 1'], [], [], ['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1'], ['Column 2'], [], ['Column 1'], [], [], ['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1'], ['Column 2'], [], ['Column 1'], [], [], ['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1']]
+
+        assert visdiff_right['_visdiff_match'] == ['replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace', 'replace', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace', 'replace', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace', 'replace', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace', 'replace', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace', 'replace', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace', 'replace', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace']
+        assert visdiff_right['_visdiff_index'] == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99]
+        assert visdiff_right['_mismatched_keys'] == [['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1'], ['Column 2'], [], ['Column 1'], [], [], ['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1'], ['Column 2'], [], ['Column 1'], [], [], ['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1'], ['Column 2'], [], ['Column 1'], [], [], ['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1'], ['Column 2'], [], ['Column 1'], [], [], ['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1'], ['Column 2'], [], ['Column 1'], [], [], ['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1'], ['Column 2'], [], ['Column 1'], [], [], ['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1']]
+
+    def test_replace_with_delete(self, test_container):
+        left = test_container._copy()
+        right = left._copy()
+        for ii in range(len(right)):
+            if ii%3 == 0: right[ii]['Column 1'] += 3
+            if ii%5 == 0: right[ii]['Column 2'] += 2
+        right = right[:20] + right[40:]
+
+        visdiff_left, visdiff_right = left.visual_diff(right)
+        save_html(visdiff_left, visdiff_right, TEST_OUTPUT_DIR.joinpath('replace_with_delete.html'))
+
+        assert visdiff_left['_visdiff_match'] == ['replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace', 'replace', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'replace', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace', 'replace', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace', 'replace', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace', 'replace', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace']
+        assert visdiff_left['_visdiff_index'] == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79]
+        assert visdiff_left['_mismatched_keys'] == [['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1'], ['Column 2'], [], ['Column 1'], [], [], ['Column 1', 'Column 2'], [], [], ['Column 1'], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], ['Column 2', '_visdiff_match', '_visdiff_index'], [], ['Column 1'], [], [], ['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1'], ['Column 2'], [], ['Column 1'], [], [], ['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1'], ['Column 2'], [], ['Column 1'], [], [], ['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1'], ['Column 2'], [], ['Column 1'], [], [], ['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1']]
+
+        assert visdiff_right['_visdiff_match'] == ['replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace', 'replace', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'replace', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace', 'replace', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace', 'replace', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace', 'replace', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'equal', 'replace', 'equal', 'replace', 'replace', 'equal', 'equal', 'replace']
+        assert visdiff_right['_visdiff_index'] == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99]
+        assert visdiff_right['_mismatched_keys'] == [['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1'], ['Column 2'], [], ['Column 1'], [], [], ['Column 1', 'Column 2'], [], [], ['Column 1'], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], ['Column 2', '_visdiff_index', '_mismatched_keys'], [], ['Column 1'], [], [], ['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1'], ['Column 2'], [], ['Column 1'], [], [], ['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1'], ['Column 2'], [], ['Column 1'], [], [], ['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1'], ['Column 2'], [], ['Column 1'], [], [], ['Column 1', 'Column 2'], [], [], ['Column 1'], [], ['Column 2'], ['Column 1'], [], [], ['Column 1']]
+
+
+    def test_stressing_case(self, test_container):
+        left = test_container._copy()
+        right = left._copy()
+        for ii in range(len(right)):
+            right[ii]['Column 1'] += 3
+            right[ii]['Column 2'] += 2
+        left = left[:30] + left[50:]
+        right = right[:20] + right[40:]
+
+        visdiff_left, visdiff_right = left.visual_diff(right)
+        save_html(visdiff_left, visdiff_right, TEST_OUTPUT_DIR.joinpath('stressing_case.html'))
+
+        assert visdiff_left['_visdiff_match'] == ['delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'delete', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert', 'empty_from_insert']
+        assert visdiff_left['_visdiff_index'] == [None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None]
+        assert visdiff_left['_mismatched_keys'] == [[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]
+
+        assert visdiff_right['_visdiff_match'] == ['empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'empty_from_delete', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert', 'insert']
+        assert visdiff_right['_visdiff_index'] == [None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None]
+        assert visdiff_right['_mismatched_keys'] == [[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]

tts_data_utils/test/invulnerable_data_manager/test_invulnerable_data_manager.py

@@ -0,0 +1,150 @@
+import pytest
+from unittest.mock import MagicMock, patch
+from tts_data_utils.invulnerable_data_manager.invulnerable_data_manager import InvulnerableDataManager
+from tts_data_utils.core.generic import GenericContainer
+
+# Simple concrete subclass for testing
+class MockManager(InvulnerableDataManager):
+    def _impl_init_data(self, container_cls, data, name=None, **kwargs):
+        pass
+
+# Mock Batcher for testing
+class MockBatcher:
+    NAME = "test_batcher"
+    REQUIRED_DATA = ["required_input"]
+    def __init__(self, all_input_data, **kwargs):
+        self.all_input_data = all_input_data
+
+@pytest.fixture
+def mock_dependencies():
+    """
+    Patches internal dependencies so we test Manager logic in isolation.
+    """
+    # PATCH PATHS: We must patch where the names are IMPORTED/USED, not defined.
+    with patch('tts_data_utils.invulnerable_data_manager.invulnerable_data_manager.AllDataBatch') as MockAllData, \
+         patch('tts_data_utils.invulnerable_data_manager.invulnerable_data_manager.UntaggedBatch') as MockUntagged, \
+         patch('tts_data_utils.invulnerable_data_manager.invulnerable_data_manager.exec_invulnerable') as MockExec:
+        
+        # 1. Configure AllDataBatch Mock Instance
+        mock_batch_instance = MockAllData.return_value
+        
+        # Internal store for the mock to simulate state
+        mock_batch_instance._data_store = []
+        
+        # Behavior: has_data checks the internal list
+        mock_batch_instance.has_data.side_effect = lambda name: name in mock_batch_instance._data_store
+        
+        # Behavior: set_data_one adds to the internal list
+        def set_data_side_effect(name, data):
+            mock_batch_instance._data_store.append(name)
+        mock_batch_instance.set_data_one.side_effect = set_data_side_effect
+        
+        # Behavior: get_inventory returns a dict (Code expects this method exists)
+        mock_batch_instance.get_inventory.return_value = {"mock_data": 1}
+        
+        # 2. Configure exec_invulnerable Mock
+        # Default: Return a valid container
+        valid_container = MagicMock()
+        valid_container.valid = True
+        MockExec.return_value = valid_container
+        
+        yield MockAllData, MockUntagged, MockExec
+
+@pytest.fixture
+def manager(mock_dependencies):
+    return MockManager()
+
+@pytest.fixture
+def valid_data():
+    return [{'id': 1, 'val': 'A'}]
+
+class TestDataManagerIngestion:
+    def test_init_data_success(self, manager, valid_data):
+        """Verify successful ingestion and registration."""
+        manager.init_data(GenericContainer, valid_data, name="test_data")
+        
+        # Verify the manager delegated to the internal batch storage
+        manager.all_input_data.set_data_one.assert_called()
+        
+        # Verify inventory logic
+        # The code calls .get_inventory(), which we mocked to return {"mock_data": 1}
+        inv = manager.get_input_inventory()
+        assert "mock_data" in inv
+
+    def test_init_data_none_raises(self, manager):
+        """Verify that passing None results in an error (swallowed by @invulnerable)."""
+        manager.init_data(GenericContainer, None, name="fail_data")
+        # Verify set_data_one was NOT called
+        manager.all_input_data.set_data_one.assert_not_called()
+
+    def test_duplicate_init_raises(self, manager, valid_data):
+        """Verify that initializing the same name twice is caught."""
+        # Note: The code checks duplicate on `container_cls.NAME`. 
+        # We omit 'name' kwarg here so it defaults to GenericContainer.NAME
+        # allowing the check to work correctly.
+        
+        # First init
+        manager.init_data(GenericContainer, valid_data)
+        
+        # Verify it was stored
+        assert manager.all_input_data.has_data(GenericContainer.NAME)
+
+        # Second init - should trigger ValueError internally
+        manager.init_data(GenericContainer, valid_data)
+        
+        # Verify set_data_one was called exactly once (for the first success only)
+        assert manager.all_input_data.set_data_one.call_count == 1
+
+    def test_invalid_container_swallowed(self, manager, mock_dependencies):
+        """Verify that a container that fails validation isn't registered."""
+        _, _, mock_exec = mock_dependencies
+        
+        # Configure exec_invulnerable to return an INVALID container for this test
+        invalid_container = MagicMock()
+        invalid_container.valid = False
+        mock_exec.return_value = invalid_container
+        
+        manager.init_data(GenericContainer, [], name="invalid_container")
+            
+        manager.all_input_data.set_data_one.assert_not_called()
+
+class TestDataManagerBatching:
+    def test_init_batcher_missing_requirements(self, manager):
+        """Verify behavior when required data is missing."""
+        # Ensure the mock says we DO NOT have data
+        manager.all_input_data.has_data.return_value = False
+        
+        manager.init_batcher(MockBatcher)
+        
+        # Current Code Behavior: Logs warning but returns the Batcher (does not return None).
+        # We test for existence, acknowledging the permissive behavior.
+        assert manager.get_batcher("test_batcher") is not None
+
+    def test_init_batcher_success(self, manager, valid_data, mock_dependencies):
+        """Verify batcher registers when requirements are met."""
+        _, _, mock_exec = mock_dependencies
+        
+        # 1. Satisfy requirements
+        # We manually force the mock to say "Yes, I have required_input"
+        manager.all_input_data._data_store.append("required_input")
+
+        # 2. Configure exec_invulnerable to return a valid Batcher
+        mock_batcher_instance = MagicMock()
+        mock_exec.return_value = mock_batcher_instance
+
+        manager.init_batcher(MockBatcher)
+        
+        assert manager.get_batcher("test_batcher") is not None
+        # Verify untagged batch was re-initialized (Mock called)
+        assert manager._untagged_batch is not None
+
+    def test_comprehensive_inventory(self, manager, valid_data):
+        """Verify the data map returns both inputs and outputs."""
+        manager.init_data(GenericContainer, valid_data, name="input_1")
+        
+        inventory = manager.get_data_inventory()
+        
+        # Based on our mock setup in the fixture, get_inventory returns {"mock_data": 1}
+        # This applies to both Input and Output batches (same mock class)
+        assert "mock_data" in inventory['Inputs']
+        assert "mock_data" in inventory['Outputs']

tts_data_utils/test/multimission/evr/test_evr.py

@@ -0,0 +1,29 @@
+#Standard Library Imports
+import pytest
+from datetime import datetime
+
+#Installed Library Imports
+from pathlib import Path
+import pdb
+
+#Teamtool Studio Imports
+from tts_utilities.logger import create_logger
+from tts_utilities.test_utilities import safe_test_paths
+
+#This Library Imports
+from tts_data_utils.multimission.evr import EvrContainer, EvrGapContainer
+
+logger = create_logger(f'data_utils.multimission.evr')
+TEST_INPUT_DIR, TEST_OUTPUT_DIR = safe_test_paths(Path(__file__).parent, 'test_files')
+
+class TestEvrGaps:
+    def test_gap_finder(self):
+        evrs = EvrContainer(csv_path=TEST_INPUT_DIR.joinpath('evrs_with_gaps.csv'), cast_fields=True)
+        #evrs.gaps().to_csv(TEST_OUTPUT_DIR.joinpath('expected_evr_gaps.csv'))
+        expected_evr_gaps = EvrGapContainer(csv_path=TEST_INPUT_DIR.joinpath('expected_evr_gaps.csv'), cast_fields=True)
+        evrs.gaps().to_csv(TEST_OUTPUT_DIR.joinpath('actual_evr_gaps.csv'))
+
+        expected_evr_gaps.diff(evrs.gaps())
+        assert len(expected_evr_gaps.diff(evrs.gaps()).eq('Same', False)) == 0
+        assert len(expected_evr_gaps.diff(evrs.gaps())) == 236
+

tts_data_utils/test/multimission/evr/test_files/actual_evr_gaps.csv

@@ -0,0 +1,12 @@
+Level,Begin Time (SCET),End Time (SCET),Begin Time (ERT),End Time (ERT),First Missing Index,Last Missing Index,Total Missing EVRs
+DIAGNOSTIC,2010-001T00:02:25.004012,2010-001T00:02:25.004015,2025-072T17:13:47.592001,2025-072T17:13:47.592004,888,889,2
+COMMAND,2010-001T00:02:25.004017,2010-001T00:02:25.004019,2025-072T17:13:47.592006,2025-072T17:13:47.592008,887,887,1
+COMMAND,2010-001T00:02:25.004020,2010-001T00:02:25.004022,2025-072T17:13:47.592009,2025-072T17:13:47.592011,890,890,1
+ACTIVITY_LO,2010-001T00:02:25.004024,2010-001T00:02:25.004024,2025-072T17:13:47.592013,2025-072T17:13:47.592016,888,889,2
+ACTIVITY_HI,2010-001T00:02:25.004024,2010-001T00:02:25.004024,2025-072T17:13:47.592019,2025-072T17:13:47.592021,888,888,1
+ACTIVITY_HI,2010-001T00:02:25.004024,2010-001T00:02:25.004024,2025-072T17:13:47.592021,2025-072T17:13:47.592023,890,890,1
+WARNING_LO,2010-001T00:02:25.004024,2010-001T00:02:25.004024,2025-072T17:13:47.592026,2025-072T17:13:47.592029,889,890,2
+WARNING_HI,2010-001T00:02:25.004024,2010-001T00:02:25.004024,2025-072T17:13:47.592033,2025-072T17:13:47.592035,890,890,1
+FATAL,2010-001T00:02:25.004024,2010-001T00:02:25.004024,2025-072T17:13:47.592036,2025-072T17:13:47.592039,887,888,2
+FATAL,2010-001T00:02:25.004024,2010-001T00:02:25.004024,2025-072T17:13:47.592039,2025-072T17:13:47.592042,890,891,2
+FATAL,2010-001T00:02:25.004024,2010-001T00:02:25.004024,2025-072T17:13:47.592042,2025-072T17:13:47.592044,893,893,1

tts_data_utils/test/multimission/evr/test_files/evrs_with_gaps.csv

@@ -0,0 +1,30 @@
+recordType,sessionId,sessionHost,name,module,level,eventId,vcid,dssId,fromSse,realtime,sclk,scet,ert,rct,lst,message,metadataKeywordList,metadataValuesList,metadata
+Evr,38,srl-wsts-ms-1,MATCH_BY_EXACT_EVR_NAME,time_svc,DIAGNOSTIC,1048607,0,0,FALSE,TRUE,145.00401,2010-001T00:02:25.004011,2025-072T17:13:47.592000,2025-072T17:13:47.593000,,It doesn't matter what this message is,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(886),(1220),(009606)]","{'CategorySequenceId': '886', 'SequenceId': '1220', 'TaskName': '009606'}"
+Evr,38,srl-wsts-ms-1,DONT_MATCH_BY_EXACT_EVR_NAME,time_svc,DIAGNOSTIC,1048607,0,0,FALSE,TRUE,146.00402,2010-001T00:02:25.004012,2025-072T17:13:47.592001,2025-072T17:13:47.593001,,It doesn't matter what this message is,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(886),(1220),(009606)]","{'CategorySequenceId': '887', 'SequenceId': '1221', 'TaskName': '009606'}"
+Evr,38,srl-wsts-ms-1,MATCH_BY_EVR_NAME_REGEX_C,time_svc,DIAGNOSTIC,1048649,0,0,FALSE,TRUE,149.00405,2010-001T00:02:25.004015,2025-072T17:13:47.592004,2025-072T17:13:47.593004,,It doesn't matter what this message is,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(887),(1221),(009608)]","{'CategorySequenceId': '890', 'SequenceId': '1224', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,DONT_MATCH_BY_EVR_NAME_REGEX,time_svc,DIAGNOSTIC,1048649,0,0,FALSE,TRUE,150.00406,2010-001T00:02:25.004016,2025-072T17:13:47.592005,2025-072T17:13:47.593005,,It doesn't matter what this message is,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(887),(1221),(009608)]","{'CategorySequenceId': '891', 'SequenceId': '1225', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,MATCH_BY_EVR_NAME_WITH_MESSAGE_REGEX,time_svc,COMMAND,1048650,0,0,FALSE,TRUE,151.00407,2010-001T00:02:25.004017,2025-072T17:13:47.592006,2025-072T17:13:47.593006,,Please _-_-_-_ match 0909090 this qwqwqwq message.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '886', 'SequenceId': '1226', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_A,time_svc,COMMAND,1048650,0,0,FALSE,TRUE,153.00409,2010-001T00:02:25.004019,2025-072T17:13:47.592008,2025-072T17:13:47.593008,,Please _-_-_-_ match 0909090 this qwqwqwq message.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '888', 'SequenceId': '1228', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_B,time_svc,COMMAND,1048650,0,0,FALSE,TRUE,154.0041,2010-001T00:02:25.004020,2025-072T17:13:47.592009,2025-072T17:13:47.593009,,Please _-_-_-_ match 0909090 this qwqwqwq message.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '889', 'SequenceId': '1229', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_A,time_svc,COMMAND,1048650,0,0,FALSE,TRUE,145.00412,2010-001T00:02:25.004022,2025-072T17:13:47.592011,2025-072T17:13:47.593011,,But don't match this one.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '891', 'SequenceId': '1231', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_B,time_svc,ACTIVITY_LO,1048650,0,0,FALSE,TRUE,145.00413,2010-001T00:02:25.004023,2025-072T17:13:47.592012,2025-072T17:13:47.593012,,But don't match this one.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '886', 'SequenceId': '1232', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_C,time_svc,ACTIVITY_LO,1048650,0,0,FALSE,TRUE,145.00414,2010-001T00:02:25.004024,2025-072T17:13:47.592013,2025-072T17:13:47.593013,,But don't match this one.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '887', 'SequenceId': '1233', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_C,time_svc,ACTIVITY_LO,1048650,0,0,FALSE,TRUE,145.00417,2010-001T00:02:25.004024,2025-072T17:13:47.592016,2025-072T17:13:47.593016,,But don't match this one.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '890', 'SequenceId': '1236', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_C,time_svc,ACTIVITY_LO,1048650,0,0,FALSE,TRUE,145.00418,2010-001T00:02:25.004024,2025-072T17:13:47.592017,2025-072T17:13:47.593017,,But don't match this one.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '891', 'SequenceId': '1237', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_C,time_svc,ACTIVITY_HI,1048650,0,0,FALSE,TRUE,145.00419,2010-001T00:02:25.004024,2025-072T17:13:47.592018,2025-072T17:13:47.593018,,But don't match this one.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '886', 'SequenceId': '1238', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_C,time_svc,ACTIVITY_HI,1048650,0,0,FALSE,TRUE,145.0042,2010-001T00:02:25.004024,2025-072T17:13:47.592019,2025-072T17:13:47.593019,,But don't match this one.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '887', 'SequenceId': '1239', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_C,time_svc,ACTIVITY_HI,1048650,0,0,FALSE,TRUE,145.00422,2010-001T00:02:25.004024,2025-072T17:13:47.592021,2025-072T17:13:47.593021,,But don't match this one.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '889', 'SequenceId': '1241', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_C,time_svc,ACTIVITY_HI,1048650,0,0,FALSE,TRUE,145.00424,2010-001T00:02:25.004024,2025-072T17:13:47.592023,2025-072T17:13:47.593023,,But don't match this one.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '891', 'SequenceId': '1243', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_C,time_svc,WARNING_LO,1048650,0,0,FALSE,TRUE,145.00425,2010-001T00:02:25.004024,2025-072T17:13:47.592024,2025-072T17:13:47.593024,,But don't match this one.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '886', 'SequenceId': '1244', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_C,time_svc,WARNING_LO,1048650,0,0,FALSE,TRUE,145.00426,2010-001T00:02:25.004024,2025-072T17:13:47.592025,2025-072T17:13:47.593025,,But don't match this one.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '887', 'SequenceId': '1245', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_C,time_svc,WARNING_LO,1048650,0,0,FALSE,TRUE,145.00427,2010-001T00:02:25.004024,2025-072T17:13:47.592026,2025-072T17:13:47.593026,,But don't match this one.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '888', 'SequenceId': '1246', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_C,time_svc,WARNING_LO,1048650,0,0,FALSE,TRUE,145.0043,2010-001T00:02:25.004024,2025-072T17:13:47.592029,2025-072T17:13:47.593029,,But don't match this one.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '891', 'SequenceId': '1249', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_C,time_svc,WARNING_HI,1048650,0,0,FALSE,TRUE,145.00431,2010-001T00:02:25.004024,2025-072T17:13:47.592030,2025-072T17:13:47.593030,,But don't match this one.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '886', 'SequenceId': '1250', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_C,time_svc,WARNING_HI,1048650,0,0,FALSE,TRUE,145.00432,2010-001T00:02:25.004024,2025-072T17:13:47.592031,2025-072T17:13:47.593031,,But don't match this one.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '887', 'SequenceId': '1251', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_C,time_svc,WARNING_HI,1048650,0,0,FALSE,TRUE,145.00433,2010-001T00:02:25.004024,2025-072T17:13:47.592032,2025-072T17:13:47.593032,,But don't match this one.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '888', 'SequenceId': '1252', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_C,time_svc,WARNING_HI,1048650,0,0,FALSE,TRUE,145.00434,2010-001T00:02:25.004024,2025-072T17:13:47.592033,2025-072T17:13:47.593033,,But don't match this one.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '889', 'SequenceId': '1253', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_C,time_svc,WARNING_HI,1048650,0,0,FALSE,TRUE,145.00436,2010-001T00:02:25.004024,2025-072T17:13:47.592035,2025-072T17:13:47.593035,,But don't match this one.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '891', 'SequenceId': '1255', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_C,time_svc,FATAL,1048650,0,0,FALSE,TRUE,145.00437,2010-001T00:02:25.004024,2025-072T17:13:47.592036,2025-072T17:13:47.593036,,But don't match this one.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '886', 'SequenceId': '1256', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_C,time_svc,FATAL,1048650,0,0,FALSE,TRUE,145.0044,2010-001T00:02:25.004024,2025-072T17:13:47.592039,2025-072T17:13:47.593039,,But don't match this one.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '889', 'SequenceId': '1259', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_C,time_svc,FATAL,1048650,0,0,FALSE,TRUE,145.00443,2010-001T00:02:25.004024,2025-072T17:13:47.592042,2025-072T17:13:47.593042,,But don't match this one.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '892', 'SequenceId': '1262', 'TaskName': '009608'}"
+Evr,38,srl-wsts-ms-1,NAME_AND_REGEX_MATCH_C,time_svc,FATAL,1048650,0,0,FALSE,TRUE,145.15645,2010-001T00:02:25.004024,2025-072T17:13:47.592044,2025-072T17:13:47.593044,,But don't match this one.,"[(CategorySequenceId),(SequenceId),(TaskName)]","[(888),(1222),(009608)]","{'CategorySequenceId': '894', 'SequenceId': '1264', 'TaskName': '009608'}"

tts_data_utils/test/multimission/evr/test_files/expected_evr_gaps.csv

@@ -0,0 +1,12 @@
+Level,Begin Time (SCET),End Time (SCET),Begin Time (ERT),End Time (ERT),First Missing Index,Last Missing Index,Total Missing EVRs
+DIAGNOSTIC,2010-001T00:02:25.004012,2010-001T00:02:25.004015,2025-072T17:13:47.592001,2025-072T17:13:47.592004,888,889,2
+COMMAND,2010-001T00:02:25.004017,2010-001T00:02:25.004019,2025-072T17:13:47.592006,2025-072T17:13:47.592008,887,887,1
+COMMAND,2010-001T00:02:25.004020,2010-001T00:02:25.004022,2025-072T17:13:47.592009,2025-072T17:13:47.592011,890,890,1
+ACTIVITY_LO,2010-001T00:02:25.004024,2010-001T00:02:25.004024,2025-072T17:13:47.592013,2025-072T17:13:47.592016,888,889,2
+ACTIVITY_HI,2010-001T00:02:25.004024,2010-001T00:02:25.004024,2025-072T17:13:47.592019,2025-072T17:13:47.592021,888,888,1
+ACTIVITY_HI,2010-001T00:02:25.004024,2010-001T00:02:25.004024,2025-072T17:13:47.592021,2025-072T17:13:47.592023,890,890,1
+WARNING_LO,2010-001T00:02:25.004024,2010-001T00:02:25.004024,2025-072T17:13:47.592026,2025-072T17:13:47.592029,889,890,2
+WARNING_HI,2010-001T00:02:25.004024,2010-001T00:02:25.004024,2025-072T17:13:47.592033,2025-072T17:13:47.592035,890,890,1
+FATAL,2010-001T00:02:25.004024,2010-001T00:02:25.004024,2025-072T17:13:47.592036,2025-072T17:13:47.592039,887,888,2
+FATAL,2010-001T00:02:25.004024,2010-001T00:02:25.004024,2025-072T17:13:47.592039,2025-072T17:13:47.592042,890,891,2
+FATAL,2010-001T00:02:25.004024,2010-001T00:02:25.004024,2025-072T17:13:47.592042,2025-072T17:13:47.592044,893,893,1

tts_data_utils/util.py

@@ -0,0 +1,104 @@
+import csv
+import random
+from datetime import datetime, timedelta
+
+def generate_telemetry_csv(filename="demosat_telemetry.csv"):
+    # Schema keys from EhaItem.DICT_VALID_KEYS
+    fieldnames = [
+        'recordType', 'sessionId', 'sessionHost', 'channelId', 'dssId', 
+        'vcid', 'name', 'module', 'ert', 'scet', 'rct', 'lst', 
+        'sclk', 'dn', 'dnStr', 'eu', 'status', 'dnAlarmState', 
+        'euAlarmState', 'realtime', 'type'
+    ]
+
+    # Configuration based on Channel.xml
+    channels = [
+        {"id": "EPS-0001", "name": "BATTERY_VOLTAGE", "mod": "eps", "vcid": 0},
+        {"id": "EPS-0002", "name": "BUS_CURRENT", "mod": "eps", "vcid": 0},
+        {"id": "INST-0003", "name": "CCD_TEMP", "mod": "inst", "vcid": 17},
+        {"id": "ADCS-2001", "name": "TARGET_LAT_ERR", "mod": "adcs", "vcid": 0},
+        {"id": "SYS-0002", "name": "BOOT_COUNT", "mod": "sys", "vcid": 0},
+        {"id": "ADCS-1001", "name": "EST_Q1", "mod": "adcs", "vcid": 0},
+        {"id": "ADCS-1002", "name": "EST_Q2", "mod": "adcs", "vcid": 0},
+        {"id": "ADCS-1003", "name": "EST_Q3", "mod": "adcs", "vcid": 0},
+        {"id": "ADCS-1004", "name": "EST_Q4", "mod": "adcs", "vcid": 0},
+        {"id": "TEL-0003", "name": "VCDU_COUNTER", "mod": "tel", "vcid": 17},
+    ]
+
+    base_scet = datetime(2026, 1, 25, 12, 0, 0)
+    base_sclk = 1737817192.0
+    
+    with open(filename, mode='w', newline='') as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+
+        for ch in channels:
+            current_scet = base_scet
+            current_sclk = base_sclk
+            
+            for i in range(50):
+                # 1. Create irregular timing (1-5 seconds jitter)
+                jitter = random.uniform(1.0, 5.0)
+                current_scet += timedelta(seconds=jitter)
+                current_sclk += jitter
+                ert = current_scet + timedelta(seconds=10) # 10s light delay simulation
+
+                # 2. Simulate Channel Logic & Alarm Violations
+                dn, eu, alarm = 0, 0.0, None
+                
+                if ch['id'] == "EPS-0001": # Battery Voltage
+                    # Start at 2200 DN (RED), drop to 1900 DN (NOMINAL)
+                    dn = max(1900, 2200 - (i * 8))
+                    eu = (0.00001 * (dn**2)) + (0.015 * dn) # Quadratic per XML
+                    alarm = "RED" if eu > 32.5 else ("YELLOW" if eu > 31.8 else None)
+                
+                elif ch['id'] == "EPS-0002": # Bus Current
+                    # Random spikes to simulate heaters
+                    eu = random.uniform(1.0, 2.0)
+                    if i in [10, 11, 30, 31]: eu += 3.0 # Spike
+                    dn = round(eu)
+                    alarm = "RED" if eu > 4.5 else ("YELLOW" if eu > 3.5 else None)
+                
+                elif ch['id'] == "INST-0003": # CCD Temp
+                    # Linear slope: slope 0.12, intercept -50.0
+                    dn = 500 + (i * 10) 
+                    eu = (dn * 0.12) - 50.0
+                    alarm = "RED" if eu > 45.0 else ("YELLOW" if eu > 35.0 else None)
+
+                elif ch['id'] == "TEL-0003": # VCDU Counter
+                    dn = 5000 + i
+                    eu = float(dn)
+                
+                else: # Default for Quaternions/Boots
+                    dn = round(random.uniform(0, 1)) if "EST_Q" in ch['name'] else 12
+                    eu = float(dn)
+
+                # 3. Write Row
+                writer.writerow({
+                    'recordType': 'EHA',
+                    'sessionId': 903,
+                    'sessionHost': 'GND-STN-01',
+                    'channelId': ch['id'],
+                    'dssId': 25,
+                    'vcid': ch['vcid'],
+                    'name': ch['name'],
+                    'module': ch['mod'],
+                    'ert': ert.strftime('%Y-%jT%H:%M:%S.%f')[:-3],
+                    'scet': current_scet.strftime('%Y-%jT%H:%M:%S.%f')[:-3],
+                    'rct': '',
+                    'lst': '',
+                    'sclk': current_sclk,
+                    'dn': dn,
+                    'dnStr': None,
+                    'eu': round(eu, 4),
+                    'status': None,
+                    'dnAlarmState': None,
+                    'euAlarmState': alarm,
+                    'realtime': True,
+                    'type': 'FSW'
+                })
+
+    print(f"Generated {filename} with 500 records.")
+
+if __name__ == "__main__":
+    generate_telemetry_csv()