corva-worker-python 2.0.0__py3-none-any.whl

corva_worker_python-2.0.0.dist-info/METADATA

@@ -0,0 +1,30 @@
+Metadata-Version: 2.4
+Name: corva-worker-python
+Version: 2.0.0
+Summary: SDK for interacting with Corva
+Home-page: https://github.com/corva-ai/corva-worker-python
+Author: Jordan Ambra <jordan.ambra@corva.ai>, Mohammadreza Kamyab <m.kamyab@corva.ai>
+License: Unlicensed
+Keywords: corva,worker
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.13
+Requires-Dist: numpy>=2.3.5
+Requires-Dist: redis>=7.1.0
+Requires-Dist: requests>=2.32.5
+Requires-Dist: simplejson>=3.20.2
+Requires-Dist: urllib3>=2.5.0
+Dynamic: author
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

corva_worker_python-2.0.0.dist-info/RECORD

@@ -0,0 +1,63 @@
+worker/__init__.py,sha256=xw-_4rRcZm65YTohJOUOZY7ZXE7WDGYwhwQxo3ng_t0,58
+worker/constants.py,sha256=beZCoZk6Tl32EzHCxP6WNtH4QPsG_Tay6sfZSnddYYI,1480
+worker/exceptions.py,sha256=ABjo2tay2TI40cVFSaKBE4i_ZqIC3IJZCw88qo2iLws,309
+worker/app/__init__.py,sha256=-PEDBW7xCBbbc3fE6pxA5JBKM_5SJaCaVFqvzS5aNxc,11596
+worker/app/modules/__init__.py,sha256=wJV7Jo_DSlkgDQqQRcmzplXGBG1SGfJIL9JHFi5hWgo,8871
+worker/app/modules/activity_module.py,sha256=ot1zHSBhOP76gu9RV4OWnQY1JNAq-r3nkRLtgpwdmU0,5046
+worker/app/modules/connection_module.py,sha256=qvaDgTBrf3NMA9Q1A-490fhjlhPVh_iskaMvLYK09lc,575
+worker/app/modules/depth_activity_module.py,sha256=0enhx_fhDiLRaPwVh4BDcu2KkWPzG_9Bkqt1p7RzHbQ,572
+worker/app/modules/scheduler.py,sha256=oZVLQH2p7aEQJUFXi4zL-QsGcfXGF859grI31TIt7bc,1448
+worker/app/modules/time_activity_module.py,sha256=_M3UhF3FNdjUHMJX1SMFx_weqCT087R0KP1HaoFH2d0,583
+worker/app/modules/trigger.py,sha256=OYRJCcK9wGxjvi0EcL9ZQSLZOMYR11RGLT_vynK5nF8,1438
+worker/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+worker/data/alert.py,sha256=1sxY2pQMnirmAV9mNNw8MW0dllebprlHK-XgfSfAKKc,3074
+worker/data/api.py,sha256=JSPVf1Ff6Cvkc1BfCpnOuhdRUuOSUVbZamAVe1Lyc1M,4799
+worker/data/enums.py,sha256=Owe3Kp7UrALRdl7_C829S6ULhBK_IRWUXcw_gA_2PT8,3876
+worker/data/json_encoder.py,sha256=jJcpNymA8370bgVIi_cz-EcUHjn4clC_I_t_2bK8Dqk,549
+worker/data/math.py,sha256=TSDwWZD-gY6oikhPSmpt97dcxhIFEV5wZgiZcs3FcxE,2952
+worker/data/operations.py,sha256=jv6w4Wqh75IDLeSPfkGNQ1Hu3cuom25d17dUSHlhRg4,12764
+worker/data/serialization.py,sha256=p0Rmi9n2L8TCm7sinSBKDh9xvSJe6cg3PUk_Uq_mSwY,3058
+worker/data/task_handler.py,sha256=X77REOvYqPqSOE4SLOgprwGd8kifmD9oRHXy3vNAH60,3113
+worker/data/two_way_dict.py,sha256=rR4sTF_whBrRP8-PKQRfry7gdpPhsxtBjtk44dgAHQ0,535
+worker/data/unit_conversions.py,sha256=UyFRYxTc1Jk2vN9Pfblq38_T-a_fBiZt7w2994ZbUag,154
+worker/data/wits.py,sha256=P-3vWRZ5T28m0Lj3DHQD8Bm6aTmhFjeCeTJeDrPNMEM,11324
+worker/data/activity/__init__.py,sha256=J63WWhGnkO3pYPJdhrtJfrzVS1xhjlO2EeRTdDNucd0,4807
+worker/data/activity/activity_grouping.py,sha256=eDy1RqrlLCS4BlKNBb0xM7QHDX6lAw28zMLRwGcUBrE,8779
+worker/event/__init__.py,sha256=7gYI4NcpO1snmrjZnrSkZgYs70xZwnlz4tXm0hzGs3E,1704
+worker/event/event_handler.py,sha256=Hv9zm0z8i3vQeCfn_qTbykMaLS1j6muIz1i28qQZkOs,3401
+worker/event/scheduled.py,sha256=Qrh6Upw1skQ4I8C0Q59zaWHcTSqMDlgEzvdocPEh-mM,2270
+worker/event/stream.py,sha256=FTBkY1LQGgsVTSZ8_e4C_mMCMW0MyVehrvUxHVaonX0,1639
+worker/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+worker/mixins/logging.py,sha256=vZrEiRZVmON0y9nMhao1gV8qqBkZ9zcpMEJkTYyhRGc,3234
+worker/mixins/rollbar.py,sha256=DdmTw8PpExEofoZcPHBvmmIHfbMx4yB1lsdsMubtB5c,3089
+worker/partial_rerun_merge/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+worker/partial_rerun_merge/merge.py,sha256=a8RvEio_3wSZKkCqVvbpf3l0EqxP-HL2S_MEwToiw5c,17620
+worker/partial_rerun_merge/models.py,sha256=UaGBf2HZkTitg4jgZtvVSyQKEFtwMSNMfND5bM-KpT4,2794
+worker/partial_rerun_merge/progress.py,sha256=BPhUf8_IEc2SbzDgkN9tqmYOYfh-L9qVGg4mZ25t94I,7791
+worker/state/__init__.py,sha256=X0TaxqfNqGp-lgraYgWpLHgdSDK0WlNjDYI0VWT9WOM,3069
+worker/state/mixins.py,sha256=43YuxtRUfYZ2c5Ql4wcUOOxuWNVnKevElwDxEtueNCE,4044
+worker/state/state.py,sha256=KEiw2-sB6xxyrweNE34JZ-iOD6C61sxXn3tsJvV6vLk,1267
+worker/test/__init__.py,sha256=WCh2zc6JHZuiK8lPB41_aytnSqVcpAsNj5Tio4BARAA,82
+worker/test/lambda_function_test_run.py,sha256=nbS6owG9deaHNYllxOs8klHHLR0_iSYy_jn1ZE1aB_I,6406
+worker/test/utils.py,sha256=Z7tgC_lUk2bRMlR-3tY9i5d836AdYlIY4i9WFFplheg,1479
+worker/test/local_testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+worker/test/local_testing/to_local_transfer.py,sha256=QYLGtT9fgDKsSK3x2QsaNDCMOpPZ5rpOsSnSl5s--WI,12343
+worker/wellbore/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+worker/wellbore/factory.py,sha256=xKKpa9y3yuIQDfpps8wKSA2CiLhZuYFeAhrdZ0ZwdQM,19353
+worker/wellbore/measured_depth_finder.py,sha256=I5FmkELT0_k1ajlraeBx2Q-G3noqNTAw1wEPKyUAQOw,367
+worker/wellbore/sections_mixin.py,sha256=JNeEKm_fRxMBBD7SUx7LM8lRDJjKIeIcVvcSOKt4DTY,1873
+worker/wellbore/wellbore.py,sha256=syvYszkbN40BhzxCQ8m6VmSBGPp2wQR9sJ6TcSu7WO8,11907
+worker/wellbore/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+worker/wellbore/model/ann.py,sha256=4xf-YQqyuWbMXDbH8XV0ohuga5r8SVO9lJVMgdxS-7c,3483
+worker/wellbore/model/annulus.py,sha256=U2JHQtJp6KPVlcAZMlO2MvrQf0I-cM2GlM969RBn8fg,3805
+worker/wellbore/model/drillstring.py,sha256=om3It2UVZs_WSlaVJmpNziEIlAdVlN58tGXjcK0qMDo,6441
+worker/wellbore/model/drillstring_components.py,sha256=PFs-as6b2D494gzBKnLz2jToKVghOymcVsU-4NaYamk,14769
+worker/wellbore/model/element.py,sha256=6axQ1eh3zwej_9kke9o1uXNtJv4UABMTK0VmsCDfJYQ,3231
+worker/wellbore/model/enums.py,sha256=DZm-o9ZCM1hSRHw_CcOK4Oy2Q_bmpDM7Gx3GlKQ_bvE,2828
+worker/wellbore/model/hole.py,sha256=LYikxOMJfxhc2yoKZN12pDBIl6W8apJcAGtGR3ZuvhM,10046
+worker/wellbore/model/hole_section.py,sha256=4sfP19xiY--wV1Ve2MtRoNqpya-FdPhNNmTgxLb90uk,1630
+worker/wellbore/model/riser.py,sha256=ElxvQMfdFTGGpJ1Jba_DNVue3zGAsm-05zDl1HBXwDk,824
+corva_worker_python-2.0.0.dist-info/METADATA,sha256=QTJwlYdwZWW7PqJNz6CYV6GY9K0mOcy5aSRz4X_Hv9Y,1017
+corva_worker_python-2.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+corva_worker_python-2.0.0.dist-info/top_level.txt,sha256=KrP-SOC-I1V0U33nVzMgkHR8wO-reJYASIvWVyLyFIY,7
+corva_worker_python-2.0.0.dist-info/RECORD,,

corva_worker_python-2.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

corva_worker_python-2.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+worker

worker/__init__.py

@@ -0,0 +1,5 @@
+from worker.data.api import API
+
+__all__ = [
+    "API",
+]

worker/app/__init__.py

@@ -0,0 +1,291 @@
+from typing import List, Optional
+
+from worker import constants, exceptions
+from worker.app.modules import Module
+from worker.app.modules.time_activity_module import TimeActivityModule
+from worker.data.enums import EventType, RerunMode
+from worker.data.operations import gather_data_for_period
+from worker.event import Event
+from worker.event.scheduled import ScheduledEvent
+from worker.event.stream import StreamEvent
+from worker.mixins.logging import LoggingMixin
+from worker.mixins.rollbar import RollbarMixin
+from worker.partial_rerun_merge.models import RerunMergeCacheUpdater
+from worker.state.mixins import RedisMixin
+from worker.state.state import State
+
+
+class App(RedisMixin, LoggingMixin, RollbarMixin):
+    """
+    The apps is designed to receive the events of a single asset
+    """
+
+    app_key = constants.get("global.app-key")
+    app_name = constants.get("global.app-name")
+
+    app_state_fields = {"asset_id": int, "last_processed_timestamp": int}
+
+    def __init__(self, *args, **kwargs):
+        self.module_key = None
+
+        self.event_type: EventType = None
+
+        self.asset_id = None
+        self.event = None  # event records
+
+        super().__init__(*args, **kwargs)
+
+    def load(self, event_type: EventType, event: Event):
+        """
+        :param event_type:
+        :param event: a scheduler event or wits stream; belong to one asset
+        :return:
+        """
+        self.event_type = event_type
+
+        self.asset_id = event.asset_id
+
+        self.state = self.load_state()
+
+        max_lookback_seconds = self.get_max_lookback_seconds()
+        event = self.load_event(event, max_lookback_seconds)
+
+        valid_stream_collections = self.get_valid_stream_collections()
+        self.event = self.filter_event_for_collections(
+            self.event_type, valid_stream_collections, event
+        )  # event records
+
+        self.log_event(self.event, max_lookback_seconds)  # event records
+
+    def log_event(self, event: List[dict], max_lookback_seconds: int):
+        self.debug(self.asset_id, "WITS input to {0} -> {1}".format(self.app_name, event))
+
+        if not event:
+            return
+
+        batch_size = len(event)
+        start_time = event[0].get("timestamp")
+        end_time = event[-1].get("timestamp")
+
+        self.log(
+            self.asset_id,
+            text=(
+                f"Received {batch_size} elements from {start_time} to {end_time}. "
+                f"{max_lookback_seconds} seconds of initial data are lookback."
+            ),
+        )
+
+    @staticmethod
+    def get_valid_stream_collections():
+        valid_collections = constants.get("global.valid-stream-collections", [])
+        if not isinstance(valid_collections, (str, list)):
+            raise TypeError("Incorrect type of valid-stream-collections in global constants")
+
+        if isinstance(valid_collections, str):
+            valid_collections = [valid_collections]
+        return valid_collections
+
+    def get_max_lookback_seconds(self):
+        """
+        For each module (mostly in time-base modules), the time of processing does not
+        match the last time of the event so extra data is required to look back and get
+        the data so the processing can start from where it left off.
+        :return:
+        """
+
+        time_modules = [module for module in self.get_modules() if issubclass(module, TimeActivityModule)]
+        maximum_lookback = 0
+        for module in time_modules:
+            module_lookback = constants.get(
+                "{}.{}.lookback-duration".format(self.app_key, module.module_key), default=0
+            )
+            maximum_lookback = max(module_lookback, maximum_lookback)
+
+        return maximum_lookback
+
+    def load_event(self, event: Event, max_lookback_seconds: int) -> List[dict]:
+        if self.event_type == EventType.SCHEDULER:
+            return self.load_scheduler_event(self.asset_id, event, max_lookback_seconds)
+
+        if self.event_type == EventType.STREAM:
+            return self.load_wits_stream_event(self.asset_id, event, max_lookback_seconds)
+
+        return None
+
+    @staticmethod
+    def filter_event_for_collections(event_type: EventType, valid_stream_collections: list, event: list) -> list:
+        """
+        This function filters the incoming event based on a list of valid collections
+
+        :param event_type: Type of incoming event.
+        :param valid_stream_collections: List of valid collections
+        :param event: List of data records
+        :return: List of records whose collection is one of the allowed event collections
+        """
+
+        # If event type is scheduler or event_collections is empty, return the entire event
+        if event_type == EventType.SCHEDULER or not valid_stream_collections:
+            return event
+
+        # Filtering the event based on valid collections
+        event = [record for record in event if record.get("collection") in valid_stream_collections]
+        return event
+
+    def filter_event_for_duplicates(self, event) -> list:
+        last_processed_timestamp = self.state.get("last_processed_timestamp") or 0
+
+        # If event is a single record and greater than the last processed timestamp, return the event
+        if len(event) == 1 and event[0].get("timestamp") > last_processed_timestamp:
+            return event
+
+        # If length of unique timestamps is same as length of event and the first record timestamp is also greater than
+        # last_processed_timestamp, then return the event
+        unique_timestamps = set([record.get("timestamp") for record in event])
+        if len(unique_timestamps) == len(event) and event[0].get("timestamp") > last_processed_timestamp:
+            return event
+
+        # Filtering the events for duplicates, once we identify that duplicates exist.
+        filtered_events = []
+        for each_record in event:
+            if each_record.get("timestamp") > last_processed_timestamp:
+                filtered_events.append(each_record)
+                last_processed_timestamp = each_record.get("timestamp")
+
+        return filtered_events
+
+    def load_scheduler_event(self, asset_id: int, event: ScheduledEvent, max_lookback_seconds: int) -> List[dict]:
+        """
+        To load a scheduler event and get the wits stream data
+        :param asset_id: The asset to load
+        :param event: A cleaned event
+        :param max_lookback_seconds: Maximum amount of time to look back prior to the scheduler event to cover gaps
+        :return: list of WITS data between the last processed timestamp and the final event item timestamp
+        """
+
+        start_timestamp = self.state.get("last_processed_timestamp", event[0].start_time - 1)
+        end_timestamp = event[-1].start_time
+
+        # the event is converted from scheduler to wits stream
+        scheduler_event = gather_data_for_period(
+            asset_id=asset_id,
+            start=start_timestamp - max_lookback_seconds,
+            end=end_timestamp,
+            limit=constants.get("global.query-limit"),
+            fields=constants.get("global.wits_query_fields", None),
+        )
+
+        return scheduler_event
+
+    def load_wits_stream_event(self, asset_id: int, event: StreamEvent, max_lookback_seconds: int) -> List[dict]:
+        """
+        To load a wits stream event and get more data if necessary
+        :param asset_id: The asset to load
+        :param event: A cleaned event
+        :param max_lookback_seconds: Maximum amount of time to look back prior to WITS data to cover gaps
+        :return: list of WITS data between the first event timestamp and the first timestamp
+        """
+        records = event.records
+
+        # First filtering original event for duplicates
+        records = self.filter_event_for_duplicates(records)
+
+        # If all record timestamps are before the last processed timestamp,
+        # the first timestamp will be the last processed timestamp from cache
+        if not records:
+            first_timestamp = self.state.get("last_processed_timestamp") or 0
+        else:
+            first_timestamp = records[0].get("timestamp")
+
+        if not first_timestamp:
+            return records
+
+        if max_lookback_seconds:
+            # Subtract one from the timestamp so that we don't reselect the final data item that was sent in the event
+            end_timestamp = first_timestamp - 1
+
+            previous_events = gather_data_for_period(
+                asset_id=asset_id,
+                start=first_timestamp - max_lookback_seconds,
+                end=end_timestamp,
+                limit=constants.get("global.query-limit"),
+                fields=constants.get("global.wits_query_fields", None),
+            )
+
+            records = previous_events + records
+
+        return records
+
+    @staticmethod
+    def determine_asset_id(event: list) -> int:
+        try:
+            return int(event[0]["asset_id"])
+        except Exception:
+            raise Exception(f"Event does not contain asset_id: {event}")
+
+    def load_state(self, state_key: Optional[str] = None, raise_warnings: bool = False) -> dict:
+        previous_state = super().load_state(state_key=state_key, raise_warnings=raise_warnings)
+
+        state = State(self.app_state_fields, previous_state)
+
+        if not state.get("asset_id", None):
+            state["asset_id"] = self.asset_id
+
+        return state
+
+    def get_modules(self) -> List[Module]:
+        raise NotImplementedError("No modules found")
+
+    def get_active_modules(self) -> List[Module]:
+        return [module for module in self.get_modules() if module.enabled]
+
+    def run_modules(self):
+        if not self.event:  # event records
+            return
+
+        for module_type in self.get_active_modules():
+            try:
+                module = module_type(self.state, rollbar=self.rollbar)
+            except Exception:
+                raise exceptions.Misconfigured(
+                    "Module {0} not able to initialize for asset_id {1}".format(module_type, self.asset_id)
+                )
+
+            try:
+                module.run(self.event)  # event records
+            except Exception as ex:
+                message = f"Error in module {module_type.module_key}"
+                ex.args += (message,)  # Adding message to existing Exception
+                raise
+
+        last_processed_timestamp = self.event[-1].get("timestamp")  # event records
+        self.state["last_processed_timestamp"] = last_processed_timestamp
+
+    @classmethod
+    def update_cache(cls, merger: "PartialRerunMerge") -> None:
+        """
+        This function updates the cache of the original asset with the rerun asset's cache
+
+        If in any app that is extending this class, it needs to be overridden, it can be done so.
+
+        :param merger: PartialRerunMerge object
+        :return: None
+        """
+        if merger.rerun_mode == RerunMode.HISTORICAL:
+            return
+
+        original_asset_id = merger.original_asset_id
+        rerun_asset_id = merger.rerun_asset_id
+
+        redis_handler = RedisMixin()
+        redis_handler.asset_id = original_asset_id
+
+        original_state_key = redis_handler.get_formatted_state_key(original_asset_id, cls.app_key)
+        rerun_state_key = redis_handler.get_formatted_state_key(rerun_asset_id, cls.app_key)
+
+        original_state = redis_handler.load_state(original_state_key)
+        rerun_state = redis_handler.load_state(rerun_state_key)
+
+        original_state = RerunMergeCacheUpdater.default_updater(original_state, rerun_state)
+
+        redis_handler.state = original_state
+        redis_handler.save_state(original_state_key)

worker/app/modules/__init__.py

@@ -0,0 +1,265 @@
+from typing import Union
+
+import simplejson as json
+
+from worker import constants
+from worker.data.api import API
+from worker.data.enums import RerunMode
+from worker.data.json_encoder import JsonEncoder
+from worker.mixins.logging import LoggingMixin
+from worker.mixins.rollbar import RollbarMixin
+from worker.partial_rerun_merge.models import RerunMergeCacheUpdater
+from worker.state.mixins import RedisMixin
+from worker.state.state import State
+
+
+class Module(RedisMixin, LoggingMixin, RollbarMixin):
+    """
+    This is an abstract base module that needs to be extended by an actual module.
+    """
+
+    # module_key is used for redis access and state of this module
+    app_key = constants.get("global.app-key")
+    app_name = constants.get("global.app-name")
+    module_key = "module"
+    collection = "collection"
+    module_state_fields = {"last_processed_timestamp": int}
+
+    enabled = True
+
+    def __init__(self, global_state, *args, **kwargs):
+        self.asset_id = global_state.get("asset_id")
+
+        self.global_state = global_state
+
+        super().__init__(*args, **kwargs)
+
+    def run(self, wits_stream: list):
+        """
+        :param wits_stream: a wits stream event
+        :return:
+        """
+        # load the state
+        state = self.get_state()
+        self.state = self.process_module_state(state)
+
+        # subclasses should implement their own run
+
+    def should_run_processor(self, event):
+        raise Exception("This method need to be implemented by subclasses!")
+
+    def get_state(self, state_key: [str, None] = None, raise_warnings: bool = False) -> dict:
+        current_state = super().load_state(state_key=state_key, raise_warnings=raise_warnings)
+        return State(self.module_state_fields, current_state)
+
+    @staticmethod
+    def process_module_state(state):
+        return state
+
+    def set_state(self, key, value):
+        self.state[key] = value
+
+    def load_dataset(self, event):
+        return event
+
+    def run_module(self, dataset: Union[list, dict], beginning_results: list) -> list:
+        raise Exception("Not implemented")
+
+    def get_last_exported_timestamp_from_collection(self, asset_id, query=None, less_than=None):
+        """
+        Query the module collection for this asset_id + module, sorted by timestamp descending,
+        limit 1, grab the last item's timestamp. Default to 0 if no records found.
+        @asset_id:
+        @less_than: the timestamp before which you want to get
+        """
+        if less_than:
+            query = query or ""
+            query += "AND{timestamp#lt#%s}" % less_than
+
+        worker = API()
+        last_document = worker.get(
+            path="/v1/data/corva",
+            query=query,
+            collection=self.collection,
+            asset_id=asset_id,
+            sort="{timestamp: -1}",
+            limit=1,
+        ).data
+
+        if not last_document:
+            return 0
+
+        last_document = last_document[0]
+        last_processed_timestamp = last_document.get("timestamp", 0)
+
+        return last_processed_timestamp
+
+    @staticmethod
+    def gather_first_wits_timestamp_since(asset_id: int, since: int, activity_fields=None, operator="eq") -> int:
+        """
+        Query the Wits collection for this asset_id where state in wits_states and timestamp >= since
+        """
+
+        query = "{timestamp#%s#%s}" % ("gt", since)
+
+        operator = operator.lower()
+
+        if activity_fields:
+            if operator == "eq" and isinstance(activity_fields, list):
+                operator = "in"
+
+            if operator in ("in", "nin"):
+                if not isinstance(activity_fields, list):
+                    activity_fields = [activity_fields]
+
+                # Put each state into a formatted string for querying
+                activity_fields = ["'{0}'".format(state) for state in activity_fields]
+                activity_fields = "[{0}]".format(",".join(activity_fields))
+            else:
+                activity_fields = "'{0}'".format(activity_fields)
+
+            query += "AND{data.state#%s#%s}" % (operator, activity_fields)
+
+        worker = API()
+        first_wits_since = worker.get(
+            path="/v1/data/corva", collection="wits", asset_id=asset_id, sort="{timestamp: 1}", limit=1, query=query
+        ).data
+
+        if not first_wits_since:
+            return 0
+
+        first_wits_since = first_wits_since[0]
+        first_wits_since_timestamp = first_wits_since.get("timestamp", 0)
+
+        return first_wits_since_timestamp
+
+    @staticmethod
+    def gather_maximum_timestamp(event, start, activity_fields):
+        """
+        get the maximum time stamp of a stream of data
+        :param event: a stream of data  that the majority is wits collection
+        :param start:
+        :param activity_fields:
+        :return:
+        """
+        maximum_timestamp = start
+        for data in event:
+            if data.get("collection") == "wits" and data.get("data", {}).get("state", None) in activity_fields:
+                maximum_timestamp = max(data.get("timestamp", 0), maximum_timestamp)
+
+        return maximum_timestamp
+
+    def gather_minimum_timestamp(self, asset_id: int, event: list):
+        minimum = self.get_last_exported_timestamp_from_collection(asset_id)
+
+        if not minimum:
+            minimum = event[0]["timestamp"] - 1800
+
+        return minimum
+
+    def gather_collections_for_period(self, asset_id, start, end, query=None):
+        limit = constants.get("global.query-limit")
+
+        query = query or ""
+        if query:
+            query += "AND"
+
+        query += "{timestamp#gte#%s}AND{timestamp#lte#%s}" % (start, end)
+
+        worker = API()
+        dataset = worker.get(
+            path="/v1/data/corva",
+            collection=self.collection,
+            asset_id=asset_id,
+            query=query,
+            sort="{timestamp: 1}",
+            limit=limit,
+        ).data
+
+        if not dataset:
+            return []
+
+        return dataset
+
+    def store_output(self, asset_id, output):
+        """
+        to store/post results
+        :param asset_id: asset id of the well
+        :param output: an array of json objects to be posted
+        :return: None
+        """
+
+        if not asset_id or not output or not self.collection:
+            return
+
+        output = self.format_output(output)
+
+        self.debug(asset_id, "{0} output -> {1}".format(self.module_key, output))
+
+        worker = API()
+        worker.post(path="/v1/data/corva", data=output)
+
+    def build_empty_output(self, wits: dict) -> dict:
+        """
+        Building an empty output result.
+        :param wits: one wits record
+        :return:
+        """
+        output = {
+            "timestamp": int(wits.get("timestamp")),
+            "company_id": int(wits.get("company_id")),
+            "asset_id": int(wits.get("asset_id")),
+            "provider": str(wits.get("provider", "corva")),
+            "version": 1,
+            "collection": self.collection,
+            "data": {},
+        }
+
+        enable_output_type_field = constants.get(
+            "{}.{}.enable-output-type-field".format(self.app_key, self.module_key), False
+        )
+
+        if not enable_output_type_field:
+            return output
+
+        # If a specific module enforces type field in the output collection,
+        # build_empty_output adds an additional type field to the output.
+        # This can be enforced by setting "enable-output-type-field" to True in the module app constants.
+        output.update({"type": self.module_key})
+
+        return output
+
+    @staticmethod
+    def format_output(output):
+        output = json.dumps(output, cls=JsonEncoder, ignore_nan=True)
+        return output
+
+    @classmethod
+    def update_cache(cls, merger: "PartialRerunMerge") -> None:
+        """
+        This function updates the cache of the original asset with the rerun asset's cache
+
+        If in any app that is extending this class, it needs to be overridden, it can be done so.
+
+        :param merger: PartialRerunMerge object
+        :return: None
+        """
+        if merger.rerun_mode == RerunMode.HISTORICAL:
+            return
+
+        original_asset_id = merger.original_asset_id
+        rerun_asset_id = merger.rerun_asset_id
+
+        redis_handler = RedisMixin()
+        redis_handler.asset_id = original_asset_id
+
+        original_state_key = redis_handler.get_formatted_state_key(original_asset_id, cls.app_key, cls.module_key)
+        rerun_state_key = redis_handler.get_formatted_state_key(rerun_asset_id, cls.app_key, cls.module_key)
+
+        original_state = redis_handler.load_state(original_state_key)
+        rerun_state = redis_handler.load_state(rerun_state_key)
+
+        original_state = RerunMergeCacheUpdater.default_updater(original_state, rerun_state)
+
+        redis_handler.state = original_state
+        redis_handler.save_state(original_state_key)