corva-worker-python 2.0.0__py3-none-any.whl

worker/data/activity/activity_grouping.py

@@ -0,0 +1,242 @@
+import itertools
+from collections import OrderedDict
+from typing import List, Union
+
+from worker.data import operations
+from worker.data.activity import Activity
+from worker.data.math import split_zip_edges
+from worker.data.serialization import serialization
+from worker.data.wits import WITS
+
+
+@serialization
+class ActivityGroup:
+    SERIALIZED_VARIABLES = {"activity": Activity, "start": int, "end": int}
+
+    def __init__(self, activity: Activity, start: int, end: int, **kwargs):
+        self.activity = activity
+        if end >= start:
+            self.start = start
+            self.end = end
+
+    @property
+    def duration(self) -> int:
+        """
+        :return: duration in seconds
+        """
+        return self.end - self.start + 1
+
+    @classmethod
+    def merge(
+        cls,
+        grp1: "ActivityGroup",
+        grp2: "ActivityGroup",
+        max_duration_gap: int = None,
+        resultant_activity: Activity = None,
+    ) -> Union["ActivityGroup", None]:
+        """
+        Merging two groups of activities into one.
+        :param grp1:
+        :param grp2:
+        :param max_duration_gap: allowed gap between groups
+        :param resultant_activity: if provided it will be used as the output activity, otherwise the activity
+        of both groups are checked and if they are only the same it will continue the merging
+        :return:
+        """
+        min_start = min(grp1.start, grp2.start)
+        max_end = max(grp1.end, grp2.end)
+
+        if max_duration_gap is not None:
+            max_start = max(grp1.start, grp2.start)
+            min_end = min(grp1.end, grp2.end)
+
+            # if the gap between two activities are greater than the threshold, return None
+            if max_start - min_end > max_duration_gap:
+                return None
+
+        if not resultant_activity:
+            if grp1.activity != grp2.activity:
+                return None
+            resultant_activity = grp1.activity
+
+        return ActivityGroup(resultant_activity, min_start, max_end)
+
+    @staticmethod
+    def has_overlap(group1: "ActivityGroup", group2: "ActivityGroup") -> bool:
+        """
+        Checking if two activity groups has overlap regardless of their precedence
+        :param group1:
+        :param group2:
+        :return:
+        """
+        if not all([group1, group2]):
+            return False
+
+        max_start = max(group1.start, group2.start)
+        min_end = min(group1.end, group2.end)
+
+        return max_start <= min_end
+
+    def __eq__(self, other):
+        if not isinstance(other, ActivityGroup):
+            return False
+
+        return operations.equal(self, other, list(self.SERIALIZED_VARIABLES.keys()))
+
+    def __repr__(self):
+        return f"{self.activity.value:<25}: {self.start}-{self.end} -> duration={self.duration:>4}"
+
+
+"""
+In 1-second data frequency sometimes some activities will be added that can be
+part of the bounding activities. For instance, you might be Reaming Down and
+for a second the bit depth is the same as the previous timestamp and the activity
+is Rotary Off Bottom. This will then be grouped into Reaming Down group.
+"""
+NEUTRAL_ACTIVITIES = OrderedDict(
+    [
+        (Activity.RUN_IN_HOLE, [Activity.STATIC_OFF_BOTTOM]),
+        (Activity.PULL_OUT_OF_HOLE, [Activity.STATIC_OFF_BOTTOM]),
+        (Activity.WASHING_DOWN, [Activity.CIRCULATING]),
+        (Activity.WASHING_UP, [Activity.CIRCULATING]),
+        (Activity.DRY_REAMING_DOWN, [Activity.DRY_ROTARY_OFF_BOTTOM]),
+        (Activity.DRY_REAMING_UP, [Activity.DRY_ROTARY_OFF_BOTTOM]),
+        (Activity.REAMING_DOWN, [Activity.ROTARY_OFF_BOTTOM]),
+        (Activity.REAMING_UP, [Activity.ROTARY_OFF_BOTTOM]),
+        (Activity.STATIC_OFF_BOTTOM, [Activity.RUN_IN_HOLE, Activity.PULL_OUT_OF_HOLE]),
+        (Activity.CIRCULATING, [Activity.WASHING_DOWN, Activity.WASHING_UP]),
+        (Activity.DRY_ROTARY_OFF_BOTTOM, [Activity.DRY_REAMING_DOWN, Activity.DRY_REAMING_UP]),
+        (Activity.ROTARY_OFF_BOTTOM, [Activity.REAMING_DOWN, Activity.REAMING_UP]),
+    ]
+)
+
+
+class ActivityGrouping:
+    def __init__(
+        self,
+        min_group_duration: int = 3,
+        apply_neutral_grouping: bool = True,
+        apply_gap_filling: bool = True,
+        merging_dict: dict = None,
+        time_step=None,
+    ):
+        """
+        :param min_group_duration: minimum duration of each group
+        :param apply_neutral_grouping:
+        :param apply_gap_filling:
+        :param merging_dict: A dictionary representing the activities to be merged. For instance in the following
+        example the Pull Out of Hole, Washing Up and Reaming Up are merged into one group and the activity of the
+        whole group is PULL_OUT_OF_HOLE.
+        {
+            Activity.PULL_OUT_OF_HOLE : [Activity.PULL_OUT_OF_HOLE, Activity.WASHING_UP, Activity.REAMING_UP],
+            ...
+        }
+        """
+        self.min_group_duration = min_group_duration
+        self.apply_neutral_grouping = apply_neutral_grouping
+        self.apply_gap_filling = apply_gap_filling
+        self.merging_dict = merging_dict
+        self.time_step = time_step
+
+    def group(self, wits_records: List[WITS]) -> List[ActivityGroup]:
+        """
+        to categorize a stream of wits data into groups of similar activities
+        :param wits_records: wits records
+        :return:
+        """
+        timestamps = [wits.timestamp for wits in wits_records]
+        activities = [wits.state for wits in wits_records]
+
+        if self.apply_neutral_grouping:
+            activities = perform_neutral_grouping(activities)
+
+        if not self.time_step:
+            self.time_step = operations.compute_time_step(timestamps, percent=95)
+
+        unique_activities = unique_everseen(activities)
+        groups = []
+        for activity in unique_activities:
+            # Finding timestamps of matching activities
+            valid_timestamps = [timestamps[i] for i in range(len(timestamps)) if activities[i] == activity]
+            # Grouping and finding the edges of the activities
+            edges = split_zip_edges(
+                valid_timestamps, separation_length=self.time_step, min_segment_length=self.min_group_duration
+            )
+            this_groups = [ActivityGroup(activity=activity, start=start, end=end) for (start, end) in edges]
+            groups.extend(this_groups)
+
+        groups.sort(key=lambda x: x.start)
+
+        if self.merging_dict:
+            # 1. mapping the activities to their merging activities
+            for grp in groups:
+                grp.activity = self.merge_activity_map(grp.activity)
+
+            # 2. merging adjacent activities with similar merging activities
+            index = 1
+            while index < len(groups):
+                grp1, grp2 = groups[index - 1 : index + 1]
+                merged_group = ActivityGroup.merge(grp1, grp2, max_duration_gap=max(self.time_step, 2))
+                if not merged_group:
+                    index += 1
+                    continue
+                # update one and remove the next
+                groups[index - 1] = merged_group
+                groups.pop(index)
+
+        if self.apply_gap_filling:
+            groups = perform_gap_filling(groups)
+
+        return groups
+
+    def merge_activity_map(self, activity):
+        """
+        Map the constituent activity to the group activity
+        :param activity:
+        :return:
+        """
+        if not self.merging_dict:
+            return activity
+
+        for merging_group, constituent_activities in self.merging_dict.items():
+            if activity in constituent_activities:
+                return merging_group
+
+        return activity
+
+
+def perform_neutral_grouping(activities: List[Activity]):
+    for bound_activity, neutral_activities in NEUTRAL_ACTIVITIES.items():
+        for i in range(1, len(activities) - 1):
+            if bound_activity == activities[i - 1] == activities[i + 1] and activities[i] in neutral_activities:
+                activities[i] = bound_activity
+    return activities
+
+
+def perform_gap_filling(groups: List[ActivityGroup]):
+    # TODO something to consider
+    return groups
+
+
+def unique_everseen(iterable, key=None):
+    """
+    from: https://docs.python.org/3.3/library/itertools.html
+    List unique elements, preserving order. Remember all elements ever seen.
+    :param iterable:
+    :param key:
+    :return:
+    Example:
+    unique_everseen('AAAABBBCCDAABBB') --> A B C D
+    unique_everseen('ABBCcAD', str.lower) --> A B C D
+    """
+    seen = set()
+    if key is None:
+        for element in itertools.filterfalse(seen.__contains__, iterable):
+            seen.add(element)
+            yield element
+    else:
+        for element in iterable:
+            k = key(element)
+            if k not in seen:
+                seen.add(k)
+                yield element

worker/data/alert.py

@@ -0,0 +1,89 @@
+import simplejson as json
+
+from worker import API
+from worker.data.json_encoder import JsonEncoder
+from worker.mixins.logging import LoggingMixin
+from worker.mixins.rollbar import RollbarMixin
+
+
+class Alert(LoggingMixin, RollbarMixin):
+    """
+    Alert class can be used to trigger alerts from a data app.
+    An event based alert has to be created on the Corva UI with a unique identifier.
+
+    How to use:
+    1. Create an instance of Alert with the same unique identifier and asset_id.
+    2. Call trigger_alert and pass the required parameters to trigger the alert.
+    3. To trigger an alert on Corva, the well must be active and visible to the user. (Current rules)
+    """
+
+    def __init__(self, asset_id: int, identifier: str, last_trigger_timestamp=None, *args, **kwargs):
+        self.asset_id = asset_id
+        self.identifier = identifier
+        self.last_trigger_timestamp = last_trigger_timestamp
+
+        super().__init__(*args, **kwargs)
+
+    def trigger_alert(self, timestamp: int, timestamp_read: int, context: dict):
+        """
+        Function used to trigger the alert for the unique identifier.
+
+        :param timestamp: timestamp at which the event for the alert is detected
+        :param timestamp_read: timestamp at which the wits record was read by the source app
+        :param context: A dict of data to be sent to the alerts engine
+        :return:
+        """
+
+        context = context or {}
+        context.update(
+            {
+                "identifier": self.identifier,
+                "asset_id": self.asset_id,
+                "timestamp": timestamp,
+                "timestamp_read": timestamp_read,
+            }
+        )
+
+        api = API()
+        try:
+            trigger = api.post(
+                "/v1/alerts/definitions/trigger/", data=json.dumps(context, cls=JsonEncoder, ignore_nan=True)
+            ).data
+            self.debug(self.asset_id, f"Triggered alert with context -> {context} and received response {trigger}")
+            self.last_trigger_timestamp = timestamp
+
+            return trigger
+        except Exception as ex:
+            message = f"Error while triggering alert for context {context} with exception {ex}"
+            self.debug(self.asset_id, f"Failed to trigger alert with context -> {context}")
+            self.track_error(message)
+
+
+class Alerter:
+    alerter = None
+    asset_id = None
+
+    @classmethod
+    def set_asset_id(cls, asset_id):
+        cls.asset_id = asset_id
+
+    @classmethod
+    def get_alerter(cls):
+        if cls.alerter is None:
+            cls.alerter = Alert(0, "")
+
+        return cls.alerter
+
+    @classmethod
+    def trigger_alert(
+        cls, identifier: str, message: str, timestamp: int = None, timestamp_read: int = None, context: dict = None
+    ):
+        if not cls.asset_id:
+            return
+
+        context = context or {}
+        context.update({"message": message})
+
+        cls.get_alerter().asset_id = cls.asset_id
+        cls.get_alerter().identifier = identifier
+        cls.get_alerter().trigger_alert(timestamp, timestamp_read, context)

worker/data/api.py

@@ -0,0 +1,155 @@
+import os
+
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3 import Retry
+
+from worker import exceptions
+
+
+class API(object):
+    HTTP_METHODS = ("get", "post", "patch", "put", "delete")
+    options = {"api_url": "API_ROOT_URL", "api_key": "API_KEY", "app_name": "APP_NAME"}
+
+    def __init__(self, *args, **kwargs):
+        self.configure(self.options, kwargs)
+
+    def configure(self, options, values):
+        for attribute, environment_key in options.items():
+            value = values.pop(attribute, os.getenv(environment_key, None))
+            if not value:
+                error = "No {0} parameter or {1} environment variable defined".format(attribute, environment_key)
+                raise exceptions.Misconfigured(error)
+
+            setattr(self, attribute, value)
+
+    def get(self, *args, **kwargs):
+        return self.call("get", *args, **kwargs)
+
+    def post(self, *args, **kwargs):
+        return self.call("post", *args, **kwargs)
+
+    def patch(self, *args, **kwargs):
+        return self.call("patch", *args, **kwargs)
+
+    def put(self, *args, **kwargs):
+        return self.call("put", *args, **kwargs)
+
+    def delete(self, *args, **kwargs):
+        return self.call("delete", *args, **kwargs)
+
+    def call(self, method, path, **kwargs):
+        content_type = kwargs.pop("content_type", "application/json")
+        headers = {
+            "Authorization": "API {0}".format(self.api_key),
+            "Content-Type": content_type,
+            "X-Corva-App": self.app_name,
+        }
+
+        method = method.lower()
+        if method not in self.HTTP_METHODS:
+            raise exceptions.APIError("Invalid HTTP method {0}".format(method))
+
+        retry_count = kwargs.pop("retry_count", 5)
+        retry_strategy = Retry(
+            total=retry_count,
+            status_forcelist=[408, 429, 500, 502, 503, 504],
+            allowed_methods=["GET", "POST", "PATCH", "PUT", "DELETE"],
+            backoff_factor=0.3,
+            raise_on_status=False,
+        )
+        adapter = HTTPAdapter(max_retries=retry_strategy)
+        http = requests.Session()
+        http.mount("http://", adapter)
+        http.mount("https://", adapter)
+        http_method = getattr(http, method)
+
+        data = kwargs.pop("data", None)
+        if not path.startswith(self.api_url):
+            path = "{0}{1}".format(self.api_url, path)
+
+        response = http_method(url=path, data=data, params=kwargs, headers=headers)
+
+        # closing the http connection after a request
+        http.close()
+
+        asset_id = kwargs.get("asset_id", "Unknown") or "Unknown"
+
+        if response.status_code == 401:
+            raise exceptions.APIError("Unable to reach Corva API")
+
+        if response.status_code == 403:
+            raise exceptions.Forbidden("No access to asset {0}".format(asset_id))
+
+        if response.status_code == 404:
+            raise exceptions.NotFound("Not Found")
+
+        if not response.ok:
+            raise exceptions.APIError(f"{response.status_code} - {response.reason}")
+
+        result = Result(response, **kwargs)
+        return result
+
+    def get_by_id(self, path, **kwargs):
+        """
+        Get a document with the given kwargs ('collection', 'id')
+        :param path: API path
+        :param kwargs: 'collection' and 'id' kwargs are required
+        :return: a result object
+        """
+        collection = kwargs.pop("collection", None)
+        component_id = kwargs.pop("id", None)
+        path = "{0}{1}{2}/{3}".format(self.api_url, path, collection, component_id)
+
+        result = self.get(path)
+        return result
+
+
+class Result(object):
+    def __init__(self, response, **kwargs):
+        self.response = response
+        self.params = kwargs
+        self.data = None
+
+        try:
+            self.data = response.json()
+        except Exception:
+            if not (response.status_code == 200 and response.content == b""):
+                raise exceptions.APIError("Invalid API response")
+
+    def __repr__(self):
+        return repr(self.data)
+
+    def __iter__(self):
+        return iter(self.data)
+
+    @property
+    def status(self):
+        return self.response.status_code
+
+    @property
+    def count(self):
+        if not self.data:
+            return 0
+
+        if isinstance(self.data, list):
+            return len(self.data)
+
+        if isinstance(self.data, dict):
+            return 1
+
+        return 0
+
+
+def convert_python_sdk_api_to_worker_api(python_sdk_api) -> API:
+    """
+    In cases that the python sdk api is used,
+    this function converts it to worker api.
+
+    Args:
+        python_sdk_api (Api): Corva Python SDK Api
+
+    Returns:
+        API: Worker API object
+    """
+    return API(api_url=python_sdk_api.api_url, api_key=python_sdk_api.api_key, app_name=python_sdk_api.app_key)

worker/data/enums.py

@@ -0,0 +1,141 @@
+from enum import Enum, auto
+from typing import List
+
+
+class LambdaStates(Enum):
+    """
+    An enumeration of possible states for a Lambda function.
+
+    Attributes:
+        TIMED_OUT (str): The Lambda function timed out.
+        SUCCEEDED (str): The Lambda function completed successfully.
+        FAILED (str): The Lambda function failed to complete.
+    """
+
+    TIMED_OUT = "Lambda timed out."
+    SUCCEEDED = "Lambda process succeeded."
+    FAILED = "Lambda process failed."
+
+
+class EventType(Enum):
+    """
+    Enum class representing the different types of events that can be handled.
+    The values of this enum are used as the keys in the event dictionary.
+    For instance in the worker apps, the 'event-type' in the constants file
+    should be either 'wits_stream' or 'scheduler'. For partial rerun merge
+    events, the 'event_type' node is set to 'partial-well-rerun-merge'.
+    """
+
+    STREAM = "wits_stream"
+    SCHEDULER = "scheduler"
+    PARTIAL_RERUN = "partial-well-rerun-merge"
+    TASK = "task_event"
+    GENERIC = "generic_event"
+
+
+class ChannelStatus(Enum):
+    """
+    Enum class representing the status of a channel in WITS data.
+    """
+
+    ON = "on"
+    OFF = "off"
+    MISSING = "missing"
+
+
+class DataStatus(Enum):
+    """
+    Enum representing the status of data.
+    """
+
+    VALID = "valid"
+    MISSING = "missing"
+    OVERRIDDEN = "overridden"
+
+
+class Environment(Enum):
+    """
+    An enumeration of the different environments.
+    """
+
+    QA = "qa"
+    STAGING = "staging"
+    PRODUCTION = "production"
+    LOCAL = "local"
+
+
+class CollectionRecordDataScope(Enum):
+    """
+    This enumeration is used to indicate the type of the
+    collection record scope.
+    """
+
+    BHA_OR_CASING = auto()  # Active BHA or Casing
+    SINCE_START = auto()  # Since the start of the well
+    FORMATION = auto()  # Active Formation
+    FROM_LAST_CASING = auto()  # Since the last casing
+    BHA = auto()  # Active BHA
+    CURRENT = auto()  # A few minutes ago to current
+    SEMI_CURRENT = auto()  # A couple of hours ago to current instant
+
+    @classmethod
+    def current_modes(cls) -> List["CollectionRecordDataScope"]:
+        """
+        Gets the items that are considered as current mode.
+        """
+        return [
+            cls.CURRENT,
+            cls.SEMI_CURRENT,
+        ]
+
+    @classmethod
+    def not_current_modes(cls) -> List["CollectionRecordDataScope"]:
+        """
+        Gets the items that are not considered as current mode.
+        """
+        return [item for item in cls.__members__.values() if item not in cls.current_modes()]
+
+
+class CountOfCollectionRecord(Enum):
+    """
+    An enumeration representing the count of collection records.
+
+    Attributes:
+        ONE_PER_BHA_OR_CASING: Represents one record per BHA or casing.
+        ONE_PER_WELL: Represents one record per well.
+    """
+
+    ONE_PER_BHA_OR_CASING = auto()
+    ONE_PER_WELL = auto()
+
+
+class RerunMode(Enum):
+    """
+    An enumeration representing the mode of rerun.
+
+    Attributes:
+        REALTIME: When the rerun asset catches up with the original asset,
+            so the end time of the rerun asset is the same as the original asset
+        HISTORICAL: When an old portion of the original asset runs on the
+            rerun asset, so the end time of the rerun asset is not the same as
+            the end time of the original asset
+    """
+
+    REALTIME = "realtime"
+    HISTORICAL = "historical"
+
+
+class PartialRerunStatus(Enum):
+    """
+    An enumeration representing the different states of a partial rerun.
+    Note that in this project, only the MERGING, FAILED and COMPLETED statuses are used.
+    """
+
+    INITIALIZED = "initialized"
+    RUNNING = "running"
+    PENDING_MERGE = "pending_merge"
+    MERGE_INITIALIZED = "merge_initialized"
+    FAILED = "failed"
+    STOPPED = "stopped"
+    MERGING = "merging"
+    COMPLETED = "completed"

worker/data/json_encoder.py

@@ -0,0 +1,18 @@
+import numpy
+import simplejson as json
+
+
+class JsonEncoder(json.JSONEncoder):
+    """
+    This encoder can be used to convert incompatible data types to types compatible with json.dumps()
+    Use like json.dumps(output, cls=JsonEncoder)
+    """
+
+    def default(self, obj):
+        if isinstance(obj, numpy.integer):
+            return int(obj)
+        elif isinstance(obj, numpy.floating):
+            return float(obj)
+        elif isinstance(obj, numpy.ndarray):
+            return obj.tolist()
+        return json.JSONEncoder.default(self, obj)