deltacat 0.1.8__py3-none-any.whl → 0.1.11__py3-none-any.whl

deltacat/autoscaler/events/dynamodb/event_store.py

@@ -1,164 +0,0 @@
-from typing import List, Dict, Any, Optional
-
-import boto3
-from botocore.client import BaseClient
-from deltacat.autoscaler.events.compaction.workflow import COMPACTION_SESSION_PARTITIONS_COMPACTED, \
-    COMPACTION_SESSION_PARTITIONS_FAILURE
-
-from deltacat.autoscaler.events.event_store import EventStoreClient
-
-
-class DynamoDBEventStoreClient(EventStoreClient):
-    def __init__(self,
-                 table_name: str,
-                 dynamodb_client: BaseClient = None):
-        if dynamodb_client is None:
-            dynamodb_client = boto3.client("dynamodb", "us-east-1")
-
-        self.dynamodb_client = dynamodb_client
-        self.table_name = table_name
-
-    def query_events(self,
-                     trace_id: str) -> List[Optional[Dict[str, Any]]]:
-        """Query events by Trace ID
-
-        Args:
-            trace_id: Trace ID for the job
-
-        Returns: list of events that are active
-
-        """
-        return self.get_events(self._query_events(trace_id))
-
-    def query_active_events(self,
-                            trace_id: str) -> List[Optional[Dict[str, Any]]]:
-        """Query active events by Trace ID
-
-        Args:
-            trace_id: Trace ID for the job
-
-        Returns: list of events that are active
-
-        """
-        return self.get_active_events(self._query_events(trace_id))
-
-    def query_active_events_by_destination_job_table(self,
-                                                     destination_job_table: str) -> List[Optional[Dict[str, Any]]]:
-        """Query active events from the job destination table index
-        
-        Args:
-            destination_job_table: Destination table for jobs
-
-        Returns: list of active events for the particular job
-
-        """
-        result = self.dynamodb_client.query(
-            TableName=self.table_name,
-            IndexName="destinationTable.timestamp",
-            ScanIndexForward=False,  # descending order traversal
-            KeyConditions={
-                "destinationTable": {
-                    "AttributeValueList": [
-                        {
-                            "S": destination_job_table
-                        },
-                    ],
-                    "ComparisonOperator": "EQ"
-                },
-            },
-        )
-        return self.get_active_events(result)
-
-    def query_active_events_by_event_name(self,
-                                          event_name: str) -> List[Optional[Dict[str, Any]]]:
-        """Query active events from the event name index
-
-        Args:
-            event_name: Name of the job event state
-
-        Returns: list of active events for the particular event name
-
-        """
-        result = self.dynamodb_client.query(
-            TableName=self.table_name,
-            IndexName="eventName.timestamp",
-            KeyConditions={
-                "eventName": {
-                    "AttributeValueList": [
-                        {
-                            "S": event_name
-                        },
-                    ],
-                    "ComparisonOperator": "EQ"
-                },
-            }
-        )
-        return self.get_active_events(result)
-
-    def get_compacted_partition_ids(self, trace_id: str) -> List[str]:
-        items = self._get_completed_partition_events(trace_id)
-        partition_id_list = [partition_id for event in items
-                             if "stateDetailMetadata" in event
-                             for partition_id in event["stateDetailMetadata"]["M"].keys()]
-        return partition_id_list
-
-    def get_failed_partition_ids(self, trace_id: str) -> List[str]:
-        items = self._get_failed_partition_events(trace_id)
-        partition_id_list = [partition_id for event in items
-                             if "stateDetailMetadata" in event
-                             for partition_id in event["stateDetailMetadata"]["M"].keys()]
-        return partition_id_list
-
-    @staticmethod
-    def get_events(query_result: Dict[str, Any]) -> List[Dict[str, Any]]:
-        """Gets a filtered list of active job state events
-
-        Args:
-            query_result: list of job state events
-
-        Returns: a filtered list of active job state events
-
-        """
-        return query_result["Items"]
-
-    @staticmethod
-    def get_active_events(query_result: Dict[str, Any]) -> List[Dict[str, Any]]:
-        """Gets a filtered list of active job state events
-
-        Args:
-            query_result: list of job state events
-
-        Returns: a filtered list of active job state events
-
-        """
-        return [item for item in query_result["Items"] if item.get("active")]
-
-    def _get_completed_partition_events(
-            self,
-            trace_id: str) -> List[Dict[str, Any]]:
-        return [item for item in self.query_active_events(trace_id)
-                if item["eventName"]["S"] == COMPACTION_SESSION_PARTITIONS_COMPACTED]
-
-    def _get_failed_partition_events(
-            self,
-            trace_id: str) -> List[Dict[str, Any]]:
-        return [item for item in self.query_active_events(trace_id)
-                if item["eventName"]["S"] == COMPACTION_SESSION_PARTITIONS_FAILURE]
-
-    def _query_events(self, trace_id: str):
-        return self.dynamodb_client.query(
-            TableName=self.table_name,
-            IndexName="traceId.timestamp",
-            ScanIndexForward=False,  # descending order traversal
-            KeyConditions={
-                "traceId": {
-                    "AttributeValueList": [
-                        {
-                            "S": trace_id
-                        },
-                    ],
-                    "ComparisonOperator": "EQ"
-                },
-            },
-        )
-

deltacat/autoscaler/events/event_store.py

@@ -1,55 +0,0 @@
-from typing import List, Dict, Any, Optional
-
-
-class EventStoreClient:
-    def query_events(self,
-                     trace_id: str) -> List[Optional[Dict[str, Any]]]:
-        """Query active events by Trace ID
-
-        Args:
-            trace_id: Trace ID for the job
-
-        Returns: list of events that are active
-
-        """
-        raise NotImplementedError("Method not implemented")
-
-    def query_active_events_by_destination_job_table(self,
-                                                     destination_job_table: str) -> List[Optional[Dict[str, Any]]]:
-        """Query active events from the job destination table
-        
-        Args:
-            destination_job_table: Destination table for jobs
-
-        Returns: list of active events for the particular job
-
-        """
-        raise NotImplementedError("Method not implemented")
-
-    def query_active_events_by_event_name(self,
-                                          event_name: str) -> List[Optional[Dict[str, Any]]]:
-        """Query active events from the event name index
-
-        Args:
-            event_name: Name of the job event state
-
-        Returns: list of active events for the particular event name
-
-        """
-        raise NotImplementedError("Method not implemented")
-
-    def get_compacted_partition_ids(self, trace_id: str) -> List[str]:
-        """Retrieve all compacted partition IDs.
-
-        Returns: list of all compacted partition IDs
-
-        """
-        raise NotImplementedError("Method not implemented")
-
-    def get_failed_partition_ids(self, trace_id: str) -> List[str]:
-        """Retrieve all partition IDs that failed compaction.
-
-        Returns: list of all failed partition IDs
-
-        """
-        raise NotImplementedError("Method not implemented")

deltacat/autoscaler/events/exceptions.py

@@ -1,6 +0,0 @@
-class EventNotFoundException(Exception):
-    pass
-
-
-class WorkflowException(Exception):
-    pass

deltacat/autoscaler/events/processor.py

@@ -1,177 +0,0 @@
-import logging
-
-import time
-from typing import Dict, Callable, Union, Tuple, Optional, List, Any
-
-from botocore.exceptions import BotoCoreError
-from deltacat.autoscaler.events.EventWorkflow import EventWorkflow
-from deltacat.autoscaler.events.compaction.workflow import CompactionWorkflow
-from deltacat.autoscaler.events.states import States
-from ray.autoscaler._private.event_system import EventPublisher
-
-from deltacat import logs
-from deltacat.autoscaler.events.event_store import EventStoreClient
-from deltacat.autoscaler.events.exceptions import EventNotFoundException, WorkflowException
-
-logging.basicConfig(level=logging.INFO)
-logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
-
-QUERY_EVENTS_MAX_RETRY_COUNTER = 10
-SLEEP_PERIOD_SECONDS = 20
-
-
-# TODO: Make this the primary open-source Job Run Event Handler / Dispatcher?
-#  Might be worth porting over some features of the Job Event Daemon (Java) to here
-
-class EventProcessor:
-    def __init__(self,
-                 events_publisher: EventPublisher,
-                 event_store: EventStoreClient,
-                 workflow: CompactionWorkflow):
-        """
-
-        Args:
-            events_publisher: Events manager for publishing events through a cloud provider
-            event_store: High-level API client for the Event Store database
-            workflow: Workflow of job states
-        """
-        self.event_publisher = events_publisher
-        self.event_store = event_store
-        self.workflow = workflow
-
-    def run(self):
-        """Polls the event store and handles state transitions based on incoming events.
-
-        This function will dispatch the STARTED event when first executed.
-        The event listener will only listen to event states from STARTED and onwards.
-
-        Event states before STARTED (i.e. NEW, DISPATCHED) are emitted from the Event Daemon (Java).
-        """
-        logger.info(f"Starting workflow...!")
-        compaction_workflow.start_workflow()
-
-        trace_id = self.event_publisher.event_base_params["traceId"]
-        dest_provider = self.event_publisher.event_base_params["destinationTable"]["owner"]
-        dest_table = self.event_publisher.event_base_params["destinationTable"]["name"]
-        expiry_timestamp = self.event_publisher.event_base_params["expirationTimestamp"]
-
-        retry_ctr = 0
-        while round(time.time() * 1000) < expiry_timestamp and retry_ctr < QUERY_EVENTS_MAX_RETRY_COUNTER:
-            logger.debug(f"Polling latest job states for trace_id: {trace_id}, "
-                        f"provider: {dest_provider} and table: {dest_table}...")
-
-            try:
-                events = self.event_store.query_events(trace_id)
-
-                # Latest non-active / active event must be checked for the completed state.
-                latest_state, latest_state_sequence = self.get_latest_event(events, trace_id)
-                if latest_state == States.COMPLETED.name:
-                    logger.info("Completed Ray job! Exiting.")
-                    break
-
-                # Latest active event must be checked for the next state transition
-                latest_active_state, latest_active_state_sequence = self.get_latest_active_event(events, trace_id)
-
-                # Uncomment for testing on non-active events to test specific steps of workflows
-                # latest_active_state, latest_active_state_sequence = get_latest_event(events, trace_id)
-
-                self.workflow.to_next_state(latest_active_state, latest_active_state_sequence)
-
-            except WorkflowException as e:
-                self.workflow.workflow_failure(error_message=str(e))
-            except EventNotFoundException as e:
-                logger.debug(e)
-            except BotoCoreError as e:
-                logger.error(e)
-                retry_ctr += 1
-
-            time.sleep(SLEEP_PERIOD_SECONDS)
-
-        if retry_ctr == QUERY_EVENTS_MAX_RETRY_COUNTER:
-            # TODO: Dispatch timeout event for IN_PROGRESS
-            logger.error(f"Failed to fetch events for {trace_id} after "
-                         f"{QUERY_EVENTS_MAX_RETRY_COUNTER} attempts")
-
-    def get_latest_event(self,
-                         events: List[Dict[str, Any]],
-                         trace_id: str) -> Tuple[Optional[str], int]:
-        """
-
-        Args:
-            events: Job events which may be active or non-active
-            trace_id: Trace ID for a Ray Job
-
-        Returns: tuple of state name (str) and the state sequence (int)
-
-        """
-        latest_event = self.get_latest_sorted_event(events)
-        if latest_event is None:
-            raise EventNotFoundException(f"No events found for Ray job: {trace_id}")
-
-        latest_state, latest_state_sequence = latest_event["state"]["S"], int(latest_event["stateSequence"]["N"])
-        return latest_state, latest_state_sequence
-
-    def get_latest_active_event(self,
-                                events: List[Dict[str, Any]],
-                                trace_id: str) -> Tuple[Optional[str], int]:
-        """
-
-        Args:
-            events: Job events which may be active or non-active
-            trace_id: Trace ID for a Ray Job
-
-        Returns: tuple of state name (str) and the state sequence (int)
-
-        """
-        active_events = [x for x in events if x.get("active")]
-        latest_event = self.get_latest_sorted_event(active_events)
-        if latest_event is None:
-            raise EventNotFoundException(f"No events found for Ray job: {trace_id}")
-
-        latest_state, latest_state_sequence = latest_event["state"]["S"], int(latest_event["stateSequence"]["N"])
-        return latest_state, latest_state_sequence
-
-    @staticmethod
-    def get_latest_sorted_event(items: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
-        """Get the latest event from the event store,
-
-        Sorted by order of higher precedence - State, State Sequence ID, and finally Timestamp
-
-        Args:
-            items: list of job events as DynamoDB items
-
-        Returns:
-            event: job event sorted by State, State Sequence ID, and finally Timestamp
-        """
-        #
-        latest_events = sorted(items,
-                               key=lambda x: (States[x["state"]["S"]].value,
-                                              int(x["stateSequence"]["N"]),
-                                              int(x["timestamp"]["N"])),
-                               reverse=True)
-        return latest_events[0] if latest_events else None
-
-    @staticmethod
-    def to_next_state(event_state: str,
-                      event_state_sequence: int,
-                      state_transition_map: Dict[str, Union[Callable[[], None], Dict]]):
-        """Reads the state_transition_map to execute the callback for the given event state and state sequence.
-
-        Args:
-            event_state: name of the job event state
-            event_state_sequence: ID of the job event state sequence
-            state_transition_map: A mapping of event states to callbacks or a dictionary of callbacks.
-
-        """
-        transition_cb = state_transition_map.get(event_state)
-
-        if transition_cb is None:
-            return
-
-        if isinstance(transition_cb, dict):
-            transition_sequence_cb: Callable[[], None] = transition_cb.get(event_state_sequence)
-            if transition_sequence_cb and callable(transition_sequence_cb):
-                logger.info(f"Calling function for {event_state} and sequence {event_state_sequence}")
-                transition_sequence_cb(event_state, event_state_sequence)
-        elif callable(transition_cb):
-            transition_cb(event_state, event_state_sequence)

deltacat/autoscaler/events/session_manager.py

@@ -1,25 +0,0 @@
-import uuid
-from typing import Optional
-
-
-PARENT_SESSION_ID_KEY = "rayParentSessionId"
-SESSION_ID_KEY = "raySessionId"
-
-
-class SessionManager:
-    def __init__(self,
-                 session_id: Optional[str] = None):
-        """Manages Ray sessions.
-
-        Args:
-            session_id: Overrideable Session ID for this Ray app instance.
-             If not provided, a Session ID is newly generated.
-        """
-        if not session_id:
-            session_id = str(uuid.uuid4())
-
-        self._session_id = session_id
-
-    @property
-    def session_id(self):
-        return self._session_id

deltacat/autoscaler/events/states.py

@@ -1,88 +0,0 @@
-from enum import Enum, auto
-
-from ray.autoscaler._private.event_system import EventSequence, CreateClusterEvent, StateEvent
-
-
-class States(Enum):
-    UNKNOWN = None
-    NEW = 1
-    DISPATCHED = 2
-    STARTED = 3
-    IN_PROGRESS = 4
-    COMPLETED = 5
-
-
-class RayJobRequestEvent(StateEvent):
-    """Events for a new Ray job request.
-    """
-    @property
-    def state(self) -> str:
-        return States.NEW.name
-
-    new_request_delivered = auto()
-
-
-class ScriptStartedEvent(StateEvent):
-    """Events to track for Ray scripts that are executed.
-    """
-    @property
-    def state(self) -> str:
-        return States.STARTED.name
-
-    start_initializing = auto()
-
-
-class ScriptInProgressEvent(StateEvent):
-    """Events tracking Ray app execution progress.
-    """
-    @property
-    def state(self) -> str:
-        return States.IN_PROGRESS.name
-
-    in_progress = auto()
-
-
-class ScriptInProgressCustomEvent(EventSequence):
-    """Custom, user-defined events to track during execution of Ray scripts.
-    """
-    def __init__(self, event_name: str, state_sequence: int):
-        self.event_name = event_name
-        self.state_sequence = state_sequence
-
-    @property
-    def state(self) -> str:
-        return States.IN_PROGRESS.name
-
-    @property
-    def name(self) -> str:
-        return self.event_name
-
-    @property
-    def value(self) -> int:
-        # the state sequence number in 1-based indexing
-        return self.state_sequence + 1
-
-
-class ScriptCompletedEvent(StateEvent):
-    """Event marking the start of Ray app execution.
-    """
-    @property
-    def state(self) -> str:
-        return States.COMPLETED.name
-
-    completed = auto()
-
-
-class ScriptFailureEvent(StateEvent):
-    """Event marking the failure of Ray app execution.
-    """
-    @property
-    def state(self) -> str:
-        return States.IN_PROGRESS.name
-
-    failed = auto()
-
-
-event_enums = [CreateClusterEvent, ScriptStartedEvent, ScriptInProgressEvent, ScriptCompletedEvent]
-event_enum_values = [sequence for event in event_enums
-                     for sequence in event.__members__.values()]

deltacat/autoscaler/events/workflow.py

@@ -1,54 +0,0 @@
-import logging
-from abc import abstractmethod, ABC
-from typing import Dict, Union, Callable
-
-from deltacat import logs
-
-logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
-StateTransitionCallback = Callable[[], None]
-StateTransitionMap = Dict[str, Union[StateTransitionCallback, Dict]]
-
-
-class EventWorkflow(ABC):
-    @property
-    @abstractmethod
-    def state_transition_map(self) -> StateTransitionMap:
-        raise NotImplementedError("Method not implemented")
-
-    @abstractmethod
-    def _build_state_transitions(self) -> StateTransitionMap:
-        """Builds a mapping of event states to state transitioning callbacks, or
-        a dictionary of state transitioning callbacks.
-
-        If an event has state sequences, a dictionary of callbacks is provided
-        with sequences as keys and callback functions as values.
-
-        Returns: a map of event states to callbacks or a dictionary of callbacks
-        """
-        raise NotImplementedError("Method not implemented")
-
-    def to_next_state(self,
-                      event_state: str,
-                      event_state_sequence: int):
-        """Reads the state_transition_map to execute the callback for the given event state and state sequence.
-
-        Args:
-            event_state: name of the job event state
-            event_state_sequence: ID of the job event state sequence
-
-        """
-        transition_cb: Union[StateTransitionCallback, Dict[int, StateTransitionCallback]] = \
-            self.state_transition_map.get(event_state)
-
-        if transition_cb is None:
-            logger.debug(f"No callback found for state: {event_state}, "
-                         f"sequence ID: {event_state_sequence}")
-            return
-
-        if isinstance(transition_cb, dict):
-            transition_sequence_cb: Callable[[], None] = transition_cb.get(event_state_sequence)
-            if transition_sequence_cb and callable(transition_sequence_cb):
-                logger.info(f"Calling function for {event_state} and sequence {event_state_sequence}")
-                transition_sequence_cb()
-        elif callable(transition_cb):
-            transition_cb()