firexapp 5.7.162__py3-none-any.whl

firex_blaze/__init__.py

@@ -0,0 +1 @@
+from firexapp import __version__

firex_blaze/__main__.py

@@ -0,0 +1,109 @@
+import argparse
+import logging
+import logging.handlers
+import os
+import sys
+import signal
+
+# Prevent dependencies from taking module loading hit of pkg_resources.
+sys.modules["pkg_resources"] = type('noop', (object,), {})
+
+
+from firexapp.events.model import FireXRunMetadata
+
+from firex_blaze.blaze_event_consumer import BlazeKafkaSenderThread
+from firex_blaze.fast_blaze_helper import get_blaze_dir
+from firex_blaze.blaze_helper import BlazeSenderConfig, get_blaze_events_file, celery_app_from_logs_dir
+
+logger = logging.getLogger(__name__)
+
+
+def _parse_blaze_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--instance_name", help="Name of the blaze instance")
+    parser.add_argument("--logs_dir", help="Logs directory for the run to keep task data for.",
+                        required=True)
+    parser.add_argument("--uid", help="FireX UID for the run to keep task data for.",
+                        required=True)
+    parser.add_argument("--firex_requester", help="Requester username, if different than running user",
+                        required=True)
+    parser.add_argument('--broker_recv_ready_file', help='File to create immediately before capturing celery events.',
+                        default=None)
+    parser.add_argument('--logs_url', help='Webserver used from which logs can be accessed.',
+                        default=None)
+
+    parser.add_argument('--kafka_topic', help='Topic use for the Kafka bus', required=True)
+    parser.add_argument('--bootstrap_servers', help='Comma seperated list of Kafka bootrap servers.', required=True)
+    parser.add_argument('--security_protocol', help='Protocol used to communicate with brokers. '
+                                                    'Valid values are: PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL. '
+                                                    'Default: PLAINTEXT.',
+                        default='PLAINTEXT')
+
+    # SASL-SSL OAuth 2.0 arguments
+    parser.add_argument('--sasl_mechanism', help='SASL mechanism (e.g., OAUTHBEARER).',
+                        default=None)
+    parser.add_argument('--sasl_oauthbearer_method', help='OAuth bearer method (e.g., oidc).',
+                        default=None)
+    parser.add_argument('--sasl_oauthbearer_client_id', help='OAuth client ID.',
+                        default=None)
+    parser.add_argument('--sasl_oauthbearer_client_secret', help='OAuth client secret.',
+                        default=None)
+    parser.add_argument('--sasl_oauthbearer_token_endpoint_url', help='OAuth token endpoint URL.',
+                        default=None)
+    parser.add_argument('--ssl_ca_location', help='CA certificate location for SSL verification.',
+                        default=None)
+
+    return parser.parse_args()
+
+
+def init_blaze():
+    args = _parse_blaze_args()
+
+    run_metadata = FireXRunMetadata(args.uid, args.logs_dir, None, None, args.firex_requester)
+
+    blaze_dir = get_blaze_dir(run_metadata.logs_dir, args.instance_name)
+    os.makedirs(blaze_dir, exist_ok=True)
+    logging.basicConfig(
+        level=logging.DEBUG,
+        handlers=[
+            logging.handlers.RotatingFileHandler(
+                os.path.join(blaze_dir, 'blaze.log'),
+                maxBytes=20*1024*1024, # Max size of a log file (20 MB)
+                backupCount=1,
+            )
+        ],
+        format='[%(asctime)s][%(levelname)s][%(name)s]: %(message)s',
+        datefmt="%Y-%m-%d %H:%M:%S")
+
+    logging.getLogger('kafka.producer').setLevel(logging.INFO)
+    logger.info(f'Starting Blaze with args: {args}')
+
+    signal.signal(signal.SIGTERM, lambda _, __: sys.exit(1))
+
+    celery_app = celery_app_from_logs_dir(run_metadata.logs_dir)
+    blaze_sender_config = BlazeSenderConfig(
+        kafka_topic=args.kafka_topic,
+        kafka_bootstrap_servers=args.bootstrap_servers.split(','),
+        max_kafka_connection_retries=2,
+        security_protocol=args.security_protocol,
+        sasl_mechanism=args.sasl_mechanism,
+        sasl_oauthbearer_method=args.sasl_oauthbearer_method,
+        sasl_oauthbearer_client_id=args.sasl_oauthbearer_client_id,
+        sasl_oauthbearer_client_secret=args.sasl_oauthbearer_client_secret,
+        sasl_oauthbearer_token_endpoint_url=args.sasl_oauthbearer_token_endpoint_url,
+        ssl_ca_location=args.ssl_ca_location,
+    )
+    recording_file = get_blaze_events_file(run_metadata.logs_dir, args.instance_name)
+    return celery_app, run_metadata, args.broker_recv_ready_file, blaze_sender_config, args.logs_url, recording_file
+
+
+def main():
+    celery_app, run_metadata, receiver_ready_file, blaze_sender_config, logs_url, recording_file = init_blaze()
+    BlazeKafkaSenderThread(
+        celery_app, run_metadata, blaze_sender_config, logs_url,
+        receiver_ready_file=receiver_ready_file,
+        recording_file=recording_file).run()
+
+
+if __name__ == '__main__':
+    main()

firex_blaze/blaze_event_consumer.py

@@ -0,0 +1,284 @@
+"""
+Process events from Celery and put them on a kafka bus.
+"""
+
+import logging
+import json
+import time
+from getpass import getuser
+from typing import Optional, Any
+
+from confluent_kafka import Producer, KafkaException
+from confluent_kafka.admin import AdminClient
+
+from firexapp.events.broker_event_consumer import BrokerEventConsumerThread
+from firexapp.events.model import FireXRunMetadata, COMPLETE_RUNSTATES, RunStates
+
+from firex_blaze.blaze_helper import BlazeSenderConfig, KAFKA_EVENTS_FILE_DELIMITER
+
+logger = logging.getLogger(__name__)
+
+TASK_EVENT_TO_STATE = {
+    'task-started-info': 'STARTED',
+    RunStates.FAILED.to_celery_event_type(): 'FAILURE',
+    RunStates.SUCCEEDED.to_celery_event_type(): 'SUCCESS',
+    RunStates.REVOKED.to_celery_event_type(): 'REVOKED',
+    RunStates.REVOKE_COMPLETED.to_celery_event_type(): 'REVOKED',
+    # historically were mapped but never sent.
+    # 'task-sent': 'PENDING',
+    # 'task-received': 'RECEIVED',
+    # 'task-started': 'STARTED',
+    # 'task-rejected': 'REJECTED',
+    # 'task-retried': 'RETRY',
+}
+
+BLAZE_SEND_EVENT_TYPES = tuple(
+    list(TASK_EVENT_TO_STATE.keys()) + ['task-completed', 'task-results', 'task-instrumentation', 'task-args']
+)
+
+
+def format_kafka_message(firex_id, event_data, uuid, logs_url, submitter=getuser(), firex_requester=None) -> dict[str, Any]:
+    return {'FIREX_ID': firex_id,
+            'SUBMITTER': submitter,
+            'FIREX_REQUESTER': firex_requester,
+            'LOGS_URL': logs_url,               # Shouldn't be required, but Lumens needs it!
+            'EVENTS': [{'DATA': event_data,
+                        'UUID': uuid}]}
+
+
+
+def send_kafka_mssg(kafka_producer: Producer, kafka_mssg: dict[str, Any], kafka_topic: str, firex_id: str,
+                    partition: Optional[int] = None):
+    """Send message using confluent-kafka Producer."""
+    try:
+        kafka_producer.produce(
+            topic=kafka_topic,
+            value=json.dumps(kafka_mssg).encode('ascii'),
+            key=firex_id.encode('ascii'),
+        )
+        # Poll to handle any internal events and queued delivery reports (in case of delivery callback functions)
+        kafka_producer.poll(0)
+    except BufferError:
+        # Buffer is full, wait for messages to be delivered
+        logger.warning('Kafka producer buffer full, waiting for delivery...')
+        kafka_producer.poll(1)
+        # Retry the send
+        kafka_producer.produce(
+            topic=kafka_topic,
+            value=json.dumps(kafka_mssg).encode('ascii'),
+            key=firex_id.encode('ascii'),
+        )
+        kafka_producer.poll(0)
+    except KafkaException as e:
+        logger.error(f'Failed to send Kafka message: {e}')
+
+
+def get_basic_event(name, event_type, timestamp=None, event_timestamp=time.time()):
+    if timestamp is None:
+        timestamp = event_timestamp
+
+    event_data = {'name': name,
+                  'type': event_type,
+                  'timestamp': timestamp,               # Shouldn't be required, but Lumens needs it!
+                  'event_timestamp': event_timestamp}
+
+    # Not all types map to states (e.g. task-results), so only populate state for some event types.
+    if event_type in TASK_EVENT_TO_STATE:
+        event_data['state'] = TASK_EVENT_TO_STATE[event_type]
+
+    return event_data
+
+
+class NoNameForEvent(Exception):
+    pass
+
+
+class KafkaSenderThread(BrokerEventConsumerThread):
+
+    def __init__(
+        self,
+        celery_app,
+        run_metadata: FireXRunMetadata,
+        config: BlazeSenderConfig,
+        max_retry_attempts: Optional[int] = None,
+        receiver_ready_file: Optional[str] = None,
+        recording_file: Optional[str] = None,
+        partition: Optional[int] = None,
+    ):
+
+        super().__init__(celery_app, max_retry_attempts, receiver_ready_file)
+        self.firex_id = run_metadata.firex_id
+        self.kafka_topic = config.kafka_topic
+        self.recording_file = recording_file
+        self.partition = partition
+
+        # Connect to bootstrap servers and get a Producer instance
+        self.producer = self.get_kafka_producer(config)
+        logger.debug (f'Kafka producer created for topic {self.kafka_topic}')
+        self.root_task = {'uuid': None, 'is_complete': False}
+
+    @classmethod
+    def get_kafka_producer(cls, config: BlazeSenderConfig) -> Producer:
+        """Create confluent-kafka Producer with retry logic."""
+        _retries = 0
+
+        # Build configuration dictionary for confluent-kafka
+        producer_config = {
+            'bootstrap.servers': ','.join(config.kafka_bootstrap_servers)
+                if isinstance(config.kafka_bootstrap_servers, list)
+                else config.kafka_bootstrap_servers,
+            'security.protocol': config.security_protocol,
+        }
+
+        # Add SASL-SSL OAuth parameters
+        if config.sasl_mechanism:
+            producer_config['sasl.mechanism'] = config.sasl_mechanism
+        if config.sasl_oauthbearer_method:
+            producer_config['sasl.oauthbearer.method'] = config.sasl_oauthbearer_method
+        if config.sasl_oauthbearer_client_id:
+            producer_config['sasl.oauthbearer.client.id'] = config.sasl_oauthbearer_client_id
+        if config.sasl_oauthbearer_client_secret:
+            producer_config['sasl.oauthbearer.client.secret'] = config.sasl_oauthbearer_client_secret
+        if config.sasl_oauthbearer_token_endpoint_url:
+            producer_config['sasl.oauthbearer.token.endpoint.url'] = config.sasl_oauthbearer_token_endpoint_url
+        if config.ssl_ca_location:
+            producer_config['ssl.ca.location'] = config.ssl_ca_location
+
+        while True:
+            try:
+                # Create the producer - it will handle connection and authentication
+                producer = Producer(producer_config)
+                logger.info('Successfully created Kafka producer')
+                return producer
+
+            except KafkaException as e:
+                if _retries < config.max_kafka_connection_retries:
+                    _retries += 1
+                    logger.exception(e)
+                    logger.warning(f'Retrying connecting to bootstrap servers '
+                                   f'[retry {_retries}/{config.max_kafka_connection_retries}]')
+                    time.sleep(min(2 ** _retries, 30))  # Exponential backoff
+                else:
+                    logger.error(f'Failed to connect to Kafka after {config.max_kafka_connection_retries} retries')
+                    raise
+
+    def _is_root_complete(self):
+        return self.root_task['is_complete']
+
+    def _update_root_task(self, event):
+        if (
+            event.get('type') == 'task-received'
+            and 'root_id' in event
+            and self.root_task['uuid'] is None
+        ):
+            self.root_task['uuid'] = event['root_id']
+
+        if (
+            event['uuid'] == self.root_task['uuid']
+            # crazy things can happen with the celery task state model;
+            # avoid switching out of completed.
+            and RunStates.is_complete_state(event.get('type'))
+        ):
+            self.root_task['is_complete'] = True
+
+    def _send_celery_event_to_kafka(self, celery_event: dict[str, Any]) -> list[dict[str, Any]]:
+        raise NotImplementedError("Subclasses must implement sending.")
+
+    def _on_celery_event(self, event):
+        if 'uuid' not in event:
+            return
+
+        self._update_root_task(event)
+
+        sent_kafka_events = self._send_celery_event_to_kafka(event)
+
+        if sent_kafka_events and self.recording_file:
+            # Append the event to the recording file.
+            with open(self.recording_file, "a") as rec:
+                for e in sent_kafka_events:
+                    event_data_str = json.dumps(e, sort_keys=True, indent=2)
+                    rec.write(event_data_str + KAFKA_EVENTS_FILE_DELIMITER)
+
+    def _on_cleanup(self):
+        """Flush and close the producer with proper timeout handling."""
+        logger.info('Flushing Kafka producer...')
+
+        # Flush with timeout - returns number of messages still in queue
+        remaining = self.producer.flush(timeout=120)
+
+        if remaining > 0:
+            logger.warning(f'{remaining} messages were not delivered before timeout')
+        else:
+            logger.info('All messages successfully delivered to Kafka')
+
+
+class BlazeKafkaSenderThread(KafkaSenderThread):
+    """Captures Celery events and puts them on a Kafka bus."""
+
+    def __init__(self,
+                 celery_app,
+                 run_metadata: FireXRunMetadata,
+                 config: BlazeSenderConfig,
+                 logs_url: str,
+                 max_retry_attempts: Optional[int] = None,
+                 receiver_ready_file: Optional[str] = None,
+                 recording_file: Optional[str] = None,
+    ):
+
+        super().__init__(
+            celery_app, run_metadata, config, max_retry_attempts,
+            receiver_ready_file, recording_file)
+
+        self.submitter = getuser()
+        self.firex_requester = run_metadata.firex_requester
+        self.firex_id = run_metadata.firex_id
+        self.logs_url = logs_url
+        self.kafka_topic = config.kafka_topic
+        self.uuid_to_task_name_mapping : dict[str, str] = {}
+
+    def _get_kafka_event(self, event: dict[str, Any]) -> dict[str, Any]:
+        uuid = event.pop('uuid')
+        if uuid not in self.uuid_to_task_name_mapping and 'long_name' in event:
+            self.uuid_to_task_name_mapping[uuid] = event['long_name']
+
+        if uuid in self.uuid_to_task_name_mapping:
+            task_name = self.uuid_to_task_name_mapping[uuid]
+        else:
+            # No need to produce this event since it won't be processed by Lumens anyways
+            raise NoNameForEvent(f'No task name found for {event}; can not send the event')
+
+        # Remove result since we only should report firex_result, not the native result
+        event.pop('result', None)
+
+        basic_event_data = get_basic_event(
+            name=task_name,
+            event_type=event.get('type'),
+            timestamp=event['timestamp'],
+            # Add the event_timestamp (copy of the local_received), since the native timestamp that
+            # Celery provides is broken (its local time instead of UTC, and utcoffset is inaccurate).
+            # This piece of -redundant- data is just because Lumens can't make local_received query-able
+            event_timestamp=event['local_received'])
+
+        return format_kafka_message(
+            firex_id=self.firex_id,
+            event_data=event | basic_event_data,
+            uuid=uuid,
+            logs_url=self.logs_url,
+            submitter=self.submitter,
+            firex_requester=self.firex_requester)
+
+    def _send_celery_event_to_kafka(self, celery_event: dict[str, Any]) -> list[dict[str, Any]]:
+        if celery_event.get('type') in BLAZE_SEND_EVENT_TYPES:
+            try:
+                kafka_event = self._get_kafka_event(celery_event)
+            except NoNameForEvent as e:
+                logger.exception(e)
+            else:
+                send_kafka_mssg(kafka_producer=self.producer,
+                                kafka_mssg=kafka_event,
+                                kafka_topic=self.kafka_topic,
+                                firex_id=self.firex_id,
+                                partition=self.partition)
+                return [kafka_event]
+
+        return []

firex_blaze/blaze_helper.py

@@ -0,0 +1,67 @@
+"""
+    Utility functions for the firex_blaze package.
+"""
+import os
+from dataclasses import dataclass
+import json
+from typing import Optional
+
+from celery.app.base import Celery
+
+from firexapp.events.event_aggregator import FireXEventAggregator
+from firexapp.broker_manager.broker_factory import RedisManager
+from firex_blaze.fast_blaze_helper import get_blaze_dir
+
+KAFKA_EVENTS_FILE_DELIMITER = '--END_OF_EVENT--'
+
+
+@dataclass
+class BlazeSenderConfig:
+    kafka_topic: str
+    kafka_bootstrap_servers: list[str]
+    max_kafka_connection_retries: int
+    security_protocol: str = 'PLAINTEXT'
+    # SASL-SSL OAuth 2.0 parameters
+    sasl_mechanism: Optional[str] = None
+    sasl_oauthbearer_method: Optional[str] = None
+    sasl_oauthbearer_client_id: Optional[str] = None
+    sasl_oauthbearer_client_secret: Optional[str] = None
+    sasl_oauthbearer_token_endpoint_url: Optional[str] = None
+    ssl_ca_location: Optional[str] = None
+
+
+def get_blaze_events_file(logs_dir, instance_name=None):
+    return os.path.join(get_blaze_dir(logs_dir, instance_name), 'kafka_events.json')
+
+
+def get_kafka_events(logs_dir, instance_name=None):
+    import gzip
+    real_rec = os.path.realpath(get_blaze_events_file(logs_dir, instance_name))
+    if real_rec.endswith('.gz'):
+        with gzip.open(real_rec, 'rt', encoding='utf-8') as rec:
+            all_text = rec.read()
+    else:
+        with open(real_rec) as rec:
+            all_text = rec.read()
+    event_records = all_text.split(sep=KAFKA_EVENTS_FILE_DELIMITER)
+    return [
+        json.loads(e) for e in event_records
+        if e
+    ]
+
+
+def aggregate_blaze_kafka_msgs(firex_id, kafka_msgs):
+    event_aggregator = FireXEventAggregator()
+    for kafka_event in kafka_msgs:
+        if kafka_event['FIREX_ID'] == firex_id:
+            inner_event = kafka_event['EVENTS'][0]
+            celery_event = dict(inner_event['DATA'])
+            celery_event['uuid'] = inner_event['UUID']
+            event_aggregator.aggregate_events([celery_event])
+
+    return event_aggregator.tasks_by_uuid
+
+
+def celery_app_from_logs_dir(logs_dir):
+    return Celery(broker=RedisManager.get_broker_url_from_logs_dir(logs_dir),
+                  accept_content=['pickle', 'json'])

firex_blaze/blaze_launcher.py

@@ -0,0 +1,144 @@
+import os
+from psutil import Process, TimeoutExpired
+import subprocess
+import time
+
+from firexapp.submit.install_configs import FireXInstallConfigs
+from firexapp.submit.submit import OptionalBoolean
+from firexapp.submit.tracking_service import TrackingService
+from firexapp.common import qualify_firex_bin, select_env_vars
+from firexapp.submit.console import setup_console_logging
+
+from firex_blaze.fast_blaze_helper import get_blaze_dir
+from firexapp.discovery import PkgVersionInfo
+
+
+logger = setup_console_logging(__name__)
+
+
+class FireXBlazeLauncher(TrackingService):
+
+    instance_name = 'blaze'
+
+    def __init__(self):
+        self.broker_recv_ready_file = None
+        self.is_ready_for_tasks = False
+        self.stdout_file = None
+        self.start_time = None
+
+    def extra_cli_arguments(self, arg_parser):
+
+        arg_parser.add_argument('--disable_blaze', '-disable_blaze',
+                                help='Disable blaze data collection', default=None, const=True, nargs='?',
+                                action=OptionalBoolean)
+
+        arg_parser.add_argument('--blaze_logs_url',
+                                help='Server URL from which logs can be fetched.',
+                                default=None)
+
+        # TODO: consider sensible default values, or not launching subprocess when these required args aren't supplied.
+        arg_parser.add_argument('--blaze_kafka_topic',
+                                help="Topic used for Blaze's Kafka bus",
+                                default=None)
+
+        arg_parser.add_argument('--blaze_bootstrap_servers',
+                                help='Comma separated list of Kafka bootrap servers.',
+                                default=None)
+
+        arg_parser.add_argument('--blaze_security_protocol',
+                                help='Protocol used to communicate with brokers. '
+                                     'Valid values are: PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL.',
+                                default='PLAINTEXT')
+
+        # SASL-SSL OAuth 2.0 arguments
+        arg_parser.add_argument('--blaze_sasl_mechanism',
+                                help='SASL mechanism to use (e.g., OAUTHBEARER).')
+        arg_parser.add_argument('--blaze_sasl_oauthbearer_method',
+                                help='OAuth bearer method (e.g., oidc).')
+        arg_parser.add_argument('--blaze_sasl_oauthbearer_client_id',
+                                help='OAuth client ID.')
+        arg_parser.add_argument('--blaze_sasl_oauthbearer_client_secret',
+                                help='OAuth client secret.')
+        arg_parser.add_argument('--blaze_sasl_oauthbearer_token_endpoint_url',
+                                help='OAuth token endpoint URL.')
+        arg_parser.add_argument('--blaze_ssl_ca_location',
+                                help='CA certificate location for SSL verification.')
+
+    @classmethod
+    def _create_blaze_command(cls, uid, args, broker_recv_ready_file):
+        cmd = [qualify_firex_bin("firex_blaze"),
+               "--uid", str(uid),
+               "--firex_requester", uid.firex_requester,
+               "--logs_dir", uid.logs_dir,
+               "--broker_recv_ready_file", broker_recv_ready_file,
+               '--logs_url', uid.logs_url,
+               '--kafka_topic', args.blaze_kafka_topic,
+               '--bootstrap_servers', args.blaze_bootstrap_servers,
+               '--instance_name', cls.instance_name,
+               '--security_protocol', args.blaze_security_protocol]
+
+        # SASL-SSL OAuth arguments
+        if hasattr(args, 'blaze_sasl_mechanism') and args.blaze_sasl_mechanism:
+            cmd += ['--sasl_mechanism', args.blaze_sasl_mechanism]
+        if hasattr(args, 'blaze_sasl_oauthbearer_method') and args.blaze_sasl_oauthbearer_method:
+            cmd += ['--sasl_oauthbearer_method', args.blaze_sasl_oauthbearer_method]
+        if hasattr(args, 'blaze_sasl_oauthbearer_client_id') and args.blaze_sasl_oauthbearer_client_id:
+            cmd += ['--sasl_oauthbearer_client_id', args.blaze_sasl_oauthbearer_client_id]
+        if hasattr(args, 'blaze_sasl_oauthbearer_client_secret') and args.blaze_sasl_oauthbearer_client_secret:
+            cmd += ['--sasl_oauthbearer_client_secret', args.blaze_sasl_oauthbearer_client_secret]
+        if hasattr(args, 'blaze_sasl_oauthbearer_token_endpoint_url') and args.blaze_sasl_oauthbearer_token_endpoint_url:
+            cmd += ['--sasl_oauthbearer_token_endpoint_url', args.blaze_sasl_oauthbearer_token_endpoint_url]
+        if hasattr(args, 'blaze_ssl_ca_location') and args.blaze_ssl_ca_location:
+            cmd += ['--ssl_ca_location', args.blaze_ssl_ca_location]
+
+        return cmd
+
+    def start(self, args, install_configs: FireXInstallConfigs, uid=None, **kwargs) -> {}:
+        super().start(args, install_configs, uid=uid, **kwargs)
+        sufficient_args = uid.logs_url and args.blaze_kafka_topic and args.blaze_bootstrap_servers
+        if args.disable_blaze or not sufficient_args:
+            if args.disable_blaze:
+                logger.debug("Blaze disabled; will not launch subprocess.")
+            if not sufficient_args:
+                logger.warning("Blaze did not receive sufficient arguments; will not launch subprocess.")
+            self.is_ready_for_tasks = True
+            return {}
+
+        blaze_debug_dir = get_blaze_dir(uid.logs_dir, instance_name=self.instance_name)
+        os.makedirs(blaze_debug_dir, exist_ok=True)
+        self.broker_recv_ready_file = os.path.join(blaze_debug_dir, 'blaze_celery_recvr_ready')
+        self.stdout_file = os.path.join(blaze_debug_dir, 'blaze.stdout')
+
+        self.start_time = time.time()
+        with open(self.stdout_file, 'w+') as f:
+            pid = subprocess.Popen(
+                self._create_blaze_command(uid, args, self.broker_recv_ready_file),
+                stdout=f,
+                stderr=subprocess.STDOUT,
+                close_fds=True,
+                env=select_env_vars(['PATH']),
+                cwd=blaze_debug_dir,
+            ).pid
+
+        try:
+            Process(pid).wait(0.1)
+        except TimeoutExpired:
+            logger.debug("Started background FireXBlaze with pid %s" % pid)
+        else:
+            logger.error("Failed to start FireXBlaze -- task data will not be put on Kafka bus.")
+
+        return {}
+
+    def ready_for_tasks(self, **kwargs) -> bool:
+        if not self.is_ready_for_tasks:
+            self.is_ready_for_tasks = os.path.isfile(self.broker_recv_ready_file)
+            if self.is_ready_for_tasks:
+                logger.debug("Blaze up after %.2f s" % (time.time() - self.start_time))
+
+        return self.is_ready_for_tasks
+
+    @staticmethod
+    def get_pkg_version_info() -> PkgVersionInfo:
+        import firex_blaze
+        return PkgVersionInfo(pkg='firex-blaze',
+                              version=firex_blaze.__version__)

firex_blaze/fast_blaze_helper.py

@@ -0,0 +1,11 @@
+# LOAD NO SLOW MODULES HERE!
+# This is to keep loading the launcher module fast.
+import os
+
+from firexapp.submit.uid import Uid
+
+
+def get_blaze_dir(logs_dir, instance_name=None):
+    if instance_name is None:
+        instance_name = 'blaze'
+    return os.path.join(logs_dir, Uid.debug_dirname, instance_name)

firex_bundle_ci/__init__.py

@@ -0,0 +1,5 @@
+"""CI-oriented FireX tasks distributed by the owning firexapp package."""
+
+from firexapp import __version__
+
+__all__ = ["__version__"]