nostromo 0.dev0__tar.gz

nostromo-0.dev0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Danila Ganchar
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

nostromo-0.dev0/PKG-INFO

@@ -0,0 +1,26 @@
+Metadata-Version: 2.1
+Name: nostromo
+Version: 0.dev0
+Summary: pipeline builder, runner, process manager, background jobs, job scheduling
+License: MIT
+Author: Danila Ganchar
+Requires-Python: >=3.9,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: bigtree (>=0.22.3,<0.23.0)
+Requires-Dist: inject (==4.1)
+Requires-Dist: psutil (>=6.1.0,<7.0.0)
+Requires-Dist: pypyr (==5.9.1)
+Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
+Requires-Dist: redis (==5.0.1)
+Requires-Dist: saq[redis] (>=0.18.3,<0.19.0)
+Requires-Dist: setuptools (>=70.2.0,<71.0.0)
+Requires-Dist: textual (==0.85.2)
+Description-Content-Type: text/markdown
+
+

nostromo-0.dev0/nostromo/models/pipeline.py

@@ -0,0 +1,52 @@
+import json
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta, UTC
+from typing import List, Union
+
+
+@dataclass
+class PipelineTaskGroup:
+    name: str
+    children: List[Union['PipelineTask']] = field(default_factory=list)
+
+
+@dataclass
+class PipelineTask:
+    name: str
+    children: List[Union['PipelineTaskGroup', 'PipelineTask']] = field(default_factory=list)
+
+
+class PipelineRunStatus:
+    KILLED = 'killed'
+    RUNNING = 'running'
+
+
+@dataclass
+class Pipeline:
+    name: str
+    pipeline_dir: str = ''
+    schedule: str = ''
+
+
+@dataclass
+class PipelineRun:
+    pipeline_name: str
+    logs_dir: str
+    started_at: datetime
+    finished_at: Union[datetime, str] = ''
+    pid: int = 0
+    status: str = PipelineRunStatus.RUNNING
+
+    @property
+    def duration(self) -> timedelta:
+        if self.finished_at:
+            return self.finished_at - self.started_at
+        return datetime.now(UTC) - self.started_at
+
+    def to_json(self) -> str:
+        data = self.__dict__
+        for key in ('started_at', 'finished_at'):
+            value = getattr(self, key)
+            if value:
+                data[key] = value.isoformat()
+        return json.dumps(data)

nostromo-0.dev0/nostromo/protocols/executor.py

@@ -0,0 +1,32 @@
+from abc import ABCMeta, abstractmethod
+from typing import Protocol
+
+
+class ExecutorProtocol(Protocol, metaclass=ABCMeta):
+    @abstractmethod
+    async def is_scheduler_running(self) -> bool:
+        pass
+
+    @abstractmethod
+    async def set_scheduler_running(self, is_scheduler_running: bool) -> None:
+        pass
+
+    @abstractmethod
+    async def send_to_executor(self, cmd: str):
+        pass
+
+    @abstractmethod
+    async def wait_and_exec_commands(self):
+        pass
+
+    @abstractmethod
+    async def action_pause_pipeline(self):
+        pass
+
+    @abstractmethod
+    async def action_kill_pipeline(self):
+        pass
+
+    @abstractmethod
+    async def wait_and_exec_actions(self):
+        pass

nostromo-0.dev0/nostromo/protocols/path.py

@@ -0,0 +1,45 @@
+import abc
+from pathlib import Path
+from typing import Protocol
+
+
+class PathProtocol(Protocol, metaclass=abc.ABCMeta):
+    @property
+    @abc.abstractmethod
+    def root(self) -> Path:
+        pass
+
+    @property
+    @abc.abstractmethod
+    def scripts(self) -> Path:
+        pass
+
+    @property
+    @abc.abstractmethod
+    def env_file(self) -> Path:
+        pass
+
+    @property
+    @abc.abstractmethod
+    def storage(self) -> Path:
+        pass
+
+    @property
+    @abc.abstractmethod
+    def pipelines(self) -> Path:
+        pass
+
+    @property
+    @abc.abstractmethod
+    def logs(self) -> Path:
+        pass
+
+    @property
+    @abc.abstractmethod
+    def pipelines_logs(self) -> Path:
+        pass
+
+    @property
+    @abc.abstractmethod
+    def scheduler_logs(self) -> Path:
+        pass

nostromo-0.dev0/nostromo/protocols/pipelines.py

@@ -0,0 +1,46 @@
+import abc
+from typing import Protocol, List, Dict
+
+from ..models.pipeline import Pipeline, PipelineTask, PipelineTaskGroup, PipelineRun
+
+
+class PipelinesProtocol(Protocol, metaclass=abc.ABCMeta):
+    @property
+    @abc.abstractmethod
+    def entrypoints(self) -> List[str]:
+        pass
+
+    @abc.abstractmethod
+    def get_scripts_by_entrypoint(self, entrypoint: str) -> List[str]:
+        pass
+
+    @abc.abstractmethod
+    def get_nostromo_env(self) -> Dict[str, str]:
+        pass
+
+    @abc.abstractmethod
+    def get_task_names(self, pipeline: Pipeline) -> List[str]:
+        pass
+
+    @abc.abstractmethod
+    def get_group_names(self, pipeline: Pipeline) -> List[str]:
+        pass
+
+    @abc.abstractmethod
+    def add_task_to_group(self, group_name: str, task: PipelineTask) -> PipelineTaskGroup:
+        pass
+
+    def get_pipelines(self) -> List[Pipeline]:
+        pass
+
+    async def run_pipeline_by_name(self, name: str) -> PipelineRun:
+        pass
+
+    def get_pipeline_by_name(self, name: str) -> Pipeline:
+        pass
+
+    async def get_last_pipeline_run(self, pipeline_name: str) -> PipelineRun:
+        pass
+
+    async def kill_pipeline(self, run: PipelineRun) -> PipelineRun:
+        pass

nostromo-0.dev0/nostromo/protocols/ui_log.py

@@ -0,0 +1,24 @@
+import abc
+from typing import Protocol
+
+from textual.scroll_view import ScrollView
+
+
+class UILogProtocol(Protocol, metaclass=abc.ABCMeta):
+    @property
+    @abc.abstractmethod
+    def log(self) -> ScrollView:
+        pass
+
+    @abc.abstractmethod
+    def info(self, message: str, title: str = '') -> None:
+        pass
+
+    @abc.abstractmethod
+    def warning(self, message: str, title: str = '') -> None:
+        pass
+
+    @abc.abstractmethod
+    def error(self, message: str, title: str = '') -> None:
+        pass
+

nostromo-0.dev0/nostromo/screens/logs.py

@@ -0,0 +1,15 @@
+import inject
+from textual.app import ComposeResult
+from textual.screen import Screen
+from textual.widgets import Footer
+
+from nostromo.protocols.ui_log import UILogProtocol
+
+
+class LogsScreen(Screen):
+    _log: UILogProtocol = inject.attr(UILogProtocol)
+    BINDINGS = [('ctrl+l', 'app.pop_screen', 'Close Logs')]
+
+    def compose(self) -> ComposeResult:
+        yield self._log.log
+        yield Footer()

nostromo-0.dev0/nostromo/screens/pipeline_builder.py

@@ -0,0 +1,11 @@
+from textual.app import ComposeResult
+from textual.screen import Screen
+
+from nostromo.widgets.pipeline_builder import PipelineBuilderWidget
+
+
+class PipelineBuilderScreen(Screen):
+    BINDINGS = [('ctrl+n', 'app.pop_screen', 'Close Pipeline Builder')]
+
+    def compose(self) -> ComposeResult:
+        yield PipelineBuilderWidget()

nostromo-0.dev0/nostromo/services/local_path.py

@@ -0,0 +1,55 @@
+import abc
+import os
+from pathlib import Path, PosixPath
+
+from nostromo.protocols.path import PathProtocol
+
+
+class LocalPath(PathProtocol, metaclass=abc.ABCMeta):
+    def __init__(self):
+        nostromo_path = os.environ.get('NOSTROMO_HOME')
+        if not nostromo_path:
+            nostromo_path = '~/.nostromo'
+
+        self._root = PosixPath(nostromo_path).expanduser()
+
+        self._scripts = PosixPath(os.path.join(str(self._root), 'scripts'))
+        self._storage = PosixPath(os.path.join(str(self._root), '.storage'))
+        self._env_file = PosixPath(os.path.join(str(self._storage), 'env.yml'))
+        self._pipelines = PosixPath(os.path.join(str(self._storage), 'pipelines'))
+
+        self._logs = PosixPath(os.path.join(str(self._storage), 'logs'))
+        self._pipelines_logs = PosixPath(os.path.join(str(self._logs), 'pipelines'))
+        self._scheduler_logs = PosixPath(os.path.join(str(self._logs), 'scheduler'))
+
+    @property
+    def root(self) -> Path:
+        return self._root
+
+    @property
+    def scripts(self) -> Path:
+        return self._scripts
+
+    @property
+    def env_file(self) -> Path:
+        return self._env_file
+
+    @property
+    def storage(self) -> Path:
+        return self._storage
+
+    @property
+    def pipelines(self) -> Path:
+        return self._pipelines
+
+    @property
+    def logs(self) -> Path:
+        return self._logs
+
+    @property
+    def pipelines_logs(self) -> Path:
+        return self._pipelines_logs
+
+    @property
+    def scheduler_logs(self) -> Path:
+        return self._scheduler_logs

nostromo-0.dev0/nostromo/services/pypyr_pipelines.py

@@ -0,0 +1,176 @@
+import codecs
+import json
+import os
+import subprocess
+from datetime import datetime, UTC, timezone
+from typing import List, Dict
+
+import inject
+import psutil
+import yaml
+from dotenv import dotenv_values
+from psutil import NoSuchProcess
+from redis.asyncio import Redis
+
+from ..models.pipeline import Pipeline, PipelineTask, PipelineTaskGroup, PipelineRun, PipelineRunStatus
+from ..protocols.path import PathProtocol
+from ..protocols.pipelines import PipelinesProtocol
+
+
+class PypyrPipelines(PipelinesProtocol):
+    _paths = inject.attr(PathProtocol)
+
+    def __init__(self, redis: Redis):
+        self._redis = redis
+
+    @property
+    def logs_dt_format(self) -> str:
+        return '%Y-%m-%d-%H_%M_%S'
+
+    @property
+    def entrypoints(self) -> List[str]:
+        entrypoints = []
+        for entrypoint in (
+            'sh',
+            'pyenv',
+            'docker',
+        ):
+            try:
+                subprocess.check_output(['which', entrypoint])
+                entrypoints.append(entrypoint)
+            except subprocess.CalledProcessError:
+                continue
+
+        return entrypoints
+
+    def get_task_names(self, pipeline: Pipeline) -> List[str]:
+        pass
+
+    def get_group_names(self, pipeline: Pipeline) -> List[str]:
+        pass
+
+    def add_task_to_group(self, group_name: str, task: PipelineTask) -> PipelineTaskGroup:
+        pass
+
+    def get_scripts_by_entrypoint(self, entrypoint: str) -> List[str]:
+        result = []
+        if entrypoint == 'docker':
+            images = subprocess.check_output(['docker', 'images', '--format=json'])
+            images = images.decode().split('\n')
+
+            for image_str in images:
+                if not image_str:
+                    continue
+
+                image = json.loads(image_str)
+                result.append(f'{image["Repository"]}:{image["Tag"]}')
+        else:
+            endswith = '.py' if entrypoint in ('pyenv', ) else '.sh'
+            for file in os.scandir(self._paths.scripts):
+                if file.name.endswith(endswith):
+                    result.append(file.name)
+
+        return sorted(result)
+
+    def get_nostromo_env(self) -> Dict[str, str]:
+        config = dotenv_values(str(self._paths.env_file))
+        config['NOSTOMO_HOME'] = os.environ.get('NOSTOMO_HOME')
+        return dict(sorted(config.items(), key=lambda x: x[0]))
+
+    def _load_pipeline_from_yml(self, folder_name: str) -> Pipeline:
+        full_dir = os.path.join(self._paths.pipelines, folder_name)
+        with codecs.open(os.path.join(full_dir, '.pipeline.yaml')) as stream:
+            params = yaml.safe_load(stream) or {}
+            params['pipeline_dir'] = full_dir
+            params['name'] = folder_name
+            return Pipeline(**params)
+
+    def get_pipelines(self) -> List[Pipeline]:
+        return [
+            self._load_pipeline_from_yml(p)
+            for p in os.listdir(self._paths.pipelines)
+        ]
+
+    def _get_pipeline_logs_dir(self, name: str) -> str:
+        return os.path.join(str(self._paths.pipelines_logs), name)
+
+    async def get_last_pipeline_run(self, pipeline_name: str) -> PipelineRun or None:
+        pattern = self._get_pipeline_runs_key(pipeline_name) + '*'
+        keys = await self._redis.scan(match=pattern.encode(), count=1)
+        if not keys[1]:
+            return
+
+        run_key = keys[1][0].decode()
+        data = await self._redis.get(run_key)
+        if not data:
+            return
+
+        record: dict = json.loads(data.decode('utf-8'))
+        for key in ('started_at', 'finished_at'):
+            if record.get(key):
+                record[key] = datetime.fromisoformat(record[key])
+
+        return PipelineRun(**record)
+
+    async def kill_pipeline(self, run: PipelineRun) -> PipelineRun:
+        try:
+            parent = psutil.Process(run.pid)
+        except NoSuchProcess:
+            run.status = PipelineRunStatus.KILLED
+            run.finished_at = datetime.now(UTC)
+            await self._save_pipeline_run(run)
+            return run
+
+        for child in parent.children(recursive=True):
+            try:
+                child.kill()
+            except NoSuchProcess:
+                pass
+
+        parent.kill()
+        run.status = PipelineRunStatus.KILLED
+        run.finished_at = datetime.now(UTC)
+        await self._save_pipeline_run(run)
+
+    def _get_pipeline_runs_key(self, pipeline_name: str) -> str:
+        return f'nostromo:pipeline_run:{pipeline_name}'
+
+    async def _save_pipeline_run(self, run: PipelineRun) -> None:
+        key = self._get_pipeline_runs_key(pipeline_name=run.pipeline_name)
+        run_dt = run.started_at.strftime(self.logs_dt_format)
+        key = f'{key}:{run_dt}'
+
+        await self._redis.set(key, run.to_json())
+
+    def get_pipeline_by_name(self, name: str) -> Pipeline:
+        return self._load_pipeline_from_yml(name)
+
+    async def run_pipeline_by_name(self, name: str) -> PipelineRun:
+        pipeline = self.get_pipeline_by_name(name)
+        pypyr_path = subprocess.check_output(
+            'which pypyr',
+            shell=True,
+            env=os.environ,
+            text=True
+        ).replace('\n', '')
+
+        run = datetime.now(UTC)
+        logs_dir = os.path.join(self._get_pipeline_logs_dir(name), run.strftime(self.logs_dt_format))
+
+        os.makedirs(logs_dir, exist_ok=True)
+        process = subprocess.Popen(
+            f'{pypyr_path} .run pipeline_log_dir={logs_dir}',
+            shell=True,
+            text=True,
+            cwd=pipeline.pipeline_dir,
+        )
+
+        pipeline_run = PipelineRun(
+            pid=process.pid,
+            logs_dir=logs_dir,
+            pipeline_name=pipeline.name,
+            started_at=run,
+        )
+
+        await self._save_pipeline_run(pipeline_run)
+        return pipeline_run

nostromo-0.dev0/nostromo/services/redis_executor.py

@@ -0,0 +1,31 @@
+from redis.asyncio import Redis
+from saq import Queue
+
+from ..protocols.executor import ExecutorProtocol
+
+
+class RedisExecutor(ExecutorProtocol):
+    def __init__(self, redis_url: str):
+        self._queue = Queue.from_url(redis_url)
+        self._client = Redis.from_url(redis_url)
+
+    async def send_to_executor(self, cmd: str):
+        return await super().send_to_executor(cmd)
+
+    async def wait_and_exec_commands(self):
+        return await super().wait_and_exec_commands()
+
+    async def is_scheduler_running(self) -> bool:
+        return await super().is_scheduler_running()
+
+    async def set_scheduler_running(self, is_scheduler_running: bool) -> None:
+        return await super().set_scheduler_running(is_scheduler_running)
+
+    async def action_pause_pipeline(self):
+        pass
+
+    async def action_kill_pipeline(self):
+        pass
+
+    async def wait_and_exec_actions(self):
+        pass

nostromo-0.dev0/nostromo/services/rich_ui_log.py

@@ -0,0 +1,52 @@
+from datetime import datetime, UTC
+
+from textual.notifications import SeverityLevel
+from textual.widgets import RichLog
+
+from nostromo.protocols.ui_log import UILogProtocol
+
+
+class RichUILogService(UILogProtocol):
+    def __init__(self, log: RichLog):
+        self._log = log
+
+    @property
+    def log(self) -> RichLog:
+        return self._log
+
+    def _get_colored_severity(self, severity: SeverityLevel) -> str:
+        color = 'blue'
+        if severity == 'warning':
+            color = 'yellow'
+        elif severity == 'error':
+            color = 'red'
+
+        if severity == 'information':
+            severity = 'info'
+
+        return f'[{color}]{severity.upper()}[/{color}] '
+
+    def _write(
+        self,
+        message: str,
+        title: str = '',
+        severity: SeverityLevel = 'information',
+    ) -> None:
+        dt = datetime.now(UTC).strftime('%Y-%m-%d %H:%M:%S')
+        msg = ''.join([
+            f'{dt} {self._get_colored_severity(severity)}',
+            f'{title}. {message}' if title else message,
+        ])
+
+        self.log.write(msg)
+        if title:
+            self.log.notify(message, title=title, severity=severity, timeout=7)
+
+    def info(self, message: str, title: str = '') -> None:
+        self._write(message, title)
+
+    def warning(self, message: str, title: str = '') -> None:
+        self._write(message, title, 'warning')
+
+    def error(self, message: str, title: str = '', notification: bool = False) -> None:
+        self._write(message, title, 'error')

nostromo-0.dev0/nostromo/tabs/pipelines.py

@@ -0,0 +1,122 @@
+from asyncio import sleep
+
+import inject
+from rich.text import Text
+from textual import work, events
+from textual.app import ComposeResult
+from textual.widget import Widget
+from textual.widgets import DataTable
+
+from nostromo.protocols.pipelines import PipelinesProtocol
+from nostromo.protocols.ui_log import UILogProtocol
+
+
+class _PipelinesTable(DataTable):
+    DEFAULT_CSS = """
+    _PipelinesTable {color: #c7c9ca; border-bottom: #1047A9; padding-bottom: 1} # FFBC40
+    _PipelinesTable > .datatable--cursor {background: #1047A9; color: #c7c9ca}
+    _PipelinesTable > .datatable--fixed-cursor {background: #1e1e1e; color: #c7c9ca}
+    _PipelinesTable > .datatable--fixed {background: #1e1e1e; color: #c7c9ca}
+    _PipelinesTable > .datatable--hover {background: #1e1e1e; color: #c7c9ca}
+    _PipelinesTable > .datatable--header {background: #1e1e1e; color: #FFBC40}
+    """
+    _log = inject.attr(UILogProtocol)
+    _pipelines = inject.attr(PipelinesProtocol)
+    BINDINGS = [
+        ('ctrl+r', 'run_pipeline()', 'Run Pipeline'),
+        ('ctrl+k', 'kill_pipeline()', 'Kill Pipeline'),
+        ('ctrl+p', 'stop_pipeline()', 'Pause Pipeline'),
+    ]
+
+    def action_run_pipeline(self):
+        self._run_pipeline_by_name()
+
+    @work(exclusive=True)
+    async def _run_pipeline_by_name(self):
+        pipeline_name = self.get_cell_at(self.cursor_coordinate)
+        self.refresh_bindings()
+        await self._pipelines.run_pipeline_by_name(pipeline_name)
+        self._log.info(pipeline_name, 'Pipeline started')
+
+    def action_kill_pipeline(self):
+        self._kill_pipeline()
+
+    @work(exclusive=True)
+    async def _kill_pipeline(self):
+        pipeline_name = self.get_cell_at(self.cursor_coordinate)
+        self._log.warning(f'Kill pipeline {pipeline_name}')
+        last_run = await self._pipelines.get_last_pipeline_run(pipeline_name)
+        if not last_run or last_run.finished_at:
+            self._log.error(f'{pipeline_name} is not running', 'Kill Error')
+            return
+
+        await self._pipelines.kill_pipeline(last_run)
+        self._log.warning(pipeline_name, 'Pipeline killed')
+
+    @property
+    def default_opacity(self) -> float:
+        return 0.7
+
+    def on_mount(self):
+        self.styles.opacity = self.default_opacity
+        if not self.rows:
+            return
+
+        self.track_pipelines()
+
+    def _on_focus(self, event: events.Focus) -> None:
+        self.styles.opacity = 1
+
+    def _on_blur(self, event: events.Blur) -> None:
+        self.styles.opacity = self.default_opacity
+
+    @work(exclusive=True)
+    async def track_pipelines(self):
+        while True:
+            for row in self.rows.values():
+                last_run = await self._pipelines.get_last_pipeline_run(row.key.value)
+                if not last_run:
+                    continue
+
+                finished = last_run.finished_at.strftime('%Y-%m-%d %H:%M:%S') if last_run.finished_at else ''
+                self.update_cell(row.key, 'Last Run', last_run.started_at.strftime('%Y-%m-%d %H:%M:%S'))
+                self.update_cell(row.key, 'Duration', str(last_run.duration).split('.')[0])
+                self.update_cell(row.key, 'Finished', finished)
+
+            await sleep(1)
+
+
+class PipelienesContent(Widget):
+    _pipelines = inject.attr(PipelinesProtocol)
+
+    @classmethod
+    def get_id(cls):
+        return 'Pipelines'
+
+    def compose(self) -> ComposeResult:
+        table = _PipelinesTable()
+        table.cursor_type = 'row'
+        table.zebra_stripes = True
+        padding = ' ' * 20
+
+        for col in (
+            'Name',
+            'Schedule',
+            'Last Run',
+            'Duration',
+            'Finished',
+        ):
+            table.add_column(Text(col, justify='center'), key=col)
+
+        for pipeline in self._pipelines.get_pipelines():
+            table.add_row(
+                pipeline.name,
+                pipeline.schedule,
+                padding,
+                '',
+                padding,
+                key=pipeline.name,
+            )
+
+        table.sort('Name')
+        yield table

nostromo-0.dev0/nostromo/tabs/scheduler.py

@@ -0,0 +1,12 @@
+from textual.app import ComposeResult
+from textual.widget import Widget
+from textual.widgets import Label
+
+
+class SchedulerContent(Widget):
+    @classmethod
+    def get_id(cls):
+        return 'Scheduler'
+
+    def compose(self) -> ComposeResult:
+        yield Label('Scheduler')

nostromo-0.dev0/nostromo/widgets/default_option_list.py

@@ -0,0 +1,103 @@
+import re
+from asyncio import sleep
+from copy import deepcopy
+from datetime import datetime, UTC
+from typing import List
+
+from textual import events, work
+from textual.reactive import reactive
+from textual.widgets import OptionList
+
+
+class DefaultOptionList(OptionList):
+    DEFAULT_CSS = """
+    DefaultOptionList {
+        layer: above;
+    }
+    """
+    _options_copy = []
+    _input_buffer = reactive('')
+    _last_input_dt = datetime.now(UTC)
+    value = reactive('')
+
+    def on_mount(self):
+        self._options_copy = deepcopy(self._options)
+        self.clear_options()
+        if self._options_copy:
+            self.add_option(self._options_copy[0])
+            self.value = self._options_copy[0].prompt
+
+    def on_key(self, event: events.Key) -> None:
+        if event.key == 'tab' and len(self._options) > 1:
+            self.action_select()
+
+        if re.fullmatch(r'[A-Za-z0-9]', event.key) or event.key == 'backspace':
+            self.clear_options()
+            if event.key == 'backspace':
+                if len(self._input_buffer) == 1:
+                    self._input_buffer = ''
+                    self.action_select()
+                    return
+
+                self._input_buffer = self._input_buffer[:-1]
+            else:
+                self._input_buffer += event.key
+
+            self.add_option(self._input_buffer)
+            self._last_input_dt = datetime.now(UTC)
+            self._find_options()
+
+    @work(exclusive=True)
+    async def _find_options(self):
+        await sleep(0.5)
+        self.clear_options()
+        for option in self._options_copy:
+            prompt = option if isinstance(option, str) else option.prompt
+            if prompt.lower().find(self._input_buffer) > -1:
+                self.add_option(option)
+
+        if self._options:
+            self.action_page_down()
+        else:
+            self.add_option(self._input_buffer)
+
+        self._input_buffer = ''
+
+    def action_select(self) -> None:
+        # disabled select
+        if len(self._options) == 1:
+            selected = self.get_option_at_index(0)
+            move = True
+            self.clear_options()
+
+            for ix, entrypoint in enumerate(self._options_copy):
+                self.add_option(entrypoint)
+                if move:
+                    self.action_page_down()
+
+                prompt = entrypoint if isinstance(entrypoint, str) else entrypoint.prompt
+                if prompt == selected.prompt:
+                    self.value = prompt
+                    move = False
+
+            return
+
+        # opened with an option
+        if self.highlighted:
+            option = self.get_option_at_index(self.highlighted)
+            self.clear_options()
+            self.add_option(option)
+            self.value = option.prompt
+            return
+
+        # init - show first
+        if self._options_copy:
+            self.clear_options()
+            self.add_option(self._options_copy[0])
+            self.value = self._options_copy[0]
+
+    def reset_options(self, options: List[str]):
+        self._options_copy = deepcopy(options)
+        self.clear_options()
+        if options:
+            self.add_option(options[0])

nostromo-0.dev0/nostromo/widgets/horizontal_form.py

@@ -0,0 +1,35 @@
+from textual.app import ComposeResult
+from textual.containers import Horizontal, Vertical
+from textual.widget import Widget
+from textual.widgets import Label, OptionList
+
+
+class HorizontalForm(Horizontal):
+    DEFAULT_CSS = """
+    HorizontalForm Label {
+        padding: 1;
+        text-align: right;
+    }
+    
+    DefaultOptionList {
+        min-height: 3;
+    }
+    """
+
+    def __init__(self, *form_items: Widget, name: str | None = None, id: str | None = None, classes: str | None = None,
+                 disabled: bool = False) -> None:
+        super().__init__(*[], name=name, id=id, classes=classes, disabled=disabled)
+        self._form_items = form_items
+
+    def compose(self) -> ComposeResult:
+        labels = Vertical(*[Label(w.name) for w in self._form_items])
+        labels.styles.max_width = max(*[len(w.name) for w in self._form_items])
+        yield labels
+
+        items = []
+        for item in self._form_items:
+            if isinstance(item, OptionList):
+                items.append(Label(item.name))
+            items.append(item)
+
+        yield Vertical(*items)

nostromo-0.dev0/nostromo/widgets/pipeline_builder.py

@@ -0,0 +1,63 @@
+import inject
+from textual import on
+from textual.validation import Regex
+from textual.widget import Widget
+from textual.widgets import Input, Button, Footer, Pretty
+
+from nostromo.protocols.path import PathProtocol
+from nostromo.protocols.pipelines import PipelinesProtocol
+from nostromo.protocols.ui_log import UILogProtocol
+from nostromo.widgets.default_option_list import DefaultOptionList
+from nostromo.widgets.horizontal_form import HorizontalForm
+
+
+class _EntrypointOptions(DefaultOptionList):
+    DEFAULT_CSS = """
+    _EntrypointOptions {
+        layer: above;
+    }
+    """
+    _pipelines = inject.attr(PipelinesProtocol)
+
+    def _reset_scripts_options(self):
+        script_options: DefaultOptionList = self.parent.parent.query_one('#ScriptOptions')
+        script_options.clear_options()
+        options = self._pipelines.get_scripts_by_entrypoint(self.value)
+        script_options.reset_options(options)
+
+    def action_select(self) -> None:
+        super().action_select()
+        self._reset_scripts_options()
+
+
+class PipelineBuilderWidget(Widget):
+    _pipelines = inject.attr(PipelinesProtocol)
+    _paths = inject.attr(PathProtocol)
+
+    def compose(self) -> None:
+        yield Footer()
+        yield HorizontalForm(
+            _EntrypointOptions(*self._pipelines.entrypoints, name='entrypoint'),
+            DefaultOptionList(id='ScriptOptions', name='ScriptOptions'),
+            Input(placeholder='calculate_events', name='Task Name', validators=[
+                Regex('^[a-z0-9_]*$', failure_description='Task name does not match regular expression "^[a-z0-9_]*$"')
+            ]),
+            # Input(placeholder='--param1=value1 --param2=value2', name='Parameters'),
+            RunCmdButton('Run', variant='success', name=''),
+        )
+        yield Pretty([])
+
+    @on(Input.Changed)
+    def show_invalid_reasons(self, event: Input.Changed) -> None:
+        # Updating the UI to show the reasons why validation failed
+        if not event.validation_result.is_valid:
+            self.query_one(Pretty).update(event.validation_result.failure_descriptions)
+        else:
+            self.query_one(Pretty).update([])
+
+
+class RunCmdButton(Button):
+    _log = inject.attr(UILogProtocol)
+
+    def on_button_pressed(self, event: Button.Pressed) -> None:
+        self._log.info('test', 'Run Script')

nostromo-0.dev0/pyproject.toml

@@ -0,0 +1,25 @@
+[build-system]
+requires = ["poetry-core==1.9.1"]
+build-backend = "poetry.core.masonry.api"
+
+
+[tool.poetry]
+name = "nostromo"
+version = "0.dev.0"
+description = "pipeline builder, runner, process manager, background jobs, job scheduling"
+authors = ["Danila Ganchar"]
+license = "MIT"
+readme = "README.md"
+
+
+[tool.poetry.dependencies]
+python = "^3.9"
+textual = "0.85.2"
+pyyaml = "^6.0.1"
+setuptools = "^70.2.0"
+inject = "4.1"
+pypyr = "5.9.1"
+bigtree = "^0.22.3"
+psutil = "^6.1.0"
+redis = "5.0.1"
+saq = {extras = ["redis"], version = "^0.18.3"}