dron 0.1.20241008__py3-none-any.whl → 0.2.20251011__py3-none-any.whl

dron/dron.py

@@ -1,16 +1,15 @@
 from __future__ import annotations
 
 import importlib.util
-import os
-import shlex
 import sys
 from collections import OrderedDict
+from collections.abc import Iterable, Iterator
+from concurrent.futures import ProcessPoolExecutor
 from difflib import unified_diff
 from itertools import tee
 from pathlib import Path
-from subprocess import check_call, run
-from tempfile import TemporaryDirectory
-from typing import Iterable, Iterator, NamedTuple, Union
+from subprocess import check_call
+from typing import NamedTuple
 
 import click
 
@@ -35,9 +34,6 @@ DRON_UNITS_DIR = DRON_DIR / 'units'
 DRON_UNITS_DIR.mkdir(parents=True, exist_ok=True)
 
 
-DRONTAB = DRON_DIR / 'drontab.py'
-
-
 def verify_units(pre_units: list[tuple[UnitName, Body]]) -> None:
     # need an inline import here in case we modify this variable from cli/tests
     from .common import VERIFY_UNITS
@@ -60,7 +56,7 @@ def verify_unit(*, unit_name: UnitName, body: Body) -> None:
     return verify_units([(unit_name, body)])
 
 
-def write_unit(*, unit: Unit, body: Body, prefix: Path=DRON_UNITS_DIR) -> None:
+def write_unit(*, unit: Unit, body: Body, prefix: Path = DRON_UNITS_DIR) -> None:
     unit_file = prefix / unit
 
     logger.info(f'writing unit file: {unit_file}')
@@ -146,11 +142,12 @@ class Add(NamedTuple):
         return self.unit_file.name
 
 
-Action = Union[Update, Delete, Add]
+Action = Update | Delete | Add
 Plan = Iterable[Action]
 
 # TODO ugh. not sure how to verify them?
 
+
 def compute_plan(*, current: State, pending: State) -> Plan:
     # eh, I feel like i'm reinventing something already existing here...
     currentd = OrderedDict((x.unit_file, unwrap(x.body)) for x in current)
@@ -178,6 +175,7 @@ def apply_state(pending: State) -> None:
     current = list(managed_units(with_body=True))
 
     pending_units = {s.unit_file.name for s in pending}
+
     def is_always_running(unit_path: Path) -> bool:
         name = unit_path.stem
         has_timer = f'{name}.timer' in pending_units
@@ -198,7 +196,7 @@ def apply_state(pending: State) -> None:
         elif isinstance(a, Update):
             _updates.append(a)
         else:
-            raise AssertionError("Can't happen", a)
+            raise TypeError("Can't happen", a)
 
     if len(deletes) == len(current) and len(deletes) > 0:
         msg = "Trying to delete all managed jobs"
@@ -213,10 +211,12 @@ def apply_state(pending: State) -> None:
 
     for u in _updates:
         unit = a.unit
-        diff: Diff = list(unified_diff(
-            u.old_body.splitlines(keepends=True),
-            u.new_body.splitlines(keepends=True),
-        ))
+        diff: Diff = list(
+            unified_diff(
+                u.old_body.splitlines(keepends=True),
+                u.new_body.splitlines(keepends=True),
+            )
+        )
         if len(diff) == 0:
             nochange.append(u)
         else:
@@ -231,15 +231,14 @@ def apply_state(pending: State) -> None:
     for a in deletes:
         if IS_SYSTEMD:
             # TODO stop timer first?
-            check_call(_systemctl('stop'   , a.unit))
+            check_call(_systemctl('stop', a.unit))
             check_call(_systemctl('disable', a.unit))
         else:
             launchd.launchctl_unload(unit=Path(a.unit).stem)
     for a in deletes:
         (DRON_UNITS_DIR / a.unit).unlink()
 
-
-    for (u, diff) in updates:
+    for u, diff in updates:
         unit = u.unit
         unit_file = u.unit_file
         logger.info(f'updating {unit}')
@@ -297,47 +296,16 @@ def manage(state: State) -> None:
 Error = str
 # TODO perhaps, return Plan or error instead?
 
-# eh, implicit convention that only one state will be emitted. oh well
-def lint(tabfile: Path) -> Iterator[Exception | State]:
-    linters = [
-        [sys.executable, '-m', 'mypy', '--no-incremental', '--check-untyped', str(tabfile)],
-    ]
-
-    ldir = tabfile.parent
-    # TODO not sure if should always lint in temporary dir to prevent turds?
-
-    dron_dir = str(Path(__file__).resolve().absolute().parent)
-    dtab_dir = drontab_dir()
-
-    # meh.
-    def extra_path(variable: str, path: str, env) -> dict[str, str]:
-        vv = env.get(variable)
-        pp = path + ('' if vv is None else ':' + vv)
-        return {**env, variable: pp}
-
-    errors = []
-    for l in linters:
-        scmd = ' '.join(map(shlex.quote, l))
-        logger.info(f'Running: {scmd}')
-        with TemporaryDirectory() as td:
-            env = {**os.environ}
-            env = extra_path('MYPYPATH'  , dtab_dir, env)
-
-            r = run(l, cwd=str(ldir), env=env, check=False)
-        if r.returncode == 0:
-            logger.info('OK')
-            continue
-        else:
-            logger.error(f'FAIL: code: {r.returncode}')
-            errors.append('error')
-    if len(errors) > 0:
-        yield RuntimeError('Python linting failed!')
-        return
 
-    # TODO just add options to skip python lint? so it always goes through same code paths
+# eh, implicit convention that only one state will be emitted. oh well
+# FIXME rename from lint? just use compileall or something as a syntax check?
+def lint(tab_module: str) -> Iterator[Exception | State]:
+    # TODO tbh compileall is pointless
+    # - we can't find out source names property without importing
+    # - we'll find out about errors during importing anyway
 
     try:
-        jobs = load_jobs(tabfile=tabfile, ppath=Path(dtab_dir))
+        jobs = load_jobs(tab_module)
     except Exception as e:
         # TODO could add better logging here? 'i.e. error while loading jobs'
         logger.exception(e)
@@ -354,101 +322,42 @@ def lint(tabfile: Path) -> Iterator[Exception | State]:
     yield state
 
 
-def test_do_lint(tmp_path: Path) -> None:
-    import pytest
-
-
-    def ok(body: str) -> None:
-        tpath = Path(tmp_path) / 'drontab.py'
-        tpath.write_text(body)
-        do_lint(tabfile=tpath)
-
-    def fails(body: str) -> None:
-        with pytest.raises(Exception):
-            ok(body)
-
-    fails(body='''
-    None.whatever
-    ''')
-
-    # no jobs
-    fails(body='''
-    ''')
-
-    ok(body='''
-def jobs():
-    yield from []
-''')
-
-    ok(body='''
-from dron.api import job
-def jobs():
-    yield job(
-        'hourly',
-        ['/bin/echo', '123'],
-        unit_name='unit_test',
-    )
-''')
-
-    from .systemd import _is_missing_systemd
-
-    if not _is_missing_systemd():
-        from .cli import _drontab_example
-
-        # this test doesn't work without systemd yet, because launchd adapter doesn't support unquoted commands, at least yet..
-        example = _drontab_example()
-        # ugh. some hackery to make it find the executable..
-        echo = " '/bin/echo"
-        example = example.replace(" 'linkchecker", echo).replace(" '/home/user/scripts/run-borg", echo).replace(" 'ping", " '/bin/ping")
-        ok(body=example)
-
-
-def do_lint(tabfile: Path) -> State:
-    eit, vit = tee(lint(tabfile))
-    errors = [r for r in eit if     isinstance(r, Exception)]
+def do_lint(tab_module: str) -> State:
+    eit, vit = tee(lint(tab_module))
+    errors = [r for r in eit if isinstance(r, Exception)]
     values = [r for r in vit if not isinstance(r, Exception)]
     assert len(errors) == 0, errors
     [state] = values
     return state
 
 
-def drontab_dir() -> str:
-    # meeh
-    return str(DRONTAB.resolve().absolute().parent)
+def _import_jobs(tab_module: str) -> list[Job]:
+    module = importlib.import_module(tab_module)
+    jobs_gen = getattr(module, 'jobs')  # get dynamically to make type checking happy
+    return list(jobs_gen())
 
 
-def load_jobs(tabfile: Path, ppath: Path) -> Iterator[Job]:
-    pp = str(ppath)
-    sys.path.insert(0, pp)
-    try:
-        spec = importlib.util.spec_from_file_location(tabfile.name, tabfile)
-        assert spec is not None, tabfile
-        loader = spec.loader
-        assert loader is not None, (tabfile, spec)
-        module = importlib.util.module_from_spec(spec)
-        loader.exec_module(module)
-    finally:
-        sys.path.remove(pp)  # extremely meh..
-
-    jobs = module.jobs
+def load_jobs(tab_module: str) -> Iterator[Job]:
+    # actually import in a separate process to avoid mess with polluting sys.modules
+    # shouldn't be a problem in most cases, but it was annoying during tests
+    with ProcessPoolExecutor(max_workers=1) as pool:
+        jobs = pool.submit(_import_jobs, tab_module).result()
+
     emitted: dict[str, Job] = {}
-    for job in jobs():
+    for job in jobs:
         assert isinstance(job, Job), job  # just in case for dumb typos
         assert job.unit_name not in emitted, (job, emitted[job.unit_name])
         yield job
         emitted[job.unit_name] = job
 
 
-def apply(tabfile: Path) -> None:
-    state = do_lint(tabfile)
+def apply(tab_module: str) -> None:
+    # TODO rename do_lint to get_state?
+    state = do_lint(tab_module)
     manage(state=state)
 
 
-get_entries_for_monitor = (
-    systemd.get_entries_for_monitor
-    if IS_SYSTEMD else
-    launchd.get_entries_for_monitor
-)
+get_entries_for_monitor = systemd.get_entries_for_monitor if IS_SYSTEMD else launchd.get_entries_for_monitor
 
 
 def main() -> None:

dron/launchd.py

@@ -7,11 +7,12 @@ import re
 import shlex
 import sys
 import textwrap
+from collections.abc import Iterator, Sequence
 from datetime import timedelta
 from pathlib import Path
 from subprocess import PIPE, Popen, check_call, check_output
 from tempfile import TemporaryDirectory
-from typing import Any, Iterator, Sequence
+from typing import Any
 
 from .api import (
     OnCalendar,
@@ -84,16 +85,14 @@ def launchctl_reload(*, unit: Unit, unit_file: UnitFile) -> None:
 
 
 def launchd_wrapper(*, job: str, on_failure: list[str]) -> list[str]:
-    # fmt: off
     return [
         sys.executable,
-        '-m',
-        'dron.launchd_wrapper',
+        '-B',  # do not write byte code, otherwise it shits into dron directory if we're using editable install
+        '-m', 'dron.launchd_wrapper',
         *itertools.chain.from_iterable(('--notify', n) for n in on_failure),
         '--job', job,
         '--',
-    ]
-    # fmt: on
+    ]  # fmt: skip
 
 
 def remove_launchd_wrapper(cmd: str) -> str:
@@ -109,7 +108,7 @@ def plist(
     unit_name: str,
     command: Command,
     on_failure: Sequence[OnFailureAction],
-    when: When | None=None,
+    when: When | None = None,
 ) -> str:
     # TODO hmm, kinda mirrors 'escape' method, not sure
     cmd: Sequence[str]
@@ -136,17 +135,21 @@ def plist(
     else:
         assert isinstance(when, OnCalendar), when
         # https://www.freedesktop.org/software/systemd/man/systemd.time.html#
+        # fmt: off
         seconds = {
             'minutely': 60,
             'hourly'  : 60 * 60,
             'daily'   : 60 * 60 * 24,
         }.get(when)
+        # fmt: on
         if seconds is None:
             # ok, try systemd-like spec..
+            # fmt: off
             specs = [
                 (re.escape('*:0/')   + r'(\d+)', 60),
-                (re.escape('*:*:0/') + r'(\d+)', 1 ),
+                (re.escape('*:*:0/') + r'(\d+)', 1),
             ]
+            # fmt: on
             for rgx, mult in specs:
                 m = re.fullmatch(rgx, when)
                 if m is not None:
@@ -159,23 +162,24 @@ def plist(
             assert m is not None, when
             hh = m.group(1)
             mm = m.group(2)
-            mschedule = '\n'.join([
-                '<key>StartCalendarInterval</key>',
-                '<dict>',
-                '<key>Hour</key>'  , f'<integer>{int(hh)}</integer>',
-                '<key>Minute</key>', f'<integer>{int(mm)}</integer>',
-                '</dict>',
-            ])
+            mschedule = '\n'.join(
+                [
+                    '<key>StartCalendarInterval</key>',
+                    '<dict>',
+                    '<key>Hour</key>',
+                    f'<integer>{int(hh)}</integer>',
+                    '<key>Minute</key>',
+                    f'<integer>{int(mm)}</integer>',
+                    '</dict>',
+                ]
+            )
         else:
             mschedule = '\n'.join(('<key>StartInterval</key>', f'<integer>{seconds}</integer>'))
 
     assert mschedule != '', unit_name
 
     # meh.. not sure how to reconcile it better with systemd
-    on_failure = [
-        x.replace('--job %n', f'--job {unit_name}') + ' --stdin'
-        for x in on_failure
-    ]
+    on_failure = [x.replace('--job %n', f'--job {unit_name}') + ' --stdin' for x in on_failure]
 
     # attempt to set argv[0] properly
     # hmm I was hoping it would make desktop notifications ('background service added' nicer)
@@ -252,14 +256,14 @@ def launchd_state(*, with_body: bool) -> Iterator[LaunchdUnitState]:
                 periodic_schedule = extras.get('run interval')
                 calendal_schedule = 'com.apple.launchd.calendarinterval' in unwrap(all_props)
 
-                schedule: str | None = None
+                schedule: str | None
                 if periodic_schedule is not None:
                     schedule = 'every ' + periodic_schedule
                 elif calendal_schedule:
                     # TODO parse properly
                     schedule = 'calendar'
                 else:
-                    # NOTE: seems like keepalive attribute isn't present in launcd dumpstate output
+                    # NOTE: seems like keepalive attribute isn't present in launchd dumpstate output
                     schedule = 'always'
 
                 yield LaunchdUnitState(
@@ -302,15 +306,19 @@ def verify_unit(*, unit_name: str, body: str) -> None:
     with TemporaryDirectory() as tdir:
         tfile = Path(tdir) / unit_name
         tfile.write_text(body)
-        check_call([
-            'plutil', '-lint',
-            '-s',  # silent on success
-            tfile,
-        ])
+        check_call(
+            [
+                'plutil',
+                '-lint',
+                '-s',  # silent on success
+                tfile,
+            ]
+        )
 
 
 def cmd_past(unit: Unit) -> None:
     sub = fqn('dron.' + unit)
+    # fmt: off
     cmd = [
         # todo maybe use 'stream'??
         'log', 'show', '--info',
@@ -323,8 +331,10 @@ def cmd_past(unit: Unit) -> None:
         '--style', 'ndjson',
         '--color', 'always',
     ]
+    # fmt: on
     with Popen(cmd, stdout=PIPE, encoding='utf8') as p:
-        out = p.stdout; assert out is not None
+        out = p.stdout
+        assert out is not None
         for line in out:
             j = json.loads(line)
             if j.get('finished') == 1:
@@ -402,14 +412,16 @@ def get_entries_for_monitor(managed: State, *, params: MonitorParams) -> list[Mo
 
         pid = s.pid
 
-        entries.append(MonitorEntry(
-            unit=name,
-            status=status,
-            left='n/a',
-            next='n/a',
-            schedule=schedule,
-            command=command,
-            pid=pid,
-            status_ok=status_ok,
-        ))
+        entries.append(
+            MonitorEntry(
+                unit=name,
+                status=status,
+                left='n/a',
+                next='n/a',
+                schedule=schedule,
+                command=command,
+                pid=pid,
+                status_ok=status_ok,
+            )
+        )
     return entries

dron/launchd_wrapper.py

@@ -1,17 +1,26 @@
 #!/usr/bin/env python3
 import argparse
+import os
 import shlex
 import sys
+from collections.abc import Iterator
 from pathlib import Path
 from subprocess import PIPE, STDOUT, Popen
-from typing import Iterator, NoReturn
+from typing import NoReturn
 
 from loguru import logger
 
 LOG_DIR = Path('~/Library/Logs/dron').expanduser()
 
+# OSX/launchd is a piece of shit and doesn't seem possible to just set it globally everywhere?
+# this works: launchctl setenv PYTHONPYCACHEPREFIX $PYTHONPYCACHEPREFIX
+# however unclear how to set it in a way that it's running before all other agents
+# allegedly possible to use global LaunchDaemon running as root, but doesn't seem possible to execute launchctl commands as other user from launchd plist??
+PYCACHE_PATH = Path('~/.cache/pycache').expanduser()
 
-def main() -> NoReturn:
+
+# ty doesn't support NoReturn yet, see https://github.com/astral-sh/ty/issues/180
+def main() -> NoReturn:  # ty: ignore[invalid-return-type]
     p = argparse.ArgumentParser()
     p.add_argument('--notify', action='append')
     p.add_argument('--job', required=True)
@@ -30,10 +39,14 @@ def main() -> NoReturn:
 
     logger.add(log_file, rotation='100 MB')  # todo configurable? or rely on osx rotation?
 
+    env = {**os.environ}
+    if "PYTHONPYCACHEPREFIX" not in env:
+        env["PYTHONPYCACHEPREFIX"] = str(PYCACHE_PATH)
+
     # hmm, a bit crap transforming everything to stdout? but not much we can do?
     captured_log = []
     try:
-        with Popen(cmd, stdout=PIPE, stderr=STDOUT) as po:
+        with Popen(cmd, stdout=PIPE, stderr=STDOUT, env=env) as po:
             out = po.stdout
             assert out is not None
             for line in out:
@@ -50,7 +63,6 @@ def main() -> NoReturn:
         captured_log.append(str(e).encode('utf8'))
         rc = 123
 
-
     def payload() -> Iterator[bytes]:
         yield f"exit code: {rc}\n".encode()
         yield b'command: \n'

dron/monitor.py

@@ -41,7 +41,6 @@ def as_row(entry: MonitorEntry) -> dict[str, Any]:
     cols = get_columns()
     res = {k: v for k, v in asdict(entry).items() if k in cols}
 
-    v = res['status']
     color = 'green' if entry.status_ok else 'red'
     res['status'] = f'[{color}]' + res['status'] + f'[/{color}]'
 
@@ -57,8 +56,7 @@ def as_row(entry: MonitorEntry) -> dict[str, Any]:
 
 
 class MonitorApp(App):
-
-    def __init__(self, *, monitor_params: MonitorParams, refresh_every: int) -> None:
+    def __init__(self, *, monitor_params: MonitorParams, refresh_every: float) -> None:
         super().__init__()
         self.monitor_params = monitor_params
         self.refresh_every = refresh_every
@@ -126,7 +124,7 @@ class MonitorApp(App):
         columns = get_columns()
 
         def sort_key(row):
-            data = dict(zip(columns, row))
+            data = dict(zip(columns, row, strict=True))
             is_running = 'running' in data['next']
             failed = 'exit-code' in data['status']
             return (not is_running, not failed, data['unit'])

dron/notify/common.py

@@ -2,8 +2,8 @@ import argparse
 import platform
 import shlex
 import sys
+from collections.abc import Iterator
 from subprocess import PIPE, STDOUT, Popen, check_output
-from typing import Iterator
 
 IS_SYSTEMD = platform.system() != 'Darwin'  # if not systemd it's launchd
 

dron/notify/email.py

@@ -1,6 +1,6 @@
 import socket
+from collections.abc import Iterator
 from subprocess import PIPE, Popen
-from typing import Iterator
 
 from .common import get_last_systemd_log, get_parser, get_stdin
 

dron/notify/ntfy_common.py

@@ -9,7 +9,8 @@ import sys
 from typing import NoReturn
 
 
-def run_ntfy(*, job: str, backend: str) -> NoReturn:
+# ty doesn't support NoReturn yet, see https://github.com/astral-sh/ty/issues/180
+def run_ntfy(*, job: str, backend: str) -> NoReturn:  # ty: ignore[invalid-return-type]
     # TODO not sure what to do with --stdin arg here?
     # could probably use last N lines of log or something
     # TODO get last logs here?
@@ -18,7 +19,7 @@ def run_ntfy(*, job: str, backend: str) -> NoReturn:
     try:
         subprocess.check_call(['ntfy', '-b', backend, '-t', title, 'send', body])
     except Exception as e:
-        logging.exception(e)
+        logging.exception(e)  # noqa: LOG015
         # TODO fallback on email?
         sys.exit(1)
     sys.exit(0)

dron/notify/telegram.py

@@ -32,7 +32,7 @@ def main() -> None:
     try:
         send(message=body)
     except Exception as e:
-        logging.exception(e)
+        logging.exception(e)  # noqa: LOG015
         # TODO fallback on email?
         sys.exit(1)
     sys.exit(0)