vsjetengine 1.0.0__py3-none-any.whl

vsengine/__init__.py

@@ -0,0 +1,28 @@
+# vs-engine
+# Copyright (C) 2022  cid-chan
+# Copyright (C) 2025  Jaded-Encoding-Thaumaturgy
+# This project is licensed under the EUPL-1.2
+# SPDX-License-Identifier: EUPL-1.2
+"""
+vsengine - A common set of function that bridge vapoursynth with your application.
+
+Parts:
+- loops:   Integrate vsengine with your event-loop (be it GUI-based or IO-based).
+- policy:  Create new isolated cores as needed.
+- video:   Get frames or render the video. Sans-IO and memory safe.
+- vpy:     Run .vpy-scripts in your application.
+"""
+
+from vsengine.loops import *
+from vsengine.policy import *
+from vsengine.video import *
+from vsengine.vpy import *
+
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+
+try:
+    from ._version import __version__, __version_tuple__
+except ImportError:
+    __version__ = "0.0.0+unknown"
+    __version_tuple__ = (0, 0, 0, "+unknown")

vsengine/_futures.py

@@ -0,0 +1,372 @@
+# vs-engine
+# Copyright (C) 2022  cid-chan
+# Copyright (C) 2025  Jaded-Encoding-Thaumaturgy
+# This project is licensed under the EUPL-1.2
+# SPDX-License-Identifier: EUPL-1.2
+from __future__ import annotations
+
+from collections.abc import AsyncIterator, Awaitable, Callable, Generator, Iterator
+from concurrent.futures import Future
+from contextlib import AbstractAsyncContextManager, AbstractContextManager
+from functools import wraps
+from inspect import isgeneratorfunction
+from types import TracebackType
+from typing import Any, Literal, Self, overload
+
+from vsengine.loops import Cancelled, get_loop, keep_environment
+
+
+class UnifiedFuture[T](Future[T], AbstractContextManager[T, Any], AbstractAsyncContextManager[T, Any], Awaitable[T]):
+    @classmethod
+    def from_call[**P](cls, func: Callable[P, Future[T]], *args: P.args, **kwargs: P.kwargs) -> Self:
+        try:
+            future = func(*args, **kwargs)
+        except Exception as e:
+            return cls.reject(e)
+
+        return cls.from_future(future)
+
+    @classmethod
+    def from_future(cls, future: Future[T]) -> Self:
+        if isinstance(future, cls):
+            return future
+
+        result = cls()
+
+        def _receive(fn: Future[T]) -> None:
+            if (exc := future.exception()) is not None:
+                result.set_exception(exc)
+            else:
+                result.set_result(future.result())
+
+        future.add_done_callback(_receive)
+        return result
+
+    @classmethod
+    def resolve(cls, value: T) -> Self:
+        future = cls()
+        future.set_result(value)
+        return future
+
+    @classmethod
+    def reject(cls, error: BaseException) -> Self:
+        future = cls()
+        future.set_exception(error)
+        return future
+
+    # Adding callbacks
+    def add_done_callback(self, fn: Callable[[Future[T]], Any]) -> None:
+        # The done_callback should inherit the environment of the current call.
+        super().add_done_callback(keep_environment(fn))
+
+    def add_loop_callback(self, func: Callable[[Future[T]], None]) -> None:
+        def _wrapper(future: Future[T]) -> None:
+            get_loop().from_thread(func, future)
+
+        self.add_done_callback(_wrapper)
+
+    # Manipulating futures
+    @overload
+    def then[V](self, success_cb: Callable[[T], V], err_cb: None) -> UnifiedFuture[V]: ...
+    @overload
+    def then[V](self, success_cb: None, err_cb: Callable[[BaseException], V]) -> UnifiedFuture[T | V]: ...
+    def then[V](
+        self, success_cb: Callable[[T], V] | None, err_cb: Callable[[BaseException], V] | None
+    ) -> UnifiedFuture[V] | UnifiedFuture[T | V]:
+        result = UnifiedFuture[T | V]()
+
+        def _run_cb(cb: Callable[[Any], V], v: Any) -> None:
+            try:
+                r = cb(v)
+            except BaseException as e:
+                result.set_exception(e)
+            else:
+                result.set_result(r)
+
+        def _done(fn: Future[T]) -> None:
+            if (exc := self.exception()) is not None:
+                if err_cb is not None:
+                    _run_cb(err_cb, exc)
+                else:
+                    result.set_exception(exc)
+            else:
+                if success_cb is not None:
+                    _run_cb(success_cb, self.result())
+                else:
+                    result.set_result(self.result())
+
+        self.add_done_callback(_done)
+        return result
+
+    def map[V](self, cb: Callable[[T], V]) -> UnifiedFuture[V]:
+        return self.then(cb, None)
+
+    def catch[V](self, cb: Callable[[BaseException], V]) -> UnifiedFuture[T | V]:
+        return self.then(None, cb)
+
+    # Nicer Syntax
+    def __enter__(self) -> T:
+        obj = self.result()
+
+        if isinstance(obj, AbstractContextManager):
+            return obj.__enter__()
+
+        raise NotImplementedError("(async) with is not implemented for this object")
+
+    def __exit__(self, exc: type[BaseException] | None, val: BaseException | None, tb: TracebackType | None) -> None:
+        obj = self.result()
+
+        if isinstance(obj, AbstractContextManager):
+            return obj.__exit__(exc, val, tb)
+
+        raise NotImplementedError("(async) with is not implemented for this object")
+
+    async def awaitable(self) -> T:
+        return await get_loop().await_future(self)
+
+    def __await__(self) -> Generator[Any, None, T]:
+        return self.awaitable().__await__()
+
+    async def __aenter__(self) -> T:
+        result = await self.awaitable()
+
+        if isinstance(result, AbstractAsyncContextManager):
+            return await result.__aenter__()
+        if isinstance(result, AbstractContextManager):
+            return result.__enter__()
+
+        raise NotImplementedError("(async) with is not implemented for this object")
+
+    async def __aexit__(
+        self, exc: type[BaseException] | None, val: BaseException | None, tb: TracebackType | None
+    ) -> None:
+        result = await self.awaitable()
+
+        if isinstance(result, AbstractAsyncContextManager):
+            return await result.__aexit__(exc, val, tb)
+        if isinstance(result, AbstractContextManager):
+            return result.__exit__(exc, val, tb)
+
+        raise NotImplementedError("(async) with is not implemented for this object")
+
+
+class UnifiedIterator[T](Iterator[T], AsyncIterator[T]):
+    def __init__(self, future_iterable: Iterator[Future[T]]) -> None:
+        self.future_iterable = future_iterable
+
+    @classmethod
+    def from_call[**P](cls, func: Callable[P, Iterator[Future[T]]], *args: P.args, **kwargs: P.kwargs) -> Self:
+        return cls(func(*args, **kwargs))
+
+    @property
+    def futures(self) -> Iterator[Future[T]]:
+        return self.future_iterable
+
+    def run_as_completed(self, callback: Callable[[Future[T]], Any]) -> UnifiedFuture[None]:
+        state = UnifiedFuture[None]()
+
+        def _is_done_or_cancelled() -> bool:
+            if state.done():
+                return True
+            if state.cancelled():
+                state.set_exception(Cancelled())
+                return True
+            return False
+
+        def _get_next_future() -> Future[T] | None:
+            if _is_done_or_cancelled():
+                return None
+
+            try:
+                next_future = self.future_iterable.__next__()
+            except StopIteration:
+                state.set_result(None)
+                return None
+            except BaseException as e:
+                state.set_exception(e)
+                return None
+            return next_future
+
+        def _run_callbacks() -> None:
+            try:
+                while (future := _get_next_future()) is not None:
+                    # Wait for the future to finish.
+                    if not future.done():
+                        future.add_done_callback(_continuation_in_foreign_thread)
+                        return
+
+                    # Run the callback.
+                    if not _run_single_callback(future):
+                        return
+
+                    # Try to give control back to the event loop.
+                    next_cycle = get_loop().next_cycle()
+                    if not next_cycle.done():
+                        next_cycle.add_done_callback(_continuation_from_next_cycle)
+                        return
+
+                    # We do not have a real event loop here.
+                    # If the next_cycle causes an error to bubble, forward it to the state future.
+                    if next_cycle.exception() is not None:
+                        state.set_exception(next_cycle.exception())
+                        return
+            except Exception as e:
+                import traceback
+
+                traceback.print_exception(e)
+                state.set_exception(e)
+
+        def _continuation_from_next_cycle(fut: Future[None]) -> None:
+            if fut.exception() is not None:
+                state.set_exception(fut.exception())
+            else:
+                _run_callbacks()
+
+        def _continuation_in_foreign_thread(fut: Future[T]) -> None:
+            # Optimization, see below.
+            get_loop().from_thread(_continuation, fut)
+
+        def _continuation(fut: Future[T]) -> None:
+            if _run_single_callback(fut):
+                _run_callbacks()
+
+        @keep_environment
+        def _run_single_callback(fut: Future[T]) -> bool:
+            # True   => Schedule next future.
+            # False  => Cancel the loop.
+            if _is_done_or_cancelled():
+                return False
+
+            try:
+                result = callback(fut)
+            except BaseException as e:
+                state.set_exception(e)
+                return False
+            else:
+                if result is None or bool(result):
+                    return True
+                else:
+                    state.set_result(None)
+                    return False
+
+        # Optimization:
+        # We do not need to inherit any kind of environment as
+        # _run_single_callback will automatically set the environment for us.
+        get_loop().from_thread(_run_callbacks)
+        return state
+
+    def __iter__(self) -> Self:
+        return self
+
+    def __next__(self) -> T:
+        fut = self.future_iterable.__next__()
+        return fut.result()
+
+    def __aiter__(self) -> Self:
+        return self
+
+    async def __anext__(self) -> T:
+        try:
+            fut = self.future_iterable.__next__()
+        except StopIteration:
+            raise StopAsyncIteration
+        return await get_loop().await_future(fut)
+
+
+@overload
+def unified[T, **P](
+    *,
+    kind: Literal["generator"],
+) -> Callable[
+    [Callable[P, Iterator[Future[T]]]],
+    Callable[P, UnifiedIterator[T]],
+]: ...
+
+
+@overload
+def unified[T, **P](
+    *,
+    kind: Literal["future"],
+) -> Callable[
+    [Callable[P, Future[T]]],
+    Callable[P, UnifiedFuture[T]],
+]: ...
+
+
+@overload
+def unified[T, **P](
+    *,
+    kind: Literal["generator"],
+    iterable_class: type[UnifiedIterator[T]],
+) -> Callable[
+    [Callable[P, Iterator[Future[T]]]],
+    Callable[P, UnifiedIterator[T]],
+]: ...
+
+
+@overload
+def unified[T, **P](
+    *,
+    kind: Literal["future"],
+    future_class: type[UnifiedFuture[T]],
+) -> Callable[
+    [Callable[P, Future[T]]],
+    Callable[P, UnifiedFuture[T]],
+]: ...
+
+
+@overload
+def unified[T, **P](
+    *,
+    kind: Literal["auto"] = "auto",
+    iterable_class: type[UnifiedIterator[Any]] = ...,
+    future_class: type[UnifiedFuture[Any]] = ...,
+) -> Callable[
+    [Callable[P, Future[T] | Iterator[Future[T]]]],
+    Callable[P, UnifiedFuture[T] | UnifiedIterator[T]],
+]: ...
+
+
+# Implementation
+def unified[T, **P](
+    *,
+    kind: str = "auto",
+    iterable_class: type[UnifiedIterator[Any]] = UnifiedIterator[Any],
+    future_class: type[UnifiedFuture[Any]] = UnifiedFuture[Any],
+) -> Any:
+    """
+    Decorator to normalize functions returning Future[T] or Iterator[Future[T]]
+    into functions returning UnifiedFuture[T] or UnifiedIterator[T].
+    """
+
+    def _decorator_generator(func: Callable[P, Iterator[Future[T]]]) -> Callable[P, UnifiedIterator[T]]:
+        @wraps(func)
+        def _wrapped(*args: P.args, **kwargs: P.kwargs) -> UnifiedIterator[T]:
+            return iterable_class.from_call(func, *args, **kwargs)
+
+        return _wrapped
+
+    def _decorator_future(func: Callable[P, Future[T]]) -> Callable[P, UnifiedFuture[T]]:
+        @wraps(func)
+        def _wrapped(*args: P.args, **kwargs: P.kwargs) -> UnifiedFuture[T]:
+            return future_class.from_call(func, *args, **kwargs)
+
+        return _wrapped
+
+    def decorator(
+        func: Callable[P, Iterator[Future[T]]] | Callable[P, Future[T]],
+    ) -> Callable[P, UnifiedIterator[T]] | Callable[P, UnifiedFuture[T]]:
+        if kind == "auto":
+            if isgeneratorfunction(func):
+                return _decorator_generator(func)
+            return _decorator_future(func)  # type:ignore[arg-type]
+
+        if kind == "generator":
+            return _decorator_generator(func)  # type:ignore[arg-type]
+
+        if kind == "future":
+            return _decorator_future(func)  # type:ignore[arg-type]
+
+        raise NotImplementedError
+
+    return decorator

vsengine/_helpers.py

@@ -0,0 +1,34 @@
+# vs-engine
+# Copyright (C) 2022  cid-chan
+# Copyright (C) 2025  Jaded-Encoding-Thaumaturgy
+# This project is licensed under the EUPL-1.2
+# SPDX-License-Identifier: EUPL-1.2
+import contextlib
+from collections.abc import Iterator
+
+import vapoursynth as vs
+
+from vsengine.policy import ManagedEnvironment
+
+
+# Automatically set the environment within that block.
+@contextlib.contextmanager
+def use_inline(function_name: str, env: vs.Environment | ManagedEnvironment | None) -> Iterator[None]:
+    if env is None:
+        # Ensure there is actually an environment set in this block.
+        try:
+            vs.get_current_environment()
+        except Exception as e:
+            raise OSError(
+                f"You are currently not running within an environment. "
+                f"Pass the environment directly to {function_name}."
+            ) from e
+        yield
+
+    elif isinstance(env, ManagedEnvironment):
+        with env.inline_section():
+            yield
+
+    else:
+        with env.use():
+            yield

vsengine/_hospice.py

@@ -0,0 +1,121 @@
+# vs-engine
+# Copyright (C) 2022  cid-chan
+# Copyright (C) 2025  Jaded-Encoding-Thaumaturgy
+# This project is licensed under the EUPL-1.2
+# SPDX-License-Identifier: EUPL-1.2
+import gc
+import logging
+import sys
+import threading
+import weakref
+from typing import Literal
+
+from vapoursynth import Core, EnvironmentData
+
+logger = logging.getLogger(__name__)
+
+
+lock = threading.Lock()
+refctr = 0
+refnanny = dict[int, weakref.ReferenceType[EnvironmentData]]()
+cores = dict[int, Core]()
+
+stage2_to_add = set[int]()
+stage2 = set[int]()
+stage1 = set[int]()
+
+hold = set[int]()
+
+
+def admit_environment(environment: EnvironmentData, core: Core) -> None:
+    global refctr
+
+    with lock:
+        ident = refctr
+        refctr += 1
+
+    ref = weakref.ref(environment, lambda _: _add_tostage1(ident))
+    cores[ident] = core
+    refnanny[ident] = ref
+
+    logger.debug("Admitted environment %r and %r as with ID:%s.", environment, core, ident)
+
+
+def any_alive() -> bool:
+    if bool(stage1) or bool(stage2) or bool(stage2_to_add):
+        gc.collect()
+    if bool(stage1) or bool(stage2) or bool(stage2_to_add):
+        gc.collect()
+    if bool(stage1) or bool(stage2) or bool(stage2_to_add):
+        gc.collect()
+    return bool(stage1) or bool(stage2) or bool(stage2_to_add)
+
+
+def freeze() -> None:
+    logger.debug("Freezing the hospice. Cores won't be collected anyore.")
+
+    hold.update(stage1)
+    hold.update(stage2)
+    hold.update(stage2_to_add)
+    stage1.clear()
+    stage2.clear()
+    stage2_to_add.clear()
+
+
+def unfreeze() -> None:
+    stage1.update(hold)
+    hold.clear()
+
+
+def _is_core_still_used(ident: int) -> bool:
+    # There has to be the Core, CoreTimings and the temporary reference as an argument to getrefcount
+    # https://docs.python.org/3/library/sys.html#sys.getrefcount
+    return sys.getrefcount(cores[ident]) > 3
+
+
+def _add_tostage1(ident: int) -> None:
+    logger.debug("Environment has died. Keeping core for a few gc-cycles. ID:%s", ident)
+
+    with lock:
+        stage1.add(ident)
+
+
+def _collectstage1(phase: Literal["start", "stop"], _: dict[str, int]) -> None:
+    if phase != "stop":
+        return
+
+    with lock:
+        for ident in tuple(stage1):
+            if _is_core_still_used(ident):
+                logger.warning("Core is still in use. ID:%s", ident)
+                continue
+
+            stage1.remove(ident)
+            stage2_to_add.add(ident)
+
+
+def _collectstage2(phase: Literal["start", "stop"], _: dict[str, int]) -> None:
+    global stage2_to_add
+
+    if phase != "stop":
+        return
+
+    garbage = []
+    with lock:
+        for ident in tuple(stage2):
+            if _is_core_still_used(ident):
+                logger.warning("Core is still in use in stage 2. ID:%s", ident)
+                continue
+
+            stage2.remove(ident)
+            garbage.append(cores.pop(ident))
+            logger.debug("Marking core %r for collection", ident)
+
+        stage2.update(stage2_to_add)
+        stage2_to_add = set()
+
+    garbage.clear()
+
+
+gc.callbacks.append(_collectstage2)
+gc.callbacks.append(_collectstage1)

vsengine/_nodes.py

@@ -0,0 +1,116 @@
+# vs-engine
+# Copyright (C) 2022  cid-chan
+# Copyright (C) 2025  Jaded-Encoding-Thaumaturgy
+# This project is licensed under the EUPL-1.2
+# SPDX-License-Identifier: EUPL-1.2
+from collections.abc import Iterable, Iterator
+from concurrent.futures import Future
+from threading import RLock
+
+from vapoursynth import RawFrame, core
+
+
+def buffer_futures[FrameT: RawFrame](
+    futures: Iterable[Future[FrameT]], prefetch: int = 0, backlog: int | None = None
+) -> Iterator[Future[FrameT]]:
+    if prefetch == 0:
+        prefetch = core.num_threads
+    if backlog is None:
+        backlog = prefetch * 3
+    if backlog < prefetch:
+        backlog = prefetch
+
+    enum_fut = enumerate(futures)
+
+    finished = False
+    running = 0
+    lock = RLock()
+    reorder = dict[int, Future[FrameT]]()
+
+    def _request_next() -> None:
+        nonlocal finished, running
+        with lock:
+            if finished:
+                return
+
+            ni = next(enum_fut, None)
+            if ni is None:
+                finished = True
+                return
+
+            running += 1
+
+            idx, fut = ni
+            reorder[idx] = fut
+            fut.add_done_callback(_finished)
+
+    def _finished(f: Future[FrameT]) -> None:
+        nonlocal finished, running
+        with lock:
+            running -= 1
+            if finished:
+                return
+
+            if f.exception() is not None:
+                finished = True
+                return
+
+            _refill()
+
+    def _refill() -> None:
+        if finished:
+            return
+
+        with lock:
+            # Two rules: 1. Don't exceed the concurrency barrier.
+            #            2. Don't exceed unused-frames-backlog
+            while (not finished) and (running < prefetch) and len(reorder) < backlog:
+                _request_next()
+
+    _refill()
+
+    sidx = 0
+    try:
+        while (not finished) or (len(reorder) > 0) or running > 0:
+            if sidx not in reorder:
+                # Spin. Reorder being empty should never happen.
+                continue
+
+            # Get next requested frame
+            fut = reorder[sidx]
+            del reorder[sidx]
+            sidx += 1
+            _refill()
+
+            yield fut
+
+    finally:
+        finished = True
+
+
+def close_when_needed[FrameT: RawFrame](future_iterable: Iterable[Future[FrameT]]) -> Iterator[Future[FrameT]]:
+    def copy_future_and_run_cb_before(fut: Future[FrameT]) -> Future[FrameT]:
+        f = Future[FrameT]()
+
+        def _as_completed(_: Future[FrameT]) -> None:
+            try:
+                r = fut.result()
+            except Exception as e:
+                f.set_exception(e)
+            else:
+                new_r = r.__enter__()
+                f.set_result(new_r)
+
+        fut.add_done_callback(_as_completed)
+        return f
+
+    def close_fut(f: Future[FrameT]) -> None:
+        def _do_close(_: Future[FrameT]) -> None:
+            if f.exception() is None:
+                f.result().__exit__(None, None, None)
+
+        f.add_done_callback(_do_close)
+
+    for fut in future_iterable:
+        yield copy_future_and_run_cb_before(fut)
+        close_fut(fut)

vsengine/_version.py

@@ -0,0 +1,2 @@
+__version__ = "1.0.0"
+__version_tuple__ = (1, 0, 0)

vsengine/adapters/__init__.py

@@ -0,0 +1,6 @@
+# vs-engine
+# Copyright (C) 2022  cid-chan
+# Copyright (C) 2025  Jaded-Encoding-Thaumaturgy
+# This project is licensed under the EUPL-1.2
+# SPDX-License-Identifier: EUPL-1.2
+