rbx 3.18.4__tar.gz → 3.18.4.dev164__tar.gz

{rbx-3.18.4 → rbx-3.18.4.dev164}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rbx
-Version: 3.18.4
+Version: 3.18.4.dev164
 Summary: A collection of common tools for Scoota services.
 Author-email: The Scoota Engineering Team <engineering@scoota.com>
 License: 
@@ -249,19 +249,12 @@ Requires-Dist: check-manifest; extra == "buildtools"
 Requires-Dist: fabric~=3.2.0; extra == "buildtools"
 Requires-Dist: Jinja2==3.1.2; extra == "buildtools"
 Requires-Dist: twine; extra == "buildtools"
-Provides-Extra: exporter
-Requires-Dist: starlette==0.38.2; extra == "exporter"
 Provides-Extra: notifications
 Requires-Dist: google-cloud-pubsub<3,>=2.9; extra == "notifications"
 Provides-Extra: storage
 Requires-Dist: google-cloud-storage<3,>=2.1; extra == "storage"
 Provides-Extra: tasks
 Requires-Dist: google-cloud-tasks<3,>=2.7; extra == "tasks"
-Provides-Extra: toolkit
-Requires-Dist: boto3; extra == "toolkit"
-Requires-Dist: google-cloud-storage<3,>=2.1; extra == "toolkit"
-Requires-Dist: playwright~=1.51.0; extra == "toolkit"
-Requires-Dist: sh~=2.0.6; extra == "toolkit"
 Provides-Extra: web
 Requires-Dist: google-cloud-error-reporting<2,>=1.9.2; extra == "web"
 Dynamic: license-file

{rbx-3.18.4 → rbx-3.18.4.dev164}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rbx"
-version = "3.18.4"
+version = "3.18.4.dev164"
 description = "A collection of common tools for Scoota services."
 authors = [
     { name = "The Scoota Engineering Team", email = "engineering@scoota.com" }
@@ -59,9 +59,6 @@ buildtools = [
     "Jinja2==3.1.2",
     "twine",
 ]
-exporter = [
-    "starlette==0.38.2",
-]
 notifications = [
     "google-cloud-pubsub>=2.9,<3"
 ]
@@ -71,26 +68,19 @@ storage = [
 tasks = [
     "google-cloud-tasks>=2.7,<3"
 ]
-toolkit = [
-    "boto3",
-    "google-cloud-storage>=2.1,<3",
-    "playwright~=1.51.0",
-    "sh~=2.0.6",
-]
 web = [
     "google-cloud-error-reporting>=1.9.2,<2"
 ]
 
 [project.scripts]
 buildtools = "rbx.buildtools.cli:cli [buildtools]"
-toolkit = "rbx.toolkit.cli:cli [toolkit]"
 
 [project.urls]
 homepage = "https://github.com/rockabox/rbx"
 repository = "https://github.com/rockabox/rbx.git"
 
 [tool.bumpversion]
-current_version = "3.18.4"
+current_version = "3.18.4.dev164"
 commit = true
 parse = """
 	(?P<major>\\d+)\\.

rbx-3.18.4.dev164/rbx/__init__.py

@@ -0,0 +1 @@
+__version__ = "3.18.4.dev164"

{rbx-3.18.4 → rbx-3.18.4.dev164}/rbx.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rbx
-Version: 3.18.4
+Version: 3.18.4.dev164
 Summary: A collection of common tools for Scoota services.
 Author-email: The Scoota Engineering Team <engineering@scoota.com>
 License: 
@@ -249,19 +249,12 @@ Requires-Dist: check-manifest; extra == "buildtools"
 Requires-Dist: fabric~=3.2.0; extra == "buildtools"
 Requires-Dist: Jinja2==3.1.2; extra == "buildtools"
 Requires-Dist: twine; extra == "buildtools"
-Provides-Extra: exporter
-Requires-Dist: starlette==0.38.2; extra == "exporter"
 Provides-Extra: notifications
 Requires-Dist: google-cloud-pubsub<3,>=2.9; extra == "notifications"
 Provides-Extra: storage
 Requires-Dist: google-cloud-storage<3,>=2.1; extra == "storage"
 Provides-Extra: tasks
 Requires-Dist: google-cloud-tasks<3,>=2.7; extra == "tasks"
-Provides-Extra: toolkit
-Requires-Dist: boto3; extra == "toolkit"
-Requires-Dist: google-cloud-storage<3,>=2.1; extra == "toolkit"
-Requires-Dist: playwright~=1.51.0; extra == "toolkit"
-Requires-Dist: sh~=2.0.6; extra == "toolkit"
 Provides-Extra: web
 Requires-Dist: google-cloud-error-reporting<2,>=1.9.2; extra == "web"
 Dynamic: license-file

{rbx-3.18.4 → rbx-3.18.4.dev164}/rbx.egg-info/SOURCES.txt

@@ -35,12 +35,6 @@ rbx/gcp/__init__.py
 rbx/gcp/cloud_tasks.py
 rbx/gcp/pubsub.py
 rbx/gcp/storage.py
-rbx/toolkit/__init__.py
-rbx/toolkit/browser.py
-rbx/toolkit/cli.py
-rbx/toolkit/exporter.py
-rbx/toolkit/media.py
-rbx/toolkit/utils.py
 rbx/utils/__init__.py
 rbx/utils/mdm.py
 rbx/utils/vast.py

{rbx-3.18.4 → rbx-3.18.4.dev164}/rbx.egg-info/entry_points.txt

@@ -1,3 +1,2 @@
 [console_scripts]
 buildtools = rbx.buildtools.cli:cli [buildtools]
-toolkit = rbx.toolkit.cli:cli [toolkit]

{rbx-3.18.4 → rbx-3.18.4.dev164}/rbx.egg-info/requires.txt

@@ -29,9 +29,6 @@ fabric~=3.2.0
 Jinja2==3.1.2
 twine
 
-[exporter]
-starlette==0.38.2
-
 [notifications]
 google-cloud-pubsub<3,>=2.9
 
@@ -41,11 +38,5 @@ google-cloud-storage<3,>=2.1
 [tasks]
 google-cloud-tasks<3,>=2.7
 
-[toolkit]
-boto3
-google-cloud-storage<3,>=2.1
-playwright~=1.51.0
-sh~=2.0.6
-
 [web]
 google-cloud-error-reporting<2,>=1.9.2

rbx-3.18.4/rbx/__init__.py

@@ -1 +0,0 @@
-__version__ = "3.18.4"

rbx-3.18.4/rbx/toolkit/__init__.py

@@ -1,79 +0,0 @@
-__all__ = ["Options", "run"]
-
-import logging
-import tempfile
-from pathlib import Path
-from typing import NamedTuple, Optional
-
-from .browser import record, screenshot
-from .utils import upload
-
-logger = logging.getLogger(__name__)
-
-
-class Options(NamedTuple):
-    url: str
-    width: int
-    height: int
-    format: str
-    duration: Optional[int] = 0
-    path: Optional[str] = "/tmp"
-    output: Optional[str] = "."
-    filename: Optional[str] = None
-
-
-def filename(output, filename):
-    if output.startswith("s3://"):
-        return "s3://" + str(Path(output[5:]) / filename)
-    elif output.startswith("gs://"):
-        return "gs://" + str(Path(output[5:]) / filename)
-    return str(Path(output) / filename)
-
-
-async def capture(filename: str, options: Options, path: Path) -> None:
-    await record(
-        dirname=str(path),
-        filename=str(path / filename),
-        duration=options.duration,
-        height=options.height,
-        url=options.url,
-        width=options.width,
-    )
-
-
-async def screengrab(filename: str, options: Options, path: Path) -> None:
-    await screenshot(
-        filename=str(path / filename),
-        height=options.height,
-        url=options.url,
-        width=options.width,
-    )
-
-
-async def run(options: Options) -> None:
-    with tempfile.TemporaryDirectory(dir=options.path) as dirname:
-        path = Path(dirname)
-        logger.debug(f"Working directory: '{path}'")
-        if options.format == "video":
-            asset = options.filename or "video.mp4"
-            output = filename(options.output, asset)
-            if options.duration:
-                logger.info(
-                    f"Capturing '{options.url}' [{options.width}x{options.height}]"
-                    f" for {options.duration}ms to '{output}'",
-                )
-            else:
-                logger.info(
-                    f"Capturing '{options.url}' [{options.width}x{options.height}] to '{output}'",
-                )
-            await capture(filename=asset, options=options, path=path)
-        else:
-            asset = options.filename or "screenshot.png"
-            output = filename(options.output, asset)
-            logger.info(
-                f"Taking screenshot of '{options.url}' [{options.width}x{options.height}]"
-                f" to '{output}'"
-            )
-            await screengrab(filename=asset, options=options, path=path)
-
-        upload(path / asset, output)

rbx-3.18.4/rbx/toolkit/browser.py

@@ -1,200 +0,0 @@
-import datetime
-import logging
-import os
-import signal
-import subprocess
-import time
-import uuid
-
-from playwright.async_api import async_playwright
-
-logger = logging.getLogger(__name__)
-
-
-def get_free_display(base=99, max_tries=100):
-    for i in range(max_tries):
-        display = f":{base + i}"
-        if not os.path.exists(f"/tmp/.X{base + i}-lock"):
-            return display
-
-    raise RuntimeError("No free display found")
-
-
-class RecordingSession:
-    def __init__(self, dirname, width, height):
-        self.display = get_free_display()
-        self.dirname = dirname
-        self.width = width
-        self.height = height
-        self.resolution = f"{self.width}x{self.height}"
-        self.session_id = str(uuid.uuid4())[:8]
-        self.sink_name = f"rec_sink_{self.session_id}"
-        self.pulse_dir = f"{dirname}/pulseaudio"
-        self.pulse_socket = os.path.join(self.pulse_dir, "pulse-socket")
-        self.proc = []
-
-    async def __aenter__(self):
-        return self
-
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        logger.debug("Stopping processes...")
-
-        # PulseAudio needs to be terminated via the kill command.
-        subprocess.Popen(
-            ["pulseaudio", "--kill"],
-            env={
-                "PULSE_SERVER": f"unix:{self.pulse_socket}",
-                "XDG_RUNTIME_DIR": self.pulse_dir,
-            },
-        )
-
-        for proc in reversed(self.proc):
-            if proc and proc.poll() is None:
-                try:
-                    proc.send_signal(signal.SIGINT)
-                    proc.wait(timeout=3)
-                except subprocess.TimeoutExpired:
-                    proc.kill()
-
-    def start_xvfb(self):
-        logger.debug(f"Starting Xvfb [{self.display}]...")
-        # fmt: off
-        self.proc.append(
-            subprocess.Popen(
-                [
-                    "Xvfb", self.display, "-screen", "0", f"{self.resolution}x24",
-                    # These options are needed when running in a container
-                    "-ac", "-nolisten", "tcp", "-nolisten", "unix",
-                ]
-            )
-        )
-        # fmt: on
-        time.sleep(1)
-
-        logger.debug(f"Starting unclutter [{self.display}]...")
-        self.proc.append(
-            subprocess.Popen(
-                ["unclutter", "-idle", "0"],
-                env={"DISPLAY": self.display},
-            )
-        )
-
-    def start_pulseaudio(self):
-        logger.debug(f"Starting PulseAudio [{self.pulse_dir}.{self.sink_name}]...")
-        os.makedirs(self.pulse_dir, exist_ok=True)
-        os.chmod(self.pulse_dir, 0o700)  # PulseAudio requires strict permissions
-
-        subprocess.Popen(
-            [
-                "pulseaudio",
-                "--start",
-                "--disable-shm",
-                "--exit-idle-time=300",
-                f"--load=module-native-protocol-unix socket={self.pulse_socket}",
-                f"--load=module-null-sink sink_name={self.sink_name}"
-                f" sink_properties=device.description={self.sink_name}",
-            ],
-            env={
-                "DISPLAY": self.display,
-                "PULSE_NO_SIMD": "1",
-                "XDG_RUNTIME_DIR": self.pulse_dir,
-            },
-        )
-        time.sleep(2)
-
-    def start_ffmpeg(self, duration, output_path):
-        logger.debug(f"Starting FFmpeg recording [{self.display}:{self.sink_name}]...")
-        # fmt: off
-        self.proc.append(
-            subprocess.Popen(
-                [
-                    "ffmpeg", "-y",
-                    "-t", str(duration),
-                    "-video_size", self.resolution,
-                    "-f", "x11grab",
-                    "-i", f"{self.display}.0",
-                    "-f", "pulse",
-                    "-i", f"{self.sink_name}.monitor",
-                    "-c:v", "libx264",
-                    "-preset", "slow",
-                    "-crf", "8",
-                    "-pix_fmt", "yuv420p",
-                    "-c:a", "aac",
-                    "-b:a", "256k",
-                    "-movflags",
-                    "+faststart",
-                    output_path,
-                ],
-                stdout=subprocess.DEVNULL,
-                stderr=subprocess.STDOUT,
-                env={
-                    "PULSE_SERVER": f"unix:{self.pulse_socket}",
-                    "XDG_RUNTIME_DIR": self.pulse_dir,
-                },
-            )
-        )
-        # fmt: on
-
-    async def capture(self, url, duration, output_path):
-        self.start_xvfb()
-        self.start_pulseaudio()
-
-        async with async_playwright() as p:
-            logger.debug(
-                f"Capturing Chromium session [{self.display}:{self.sink_name}]..."
-            )
-            context = await p.chromium.launch_persistent_context(
-                user_data_dir=f"{self.dirname}/user-data",
-                headless=False,
-                channel="chrome",
-                args=[
-                    "--no-sandbox",
-                    "--disable-gpu",
-                    "--disable-software-rasterizer",
-                    f"--display={self.display}",
-                    "--autoplay-policy=no-user-gesture-required",
-                    "--use-fake-ui-for-media-stream",
-                    f"--alsa-output-device={self.sink_name}",
-                    "--start-fullscreen",
-                    "--window-position=0,0",
-                    f"--window-size={self.width},{self.height}",
-                ],
-                ignore_default_args=["--enable-automation"],
-                no_viewport=True,
-                env={
-                    "PULSE_SERVER": f"unix:{self.pulse_socket}",
-                    "XDG_RUNTIME_DIR": self.pulse_dir,
-                },
-            )
-            page = await context.new_page()
-            await page.goto(url, wait_until="commit")
-
-            logger.info(
-                f"Recording session [{self.display}:{self.sink_name}] for {duration}ms"
-            )
-
-            self.start_ffmpeg(duration, output_path)
-
-            await page.wait_for_timeout(duration)  # Wait for the duration
-            await context.close()
-
-
-async def record(
-    dirname: str, filename: str, duration: int, height: int, url: str, width: int
-) -> None:
-    async with RecordingSession(dirname, width, height) as session:
-        await session.capture(url, duration, filename)
-
-
-async def screenshot(filename: str, height: int, url: str, width: int) -> None:
-    async with async_playwright() as p:
-        browser = await p.chromium.launch()
-        page = await browser.new_page()
-        await page.set_viewport_size({"width": width, "height": height})
-        await page.goto(url, wait_until="networkidle")
-        await page.screenshot(path=filename)
-        await browser.close()
-
-
-def to_ms(timedelta: datetime.timedelta) -> int:
-    return round(1000 * timedelta.total_seconds())

rbx-3.18.4/rbx/toolkit/cli.py

@@ -1,108 +0,0 @@
-import asyncio
-import datetime
-import logging
-import logging.config
-import os
-
-import click
-
-from . import Options, run
-
-
-class ColourFormatter(logging.Formatter):
-    def format(self, record):
-        message = super().format(record)
-        colours = {
-            logging.INFO: {"bold": True},
-            logging.WARNING: {"fg": "yellow"},
-            logging.ERROR: {"fg": "bright_red"},
-            logging.CRITICAL: {"fg": "bright_white", "bg": "red"},
-        }
-        try:
-            message = click.style(message, **colours[record.levelno])
-        except KeyError:
-            pass
-        return message
-
-
-@click.group()
-@click.version_option(message="%(version)s")
-def cli():
-    """Toolkit Creative Converter."""
-
-
-def validate_filename(ctx, param, value):
-    if value is None:
-        return
-
-    if ctx.params["format"] == "image" and not value.endswith(".png"):
-        raise click.BadParameter(f"{param.name} must use the '.png' extension")
-    elif ctx.params["format"] == "video" and not value.endswith(".mp4"):
-        raise click.BadParameter(f"{param.name} must use the '.mp4' extension")
-
-    return value
-
-
-@cli.command(context_settings={"show_default": True})
-@click.argument("url")
-@click.option("--width", "-w", type=click.INT, default=1080)
-@click.option("--height", "-h", type=click.INT, default=1920)
-@click.option(
-    "--format",
-    type=click.Choice(["image", "video"], case_sensitive=False),
-    default="image",
-)
-@click.option("--duration", "-d", type=click.INT, default=0)
-@click.option("--path", "-p", default="/tmp")
-@click.option("--output-dir", "-o", type=click.Path(file_okay=False), default=".")
-@click.option("--filename", "-f", callback=validate_filename)
-def export(url, width, height, format, duration, path, output_dir, filename):
-    log_level = os.getenv("LOG_LEVEL", "INFO")
-    logging.config.dictConfig(
-        {
-            "version": 1,
-            "formatters": {
-                "coloured": {
-                    "class": "rbx.toolkit.cli.ColourFormatter",
-                    "format": "[%(asctime)s] %(message)s",
-                    "datefmt": "%Y-%m-%d %H:%M:%S",
-                },
-            },
-            "handlers": {
-                "console": {
-                    "level": "DEBUG",
-                    "class": "logging.StreamHandler",
-                    "formatter": "coloured",
-                },
-            },
-            "loggers": {
-                "rbx.toolkit": {
-                    "level": log_level,
-                    "handlers": ["console"],
-                    "propagate": False,
-                },
-            },
-        }
-    )
-
-    start_timestamp = datetime.datetime.now()
-
-    try:
-        asyncio.run(
-            run(
-                options=Options(
-                    url=url,
-                    width=width,
-                    height=height,
-                    format=format,
-                    duration=duration,
-                    path=path,
-                    output=output_dir,
-                    filename=filename,
-                )
-            )
-        )
-    except RuntimeError as e:
-        click.secho(e, fg="red", bold=True)
-
-    click.secho(f"Finished in {datetime.datetime.now() - start_timestamp}", bold=True)

rbx-3.18.4/rbx/toolkit/exporter.py

@@ -1,53 +0,0 @@
-import logging
-
-from starlette.applications import Starlette
-from starlette.config import Config
-from starlette.responses import JSONResponse
-from starlette.routing import Route
-
-from . import Options, run
-
-config = Config()
-logging.basicConfig(level=config("LOG_LEVEL", default="INFO"))
-
-
-async def handler(request):
-    payload = await request.json()
-
-    errors = []
-    for param in ("url", "width", "height", "format", "name"):
-        if param not in payload.keys():
-            errors.append(f"Missing required '{param}' parameter")
-
-    if payload.get("format") == "video" and "duration" not in payload.keys():
-        errors.append("Missing required 'duration' parameter")
-
-    if errors:
-        return JSONResponse({"errors": errors}, status_code=400)
-
-    ext = "mp4" if payload["format"] == "video" else "png"
-    filename = f"{payload['name']}.{ext}"
-    project_id = config("GOOGLE_CLOUD_PROJECT", default="dev-platform-eu")
-    output = f"gs://{project_id}.appspot.com/toolkit/exports/"
-
-    await run(
-        options=Options(
-            url=payload["url"],
-            width=payload["width"],
-            height=payload["height"],
-            format=payload["format"],
-            duration=int(payload.get("duration", 0)),
-            output=output,
-            filename=filename,
-        )
-    )
-
-    return JSONResponse({"path": f"{output}{filename}"})
-
-
-def create_app() -> Starlette:
-    return Starlette(
-        routes=[
-            Route("/", endpoint=handler, methods=["POST"]),
-        ]
-    )

rbx-3.18.4/rbx/toolkit/media.py

@@ -1,45 +0,0 @@
-from typing import Optional
-
-import sh
-
-
-def convert(
-    infile: str,
-    outfile: str,
-    delay: Optional[int] = 0,
-    duration: Optional[int] = 0,
-    opts: Optional[list] = None,
-):
-    """Convert `infile` to `outfile`.
-
-    For WebM to MP4 conversion, see
-        https://blog.addpipe.com/converting-webm-to-mp4-with-ffmpeg/
-
-    """
-    args = ["-y"]
-    args.extend(["-i", infile])
-    if delay:
-        args.extend(["-ss", milliseconds_to_duration(delay)])
-    if duration:
-        args.extend(["-to", milliseconds_to_duration(delay + duration)])
-    if delay or duration:
-        args.extend(["-c:v", "libx264", "-preset", "slow", "-crf", "8"])
-    if opts:
-        args.extend(opts)
-    args.append(outfile)
-    ffmpg(args)
-
-
-def ffmpg(args):
-    try:
-        sh.ffmpeg(*args)
-    except sh.ErrorReturnCode as e:
-        raise RuntimeError(
-            f"Command {e.full_cmd} exited with {e.exit_code}\n\n{e.stderr.decode()}"
-        )
-
-
-def milliseconds_to_duration(milliseconds):
-    minutes, rem = divmod(milliseconds / 1000.0, 60)
-    seconds, ms = divmod(1000 * rem, 1000)
-    return f"00:{int(minutes):02d}:{int(seconds):02d}.{int(ms)}"

rbx-3.18.4/rbx/toolkit/utils.py

@@ -1,72 +0,0 @@
-import logging
-import os
-import shutil
-from pathlib import Path
-
-import boto3
-from botocore.exceptions import ClientError
-from google.cloud import storage
-from google.cloud.exceptions import GoogleCloudError
-
-
-def upload(filename: Path, target: str):
-    if target.startswith(("gs://", "s3://")):
-        service, _, target = str(target).partition("://")
-        parts = target.split("/")
-        bucket = parts.pop(0)
-        object_name = "/".join(parts)
-        if service == "s3":
-            upload_to_s3(filename, bucket, object_name)
-        elif service == "gs":
-            upload_to_storage(filename, bucket, object_name)
-        else:
-            logging.error(f"Unknown upload service '{service}'")
-
-    else:
-        shutil.copy(filename, target)
-
-
-def upload_to_s3(filename, bucket, object_name=None):
-    """Upload a file to an S3 bucket
-
-    :param file_name: File to upload
-    :param bucket: Bucket to upload to
-    :param object_name: S3 object name. If not specified then file_name is used
-    :return: True if file was uploaded, else False
-    """
-    if object_name is None:
-        object_name = os.path.basename(filename)
-
-    s3_client = boto3.client("s3")
-
-    try:
-        s3_client.upload_file(filename, bucket, object_name)
-    except ClientError as e:
-        logging.error(e)
-        return False
-
-    return True
-
-
-def upload_to_storage(filename, bucket, object_name=None):
-    """Upload a file to a Cloud Storage bucket
-
-    :param file_name: File to upload
-    :param bucket: Bucket to upload to
-    :param object_name: object name. If not specified then file_name is used
-    :return: True if file was uploaded, else False
-    """
-    if object_name is None:
-        object_name = os.path.basename(filename)
-
-    client = storage.Client()
-    bucket = client.bucket(bucket)
-    blob = bucket.blob(object_name)
-
-    try:
-        blob.upload_from_filename(filename)
-    except GoogleCloudError as e:
-        logging.error(e)
-        return False
-
-    return True