opencue-rqd 1.11.66__tar.gz

opencue_rqd-1.11.66/.gitignore

@@ -0,0 +1,3 @@
+/build/
+/dist/
+/rqd.egg-info/

opencue_rqd-1.11.66/Dockerfile

@@ -0,0 +1,43 @@
+FROM rockylinux:8.9
+
+WORKDIR /opt/opencue
+
+# Paths to the pre-built python packages.
+# They are all defined her but no necessary used to make it simpler between the images
+ARG OPENCUE_PROTO_PACKAGE_PATH=/tmp/invalidpath
+ARG OPENCUE_RQD_PACKAGE_PATH=/tmp/invalidpath
+ARG OPENCUE_PYCUE_PACKAGE_PATH=/tmp/invalidpath
+ARG OPENCUE_PYOUTLINE_PACKAGE_PATH=/tmp/invalidpath
+ARG OPENCUE_CUESUBMIT_PACKAGE_PATH=/tmp/invalidpath
+ARG OPENCUE_CUEGUI_PACKAGE_PATH=/tmp/invalidpath
+ARG OPENCUE_CUEADMIN_PACKAGE_PATH=/tmp/invalidpath
+
+RUN yum -y install \
+  epel-release \
+  gcc \
+  time
+
+RUN dnf install -y \
+  python39 \
+  python39-devel \
+  python39-pip
+
+RUN python3.9 -m pip install --upgrade pip
+
+COPY LICENSE ./
+
+RUN mkdir /tmp/packages
+COPY ${OPENCUE_PROTO_PACKAGE_PATH} /tmp/packages/
+COPY ${OPENCUE_RQD_PACKAGE_PATH} /tmp/packages/
+
+RUN python3.9 -m pip install /tmp/packages/*.whl
+
+RUN mkdir -p /etc/opencue
+RUN echo "[Override]" > /etc/opencue/rqd.conf
+RUN echo "USE_NIMBY_PYNPUT=false" >> /etc/opencue/rqd.conf
+
+# RQD gRPC server
+EXPOSE 8444
+
+# NOTE: This shell out is needed to avoid RQD getting PID 0 which leads to leaking child processes.
+ENTRYPOINT ["/bin/bash", "-c", "set -e && rqd"]

opencue_rqd-1.11.66/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.1
+Name: opencue_rqd
+Version: 1.11.66
+Requires-Dist: future==1.0.0
+Requires-Dist: opencue-proto
+Requires-Dist: psutil==5.9.8
+Requires-Dist: pynput==1.7.6
+Provides-Extra: test
+Requires-Dist: docker==7.1.0; extra == 'test'
+Requires-Dist: loki-urllib3-client; extra == 'test'
+Requires-Dist: mock==2.0.0; extra == 'test'
+Requires-Dist: pyfakefs==5.2.3; extra == 'test'
+Requires-Dist: pytest; extra == 'test'

opencue_rqd-1.11.66/README.md

@@ -0,0 +1,59 @@
+# RQD
+
+RQD is a software client that runs on all hosts doing work for an OpenCue
+deployment.
+
+RQD's responsibilities include:
+
+- Registering the host with Cuebot.
+- Receiving instructions about what work to do.
+- Monitoring the worker processes it launches and reporting on results.
+
+RQD uses [gRPC](https://grpc.io/) to communicate with Cuebot. It also runs its
+own gRPC server, which is called by the Cuebot client to send instructions to
+RQD.
+
+## How to start rqd from source
+
+### Setup python environment
+```bash
+# Create virtual environment
+python3 -m venv OpenCue-venv
+# Activate virtual environment
+source OpenCue-venv/bin/activate
+# Change directory to Opencue source
+cd <OpenCueSourceDir>
+
+# Install compiled proto files which is needed by RQD into the virtual environment
+pip install ./proto
+
+# Install RQD into the virtual environment
+pip install ./rqd
+```
+
+### Create rqd.conf
+Example `rqd.conf` file :
+
+```ini
+[Override]
+# Variable that decided if RQD should switch to the user of the job running. Requires root
+RQD_BECOME_JOB_USER = False
+
+# Log levels for RQD
+CONSOLE_LOG_LEVEL = INFO
+FILE_LOG_LEVEL = ERROR
+
+# Number of seconds to wait before checking if the user has become idle.
+CHECK_INTERVAL_LOCKED = 60
+# Seconds of idle time required before NIMBY (Not In My BackYard) unlocks.
+MINIMUM_IDLE = 900
+
+# Whether or not to prefix each line in the log with a timestamp
+RQD_PREPEND_TIMESTAMP = 0
+```
+
+### Run rqd
+(using the above virtual environment)
+```bash
+rqd -c <path to rqd.conf>
+```

opencue_rqd-1.11.66/deploy/opencue-rqd.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=OpenCue RQD Service
+Wants=network.target
+After=network.target
+
+[Service]
+Environment=OPTIONS=
+Environment=BIN=/usr/bin
+EnvironmentFile=-/etc/sysconfig/opencue-rqd
+ExecStart=/bin/bash -c "${BIN}/rqd ${OPTIONS}"
+ExecStop=/bin/bash -c "${BIN}/rqd --exit_now
+LimitNOFILE=500000
+LimitNPROC=500000
+StandardOutput=syslog+console
+StandardError=syslog+console
+SyslogIdentifier=rqd
+
+[Install]
+WantedBy=multi-user.target

opencue_rqd-1.11.66/deploy/rqd3_init.d

@@ -0,0 +1,53 @@
+#!/bin/sh
+#
+# RQD3:    Start/stop rqd3 services
+#
+# chkconfig:    345 98 02
+# description:  Opencue RQD agent
+#
+
+# Source function library.
+. /etc/rc.d/init.d/functions
+
+IS_ON=/sbin/chkconfig
+
+RQD_PATH=/usr/local/spi/rqd3/
+
+RQD=${RQD_PATH}rqd.py
+
+start()
+{
+    [ -f /usr/local/etc/sweatbox.csh ] && echo "Refusing to start RQD3 on a sweatbox" && exit 0
+    echo -n $"Starting openrqd services:"
+    cd ${RQD_PATH}
+    daemon "${RQD}" -d
+    echo ""
+}
+
+stop()
+{
+    echo -n "Stopping openrqd services:"
+    cd ${RQD_PATH}
+    daemon "rqd/cuerqd.py" --exit_now
+    echo "Stop Request completed"
+}
+
+case "$1" in
+  start)
+        start
+        ;;
+  stop)
+        stop
+        ;;
+  restart)
+        stop
+        sleep 3
+        start
+        ;;
+  *)
+        echo $"Usage: $0 {start|stop|restart|idle_restart}"
+        exit 1
+        ;;
+esac
+
+exit 0

opencue_rqd-1.11.66/pyproject.toml

@@ -0,0 +1,43 @@
+[build-system]
+requires = [
+    "hatchling"
+]
+build-backend = "hatchling.build"
+
+[project]
+name = "opencue_rqd"
+dynamic = ["version"]
+dependencies = [
+    "opencue_proto",
+    "psutil==5.9.8",
+    "pynput==1.7.6",
+    "future==1.0.0"
+]
+
+[tool.hatch.version]
+source = "code"
+path = "../ci/generate_version_number.py"
+default-version = "0.0.0"
+
+[tool.hatch.build.targets.wheel]
+packages = ["rqd"]
+
+[project.scripts]
+rqd = "rqd.__main__:main"
+
+# --- Pytest configuration ---
+[tool.pytest.ini_options]
+minversion = "6.0" # Set to required pytest version
+testpaths = ["tests", "pytests"] # Relative path(s) where tests are located
+python_files = ["test_*.py", "*_test.py"] # Default test file pattern
+python_functions = ["test_*"] # Default test function pattern
+
+# --- Optional Test Dependencies ---
+[project.optional-dependencies]
+test = [
+    "mock==2.0.0",
+    "pyfakefs==5.2.3",
+    "pytest",
+    "docker==7.1.0",
+    "loki-urllib3-client"
+]

opencue_rqd-1.11.66/pytests/test_rqdlogging.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+#  Copyright Contributors to the OpenCue Project
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
+"""Pytests for rqd.rqdlogging"""
+
+
+import mock
+import pytest
+import opencue_proto.rqd_pb2
+from rqd.rqlogging import LokiLogger
+
+@pytest.fixture
+@mock.patch('opencue_proto.rqd_pb2_grpc.RunningFrameStub')
+@mock.patch('opencue_proto.rqd_pb2_grpc.RqdInterfaceStub')
+@mock.patch('grpc.insecure_channel', new=mock.MagicMock())
+def runFrame(stubMock, frameStubMock):
+    rf = opencue_proto.rqd_pb2.RunFrame()
+    rf.job_id = "SD6F3S72DJ26236KFS"
+    rf.job_name = "edu-trn_job-name"
+    rf.frame_id = "FD1S3I154O646UGSNN"
+    return rf
+
+# pylint: disable=redefined-outer-name
+def test_LokiLogger(runFrame):
+    ll = LokiLogger("http://localhost:3100", runFrame)
+    assert isinstance(ll, LokiLogger)
+
+def test_LokiLogger_invalid_runFrame():
+    rf = None
+
+    with pytest.raises(AttributeError) as excinfo:
+        LokiLogger("http://localhost:3100", rf)
+    assert excinfo.type == AttributeError

opencue_rqd-1.11.66/rqd/__init__.py

@@ -0,0 +1,13 @@
+#  Copyright Contributors to the OpenCue Project
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.

opencue_rqd-1.11.66/rqd/__main__.py

@@ -0,0 +1,245 @@
+#  Copyright Contributors to the OpenCue Project
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
+"""
+Initializes and starts RQD.
+
+- RQD allows the cuebot to launch frames on a remote host.
+- RQD monitors the resources on a machine.
+- Frames can be monitored or killed.
+- Status updates are sent to the cuebot every 60 seconds.
+- Nimby built into RQD allows a desktop to be used as a render machine when
+  not in use.
+- See the rqnetwork module for a description of ICE interfaces.
+
+Optional configuration file:
+----------------------------
+In /etc/opencue/rqd.conf (on Linux) or %LOCALAPPDATA%/OpenCue/rqd.conf (on Windows):
+[Override]
+OVERRIDE_CORES = 2
+OVERRIDE_PROCS = 3
+OVERRIDE_MEMORY = 1000000
+OVERRIDE_CUEBOT = cuebot1 cuebot2 cuebot3
+# True will start nimby, False will keep nimby from starting
+OVERRIDE_NIMBY = False
+# True will check and report gpu memory if cuda capable
+GPU = True
+# True will force 256mb gpu memory
+PLAYBLAST = True
+"""
+
+
+from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import division
+
+import getopt
+import logging
+import logging.handlers
+import os
+import platform
+import sys
+
+import rqd.rqconstants
+import rqd.rqcore
+import rqd.rqutil
+
+
+def reopen_stdout_stderr(log_path):
+    """
+    Flush sys.stdout and sys.stderr and then reopen the underlying
+    file descriptor for both stdout and stderr to the file at
+    LOG_PATH.  LOG_PATH is opened before stdout and stderr are closed,
+    so if LOG_PATH fails to open, stdout and stderr will remain
+    unchanged.
+    """
+    log_fd = os.open(log_path, os.O_CREAT|os.O_WRONLY|os.O_TRUNC, 0o644)
+
+    for f in [sys.stdout, sys.stderr]:
+        fileno = f.fileno()
+        f.flush()
+        os.close(fileno)
+        os.dup2(log_fd, fileno)
+
+    os.close(log_fd)
+
+
+def daemonize(log_path=None, chdir_to_root=True):
+    """
+    Daemonizes the current process. The process will be in a new session
+    with stdin reading from /dev/null and stdout and stderr writing to
+    /dev/null with all other file descriptors closed.
+
+    :type log_path: String
+    :param log_path: If LOG_PATH is provided, then stdout and stderr
+                     write to LOG_PATH. LOG_PATH may be a relative
+                     path to the process' current working directory
+                     before calling daemonize().
+    :type chdir_to_root: bool
+    :param chdir_to_root: If CHDIR_TO_ROOT is True, then the process'
+                          current working directory will be changed
+                          to / after opening the log files; if
+                          CHDIR_TO_ROOT is false, then the current
+                          working directory is not changed.
+    """
+    # pylint: disable=import-outside-toplevel
+    import resource
+
+    if hasattr(os, "devnull"):
+        dev_null = os.devnull
+    else:
+        dev_null = "/dev/null"
+
+    if not log_path:
+        log_path = dev_null
+
+    # pylint: disable=protected-access
+    if os.fork() != 0:
+        # Intentionally not using sys.exit here to avoid raising SystemExit,
+        # cleaning handlers and flushing stdio buffers
+        os._exit(0)
+
+    os.setsid()
+
+    if os.fork() != 0:
+        # Intentionally not using sys.exit here to avoid raising SystemExit,
+        # cleaning handlers and flushing stdio buffers
+        os._exit(0)
+    # pylint: enable=protected-access
+
+    os.close(sys.stdin.fileno())
+    os.open(dev_null, os.O_RDONLY)
+
+    # The log file is opened before chdir'ing so a relative path may
+    # be used and is opened before stdout and stderr are closed so any
+    # errors in opening it will appear on stderr.
+    reopen_stdout_stderr(log_path)
+
+    if chdir_to_root:
+        os.chdir('/')
+
+    max_fd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
+    if max_fd == resource.RLIM_INFINITY:
+        max_fd = 1024
+    for fd in range(3, max_fd+1):
+        try:
+            # Ice.loadSlice() function sometimes open /dev/urandom to
+            # get random numbers. If daemonize() function is called after
+            # Ice.loadSlice(), we should not close /dev/urandom. Or else
+            # the child process will get exceptions when trying to access
+            # /dev/urandom.
+            fn = os.readlink("/proc/%s/%s" % (os.getpid(), fd))
+            if fn != "/dev/urandom":
+                os.close(fd)
+        except OSError:
+            pass
+
+def setupLogging():
+    """Sets up the logging for RQD.
+    Logs to /var/log/messages"""
+    logger = logging.getLogger()
+    logger.setLevel(rqd.rqconstants.CONSOLE_LOG_LEVEL)
+    for handler in logger.handlers:
+        handler.setFormatter(logging.Formatter(rqd.rqconstants.LOG_FORMAT))
+
+    if rqd.rqconstants.FILE_LOG_LEVEL is not None:
+        if platform.system() in ('Linux', 'Darwin'):
+            if platform.system() == 'Linux':
+                syslogAddress = '/dev/log'
+            else:
+                syslogAddress = '/var/run/syslog'
+            if os.path.exists(syslogAddress):
+                logfile = logging.handlers.SysLogHandler(address=syslogAddress)
+            else:
+                logfile = logging.handlers.SysLogHandler()
+        elif platform.system() == 'Windows':
+            logfile = logging.FileHandler(os.path.expandvars('%TEMP%/openrqd.log'))
+        else:
+            logfile = logging.handlers.SysLogHandler()
+        logfile.setLevel(rqd.rqconstants.FILE_LOG_LEVEL)
+        logfile.setFormatter(logging.Formatter(rqd.rqconstants.LOG_FORMAT))
+        logger.addHandler(logfile)
+
+
+def setup_sentry():
+    """Set up sentry if a SENTRY_DSN_PATH is configured"""
+    sentry_dsn_path = rqd.rqconstants.SENTRY_DSN_PATH
+    if sentry_dsn_path is None:
+        return
+
+    # Not importing sentry at the toplevel to avoid an unecessary dependency
+    try:
+        # pylint: disable=import-outside-toplevel
+        import sentry_sdk
+        # pylint: enable=import-outside-toplevel
+        sentry_sdk.init(sentry_dsn_path)
+    except ImportError:
+        logging.warning('Sentry support disabled. SENTRY_DSN_PATH is set but '
+                        'the lib is not available')
+
+
+def usage():
+    """Prints command line syntax"""
+    usage_msg = f"""SYNOPSIS
+  {sys.argv[0]} [options]
+
+  -d | --daemon          => Run as daemon
+       --nimbyoff        => Disables nimby activation
+  -c                     => Provide an alternate config file
+                            On Linux: defaults to /etc/opencue/rqd.conf
+                            On Windows: Defaults to %LOCALAPPDATA%/OpenCue/rqd.conf
+                            Config file is optional
+"""
+    print(usage_msg, file=sys.stderr)
+
+
+def main():
+    """Entrypoint for RQD."""
+    setupLogging()
+    logger = logging.getLogger()
+
+    if platform.system() == 'Linux' and os.getuid() != 0 and \
+       rqd.rqconstants.RQD_BECOME_JOB_USER:
+        logging.critical("Please run launch as root")
+        sys.exit(1)
+
+    try:
+        opts, _ = getopt.getopt(sys.argv[1:], 'hdc:', ['help', 'daemon', 'nimbyoff', 'update'])
+    except getopt.GetoptError:
+        usage()
+        sys.exit(1)
+
+    optNimbyOff = False
+    for option, _ in opts:
+        if option in ["-h", "--help"]:
+            usage()
+            sys.exit(0)
+        if option in ["-d", "--daemon"] and platform.system() != "Windows":
+            daemonize()
+        if option in ["--nimbyoff"]:
+            optNimbyOff = True
+
+    rqd.rqutil.permissionsLow()
+
+    logger.warning('RQD Starting Up')
+
+    setup_sentry()
+
+    rqCore = rqd.rqcore.RqCore(optNimbyOff)
+    rqCore.start()
+
+
+if __name__ == "__main__":
+    main()