ominfra 0.0.0.dev137__py3-none-any.whl → 0.0.0.dev138__py3-none-any.whl

ominfra/pyremote.py

@@ -5,7 +5,6 @@ Basically this: https://mitogen.networkgenomics.com/howitworks.html
 """
 import base64
 import dataclasses as dc
-import inspect
 import json
 import os
 import platform
@@ -13,7 +12,6 @@ import pwd
 import site
 import struct
 import sys
-import textwrap
 import typing as ta
 import zlib
 
@@ -21,11 +19,120 @@ import zlib
 ##
 
 
+@dc.dataclass(frozen=True)
+class PyremoteBootstrapOptions:
+    debug: bool = False
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class PyremoteEnvInfo:
+    sys_base_prefix: str
+    sys_byteorder: str
+    sys_defaultencoding: str
+    sys_exec_prefix: str
+    sys_executable: str
+    sys_implementation_name: str
+    sys_path: ta.List[str]
+    sys_platform: str
+    sys_prefix: str
+    sys_version: str
+    sys_version_info: ta.List[ta.Union[int, str]]
+
+    platform_architecture: ta.List[str]
+    platform_machine: str
+    platform_platform: str
+    platform_processor: str
+    platform_system: str
+    platform_release: str
+    platform_version: str
+
+    site_userbase: str
+
+    os_cwd: str
+    os_gid: int
+    os_loadavg: ta.List[float]
+    os_login: ta.Optional[str]
+    os_pgrp: int
+    os_pid: int
+    os_ppid: int
+    os_uid: int
+
+    pw_name: str
+    pw_uid: int
+    pw_gid: int
+    pw_gecos: str
+    pw_dir: str
+    pw_shell: str
+
+    env_path: ta.Optional[str]
+
+
+def _get_pyremote_env_info() -> PyremoteEnvInfo:
+    os_uid = os.getuid()
+
+    pw = pwd.getpwuid(os_uid)
+
+    os_login: ta.Optional[str]
+    try:
+        os_login = os.getlogin()
+    except OSError:
+        os_login = None
+
+    return PyremoteEnvInfo(
+        sys_base_prefix=sys.base_prefix,
+        sys_byteorder=sys.byteorder,
+        sys_defaultencoding=sys.getdefaultencoding(),
+        sys_exec_prefix=sys.exec_prefix,
+        sys_executable=sys.executable,
+        sys_implementation_name=sys.implementation.name,
+        sys_path=sys.path,
+        sys_platform=sys.platform,
+        sys_prefix=sys.prefix,
+        sys_version=sys.version,
+        sys_version_info=list(sys.version_info),
+
+        platform_architecture=list(platform.architecture()),
+        platform_machine=platform.machine(),
+        platform_platform=platform.platform(),
+        platform_processor=platform.processor(),
+        platform_system=platform.system(),
+        platform_release=platform.release(),
+        platform_version=platform.version(),
+
+        site_userbase=site.getuserbase(),
+
+        os_cwd=os.getcwd(),
+        os_gid=os.getgid(),
+        os_loadavg=list(os.getloadavg()),
+        os_login=os_login,
+        os_pgrp=os.getpgrp(),
+        os_pid=os.getpid(),
+        os_ppid=os.getppid(),
+        os_uid=os_uid,
+
+        pw_name=pw.pw_name,
+        pw_uid=pw.pw_uid,
+        pw_gid=pw.pw_gid,
+        pw_gecos=pw.pw_gecos,
+        pw_dir=pw.pw_dir,
+        pw_shell=pw.pw_shell,
+
+        env_path=os.environ.get('PATH'),
+    )
+
+
+##
+
+
 _PYREMOTE_BOOTSTRAP_INPUT_FD = 100
 _PYREMOTE_BOOTSTRAP_SRC_FD = 101
 
 _PYREMOTE_BOOTSTRAP_CHILD_PID_VAR = '_OPYR_CHILD_PID'
 _PYREMOTE_BOOTSTRAP_ARGV0_VAR = '_OPYR_ARGV0'
+_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR = '_OPYR_OPTIONS_JSON'
 
 _PYREMOTE_BOOTSTRAP_ACK0 = b'OPYR000\n'
 _PYREMOTE_BOOTSTRAP_ACK1 = b'OPYR001\n'
@@ -87,7 +194,9 @@ def _pyremote_bootstrap_main(context_name: str) -> None:
 
         # Read main src from stdin
         main_z_len = struct.unpack('<I', os.read(0, 4))[0]
-        main_src = zlib.decompress(os.fdopen(0, 'rb').read(main_z_len))
+        if len(main_z := os.fdopen(0, 'rb').read(main_z_len)) != main_z_len:
+            raise EOFError
+        main_src = zlib.decompress(main_z)
 
         # Write both copies of main src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely fill
         # and block and need to be drained by pyremote_bootstrap_finalize running in parent.
@@ -110,6 +219,8 @@ def pyremote_build_bootstrap_cmd(context_name: str) -> str:
     if any(c in context_name for c in '\'"'):
         raise NameError(context_name)
 
+    import inspect
+    import textwrap
     bs_src = textwrap.dedent(inspect.getsource(_pyremote_bootstrap_main))
 
     for gl in [
@@ -136,9 +247,6 @@ def pyremote_build_bootstrap_cmd(context_name: str) -> str:
     bs_z = zlib.compress(bs_src.encode('utf-8'))
     bs_z64 = base64.encodebytes(bs_z).replace(b'\n', b'')
 
-    def dq_repr(o: ta.Any) -> str:
-        return '"' + repr(o)[1:-1] + '"'
-
     stmts = [
         f'import {", ".join(_PYREMOTE_BOOTSTRAP_IMPORTS)}',
         f'exec(zlib.decompress(base64.decodebytes({bs_z64!r})))',
@@ -153,99 +261,83 @@ def pyremote_build_bootstrap_cmd(context_name: str) -> str:
 
 
 @dc.dataclass(frozen=True)
-class PyremoteEnvInfo:
-    sys_base_prefix: str
-    sys_byteorder: str
-    sys_defaultencoding: str
-    sys_exec_prefix: str
-    sys_executable: str
-    sys_implementation_name: str
-    sys_path: ta.List[str]
-    sys_platform: str
-    sys_prefix: str
-    sys_version: str
-    sys_version_info: ta.List[ta.Union[int, str]]
-
-    platform_architecture: ta.List[str]
-    platform_machine: str
-    platform_platform: str
-    platform_processor: str
-    platform_system: str
-    platform_release: str
-    platform_version: str
-
-    site_userbase: str
-
-    os_cwd: str
-    os_gid: int
-    os_loadavg: ta.List[float]
-    os_login: ta.Optional[str]
-    os_pgrp: int
-    os_pid: int
-    os_ppid: int
-    os_uid: int
-
-    pw_name: str
-    pw_uid: int
-    pw_gid: int
-    pw_gecos: str
-    pw_dir: str
-    pw_shell: str
-
-    env_path: ta.Optional[str]
+class PyremotePayloadRuntime:
+    input: ta.BinaryIO
+    output: ta.BinaryIO
+    main_src: str
+    options: PyremoteBootstrapOptions
+    env_info: PyremoteEnvInfo
 
 
-def _get_pyremote_env_info() -> PyremoteEnvInfo:
-    os_uid = os.getuid()
+def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
+    # If json options var is not present we need to do initial finalization
+    if _PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR not in os.environ:
+        # Read second copy of main src
+        r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
+        main_src = r1.read().decode('utf-8')
+        r1.close()
+
+        # Reap boostrap child. Must be done after reading second copy of source because source may be too big to fit in
+        # a pipe at once.
+        os.waitpid(int(os.environ.pop(_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR)), 0)
+
+        # Read options
+        options_json_len = struct.unpack('<I', os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, 4))[0]
+        if len(options_json := os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, options_json_len)) != options_json_len:
+            raise EOFError
+        options = PyremoteBootstrapOptions(**json.loads(options_json.decode('utf-8')))
+
+        # If debugging, re-exec as file
+        if options.debug:
+            # Write temp source file
+            import tempfile
+            tfd, tfn = tempfile.mkstemp('-pyremote.py')
+            os.write(tfd, main_src.encode('utf-8'))
+            os.close(tfd)
+
+            # Set json options var
+            os.environ[_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR] = options_json.decode('utf-8')
+
+            # Re-exec temp file
+            os.execl(os.environ[_PYREMOTE_BOOTSTRAP_ARGV0_VAR], sys.orig_argv[0], tfn)
 
-    pw = pwd.getpwuid(os_uid)
+    else:
+        # Load options json var
+        options_json_str = os.environ.pop(_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR)
+        options = PyremoteBootstrapOptions(**json.loads(options_json_str))
 
-    os_login: ta.Optional[str]
-    try:
-        os_login = os.getlogin()
-    except OSError:
-        os_login = None
+        # Read temp source file
+        with open(sys.orig_argv[1]) as sf:
+            main_src = sf.read()
 
-    return PyremoteEnvInfo(
-        sys_base_prefix=sys.base_prefix,
-        sys_byteorder=sys.byteorder,
-        sys_defaultencoding=sys.getdefaultencoding(),
-        sys_exec_prefix=sys.exec_prefix,
-        sys_executable=sys.executable,
-        sys_implementation_name=sys.implementation.name,
-        sys_path=sys.path,
-        sys_platform=sys.platform,
-        sys_prefix=sys.prefix,
-        sys_version=sys.version,
-        sys_version_info=list(sys.version_info),
+    # Restore original argv0
+    sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
 
-        platform_architecture=list(platform.architecture()),
-        platform_machine=platform.machine(),
-        platform_platform=platform.platform(),
-        platform_processor=platform.processor(),
-        platform_system=platform.system(),
-        platform_release=platform.release(),
-        platform_version=platform.version(),
+    # Write third ack
+    os.write(1, _PYREMOTE_BOOTSTRAP_ACK2)
 
-        site_userbase=site.getuserbase(),
+    # Write env info
+    env_info = _get_pyremote_env_info()
+    env_info_json = json.dumps(dc.asdict(env_info), indent=None, separators=(',', ':'))  # noqa
+    os.write(1, struct.pack('<I', len(env_info_json)))
+    os.write(1, env_info_json.encode('utf-8'))
 
-        os_cwd=os.getcwd(),
-        os_gid=os.getgid(),
-        os_loadavg=list(os.getloadavg()),
-        os_login=os_login,
-        os_pgrp=os.getpgrp(),
-        os_pid=os.getpid(),
-        os_ppid=os.getppid(),
-        os_uid=os_uid,
+    # Setup IO
+    input = os.fdopen(_PYREMOTE_BOOTSTRAP_INPUT_FD, 'rb', 0)  # noqa
+    output = os.fdopen(os.dup(1), 'wb', 0)  # noqa
+    os.dup2(nfd := os.open('/dev/null', os.O_WRONLY), 1)
+    os.close(nfd)
 
-        pw_name=pw.pw_name,
-        pw_uid=pw.pw_uid,
-        pw_gid=pw.pw_gid,
-        pw_gecos=pw.pw_gecos,
-        pw_dir=pw.pw_dir,
-        pw_shell=pw.pw_shell,
+    # Write fourth ack
+    output.write(_PYREMOTE_BOOTSTRAP_ACK3)
 
-        env_path=os.environ.get('PATH'),
+    # Return
+    return PyremotePayloadRuntime(
+        input=input,
+        output=output,
+        main_src=main_src,
+        options=options,
+        env_info=env_info,
     )
 
 
@@ -253,12 +345,15 @@ def _get_pyremote_env_info() -> PyremoteEnvInfo:
 
 
 class PyremoteBootstrapDriver:
-    def __init__(self, main_src: str) -> None:
+    def __init__(self, main_src: str, options: PyremoteBootstrapOptions = PyremoteBootstrapOptions()) -> None:
         super().__init__()
 
         self._main_src = main_src
         self._main_z = zlib.compress(main_src.encode('utf-8'))
 
+        self._options = options
+        self._options_json = json.dumps(dc.asdict(options), indent=None, separators=(',', ':')).encode('utf-8')  # noqa
+
     #
 
     @dc.dataclass(frozen=True)
@@ -278,7 +373,7 @@ class PyremoteBootstrapDriver:
         env_info: PyremoteEnvInfo
 
     def gen(self) -> ta.Generator[ta.Union[Read, Write], ta.Optional[bytes], Result]:
-        # Read first ack
+        # Read first ack (after fork)
         yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK0)
 
         # Read pid
@@ -289,8 +384,14 @@ class PyremoteBootstrapDriver:
         yield from self._write(struct.pack('<I', len(self._main_z)))
         yield from self._write(self._main_z)
 
-        # Read second and third ack
+        # Read second ack (after writing src copies)
         yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK1)
+
+        # Write options
+        yield from self._write(struct.pack('<I', len(self._options_json)))
+        yield from self._write(self._options_json)
+
+        # Read third ack (after reaping child process)
         yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK2)
 
         # Read env info
@@ -300,7 +401,7 @@ class PyremoteBootstrapDriver:
         env_info_json = d.decode('utf-8')
         env_info = PyremoteEnvInfo(**json.loads(env_info_json))
 
-        # Read fourth ack
+        # Read fourth ack (after finalization completed)
         yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK3)
 
         # Return
@@ -343,61 +444,11 @@ class PyremoteBootstrapDriver:
                 return e.value
 
             if isinstance(go, self.Read):
-                gi = stdout.read(go.sz)
+                if len(gi := stdout.read(go.sz)) != go.sz:
+                    raise EOFError
             elif isinstance(go, self.Write):
                 gi = None
                 stdin.write(go.d)
                 stdin.flush()
             else:
                 raise TypeError(go)
-
-
-##
-
-
-@dc.dataclass(frozen=True)
-class PyremotePayloadRuntime:
-    input: ta.BinaryIO
-    output: ta.BinaryIO
-    main_src: str
-    env_info: PyremoteEnvInfo
-
-
-def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
-    # Restore original argv0
-    sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
-
-    # Read second copy of main src
-    r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
-    main_src = r1.read().decode('utf-8')
-    r1.close()
-
-    # Reap boostrap child. Must be done after reading second copy of source because source may be too big to fit in a
-    # pipe at once.
-    os.waitpid(int(os.environ.pop(_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR)), 0)
-
-    # Write third ack
-    os.write(1, _PYREMOTE_BOOTSTRAP_ACK2)
-
-    # Write env info
-    env_info = _get_pyremote_env_info()
-    env_info_json = json.dumps(dc.asdict(env_info), indent=None, separators=(',', ':'))  # noqa
-    os.write(1, struct.pack('<I', len(env_info_json)))
-    os.write(1, env_info_json.encode('utf-8'))
-
-    # Setup IO
-    input = os.fdopen(_PYREMOTE_BOOTSTRAP_INPUT_FD, 'rb', 0)  # noqa
-    output = os.fdopen(os.dup(1), 'wb', 0)  # noqa
-    os.dup2(nfd := os.open('/dev/null', os.O_WRONLY), 1)
-    os.close(nfd)
-
-    # Write fourth ack
-    output.write(_PYREMOTE_BOOTSTRAP_ACK3)
-
-    # Return
-    return PyremotePayloadRuntime(
-        input=input,
-        output=output,
-        main_src=main_src,
-        env_info=env_info,
-    )

ominfra/scripts/supervisor.py

@@ -7382,7 +7382,7 @@ class ProcessImpl(Process):
         self._backoff = 0  # backoff counter (to start_retries)
 
         self._exitstatus: ta.Optional[Rc] = None  # status attached to dead process by finish()
-        self._spawn_err: ta.Optional[str] = None  # error message attached by spawn() if any
+        self._spawn_err: ta.Optional[str] = None  # error message attached by _spawn() if any
 
     #
 
@@ -7419,12 +7419,12 @@ class ProcessImpl(Process):
 
     #
 
-    def spawn(self) -> ta.Optional[Pid]:
+    def _spawn(self) -> ta.Optional[Pid]:
         if self.pid:
             log.warning('process \'%s\' already running', self.name)
             return None
 
-        self.check_in_state(
+        self._check_in_state(
             ProcessState.EXITED,
             ProcessState.FATAL,
             ProcessState.BACKOFF,
@@ -7439,15 +7439,15 @@ class ProcessImpl(Process):
 
         self._last_start = time.time()
 
-        self.change_state(ProcessState.STARTING)
+        self._change_state(ProcessState.STARTING)
 
         try:
             sp = self._spawning.spawn()
         except ProcessSpawnError as err:
             log.exception('Spawn error')
             self._spawn_err = err.args[0]
-            self.check_in_state(ProcessState.STARTING)
-            self.change_state(ProcessState.BACKOFF)
+            self._check_in_state(ProcessState.STARTING)
+            self._change_state(ProcessState.BACKOFF)
             return None
 
         log.info("Spawned: '%s' with pid %s", self.name, sp.pid)
@@ -7480,7 +7480,7 @@ class ProcessImpl(Process):
 
     #
 
-    def change_state(self, new_state: ProcessState, expected: bool = True) -> bool:
+    def _change_state(self, new_state: ProcessState, expected: bool = True) -> bool:
         old_state = self._state
         if new_state is old_state:
             return False
@@ -7498,7 +7498,7 @@ class ProcessImpl(Process):
 
         return True
 
-    def check_in_state(self, *states: ProcessState) -> None:
+    def _check_in_state(self, *states: ProcessState) -> None:
         if self._state not in states:
             raise ProcessStateError(
                 f'Check failed for {self._config.name}: '
@@ -7507,7 +7507,7 @@ class ProcessImpl(Process):
 
     #
 
-    def _check_and_adjust_for_system_clock_rollback(self, test_time):
+    def _check_and_adjust_for_system_clock_rollback(self, test_time: float) -> None:
         """
         Check if system clock has rolled backward beyond test_time. If so, set affected timestamps to test_time.
         """
@@ -7551,8 +7551,8 @@ class ProcessImpl(Process):
         self._delay = 0
         self._backoff = 0
         self._system_stop = True
-        self.check_in_state(ProcessState.BACKOFF)
-        self.change_state(ProcessState.FATAL)
+        self._check_in_state(ProcessState.BACKOFF)
+        self._change_state(ProcessState.FATAL)
 
     def kill(self, sig: int) -> ta.Optional[str]:
         """
@@ -7562,6 +7562,7 @@ class ProcessImpl(Process):
         Return None if the signal was sent, or an error message string if an error occurred or if the subprocess is not
         running.
         """
+
         now = time.time()
 
         # If the process is in BACKOFF and we want to stop or kill it, then BACKOFF -> STOPPED. This is needed because
@@ -7569,7 +7570,7 @@ class ProcessImpl(Process):
         # blocked for a long time waiting for the retries.
         if self._state == ProcessState.BACKOFF:
             log.debug('Attempted to kill %s, which is in BACKOFF state.', self.name)
-            self.change_state(ProcessState.STOPPED)
+            self._change_state(ProcessState.STOPPED)
             return None
 
         args: tuple
@@ -7594,8 +7595,8 @@ class ProcessImpl(Process):
         self._killing = True
         self._delay = now + self._config.stop_wait_secs
         # we will already be in the STOPPING state if we're doing a SIGKILL as a result of overrunning stop_wait_secs
-        self.check_in_state(ProcessState.RUNNING, ProcessState.STARTING, ProcessState.STOPPING)
-        self.change_state(ProcessState.STOPPING)
+        self._check_in_state(ProcessState.RUNNING, ProcessState.STARTING, ProcessState.STOPPING)
+        self._change_state(ProcessState.STOPPING)
 
         kpid = int(self.pid)
         if kill_as_group:
@@ -7616,7 +7617,7 @@ class ProcessImpl(Process):
             tb = traceback.format_exc()
             fmt, args = 'unknown problem killing %s (%s):%s', (self.name, self.pid, tb)
             log.critical(fmt, *args)
-            self.change_state(ProcessState.UNKNOWN)
+            self._change_state(ProcessState.UNKNOWN)
             self._killing = False
             self._delay = 0
             return fmt % args
@@ -7638,7 +7639,7 @@ class ProcessImpl(Process):
 
         log.debug('sending %s (pid %s) sig %s', self.name, self.pid, sig_name(sig))
 
-        self.check_in_state(ProcessState.RUNNING, ProcessState.STARTING, ProcessState.STOPPING)
+        self._check_in_state(ProcessState.RUNNING, ProcessState.STARTING, ProcessState.STOPPING)
 
         try:
             try:
@@ -7659,7 +7660,7 @@ class ProcessImpl(Process):
             tb = traceback.format_exc()
             fmt, args = 'unknown problem sending sig %s (%s):%s', (self.name, self.pid, tb)
             log.critical(fmt, *args)
-            self.change_state(ProcessState.UNKNOWN)
+            self._change_state(ProcessState.UNKNOWN)
             return fmt % args
 
         return None
@@ -7697,8 +7698,8 @@ class ProcessImpl(Process):
             self._exitstatus = Rc(es)
 
             fmt, args = 'stopped: %s (%s)', (self.name, msg)
-            self.check_in_state(ProcessState.STOPPING)
-            self.change_state(ProcessState.STOPPED)
+            self._check_in_state(ProcessState.STOPPING)
+            self._change_state(ProcessState.STOPPED)
             if exit_expected:
                 log.info(fmt, *args)
             else:
@@ -7708,8 +7709,8 @@ class ProcessImpl(Process):
             # the program did not stay up long enough to make it to RUNNING implies STARTING -> BACKOFF
             self._exitstatus = None
             self._spawn_err = 'Exited too quickly (process log may have details)'
-            self.check_in_state(ProcessState.STARTING)
-            self.change_state(ProcessState.BACKOFF)
+            self._check_in_state(ProcessState.STARTING)
+            self._change_state(ProcessState.BACKOFF)
             log.warning('exited: %s (%s)', self.name, msg + '; not expected')
 
         else:
@@ -7722,18 +7723,18 @@ class ProcessImpl(Process):
             # if the process was STARTING but a system time change causes self.last_start to be in the future, the
             # normal STARTING->RUNNING transition can be subverted so we perform the transition here.
             if self._state == ProcessState.STARTING:
-                self.change_state(ProcessState.RUNNING)
+                self._change_state(ProcessState.RUNNING)
 
-            self.check_in_state(ProcessState.RUNNING)
+            self._check_in_state(ProcessState.RUNNING)
 
             if exit_expected:
                 # expected exit code
-                self.change_state(ProcessState.EXITED, expected=True)
+                self._change_state(ProcessState.EXITED, expected=True)
                 log.info('exited: %s (%s)', self.name, msg + '; expected')
             else:
                 # unexpected exit code
                 self._spawn_err = f'Bad exit code {es}'
-                self.change_state(ProcessState.EXITED, expected=False)
+                self._change_state(ProcessState.EXITED, expected=False)
                 log.warning('exited: %s (%s)', self.name, msg + '; not expected')
 
         self._pid = Pid(0)
@@ -7753,21 +7754,21 @@ class ProcessImpl(Process):
                 if self._config.auto_restart:
                     if self._config.auto_restart is RestartUnconditionally:
                         # EXITED -> STARTING
-                        self.spawn()
+                        self._spawn()
                     elif self._exitstatus not in self._config.exitcodes:
                         # EXITED -> STARTING
-                        self.spawn()
+                        self._spawn()
 
             elif state == ProcessState.STOPPED and not self._last_start:
                 if self._config.auto_start:
                     # STOPPED -> STARTING
-                    self.spawn()
+                    self._spawn()
 
             elif state == ProcessState.BACKOFF:
                 if self._backoff <= self._config.start_retries:
                     if now > self._delay:
                         # BACKOFF -> STARTING
-                        self.spawn()
+                        self._spawn()
 
         if state == ProcessState.STARTING:
             if now - self._last_start > self._config.start_secs:
@@ -7775,8 +7776,8 @@ class ProcessImpl(Process):
                 # proc.config.start_secs,
                 self._delay = 0
                 self._backoff = 0
-                self.check_in_state(ProcessState.STARTING)
-                self.change_state(ProcessState.RUNNING)
+                self._check_in_state(ProcessState.STARTING)
+                self._change_state(ProcessState.RUNNING)
                 msg = ('entered RUNNING state, process has stayed up for > than %s seconds (start_secs)' % self._config.start_secs)  # noqa
                 log.info('success: %s %s', self.name, msg)