cleepcli 1.43.10__tar.gz → 1.44.2__tar.gz

{cleepcli-1.43.10/cleepcli.egg-info → cleepcli-1.44.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cleepcli
-Version: 1.43.10
+Version: 1.44.2
 Summary: Cleep-cli helps developers to build great Cleep applications from command line.
 Home-page: https://github.com/CleepDevice/cleep-cli
 Author: Tanguy Bonneau

{cleepcli-1.43.10 → cleepcli-1.44.2}/bin/ccli

@@ -90,7 +90,8 @@ def coreget():
 
     # force coresync to synchronize installation with repo
     f = File()
-    res = f.core_sync()
+    if not f.core_sync():
+        sys.exit(1)
 
     # core modules
     if not get_core_mods():
@@ -290,12 +291,24 @@ def modssync(run_scripts):
     f = File()
 
     for module in config.MODULES_REPO_URL.keys():
-        res = f.module_sync(module)
-        if not res:
-            sys.exit(1)
-        res = f.module_run_install_scripts(module)
-        if not res:
-            sys.exit(1)
+        if run_scripts:
+            # Same order as package install: scripts + preinst, then files, then postinst
+            res = f.module_sync(module, backend=False, frontend=False, scripts=True)
+            if not res:
+                sys.exit(1)
+            res = f.module_run_install_script(module, "preinst.sh")
+            if not res:
+                sys.exit(1)
+            res = f.module_sync(module, backend=True, frontend=True, scripts=True)
+            if not res:
+                sys.exit(1)
+            res = f.module_run_install_script(module, "postinst.sh")
+            if not res:
+                sys.exit(1)
+        else:
+            res = f.module_sync(module)
+            if not res:
+                sys.exit(1)
 
     logging.info('Done')
 
@@ -899,17 +912,21 @@ def cli():
 
 
 if __name__ == '__main__':
-    def is_raspbian():
-        if platform.system()!='Linux':
+    def is_raspberry_pi_os():
+        if platform.system() != 'Linux':
             return False
-        res = subprocess.Popen(u'cat /etc/os-release | grep -i raspbian | wc -l', stdout=subprocess.PIPE, shell=True)
-        stdout = res.communicate()[0]
-        if stdout.strip() == '0':
+        try:
+            with open('/etc/os-release', 'r', encoding='utf-8') as handle:
+                content = handle.read().lower()
+        except OSError:
             return False
-        return True
+        return any(
+            token in content for token in ('raspbian', 'raspios', 'raspberry')
+        )
 
-    # execute only on raspbian
-    if is_raspbian():
+    # execute only on Raspberry Pi OS
+    if is_raspberry_pi_os():
         cli()
     else:
-        click.echo('Cleep-cli runs only on RaspberrypiOS distribution')
+        click.echo('Cleep-cli runs only on Raspberry Pi OS')
+        sys.exit(1)

{cleepcli-1.43.10 → cleepcli-1.44.2}/bin/cleep-cli

@@ -90,7 +90,8 @@ def coreget():
 
     # force coresync to synchronize installation with repo
     f = File()
-    res = f.core_sync()
+    if not f.core_sync():
+        sys.exit(1)
 
     # core modules
     if not get_core_mods():
@@ -290,12 +291,24 @@ def modssync(run_scripts):
     f = File()
 
     for module in config.MODULES_REPO_URL.keys():
-        res = f.module_sync(module)
-        if not res:
-            sys.exit(1)
-        res = f.module_run_install_scripts(module)
-        if not res:
-            sys.exit(1)
+        if run_scripts:
+            # Same order as package install: scripts + preinst, then files, then postinst
+            res = f.module_sync(module, backend=False, frontend=False, scripts=True)
+            if not res:
+                sys.exit(1)
+            res = f.module_run_install_script(module, "preinst.sh")
+            if not res:
+                sys.exit(1)
+            res = f.module_sync(module, backend=True, frontend=True, scripts=True)
+            if not res:
+                sys.exit(1)
+            res = f.module_run_install_script(module, "postinst.sh")
+            if not res:
+                sys.exit(1)
+        else:
+            res = f.module_sync(module)
+            if not res:
+                sys.exit(1)
 
     logging.info('Done')
 
@@ -899,17 +912,21 @@ def cli():
 
 
 if __name__ == '__main__':
-    def is_raspbian():
-        if platform.system()!='Linux':
+    def is_raspberry_pi_os():
+        if platform.system() != 'Linux':
             return False
-        res = subprocess.Popen(u'cat /etc/os-release | grep -i raspbian | wc -l', stdout=subprocess.PIPE, shell=True)
-        stdout = res.communicate()[0]
-        if stdout.strip() == '0':
+        try:
+            with open('/etc/os-release', 'r', encoding='utf-8') as handle:
+                content = handle.read().lower()
+        except OSError:
             return False
-        return True
+        return any(
+            token in content for token in ('raspbian', 'raspios', 'raspberry')
+        )
 
-    # execute only on raspbian
-    if is_raspbian():
+    # execute only on Raspberry Pi OS
+    if is_raspberry_pi_os():
         cli()
     else:
-        click.echo('Cleep-cli runs only on RaspberrypiOS distribution')
+        click.echo('Cleep-cli runs only on Raspberry Pi OS')
+        sys.exit(1)

{cleepcli-1.43.10 → cleepcli-1.44.2}/cleepcli/ci.py

@@ -12,7 +12,7 @@ import urllib.parse
 from . import config
 from .console import Console
 from .check import Check
-from .cleepapi import resolve_rpc_url
+from .cleepapi import local_socket_available, request_over_local_socket, resolve_rpc_url
 import subprocess
 
 requests.packages.urllib3.disable_warnings()
@@ -284,6 +284,8 @@ class Ci:
         """
         POST JSON to Cleep /command (self-signed TLS accepted).
         """
+        if local_socket_available():
+            return request_over_local_socket('POST', self.command_url, json_body=payload)
         return requests.post(self.command_url, json=payload, verify=False)
 
     def _rpc_get(self, url):

{cleepcli-1.43.10 → cleepcli-1.44.2}/cleepcli/cleepapi.py

@@ -17,6 +17,9 @@ DEFAULT_RPC_URLS = (
     'https://127.0.0.1:443',
     'http://127.0.0.1:80',
 )
+# Local commands (frontend restart, docs, CI) use this socket so they
+# stay available when TCP requires the Cleep password.
+LOCAL_SOCKET_PATH = '/run/cleep/rpc.sock'
 
 
 def resolve_rpc_url(urls=None, timeout=3.0):
@@ -54,6 +57,144 @@ def resolve_rpc_url(urls=None, timeout=3.0):
     raise Exception('Unable to reach Cleep RPC (tried %s): %s' % (', '.join(candidates), last_error))
 
 
+def local_socket_available(path=None):
+    """
+    Return True when the local Cleep RPC socket exists.
+    """
+    return os.path.exists(path or LOCAL_SOCKET_PATH)
+
+
+def _http_path(url):
+    """
+    Return the path and query of an RPC URL.
+    """
+    parts = urllib.parse.urlsplit(url)
+    path = parts.path or '/'
+    if parts.query:
+        path += '?' + parts.query
+    return path
+
+
+def _read_http_response(sock):
+    """
+    Read one HTTP/1.1 response from a socket.
+
+    Returns:
+        tuple: status code (int), body (bytes)
+    """
+    data = b''
+    while b'\r\n\r\n' not in data:
+        chunk = sock.recv(4096)
+        if not chunk:
+            break
+        data += chunk
+    if b'\r\n\r\n' not in data:
+        raise Exception('Incomplete HTTP response from local socket')
+
+    header_blob, rest = data.split(b'\r\n\r\n', 1)
+    lines = header_blob.split(b'\r\n')
+    status_code = int(lines[0].split()[1])
+    headers = {}
+    for line in lines[1:]:
+        if b':' not in line:
+            continue
+        name, value = line.split(b':', 1)
+        headers[name.decode('ascii').lower()] = value.strip().decode('ascii')
+
+    if headers.get('transfer-encoding', '').lower() == 'chunked':
+        body = _read_chunked(sock, rest)
+    elif 'content-length' in headers:
+        length = int(headers['content-length'])
+        while len(rest) < length:
+            chunk = sock.recv(4096)
+            if not chunk:
+                break
+            rest += chunk
+        body = rest[:length]
+    else:
+        while True:
+            chunk = sock.recv(4096)
+            if not chunk:
+                break
+            rest += chunk
+        body = rest
+    return status_code, body
+
+
+def _read_chunked(sock, pending):
+    """
+    Decode a chunked HTTP body.
+    """
+    data = pending
+    body = b''
+    while True:
+        while b'\r\n' not in data:
+            chunk = sock.recv(4096)
+            if not chunk:
+                return body
+            data += chunk
+        size_line, data = data.split(b'\r\n', 1)
+        size = int(size_line.split(b';', 1)[0], 16)
+        if size == 0:
+            return body
+        while len(data) < size + 2:
+            chunk = sock.recv(4096)
+            if not chunk:
+                return body
+            data += chunk
+        body += data[:size]
+        data = data[size + 2:]
+
+
+def request_over_local_socket(method, url, json_body=None, timeout=30.0, path=None):
+    """
+    Send one HTTP request through the local Cleep Unix socket.
+
+    Args:
+        method (str): HTTP method
+        url (str): request URL, used for the path when path is omitted
+        json_body (dict): JSON body for POST
+        timeout (float): socket timeout in seconds
+        path (str): socket path
+
+    Returns:
+        requests.Response
+    """
+    import socket
+    from requests import Response
+
+    socket_path = path or LOCAL_SOCKET_PATH
+    http_path = _http_path(url)
+    body = b''
+    headers = [
+        '%s %s HTTP/1.1' % (method, http_path),
+        'Host: localhost',
+        'Accept: application/json',
+        'Connection: close',
+    ]
+    if json_body is not None:
+        body = json.dumps(json_body).encode('utf-8')
+        headers.append('Content-Type: application/json')
+        headers.append('Content-Length: %s' % len(body))
+    raw = ('\r\n'.join(headers) + '\r\n\r\n').encode('ascii') + body
+
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    sock.settimeout(timeout)
+    try:
+        sock.connect(socket_path)
+        sock.sendall(raw)
+        status_code, payload = _read_http_response(sock)
+    finally:
+        sock.close()
+
+    response = Response()
+    response.status_code = status_code
+    response._content = payload
+    response.url = 'http://localhost' + http_path
+    response.encoding = 'utf-8'
+    return response
+
+
 class CleepApi():
     """
     Cleep api helper
@@ -81,9 +222,10 @@ class CleepApi():
 
         cmd = '/bin/systemctl restart cleep'
         c = Console()
-        resp = c.command(cmd)
+        # Maximum wait. systemctl returns as soon as the service has restarted.
+        resp = c.command(cmd, timeout=60)
         self.logger.debug('Systemctl resp: %s' % resp)
-        if resp['error'] or resp['killed']:
+        if resp['killed'] or resp['returncode'] != 0:
             self.logger.error('Error restarting cleep backend')
             return False
 
@@ -95,7 +237,11 @@ class CleepApi():
         """
         self.logger.info('Restarting frontend')
         data = {'to':'developer', 'command':'restart_frontend'}
-        self.__post(self.command_url, data)
+        result = self.__post(self.command_url, data)
+        if not result or result[0] != 200:
+            self.logger.error('Error restarting cleep frontend')
+            return False
+        return True
 
     def get_documentation(self, module_name):
         """
@@ -132,7 +278,7 @@ class CleepApi():
         (status_code, resp) = self.__get(url)
 
         if status_code != 200:
-            raise Exception("Unable to call cleep %s endpoint", url)
+            raise Exception("Unable to call cleep %s endpoint" % url)
         return resp
 
     def __post(self, url, data):
@@ -152,7 +298,10 @@ class CleepApi():
         """
         try:
             self.logger.debug("POST url: %s", url)
-            resp = requests.post(url, json=data, verify=False)
+            if local_socket_available():
+                resp = request_over_local_socket('POST', url, json_body=data)
+            else:
+                resp = requests.post(url, json=data, verify=False)
             resp_data = resp.json()
             self.logger.debug('Response[%s]: %s', resp.status_code, resp_data)
             return (resp.status_code, resp_data)
@@ -178,7 +327,10 @@ class CleepApi():
         """
         try:
             self.logger.debug("GET url: %s", url)
-            resp = requests.get(url, verify=False)
+            if local_socket_available():
+                resp = request_over_local_socket('GET', url)
+            else:
+                resp = requests.get(url, verify=False)
             resp_data = resp.json()
             self.logger.debug('Response[%s]: %s', resp.status_code, resp_data)
             return (resp.status_code, resp_data)

{cleepcli-1.43.10 → cleepcli-1.44.2}/cleepcli/console.py

@@ -14,6 +14,9 @@ import signal
 import logging
 import re
 
+ON_POSIX = os.name == "posix"
+
+
 class EndlessConsole(Thread):
     """
     Helper class to execute long command line (system update...)
@@ -322,65 +325,52 @@ class Console():
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             close_fds=True,
-            preexec_fn=os.setsid
+            preexec_fn=os.setsid if ON_POSIX else None,
         )
         pid = proc.pid
-
-        # wait for end of command line
-        done = False
-        start = time.time()
         killed = False
-        return_code = None
-        while not done:
-            # read stds to avoid deadlocks
-            (stdout, stderr) = proc.communicate()
-            if stdout:
-                result['stdout'] += self.__process_lines(stdout.split(b'\n'))
-            if stderr:
-                result['stderr'] += self.__process_lines(stderr.split(b'\n'))
-
-            # check if command has finished
-            proc.poll()
-            if proc.returncode is not None:
-                # command terminated
-                self.logger.trace('Command terminated with returncode %s' % proc.returncode)
-                return_code = proc.returncode
-                self.last_return_code = return_code
-                done = True
-                break
-
-            # check timeout
-            if time.time() > (start + timeout):
-                # timeout is over, kill command
-                pgid = os.getpgid(pid)
-                self.logger.debug('Timeout over, kill command for PID=%s PGID=%s' % (pid, pgid))
-                try:
-                    if ON_POSIX:
-                        os.killpg(pgid, signal.SIGKILL)
-                    else: # pragma: no cover
-                        proc.kill()
-                except Exception: # pragma: no cover
-                    pass
-                killed = True
-                break
+        stdout = b""
+        stderr = b""
 
-            # pause
-            time.sleep(0.125)
-
-        # prepare result
-        result['returncode'] = return_code
-        if not killed:
-            result['error'] = len(result['stderr']) > 0
-        self.logger.trace('Result: %s' % result)
+        try:
+            stdout, stderr = proc.communicate(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            self.logger.debug("Timeout over, kill command for PID=%s" % pid)
+            try:
+                if ON_POSIX:
+                    os.killpg(os.getpgid(pid), signal.SIGKILL)
+                else:  # pragma: no cover
+                    proc.kill()
+            except Exception:  # pragma: no cover
+                pass
+            killed = True
+            try:
+                stdout, stderr = proc.communicate(timeout=1.0)
+            except Exception:  # pragma: no cover
+                stdout, stderr = b"", b""
+
+        if stdout:
+            result["stdout"] = self.__process_lines(stdout.split(b"\n"))
+        if stderr:
+            result["stderr"] = self.__process_lines(stderr.split(b"\n"))
+
+        return_code = proc.returncode
+        self.last_return_code = return_code
+        result["returncode"] = return_code
+        result["killed"] = killed
+        # Prefer return code over stderr: tools like systemctl often write to stderr on success
+        result["error"] = killed or return_code not in (0, None)
+        self.logger.trace("Command terminated with returncode %s" % return_code)
+        self.logger.trace("Result: %s" % result)
 
         # make sure all stds are closed
         try:
             proc.stdout.close()
-        except Exception: # pragma: no cover
+        except Exception:  # pragma: no cover
             pass
         try:
             proc.stderr.close()
-        except Exception: # pragma: no cover
+        except Exception:  # pragma: no cover
             pass
 
         # trigger callback (used for delayed command)

{cleepcli-1.43.10 → cleepcli-1.44.2}/cleepcli/file.py

@@ -31,37 +31,48 @@ class File():
             'MEDIA_DST': config.MEDIA_DST,
             'CONFIG_DIR': config.CONFIG_DIR,
         }
+        variables['MODULES_DST'] = config.MODULES_DST
         cmd = """
+set -e
 mkdir -p "%(HTML_DST)s/"
-mkdir -p "%(CORE_DST)s/modules"
+mkdir -p "%(CORE_DST)s"
+mkdir -p "%(MODULES_DST)s"
 mkdir -p "%(MEDIA_DST)s/modules"
 mkdir -p "%(CONFIG_DIR)s"
+# Python imports cleep.modules from CORE_DST/modules; it must point at the live apps tree
+if [ -e "%(CORE_DST)s/modules" ] && [ ! -L "%(CORE_DST)s/modules" ]; then
+    rm -rf "%(CORE_DST)s/modules"
+fi
+ln -sfn "%(MODULES_DST)s" "%(CORE_DST)s/modules"
 rsync -av "%(REPO_DIR)s/cleep/" "%(CORE_DST)s/" --exclude "/tests/" --exclude "modules" --exclude "*__pycache__*" --delete --exclude "*.pyc" --keep-dirlinks
 rsync -av "%(REPO_DIR)s/html/" "%(HTML_DST)s/" --delete --exclude "js/modules/" --exclude "*node_modules*"
 rsync -av "%(REPO_DIR)s/bin/cleep" "%(BIN_DST)s/cleep"
 rsync -av "%(REPO_DIR)s/medias/sounds" "%(MEDIA_DST)s/sounds/" --delete
     """ % variables
         self.logger.debug('Coresync cmd: %s' % cmd)
-        resp = c.command(cmd, 15)
+        resp = c.command(cmd, 120)
         self.logger.debug('Coresync stdout:\n%s' % '\n'.join(resp['stdout']))
         if resp['stderr']:
             self.logger.debug('Coresync stderr:\n%s' % '\n'.join(resp['stderr']))
-        if resp['error'] or resp['killed']:
+        if resp['killed'] or resp['returncode'] != 0:
             self.logger.error('Error occured while sync core content: %s' % ('killed' if resp['killed'] else resp['stderr']))
             return False
 
         return True
 
-    def module_sync(self, module_name):
-        """ 
+    def module_sync(self, module_name, backend=True, frontend=True, scripts=True):
+        """
         Copy specified module content to valid execution location
+
+        Args:
+            module_name (str): module folder name under MODULES_SRC
+            backend (bool): sync backend/ when True
+            frontend (bool): sync frontend/ when True
+            scripts (bool): sync scripts/ when True
         """
         if not os.path.exists(os.path.join(config.MODULES_SRC, module_name)):
             self.logger.error('Module "%s" doesn\'t exist [%s]' % (module_name, os.path.join(config.MODULES_SRC, module_name)))
             return False
-    
-        if not os.path.exists(os.path.join(config.MODULES_DST, module_name)):
-            os.makedirs(os.path.join(config.MODULES_DST, module_name))
 
         mod_backend_src = os.path.join(config.MODULES_SRC, module_name, 'backend/')
         mod_backend_dst = os.path.join(config.MODULES_DST, module_name)
@@ -70,61 +81,79 @@ rsync -av "%(REPO_DIR)s/medias/sounds" "%(MEDIA_DST)s/sounds/" --delete
         mod_scripts_src = os.path.join(config.MODULES_SRC, module_name, 'scripts/')
         mod_scripts_dst = os.path.join(config.MODULES_SCRIPTS_DST, module_name)
 
-        c = Console()
-        cmd = """ 
+        parts = []
+        if backend:
+            parts.append("""
 if [ -d "%(BACKEND_SRC)s" ]; then
     /bin/mkdir -p "%(BACKEND_DST)s"
     rsync -av "%(BACKEND_SRC)s" "%(BACKEND_DST)s" --delete --exclude "*.pyc" --exclude "*__pycache__*"
 fi
+""" % {'BACKEND_SRC': mod_backend_src, 'BACKEND_DST': mod_backend_dst})
+        if frontend:
+            parts.append("""
 if [ -d "%(FRONTEND_SRC)s" ]; then
     /bin/mkdir -p "%(FRONTEND_DST)s"
     rsync -av "%(FRONTEND_SRC)s" "%(FRONTEND_DST)s" --delete --exclude "*node_modules*"
 fi
+""" % {'FRONTEND_SRC': mod_frontend_src, 'FRONTEND_DST': mod_frontend_dst})
+        if scripts:
+            parts.append("""
 if [ -d "%(SCRIPTS_SRC)s" ]; then
     /bin/mkdir -p "%(SCRIPTS_DST)s"
     rsync -av "%(SCRIPTS_SRC)s" "%(SCRIPTS_DST)s" --delete --exclude "*__pycache__*"
 fi
-    """ % { 
-            'BACKEND_SRC': mod_backend_src, 'BACKEND_DST': mod_backend_dst,
-            'FRONTEND_SRC': mod_frontend_src, 'FRONTEND_DST': mod_frontend_dst,
-            'SCRIPTS_SRC': mod_scripts_src, 'SCRIPTS_DST': mod_scripts_dst
-        }
+""" % {'SCRIPTS_SRC': mod_scripts_src, 'SCRIPTS_DST': mod_scripts_dst})
+
+        if not parts:
+            return True
+
+        c = Console()
+        cmd = "set -e\n" + "\n".join(parts)
         self.logger.debug('Modsync cmd: %s' % cmd)
-        resp = c.command(cmd, 15) 
+        resp = c.command(cmd, 120)
 
         self.logger.debug('Modsync stdout:\n%s' % '\n'.join(resp['stdout']))
         if resp['stderr']:
             self.logger.debug('Modsync stderr:\n%s' % '\n'.join(resp['stderr']))
-        if resp['error'] or resp[u'killed']:
-            self.logger.error(u'Error occured while sync module content: %s' % (u'killed' if resp[u'killed'] else resp[u'stderr']))
+        if resp['killed'] or resp['returncode'] != 0:
+            self.logger.error('Error occured while sync module content: %s' % ('killed' if resp['killed'] else resp['stderr']))
             return False
 
         return True
 
-    def module_run_install_scripts(self, module_name):
+    def module_run_install_script(self, module_name, script_name):
         """
-        Run module installation scripts
+        Run one module installation script from the synced scripts directory
+
+        Args:
+            module_name (str): module name
+            script_name (str): script filename (preinst.sh, postinst.sh, ...)
         """
         if not os.path.exists(os.path.join(config.MODULES_SRC, module_name)):
             self.logger.error('Module "%s" doesn\'t exist [%s]' % (module_name, os.path.join(config.MODULES_SRC, module_name)))
             return False
 
-        preinst_script = os.path.join(config.MODULES_SCRIPTS_DST, module_name, "preinst.sh")
-        postinst_script = os.path.join(config.MODULES_SCRIPTS_DST, module_name, "postinst.sh")
-        console = Console()
-
-        if os.path.exists(preinst_script):
-            self.logger.info('Running %s scripts', preinst_script)
-            resp = console.command(preinst_script, timeout=300)
-            if resp["returncode"] != 0:
-                self.logger.error('Error running %s [%s]: %s', preinst_script, resp["returncode"], ''.join(resp["stderr"]))
-                return False
-
-        if os.path.exists(postinst_script):
-            self.logger.info('Running %s scripts', postinst_script)
-            resp = console.command(postinst_script, timeout=300)
-            if resp["returncode"] != 0:
-                self.logger.error('Error running %s [%s]: %s', postinst_script, resp["returncode"], ''.join(resp["stderr"]))
-                return False
-
+        script_path = os.path.join(config.MODULES_SCRIPTS_DST, module_name, script_name)
+        if not os.path.exists(script_path):
+            return True
+
+        self.logger.info('Running %s', script_path)
+        resp = Console().command(script_path, timeout=300)
+        if resp["killed"] or resp["returncode"] != 0:
+            self.logger.error(
+                'Error running %s [%s]: %s',
+                script_path,
+                resp["returncode"],
+                ''.join(resp["stderr"]),
+            )
+            return False
         return True
+
+    def module_run_install_scripts(self, module_name):
+        """
+        Run module installation scripts (preinst then postinst)
+        """
+        return (
+            self.module_run_install_script(module_name, "preinst.sh")
+            and self.module_run_install_script(module_name, "postinst.sh")
+        )