jolt 0.9.123__py3-none-any.whl → 0.9.435__py3-none-any.whl

jolt/common_pb2.py

@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: jolt/common.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x11jolt/common.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"-\n\x06\x44igest\x12\x16\n\talgorithm\x18\x01 \x01(\tR\x03\x61lg\x12\x0b\n\x03hex\x18\x02 \x01(\t\"o\n\x07LogLine\x12\x18\n\x05level\x18\x01 \x01(\x0e\x32\t.LogLevel\x12(\n\x04time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07\x63ontext\x18\x03 \x01(\t\x12\x0f\n\x07message\x18\x04 \x01(\t\"&\n\x08Property\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\")\n\x08Platform\x12\x1d\n\nproperties\x18\x01 \x03(\x0b\x32\t.Property\"\x83\x01\n\x04Task\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x10\n\x08instance\x18\x03 \x01(\t\x12\r\n\x05taint\x18\x04 \x01(\t\x12\x1d\n\nproperties\x18\x05 \x03(\x0b\x32\t.Property\x12\x1b\n\x08platform\x18\x06 \x01(\x0b\x32\t.Platform\"M\n\tTaskError\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x10\n\x08location\x18\x02 \x01(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\x12\x0f\n\x07\x64\x65tails\x18\x04 \x01(\t\"5\n\x04\x46ile\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0e\n\x06\x64igest\x18\x02 \x01(\t\x12\x0f\n\x07\x63ontent\x18\x03 \x01(\t\"\xf1\x01\n\x07Project\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\"\n\x05paths\x18\x02 \x03(\x0b\x32\x13.Project.SystemPath\x12 \n\x07recipes\x18\x03 \x03(\x0b\x32\x0f.Project.Recipe\x12$\n\tresources\x18\x04 \x03(\x0b\x32\x11.Project.Resource\x1a\x1a\n\nSystemPath\x12\x0c\n\x04path\x18\x01 \x01(\t\x1a\'\n\x08Resource\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x61lias\x18\x02 \x01(\t\x1a\'\n\x06Recipe\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0f\n\x07workdir\x18\x03 \x01(\t\"\x8e\x01\n\tWorkspace\x12\x0f\n\x07rootdir\x18\x01 \x01(\t\x12\x10\n\x08\x63\x61\x63hedir\x18\x02 \x01(\t\x12\x14\n\x05\x66iles\x18\x03 \x03(\x0b\x32\x05.File\x12\x1a\n\x08projects\x18\x04 \x03(\x0b\x32\x08.Project\x12\x0c\n\x04name\x18\x05 \x01(\t\x12\x0c\n\x04tree\x18\x06 \x01(\t\x12\x10\n\x08\x62uilddir\x18\x07 \x01(\t\"i\n\x06\x43lient\x12\x0f\n\x07version\x18\x01 \x01(\t\x12\x0c\n\x04tree\x18\x02 \x01(\t\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x0b\n\x03url\x18\x04 \x01(\t\x12\x14\n\x0crequirements\x18\x05 \x03(\t\x12\x0b\n\x03nix\x18\x06 \x01(\x08\"\x99\x02\n\x10\x42uildEnvironment\x12\x17\n\x06\x63lient\x18\x01 \x01(\x0b\x32\x07.Client\x12\x1d\n\tworkspace\x18\x02 \x01(\x0b\x32\n.Workspace\x12\x1d\n\nparameters\x18\x03 \x03(\x0b\x32\t.Property\x12\x1f\n\x17task_default_parameters\x18\x04 \x03(\t\x12+\n\x05tasks\x18\x05 \x03(\x0b\x32\x1c.BuildEnvironment.TasksEntry\x12\x1b\n\x08loglevel\x18\x06 \x01(\x0e\x32\t.LogLevel\x12\x0e\n\x06\x63onfig\x18\x07 \x01(\t\x1a\x33\n\nTasksEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x14\n\x05value\x18\x02 \x01(\x0b\x32\x05.Task:\x02\x38\x01*k\n\x08LogLevel\x12\r\n\tEXCEPTION\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07VERBOSE\x10\x02\x12\n\n\x06STDOUT\x10\x03\x12\x08\n\x04INFO\x10\x04\x12\x0b\n\x07WARNING\x10\x05\x12\t\n\x05\x45RROR\x10\x06\x12\n\n\x06STDERR\x10\x07*_\n\x0b\x42uildStatus\x12\x12\n\x0e\x42UILD_ACCEPTED\x10\x00\x12\x12\n\x0e\x42UILD_REJECTED\x10\x01\x12\x13\n\x0f\x42UILD_COMPLETED\x10\x03\x12\x13\n\x0f\x42UILD_CANCELLED\x10\x04*\xd4\x01\n\nTaskStatus\x12\x0f\n\x0bTASK_QUEUED\x10\x00\x12\x10\n\x0cTASK_RUNNING\x10\x01\x12\x0f\n\x0bTASK_FAILED\x10\x02\x12\x0f\n\x0bTASK_PASSED\x10\x03\x12\x11\n\rTASK_UNSTABLE\x10\x04\x12\x13\n\x0fTASK_DOWNLOADED\x10\x05\x12\x11\n\rTASK_UPLOADED\x10\x06\x12\x10\n\x0cTASK_SKIPPED\x10\x07\x12\x12\n\x0eTASK_CANCELLED\x10\x08\x12\x0e\n\nTASK_ERROR\x10\t\x12\x10\n\x0cTASK_CREATED\x10\nB\x0eZ\x0cpkg/protocolb\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'jolt.common_pb2', _globals)
+if _descriptor._USE_C_DESCRIPTORS == False:
+  _globals['DESCRIPTOR']._options = None
+  _globals['DESCRIPTOR']._serialized_options = b'Z\014pkg/protocol'
+  _globals['_BUILDENVIRONMENT_TASKSENTRY']._options = None
+  _globals['_BUILDENVIRONMENT_TASKSENTRY']._serialized_options = b'8\001'
+  _globals['_LOGLEVEL']._serialized_start=1345
+  _globals['_LOGLEVEL']._serialized_end=1452
+  _globals['_BUILDSTATUS']._serialized_start=1454
+  _globals['_BUILDSTATUS']._serialized_end=1549
+  _globals['_TASKSTATUS']._serialized_start=1552
+  _globals['_TASKSTATUS']._serialized_end=1764
+  _globals['_DIGEST']._serialized_start=54
+  _globals['_DIGEST']._serialized_end=99
+  _globals['_LOGLINE']._serialized_start=101
+  _globals['_LOGLINE']._serialized_end=212
+  _globals['_PROPERTY']._serialized_start=214
+  _globals['_PROPERTY']._serialized_end=252
+  _globals['_PLATFORM']._serialized_start=254
+  _globals['_PLATFORM']._serialized_end=295
+  _globals['_TASK']._serialized_start=298
+  _globals['_TASK']._serialized_end=429
+  _globals['_TASKERROR']._serialized_start=431
+  _globals['_TASKERROR']._serialized_end=508
+  _globals['_FILE']._serialized_start=510
+  _globals['_FILE']._serialized_end=563
+  _globals['_PROJECT']._serialized_start=566
+  _globals['_PROJECT']._serialized_end=807
+  _globals['_PROJECT_SYSTEMPATH']._serialized_start=699
+  _globals['_PROJECT_SYSTEMPATH']._serialized_end=725
+  _globals['_PROJECT_RESOURCE']._serialized_start=727
+  _globals['_PROJECT_RESOURCE']._serialized_end=766
+  _globals['_PROJECT_RECIPE']._serialized_start=768
+  _globals['_PROJECT_RECIPE']._serialized_end=807
+  _globals['_WORKSPACE']._serialized_start=810
+  _globals['_WORKSPACE']._serialized_end=952
+  _globals['_CLIENT']._serialized_start=954
+  _globals['_CLIENT']._serialized_end=1059
+  _globals['_BUILDENVIRONMENT']._serialized_start=1062
+  _globals['_BUILDENVIRONMENT']._serialized_end=1343
+  _globals['_BUILDENVIRONMENT_TASKSENTRY']._serialized_start=1292
+  _globals['_BUILDENVIRONMENT_TASKSENTRY']._serialized_end=1343
+# @@protoc_insertion_point(module_scope)

jolt/common_pb2_grpc.py

@@ -0,0 +1,4 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+

jolt/config.py

@@ -1,10 +1,12 @@
 from configparser import ConfigParser, NoOptionError, NoSectionError
+from urllib.parse import urlparse
 import os
+import re
 
+from jolt import common_pb2 as common_pb
 from jolt import filesystem as fs
 from jolt import utils
 from jolt.error import raise_error_if
-from jolt.manifest import ManifestExtension, ManifestExtensionRegistry
 
 
 _workdir = os.getcwd()
@@ -13,13 +15,18 @@ _workdir = os.getcwd()
 if os.getenv("JOLT_CONFIG_PATH"):
     location = fs.path.join(os.getenv("JOLT_CONFIG_PATH"), "config")
     location_user = fs.path.join(os.getenv("JOLT_CONFIG_PATH"), "user")
+    location_overlay = os.getenv("JOLT_CONFIG_OVERLAY")
+    if location_overlay:
+        location_overlay = fs.path.join(os.getenv("JOLT_CONFIG_PATH"), location_overlay)
 elif os.name == "nt":
     appdata = os.getenv("APPDATA", fs.path.join(fs.userhome(), "AppData", "Roaming"))
     location = fs.path.join(appdata, "Jolt", "config")
     location_user = fs.path.join(appdata, "Jolt", "user")
+    location_overlay = os.getenv("JOLT_CONFIG_OVERLAY")
 else:
     location = fs.path.join(fs.userhome(), ".config", "jolt", "config")
     location_user = fs.path.join(fs.userhome(), ".config", "jolt", "user")
+    location_overlay = os.getenv("JOLT_CONFIG_OVERLAY")
 
 
 class ConfigFile(ConfigParser):
@@ -37,10 +44,10 @@ class ConfigFile(ConfigParser):
             if not self.has_section("jolt"):
                 self.add_section("jolt")
 
-    def save(self):
-        if self._location is None:
+    def save(self, path=None):
+        if self._location is None and path is None:
             return
-        with open(self._location, 'w') as configfile:
+        with open(path or self._location, 'w') as configfile:
             super().write(configfile)
 
     def delete(self, section, key):
@@ -122,14 +129,16 @@ class Config(object):
         for config in self.configs():
             config.load()
 
-    def save(self):
-        for config in self.configs():
-            config.save()
+    def save(self, path=None):
+        for name, config in self._configs:
+            config.save(os.path.join(path, f"{name}.conf") if path else None)
 
 
 _config = Config()
 _config.add_file("global", location)
 _config.add_file("user", location_user)
+if location_overlay:
+    _config.add_file("overlay", location_overlay)
 _manifest = _config.add_file("manifest", None)
 # Note: cli configs are added next to last in the chain,
 # before manifest, and can therefore not override the
@@ -144,30 +153,51 @@ def get(section, key, default=None, expand=True, alias=None):
 
 
 def getint(section, key, default=None, alias=None):
-    return int(get(section, key, default=default, alias=alias))
+    value = get(section, key, default=default, alias=alias)
+    if value is not None:
+        try:
+            return int(value)
+        except ValueError:
+            raise_error_if(True, "Config: value '{0}' invalid for '{1}.{2}', expected integer", value, section, key)
+    return None
 
 
 def getsize(section, key, default=None, alias=None):
-    units = {"B": 1, "K": 1024, "M": 1024**2, "G": 1024**3, "T": 1024**4}
+    units = {
+        None: 1,
+        "B": 1,
+
+        "K": 1000,
+        "M": 1000**2,
+        "G": 1000**3,
+        "T": 1000**4,
+        "P": 1000**5,
+        "E": 1000**6,
+
+        "Ki": 1024,
+        "Mi": 1024**2,
+        "Gi": 1024**3,
+        "Ti": 1024**4,
+        "Pi": 1024**5,
+        "Ei": 1024**6,
+    }
+
     value = get(section, key, default=None, alias=alias)
     if value is None:
         if type(default) is int:
             return default
         else:
             value = str(default)
-    value = value.strip()
-    value = value.split()
-    if len(value) == 1 and value[0][-1] in units:
-        size, unit = value[0][:-1], value[0][-1]
-    else:
-        raise_error_if(
-            len(value) != 2,
-            "Config: size '{2}' invalid for '{0}.{1}', expected '<size> <unit>'", value, section, key)
-        size, unit = value[0], value[1]
+
+    m = re.search(r"^(0|[1-9][0-9]*) ?([KMGTPE]i?)?B?$", value)
+    raise_error_if(
+        not m,
+        "Config: size '{0}' invalid for '{1}.{2}', expected '<size> <unit>'", value, section, key)
     raise_error_if(
-        unit not in units,
-        "Config: unit invalid for '{0}.{1}', expected [B,K,M,G,T]", section, key)
-    return int(size) * units[unit]
+        m[2] not in units,
+        "Config: unit invalid for '{0}.{1}', expected [B,K,M,G,T,P,E,Mi,Gi,Ti,Pi,Ei]", section, key)
+
+    return int(m[1]) * units.get(m[2], 1)
 
 
 def getfloat(section, key, default=None, alias=None):
@@ -179,6 +209,13 @@ def getboolean(section, key, default=None, alias=None):
     return value is not None and str(value).lower() in ["true", "yes", "on", "1"]
 
 
+def geturi(section, key, default=None, alias=None):
+    value = get(section, key, default=default, alias=alias)
+    if value is None:
+        return None
+    return urlparse(value)
+
+
 def get_jolthome():
     if os.name == "nt":
         return fs.path.join(os.getenv("LOCALAPPDATA", fs.path.join(fs.userhome(), "AppData", "Local")), "Jolt")
@@ -187,7 +224,7 @@ def get_jolthome():
 
 
 def get_logpath():
-    return get_jolthome()
+    return get("jolt", "logpath", get_jolthome())
 
 
 def get_cachedir():
@@ -209,6 +246,23 @@ def get_shell():
     return get("jolt", "shell", cmd)
 
 
+def get_keep_going():
+    return getboolean("params", "keep_going", False)
+
+
+def is_incremental_build():
+    """
+    Whether to enable incremental builds.
+
+    If disabled, all build directories are deleted upon completion of tasks.
+    """
+    return getboolean("jolt", "incremental_dirs", True)
+
+
+def set_keep_going(value=False):
+    return set("params", "keep_going", str(value).lower())
+
+
 def set(section, key, value, alias=None):
     _config.set(section, key, value, alias or "user")
 
@@ -229,8 +283,8 @@ def load_or_set(file_or_str):
         _config.set(section_key[0], section_key[1], key_value[1], alias="cli")
 
 
-def save():
-    _config.save()
+def save(path=None):
+    _config.save(path)
 
 
 def delete(key, alias=None):
@@ -263,25 +317,28 @@ def split(string):
     return section, key
 
 
-class ConfigExtension(ManifestExtension):
-    def export_manifest(self, manifest, task):
-        manifest.config = get("network", "config", "", expand=False)
+def import_config(snippet: str):
+    """ Apply extra configuration for the worker, provided by the client. """
+    _manifest.read_string(snippet)
+    from jolt.loader import JoltLoader
+    JoltLoader.get().load_plugins()
+
 
-        for key, value in options("params"):
-            p = manifest.create_parameter()
-            p.key = "config." + key
-            p.value = value
+def export_config():
+    """ Get extra configuration for the worker. """
+    return get("network", "config", "", expand=False)
 
-    def import_manifest(self, manifest):
-        if manifest.config:
-            _manifest.read_string(manifest.config)
-            from jolt.loader import JoltLoader
-            JoltLoader.get().load_plugins()
 
-        for param in manifest.parameters:
-            if param.key.startswith("config."):
-                set("params", param.key.split(".", 1)[1], param.value)
+def import_params(params: dict):
+    """ Apply user-defined parameters (-c params.key=value). """
+    for key, value in params.items():
+        if key.startswith("config."):
+            set("params", key.split(".", 1)[1], value)
 
 
-# High priority so that plugins are loaded before resources are acquired.
-ManifestExtensionRegistry.add(ConfigExtension(), -10)
+def export_params():
+    """ Get user-defined parameters (-c params.key=value). """
+    parameters = []
+    for key, value in options("params"):
+        parameters.append(common_pb.Property(key="config." + key, value=value))
+    return parameters

jolt/error.py

@@ -4,17 +4,30 @@ from jolt import utils
 
 class JoltError(Exception):
     def __init__(self, *args, **kwargs):
-        super(JoltError, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
 
 class JoltCommandError(JoltError):
     def __init__(self, what, stdout=[], stderr=[], returncode=None, *args, **kwargs):
-        super(JoltCommandError, self).__init__(what, *args, **kwargs)
+        super().__init__(what, *args, **kwargs)
         self.stdout = stdout
         self.stderr = stderr
         self.returncode = returncode
 
 
+class JoltTimeoutError(JoltError):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def __str__(self):
+        return super().__str__() or "Timeout"
+
+
+class LoggedJoltError(JoltError):
+    def __init__(self, exc):
+        self.exc = exc
+
+
 def raise_error(msg, *args, **kwargs):
     raise JoltError(msg.format(*args, **kwargs))
 
@@ -23,8 +36,10 @@ def raise_task_error(task, msg, *args, **kwargs):
     if task:
         with utils.ignore_exception():
             with task.report() as report:
-                report.add_error("Error", inspection.getfile(task.__class__),
-                                 msg.format(*args, **kwargs))
+                report.add_error(
+                    kwargs.pop("type", "Error"),
+                    inspection.getfile(task.__class__),
+                    msg.format(*args, **kwargs))
         raise_error(msg + " (" + str(task) + ")", *args, **kwargs)
     else:
         raise_error(msg, *args, **kwargs)

jolt/filesystem.py

@@ -88,10 +88,10 @@ def move(src, dst):
 
 def onerror_warning(func, path, exc_info):
     from jolt import log
-    if type(exc_info[1]) is OSError:
+    if hasattr(exc_info[1], "strerror"):
         msg = exc_info[1].strerror
     else:
-        msg = "Reason unknown"
+        msg = "Reason unknown: " + str(exc_info[1])
     if os.path.exists(path):
         log.warning("Could not remove file or directory: {} ({})", path, msg)
 
@@ -238,7 +238,3 @@ def scandir(scanpath, filterfn=lambda path: path[0] != ".", relative=False):
             for path, dirs, files in os.walk(scanpath)
             for f in files
             if filterfn(f)]
-
-
-def get_archive(path):
-    return path + ".tar.gz"