jolt 0.9.123__py3-none-any.whl → 0.9.435__py3-none-any.whl

jolt/log.py

@@ -1,7 +1,9 @@
 from __future__ import print_function
+import glob
 import os
 import re
 import sys
+import time
 import tqdm
 if os.name == "nt":
     # FIXME: Workaround to make tqdm behave correctly on Windows
@@ -23,34 +25,40 @@ from jolt import config
 from jolt.error import JoltError
 from jolt import filesystem as fs
 from jolt import colors
+from jolt import common_pb2 as common_pb
+from jolt import utils
 
 
-default_path = fs.path.join(config.get_logpath(), "jolt.log")
-logfile = config.get("jolt", "logfile", default_path)
-logsize = config.getsize("jolt", "logsize", os.environ.get("JOLT_LOGSIZE", 10 * 1024 ** 2))  # 10MiB
-logcount = config.getint("jolt", "logcount", os.environ.get("JOLT_LOGCOUNT", 1))
+current_time = datetime.now().strftime("%Y-%m-%dT%H%M%S.%f")
+logpath = config.get_logpath()
+logfile = fs.path.join(logpath, f"{current_time}.log")
+logcount = config.getint("jolt", "logcount", os.environ.get("JOLT_LOGCOUNT", 100))
+logfiles = list(sorted(glob.glob(os.path.join(logpath, "*T*.log"))))
 
 dirpath = fs.path.dirname(logfile)
 if not fs.path.exists(dirpath):
     fs.makedirs(dirpath)
-with open(logfile, "a") as f:
-    f.write("--------------------------------------------------------------------------------\n")
 
 ################################################################################
 
-ERROR = logging.ERROR
-WARNING = logging.WARNING
-INFO = logging.INFO
-VERBOSE = 15
-DEBUG = logging.DEBUG
-EXCEPTION = logging.DEBUG + 1
-STDOUT = logging.INFO + 1
-STDERR = logging.ERROR + 1
-SILENCE = STDERR + 1
+ERROR = common_pb.LogLevel.ERROR + 1
+WARNING = common_pb.LogLevel.WARNING + 1
+INFO = common_pb.LogLevel.INFO + 1
+VERBOSE = common_pb.LogLevel.VERBOSE + 1
+DEBUG = common_pb.LogLevel.DEBUG + 1
+EXCEPTION = common_pb.LogLevel.EXCEPTION + 1
+STDOUT = common_pb.LogLevel.STDOUT + 1
+STDERR = common_pb.LogLevel.STDERR + 1
+SILENCE = STDERR + 2
+
 logging.addLevelName(VERBOSE, "VERBOSE")
 logging.addLevelName(STDOUT, "STDOUT")
 logging.addLevelName(STDERR, "STDERR")
 logging.addLevelName(EXCEPTION, "EXCEPT")
+logging.addLevelName(WARNING, "WARNING")
+logging.addLevelName(INFO, "INFO")
+logging.addLevelName(ERROR, "ERROR")
+logging.addLevelName(DEBUG, "DEBUG")
 
 logging.raiseExceptions = False
 
@@ -135,13 +143,37 @@ class TqdmStream(object):
         getattr(self.stream, 'flush', lambda: None)()
 
 
+class ThreadPrefix(logging.LoggerAdapter):
+    thread_prefix = {}
+
+    def process(self, msg, kwargs):
+        tid = threading.current_thread()
+        if tid in self.thread_prefix:
+            msg = f"[{self.thread_prefix[tid]}] {msg}"
+        return msg, kwargs
+
+    def set_thread_prefix(self, prefix):
+        tid = threading.current_thread()
+        self.thread_prefix[tid] = prefix
+
+    def clear_thread_prefix(self):
+        tid = threading.current_thread()
+        if tid in self.thread_prefix:
+            del self.thread_prefix[tid]
+
+
 # silence root logger
 _root = logging.getLogger()
 _root.setLevel(logging.CRITICAL)
 
-# create jolt logger
+# create jolt logger and protect it's methods against interrupts
 _logger = logging.getLogger('jolt')
-_logger.setLevel(logging.DEBUG)
+_logger.setLevel(EXCEPTION)
+_logger.handle = utils.delay_interrupt(_logger.handle)
+_logger.log = utils.delay_interrupt(_logger.log)
+_logger.addHandler = utils.delay_interrupt(_logger.addHandler)
+_logger.removeHandler = utils.delay_interrupt(_logger.removeHandler)
+
 
 _console_formatter = ConsoleFormatter('[{levelname:>7}] {message}', '{message}')
 
@@ -149,91 +181,134 @@ if sys.stdout.isatty() and sys.stderr.isatty():
     _stdout = logging.StreamHandler(TqdmStream(sys.stdout))
 else:
     _stdout = logging.StreamHandler(sys.stdout)
-_stdout.setLevel(INFO)
 _stdout.setFormatter(_console_formatter)
 _stdout.addFilter(Filter(lambda r: r.levelno < ERROR))
+_stdout.addFilter(Filter(lambda r: r.levelno != EXCEPTION))
 
 if sys.stdout.isatty() and sys.stderr.isatty():
     _stderr = logging.StreamHandler(TqdmStream(sys.stdout))
 else:
     _stderr = logging.StreamHandler(sys.stderr)
-_stderr.setLevel(INFO)
 _stderr.setFormatter(_console_formatter)
-_stderr.addFilter(Filter(lambda r: r.levelno >= ERROR))
-_stderr.addFilter(Filter(lambda r: r.levelno != EXCEPTION))
-
-_file = logging.handlers.RotatingFileHandler(logfile, maxBytes=logsize, backupCount=logcount)
-_file.setLevel(logging.DEBUG)
-_file_formatter = Formatter('{asctime} [{levelname:>7}] {message}')
-_file.setFormatter(_file_formatter)
+_stderr.addFilter(Filter(lambda r: r.levelno >= ERROR or r.levelno == EXCEPTION))
 
 _logger.addHandler(_stdout)
 _logger.addHandler(_stderr)
-_logger.addHandler(_file)
+_logger_frontend = ThreadPrefix(_logger, {})
+
+_file_formatter = Formatter('{asctime} [{levelname:>7}] {message}')
+
+
+def start_file_log():
+    global logfiles
+
+    if len(logfiles) >= logcount:
+        outdated = logfiles[:len(logfiles) - logcount + 1]
+        logfiles = logfiles[-logcount + 1:]
+        for file in outdated:
+            os.unlink(file)
+
+    _file = logging.FileHandler(logfile)
+    _file.setLevel(EXCEPTION)
+    _file.setFormatter(_file_formatter)
+    _logger.addHandler(_file)
+
+
+def log(level, message, created=None, context=None, prefix=False):
+    created = created or time.time()
+    message = f"[{context}] {message}" if context else message
+    record = logging.LogRecord(
+        name="log",
+        level=level,
+        pathname=__file__,
+        lineno=0,
+        msg=message,
+        args={},
+        exc_info=None,
+    )
+    record.created = created
+    record.prefix = prefix
+    _logger.handle(record)
 
 
 def info(fmt, *args, **kwargs):
-    _logger.info(fmt, *args, **kwargs)
+    _logger_frontend.log(INFO, fmt, *args, **kwargs)
 
 
 def warning(fmt, *args, **kwargs):
-    _logger.warning(fmt, *args, **kwargs)
+    _logger_frontend.log(WARNING, fmt, *args, **kwargs)
 
 
 def verbose(fmt, *args, **kwargs):
-    _logger.log(VERBOSE, fmt, *args, **kwargs)
+    _logger_frontend.log(VERBOSE, fmt, *args, **kwargs)
 
 
 def debug(fmt, *args, **kwargs):
-    _logger.debug(fmt, *args, **kwargs)
+    _logger_frontend.log(DEBUG, fmt, *args, **kwargs)
 
 
 def error(fmt, *args, **kwargs):
-    _logger.error(fmt, *args, **kwargs)
+    _logger_frontend.log(ERROR, fmt, *args, **kwargs)
 
 
 def stdout(line, **kwargs):
     line = line.replace("{", "{{")
     line = line.replace("}", "}}")
-    _logger.log(STDOUT, line, extra=kwargs)
+    _logger_frontend.log(STDOUT, line, extra=kwargs)
 
 
 def stderr(line, **kwargs):
     line = line.replace("{", "{{")
     line = line.replace("}", "}}")
-    _logger.log(STDERR, line, extra=kwargs)
+    _logger_frontend.log(STDERR, line, extra=kwargs)
+
+
+def format_exception_msg(exc):
+    te = traceback.TracebackException.from_exception(exc)
 
+    if isinstance(exc, JoltError):
+        return str(exc)
 
-def exception(exc=None):
+    elif isinstance(exc, SyntaxError):
+        filename = fs.path.relpath(
+            te.filename,
+            fs.path.commonprefix([os.getcwd(), te.filename]))
+        return "SyntaxError: {} ({}, line {})".format(
+            te.text.strip(),
+            filename,
+            te.lineno)
+
+    else:
+        filename = fs.path.relpath(
+            te.stack[-1].filename,
+            fs.path.commonprefix([os.getcwd(), te.stack[-1].filename]))
+        return "{}: {} ({}, line {}, in {})".format(
+            type(exc).__name__,
+            str(exc) or te.stack[-1].line,
+            filename,
+            te.stack[-1].lineno,
+            te.stack[-1].name)
+
+
+def exception(exc=None, error=True):
     if exc:
-        te = traceback.TracebackException.from_exception(exc)
-        if isinstance(exc, JoltError):
-            _logger.error("{}", str(exc))
-        elif isinstance(exc, SyntaxError):
-            filename = fs.path.relpath(
-                te.filename,
-                fs.path.commonprefix([os.getcwd(), te.filename]))
-            _logger.error("SyntaxError: {} ({}, line {})",
-                          te.text.strip(),
-                          filename,
-                          te.lineno)
-        else:
-            filename = fs.path.relpath(
-                te.stack[-1].filename,
-                fs.path.commonprefix([os.getcwd(), te.stack[-1].filename]))
-            _logger.error("{}: {} ({}, line {}, in {})",
-                          type(exc).__name__,
-                          str(exc) or te.stack[-1].line,
-                          filename,
-                          te.stack[-1].lineno,
-                          te.stack[-1].name)
-        backtrace = traceback.format_exc().splitlines()
+        if error:
+            _logger_frontend.log(ERROR, format_exception_msg(exc))
+
+        tb = traceback.format_exception(type(exc), value=exc, tb=exc.__traceback__)
+        installdir = fs.path.dirname(__file__)
+        if any(map(lambda frame: installdir not in frame, tb[1:-1])):
+            while len(tb) > 2 and installdir in tb[1]:
+                del tb[1]
+        backtrace = "".join(tb).splitlines()
     else:
         backtrace = traceback.format_exc().splitlines()
+
     for line in backtrace:
         line = line.replace("{", "{{")
         line = line.replace("}", "}}")
-        _logger.log(EXCEPTION, line.strip())
+        line = line.strip()
+        _logger_frontend.log(EXCEPTION, line)
 
 
 def transfer(line, context):
@@ -242,6 +317,8 @@ def transfer(line, context):
     outline2 = context + line
     if line.startswith("[  ERROR]"):
         error(outline1)
+    elif line.startswith("[WARNING]"):
+        warning(outline1)
     elif line.startswith("[VERBOSE]"):
         verbose(outline1)
     elif line.startswith("[  DEBUG]"):
@@ -251,7 +328,7 @@ def transfer(line, context):
     elif line.startswith("[ EXCEPT]"):
         outline1 = outline1.replace("{", "{{")
         outline1 = outline1.replace("}", "}}")
-        _logger.log(EXCEPTION, outline1)
+        _logger_frontend.log(EXCEPTION, outline1)
     elif line.startswith("[ STDERR]"):
         stderr(outline1, prefix=True)
     elif line.startswith("[ STDOUT]"):
@@ -281,26 +358,90 @@ class _Progress(object):
     def update(self, *args, **kwargs):
         pass
 
+    def refresh(self, *args, **kwargs):
+        pass
+
+    def reset(self, *args, **kwargs):
+        pass
+
+    def set_description(self, *args, **kwargs):
+        pass
+
 
 def progress_log(desc, count, unit):
     return _Progress(desc)
 
 
 def progress(desc, count, unit, estimates=True, debug=False):
-    bar_format = '{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}]' \
-                 if not estimates else None
+    if count:
+        bar_format = '{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}]' \
+                     if not estimates else None
+    else:
+        bar_format = '{desc}{n_fmt}{unit} [{elapsed}]'
     if not debug and is_interactive() and not is_verbose():
-        p = tqdm.tqdm(total=count, unit=unit, unit_scale=True, bar_format=bar_format, dynamic_ncols=True)
-        p.set_description("[   INFO] " + desc)
+        class ProgressTqdm(tqdm.tqdm):
+            def __init__(self, *args, **kwargs):
+                super(ProgressTqdm, self).__init__(*args, **kwargs)
+
+            def set_description(self, desc=None, refresh=True):
+                return super().set_description("[   INFO] " + desc, refresh)
+
+        p = ProgressTqdm(total=count, unit=unit, unit_scale=True, bar_format=bar_format, dynamic_ncols=True)
+        p.set_description(desc)
         return p
     return progress_log(desc, count, unit)
 
 
+_level = INFO
+
+
+def level_to_pb(level):
+    """ Convert a log level to a protobuf enum value. """
+    return level - 1
+
+
+def pb_to_level(level):
+    """ Convert a protobuf enum value to a log level. """
+    return level + 1
+
+
 def set_level(level):
+    """ Set the log level for terminal output. """
+
+    if level not in [
+        DEBUG,
+        ERROR,
+        EXCEPTION,
+        INFO,
+        SILENCE,
+        STDERR,
+        STDOUT,
+        VERBOSE,
+        WARNING,
+    ]:
+        raise ValueError("invalid log level")
+
+    global _level
+    _level = level
     _stdout.setLevel(level)
     _stderr.setLevel(level)
 
 
+def set_level_pb(level):
+    """ Set the log level from a protobuf enum value. """
+    set_level(pb_to_level(level))
+
+
+def get_level():
+    global _level
+    return _level
+
+
+def get_level_pb():
+    """ Get the log level as a protobuf enum value. """
+    return level_to_pb(_level)
+
+
 def set_worker():
     _console_formatter.enable_prefixes()
 
@@ -361,6 +502,15 @@ def threadsink(level=DEBUG):
         _logger.removeHandler(handler)
 
 
+@contextmanager
+def handler(h):
+    _logger.addHandler(h)
+    try:
+        yield
+    finally:
+        _logger.removeHandler(h)
+
+
 @contextmanager
 def map_thread(thread_from, thread_to):
     tid = thread_from.ident
@@ -371,6 +521,15 @@ def map_thread(thread_from, thread_to):
         _thread_map.unmap(tid)
 
 
+@contextmanager
+def thread_prefix(prefix):
+    try:
+        _logger_frontend.set_thread_prefix(prefix)
+        yield
+    finally:
+        _logger_frontend.clear_thread_prefix()
+
+
 class _LogStream(object):
     def __init__(self):
         self.buf = ""
@@ -396,3 +555,6 @@ class _LogStream(object):
 @contextmanager
 def stream():
     yield _LogStream()
+
+
+set_level(STDOUT)

jolt/manifest.py

@@ -40,6 +40,14 @@ class _JoltProjectModule(SubElement):
     def __init__(self, elem=None):
         super(_JoltProjectModule, self).__init__('module', elem=elem)
 
+    @property
+    def path(self):
+        return self.src
+
+    @path.setter
+    def path(self, value):
+        self.src = value
+
 
 @Attribute('name')
 class _JoltProjectResource(SubElement):
@@ -70,6 +78,7 @@ class _JoltTaskError(SubElement):
 @Attribute('duration', child=True)
 @Attribute('goal', child=True)
 @Attribute('identity', child=True)
+@Attribute('instance', child=True)
 @Attribute('logstash', child=True)
 @Attribute('result', child=True)
 @Composition(_JoltAttribute, "attribute")
@@ -104,7 +113,9 @@ class _JoltNetworkParameter(SubElement):
 @Attribute("stderr", child=True, zlib=True)
 @Attribute("result", child=True)
 @Attribute("duration", child=True)
+@Attribute("name")
 @Attribute("workspace")
+@Attribute("build")
 @Attribute("version")
 @Composition(_JoltRecipe, "recipe")
 @Composition(_JoltTask, "task")
@@ -118,8 +129,10 @@ class JoltManifest(ElementTree):
         self._elem = self.getroot()
         self.path = None
 
-    @property
-    def joltdir(self):
+    def is_valid(self):
+        return self.path is not None
+
+    def get_workspace_path(self):
         if self.path is None:
             return None
         joltdir = fs.path.dirname(self.path)
@@ -127,6 +140,13 @@ class JoltManifest(ElementTree):
             joltdir = fs.path.normpath(fs.path.join(joltdir, self.workspace))
         return joltdir
 
+    def get_workspace_name(self):
+        if self.name:
+            return self.name
+        if self.path is None:
+            return None
+        return fs.path.basename(fs.path.dirname(self.path))
+
     @property
     def attrib(self):
         return self.getroot().attrib
@@ -195,6 +215,12 @@ class JoltManifest(ElementTree):
             return True
         return self.find("./task[result='FAILED']") is not None
 
+    def has_unstable(self):
+        return self.find("./task[result='UNSTABLE']") is not None
+
+    def has_tasks(self):
+        return self.find("./task") is not None
+
     def find_task(self, task):
         match = self.find("./task[@name='{0}']".format(task))
         if match is None:
@@ -215,39 +241,3 @@ class JoltManifest(ElementTree):
         for manifest_task in self.tasks:
             self._identities[manifest_task.name] = manifest_task.identity
         return self._identities
-
-    @staticmethod
-    def export(task):
-        manifest = JoltManifest()
-        ManifestExtensionRegistry.export_manifest(manifest, task)
-        return manifest
-
-    def process_import(self):
-        ManifestExtensionRegistry.import_manifest(self)
-
-
-class ManifestExtensionRegistry(object):
-    extensions = []
-
-    @staticmethod
-    def add(extension, priority=0):
-        ManifestExtensionRegistry.extensions.append((extension, priority))
-        ManifestExtensionRegistry.extensions.sort(key=lambda x: x[1])
-
-    @staticmethod
-    def export_manifest(manifest, task):
-        for extension, _ in ManifestExtensionRegistry.extensions:
-            extension.export_manifest(manifest, task)
-
-    @staticmethod
-    def import_manifest(manifest):
-        for extension, _ in ManifestExtensionRegistry.extensions:
-            extension.import_manifest(manifest)
-
-
-class ManifestExtension(object):
-    def export_manifest(self, manifest, task):
-        pass
-
-    def import_manifest(self, manifest):
-        pass

jolt/options.py

@@ -1,14 +1,39 @@
 
 class JoltOptions(object):
+    """ Jolt options that control the behavior of builds. """
+
+    debug = False
+    """ Enable debug mode. Break into debugger on exceptions. """
+
+    download = True
+    """ Enable downloading of remote artifacts, both session and persistent. """
+
+    download_session = True
+    """ Enable downloading of remote session artifacts. """
+
+    keep_going = False
+    """ Keep going with the build after a task fails. """
+
+    local = False
+    """ Disable network access. """
+
+    network = False
+    """ Distribute tasks to workers. """
+
+    upload = True
+    """ Enable uploading of artifacts. """
+
+    worker = False
+    """ Running as a worker. """
+
+    salt = None
+    """ Salt for hashing (--salt). """
+
+    jobs = 1
+    """ Number of concurrent local tasks to run (1). """
+
+    mute = False
+    """ Mute task output, until a task fails. """
+
     def __init__(self, **kwargs):
-        self.debug = False
-        self.default = []
-        self.download = True
-        self.keep_going = False
-        self.local = False
-        self.network = False
-        self.upload = True
-        self.worker = False
-        self.salt = None
-        self.jobs = 1
         self.__dict__.update(kwargs)

jolt/pkgs/abseil.py

@@ -0,0 +1,42 @@
+from jolt import attributes, BooleanParameter, Parameter, Task
+from jolt.plugins import cmake, git
+from jolt.tasks import TaskRegistry
+import os
+
+
+@attributes.requires("requires_git")
+@attributes.system
+@cmake.requires()
+@cmake.use_ninja()
+class Abseil(cmake.CMake):
+    """ Abseil C++ Common Libraries """
+
+    name = "abseil"
+    version = Parameter("20250814.1")
+    shared = BooleanParameter(False, help="Build shared libraries")
+    requires_git = ["git:url=https://github.com/abseil/abseil-cpp.git,rev={version}"]
+    srcdir = "{git[abseil-cpp]}"
+    options = [
+        "ABSL_MSVC_STATIC_RUNTIME=OFF",
+        "BUILD_SHARED_LIBS={shared[ON,OFF]}",
+        "CMAKE_CXX_STANDARD=17",
+    ]
+
+    def publish(self, artifact, tools):
+        super().publish(artifact, tools)
+        artifact.cxxinfo.crt = "Dynamic"
+        artifact.cxxinfo.incpaths.append("include")
+        artifact.cxxinfo.libpaths.append("lib")
+        if self.shared:
+            artifact.environ.LD_LIBRARY_PATH.append("lib")
+
+        with tools.cwd(artifact.path, "lib"):
+            for libfile in tools.glob("*.lib"):
+                libname, _ = os.path.splitext(libfile)
+                artifact.cxxinfo.libraries.append(libname)
+            for libfile in tools.glob("lib*.a"):
+                libname, _ = os.path.splitext(libfile)
+                artifact.cxxinfo.libraries.append(libname[3:])
+
+
+TaskRegistry.get().add_task_class(Abseil)

jolt/pkgs/asio.py

@@ -0,0 +1,25 @@
+from jolt import attributes, Parameter
+from jolt.plugins import cmake, git, autotools, pkgconfig
+from jolt.tasks import TaskRegistry
+
+
+@attributes.requires("requires_git")
+@autotools.requires()
+@pkgconfig.requires()
+@pkgconfig.to_cxxinfo("asio")
+class Asio(autotools.Autotools):
+    name = "asio"
+    version = Parameter("1.36.0", help="asio version.")
+    requires_git = ["git:url=https://github.com/chriskohlhoff/asio.git,rev=asio-{_version}"]
+    srcdir = "{git[asio]}/asio"
+
+    @property
+    def _version(self):
+        return str(self.version).replace(".", "-")
+
+    def publish(self, artifact, tools):
+        super().publish(artifact, tools)
+        artifact.environ.CMAKE_PREFIX_PATH.append(".")
+
+
+TaskRegistry.get().add_task_class(Asio)