karton-core 5.7.0__py3-none-any.whl → 5.9.0__py3-none-any.whl

karton/core/base.py

@@ -9,32 +9,16 @@ from typing import Optional, Union, cast
 from .__version__ import __version__
 from .backend import KartonBackend, KartonServiceInfo
 from .config import Config
-from .logger import KartonLogHandler
-from .task import Task
+from .logger import KartonLogHandler, TaskContextFilter
+from .task import Task, get_current_task, set_current_task
 from .utils import HardShutdownInterrupt, StrictClassMethod, graceful_killer
 
 
-class KartonBase(abc.ABC):
-    """
-    Base class for all Karton services
-
-    You can set an informative version information by setting the ``version`` class
-    attribute.
-    """
-
-    #: Karton service identity
-    identity: str = ""
-    #: Karton service version
-    version: Optional[str] = None
-    #: Include extended service information for non-consumer services
-    with_service_info: bool = False
+class ConfigMixin:
+    identity: Optional[str]
+    version: Optional[str]
 
-    def __init__(
-        self,
-        config: Optional[Config] = None,
-        identity: Optional[str] = None,
-        backend: Optional[KartonBackend] = None,
-    ) -> None:
+    def __init__(self, config: Optional[Config] = None, identity: Optional[str] = None):
         self.config = config or Config()
         self.enable_publish_log = self.config.getboolean(
             "logging", "enable_publish", True
@@ -50,25 +34,81 @@ class KartonBase(abc.ABC):
 
         self.debug = self.config.getboolean("karton", "debug", False)
 
-        if self.debug:
+        if self.debug and self.identity:
             self.identity += "-" + os.urandom(4).hex() + "-dev"
 
-        self.service_info = None
-        if self.identity is not None and self.with_service_info:
-            self.service_info = KartonServiceInfo(
-                identity=self.identity,
-                karton_version=__version__,
-                service_version=self.version,
-            )
+    @classmethod
+    def args_description(cls) -> str:
+        """Return short description for argument parser."""
+        if not cls.__doc__:
+            return ""
+        return textwrap.dedent(cls.__doc__).strip().splitlines()[0]
 
-        self.backend = backend or KartonBackend(
-            self.config, identity=self.identity, service_info=self.service_info
+    @classmethod
+    def args_parser(cls) -> argparse.ArgumentParser:
+        """
+        Return ArgumentParser for main() class method.
+
+        This method should be overridden and call super methods
+        if you want to add more arguments.
+        """
+        parser = argparse.ArgumentParser(description=cls.args_description())
+        parser.add_argument(
+            "--version", action="version", version=cast(str, cls.version)
+        )
+        parser.add_argument("--config-file", help="Alternative configuration path")
+        parser.add_argument(
+            "--identity", help="Alternative identity for Karton service"
         )
+        parser.add_argument("--log-level", help="Logging level of Karton logger")
+        parser.add_argument(
+            "--debug", help="Enable debugging mode", action="store_true", default=None
+        )
+        return parser
+
+    @classmethod
+    def config_from_args(cls, config: Config, args: argparse.Namespace) -> None:
+        """
+        Updates configuration with settings from arguments
 
-        self._log_handler = KartonLogHandler(
-            backend=self.backend, channel=self.identity
+        This method should be overridden and call super methods
+        if you want to add more arguments.
+        """
+        config.load_from_dict(
+            {
+                "karton": {
+                    "identity": args.identity,
+                    "debug": args.debug,
+                },
+                "logging": {"level": args.log_level},
+            }
         )
-        self.current_task: Optional[Task] = None
+
+    @classmethod
+    def karton_from_args(cls, args: Optional[argparse.Namespace] = None):
+        """
+        Returns Karton instance configured using configuration files
+        and provided arguments
+
+        Used by :py:meth:`KartonServiceBase.main` method
+        """
+        if args is None:
+            parser = cls.args_parser()
+            args = parser.parse_args()
+        config = Config(path=args.config_file)
+        cls.config_from_args(config, args)
+        return cls(config=config)
+
+
+class LoggingMixin:
+    config: Config
+    identity: Optional[str]
+    debug: bool
+    enable_publish_log: bool
+
+    def __init__(self, log_handler: logging.Handler, log_format: str) -> None:
+        self._log_handler = log_handler
+        self._log_format = log_format
 
     def setup_logger(self, level: Optional[Union[str, int]] = None) -> None:
         """
@@ -91,10 +131,11 @@ class KartonBase(abc.ABC):
         if not self.identity:
             raise ValueError("Can't setup logger without identity")
 
+        task_context_filter = TaskContextFilter()
         self._log_handler.setFormatter(logging.Formatter())
+        self._log_handler.addFilter(task_context_filter)
 
         logger = logging.getLogger(self.identity)
-
         if logger.handlers:
             # If logger already have handlers set: clear them
             logger.handlers.clear()
@@ -106,16 +147,15 @@ class KartonBase(abc.ABC):
 
         logger.setLevel(log_level)
         stream_handler = logging.StreamHandler()
-        stream_handler.setFormatter(
-            logging.Formatter("[%(asctime)s][%(levelname)s] %(message)s")
-        )
+        stream_handler.setFormatter(logging.Formatter(self._log_format))
+        stream_handler.addFilter(task_context_filter)
         logger.addHandler(stream_handler)
 
         if not self.debug and self.enable_publish_log:
             logger.addHandler(self._log_handler)
 
     @property
-    def log_handler(self) -> KartonLogHandler:
+    def log_handler(self) -> logging.Handler:
         """
         Return KartonLogHandler bound to this Karton service.
 
@@ -141,67 +181,54 @@ class KartonBase(abc.ABC):
         """
         return logging.getLogger(self.identity)
 
-    @classmethod
-    def args_description(cls) -> str:
-        """Return short description for argument parser."""
-        if not cls.__doc__:
-            return ""
-        return textwrap.dedent(cls.__doc__).strip().splitlines()[0]
 
-    @classmethod
-    def args_parser(cls) -> argparse.ArgumentParser:
-        """
-        Return ArgumentParser for main() class method.
+class KartonBase(abc.ABC, ConfigMixin, LoggingMixin):
+    """
+    Base class for all Karton services
 
-        This method should be overridden and call super methods
-        if you want to add more arguments.
-        """
-        parser = argparse.ArgumentParser(description=cls.args_description())
-        parser.add_argument(
-            "--version", action="version", version=cast(str, cls.version)
-        )
-        parser.add_argument("--config-file", help="Alternative configuration path")
-        parser.add_argument(
-            "--identity", help="Alternative identity for Karton service"
-        )
-        parser.add_argument("--log-level", help="Logging level of Karton logger")
-        parser.add_argument(
-            "--debug", help="Enable debugging mode", action="store_true", default=None
-        )
-        return parser
+    You can set an informative version information by setting the ``version`` class
+    attribute.
+    """
 
-    @classmethod
-    def config_from_args(cls, config: Config, args: argparse.Namespace) -> None:
-        """
-        Updates configuration with settings from arguments
+    #: Karton service identity
+    identity: str = ""
+    #: Karton service version
+    version: Optional[str] = None
+    #: Include extended service information for non-consumer services
+    with_service_info: bool = False
 
-        This method should be overridden and call super methods
-        if you want to add more arguments.
-        """
-        config.load_from_dict(
-            {
-                "karton": {
-                    "identity": args.identity,
-                    "debug": args.debug,
-                },
-                "logging": {"level": args.log_level},
-            }
+    def __init__(
+        self,
+        config: Optional[Config] = None,
+        identity: Optional[str] = None,
+        backend: Optional[KartonBackend] = None,
+    ) -> None:
+        ConfigMixin.__init__(self, config, identity)
+
+        self.service_info = None
+        if self.identity is not None and self.with_service_info:
+            self.service_info = KartonServiceInfo(
+                identity=self.identity,
+                karton_version=__version__,
+                service_version=self.version,
+            )
+
+        self.backend = backend or KartonBackend(
+            self.config, identity=self.identity, service_info=self.service_info
         )
 
-    @classmethod
-    def karton_from_args(cls, args: Optional[argparse.Namespace] = None):
-        """
-        Returns Karton instance configured using configuration files
-        and provided arguments
+        log_handler = KartonLogHandler(backend=self.backend, channel=self.identity)
+        LoggingMixin.__init__(
+            self, log_handler, log_format="[%(asctime)s][%(levelname)s] %(message)s"
+        )
 
-        Used by :py:meth:`KartonServiceBase.main` method
-        """
-        if args is None:
-            parser = cls.args_parser()
-            args = parser.parse_args()
-        config = Config(path=args.config_file)
-        cls.config_from_args(config, args)
-        return cls(config=config)
+    @property
+    def current_task(self) -> Optional[Task]:
+        return get_current_task()
+
+    @current_task.setter
+    def current_task(self, task: Optional[Task]):
+        set_current_task(task)
 
 
 class KartonServiceBase(KartonBase):

karton/core/config.py

@@ -14,6 +14,7 @@ class Config(object):
     - ``/etc/karton/karton.ini`` (global)
     - ``~/.config/karton/karton.ini`` (user local)
     - ``./karton.ini`` (subsystem local)
+    - path from ``KARTON_CONFIG_FILE`` environment variable
     - ``<path>`` optional, additional path provided in arguments
 
     It is also possible to pass configuration via environment variables.
@@ -38,6 +39,12 @@ class Config(object):
     ) -> None:
         self._config: Dict[str, Dict[str, Any]] = {}
 
+        path_from_env = os.getenv("KARTON_CONFIG_FILE")
+        if path_from_env:
+            if not os.path.isfile(path_from_env):
+                raise IOError(f"Configuration file not found in {path_from_env}")
+            self.SEARCH_PATHS = self.SEARCH_PATHS + [path_from_env]
+
         if path is not None:
             if not os.path.isfile(path):
                 raise IOError("Configuration file not found in " + path)
@@ -116,6 +123,11 @@ class Config(object):
     @overload
     def getint(self, section_name: str, option_name: str) -> Optional[int]: ...
 
+    @overload
+    def getint(
+        self, section_name: str, option_name: str, fallback: Optional[int]
+    ) -> Optional[int]: ...
+
     def getint(
         self, section_name: str, option_name: str, fallback: Optional[int] = None
     ) -> Optional[int]:
@@ -217,7 +229,7 @@ class Config(object):
         for name, value in os.environ.items():
             # Load env variables named KARTON_[section]_[key]
             # to match ConfigParser structure
-            result = re.fullmatch(r"KARTON_([A-Z0-9-]+)_([A-Z0-9_]+)", name)
+            result = re.fullmatch(r"KARTON_([A-Z0-9\-\.]+)_([A-Z0-9_]+)", name)
 
             if not result:
                 continue

karton/core/karton.py

@@ -80,13 +80,8 @@ class Producer(KartonBase):
         task.last_update = time.time()
         task.headers.update({"origin": self.identity})
 
-        # Ensure all local resources have good buckets
-        for resource in task.iterate_resources():
-            if isinstance(resource, LocalResource) and not resource.bucket:
-                resource.bucket = self.backend.default_bucket_name
-
         # Register new task
-        self.backend.register_task(task)
+        self.backend.declare_task(task)
 
         # Upload local resources
         for resource in task.iterate_resources():
@@ -137,7 +132,7 @@ class Consumer(KartonServiceBase):
         )
         if self.task_timeout is None:
             self.task_timeout = self.config.getint("karton", "task_timeout")
-        self.current_task: Optional[Task] = None
+
         self._pre_hooks: List[Tuple[Optional[str], Callable[[Task], None]]] = []
         self._post_hooks: List[
             Tuple[
@@ -170,19 +165,22 @@ class Consumer(KartonServiceBase):
         """
 
         self.current_task = task
-        self.log_handler.set_task(self.current_task)
 
-        if not self.current_task.matches_filters(self.filters):
-            self.log.info("Task rejected because binds are no longer valid.")
-            self.backend.set_task_status(self.current_task, TaskState.FINISHED)
+        if not task.matches_filters(self.filters):
+            self.log.info(
+                "Task rejected because binds are no longer valid. "
+                "Rejected ask headers: %s",
+                task.headers,
+            )
+            self.backend.set_task_status(task, TaskState.FINISHED)
             # Task rejected: end of processing
             return
 
         exception_str = None
 
         try:
-            self.log.info("Received new task - %s", self.current_task.uid)
-            self.backend.set_task_status(self.current_task, TaskState.STARTED)
+            self.log.info("Received new task - %s", task.uid)
+            self.backend.set_task_status(task, TaskState.STARTED)
 
             self._run_pre_hooks()
 
@@ -190,22 +188,22 @@ class Consumer(KartonServiceBase):
             try:
                 if self.task_timeout:
                     with timeout(self.task_timeout):
-                        self.process(self.current_task)
+                        self.process(task)
                 else:
-                    self.process(self.current_task)
+                    self.process(task)
             except (Exception, TaskTimeoutError) as exc:
                 saved_exception = exc
                 raise
             finally:
                 self._run_post_hooks(saved_exception)
 
-            self.log.info("Task done - %s", self.current_task.uid)
+            self.log.info("Task done - %s", task.uid)
         except (Exception, TaskTimeoutError):
             exc_info = sys.exc_info()
             exception_str = traceback.format_exception(*exc_info)
 
             self.backend.increment_metrics(KartonMetrics.TASK_CRASHED, self.identity)
-            self.log.exception("Failed to process task - %s", self.current_task.uid)
+            self.log.exception("Failed to process task - %s", task.uid)
         finally:
             self.backend.increment_metrics(KartonMetrics.TASK_CONSUMED, self.identity)
 
@@ -215,9 +213,10 @@ class Consumer(KartonServiceBase):
             # if an exception was caught while processing
             if exception_str is not None:
                 task_state = TaskState.CRASHED
-                self.current_task.error = exception_str
+                task.error = exception_str
 
-            self.backend.set_task_status(self.current_task, task_state)
+            self.backend.set_task_status(task, task_state)
+            self.current_task = None
 
     @property
     def _bind(self) -> KartonBind:
@@ -228,6 +227,7 @@ class Consumer(KartonServiceBase):
             filters=self.filters,
             persistent=self.persistent,
             service_version=self.__class__.version,
+            is_async=False,
         )
 
     @classmethod
@@ -338,15 +338,28 @@ class Consumer(KartonServiceBase):
         if not old_bind:
             self.log.info("Service binds created.")
         elif old_bind != self._bind:
-            self.log.info("Binds changed, old service instances should exit soon.")
+            self.log.info(
+                "Binds changed, old service instances should exit soon. "
+                "Old binds: %s "
+                "New binds: %s",
+                old_bind,
+                self._bind,
+            )
 
         for task_filter in self.filters:
             self.log.info("Binding on: %s", task_filter)
 
         with self.graceful_killer():
             while not self.shutdown:
-                if self.backend.get_bind(self.identity) != self._bind:
-                    self.log.info("Binds changed, shutting down.")
+                current_bind = self.backend.get_bind(self.identity)
+                if current_bind != self._bind:
+                    self.log.info(
+                        "Binds changed, shutting down. "
+                        "Old binds: %s "
+                        "New binds: %s",
+                        self._bind,
+                        current_bind,
+                    )
                     break
                 task = self.backend.consume_routed_task(self.identity)
                 if task:

karton/core/logger.py

@@ -2,30 +2,32 @@ import logging
 import platform
 import traceback
 import warnings
-from typing import Optional
+from typing import Any, Callable, Dict
 
 from .backend import KartonBackend
-from .task import Task
+from .task import get_current_task
 
 HOSTNAME = platform.node()
 
 
-class KartonLogHandler(logging.Handler):
+class TaskContextFilter(logging.Filter):
     """
-    logging.Handler that passes logs to the Karton backend.
+    This is a filter which injects information about current task ID to the log.
     """
 
-    def __init__(self, backend: KartonBackend, channel: str) -> None:
-        logging.Handler.__init__(self)
-        self.backend = backend
-        self.task: Optional[Task] = None
-        self.is_consumer_active: bool = True
-        self.channel: str = channel
+    def filter(self, record: logging.LogRecord) -> bool:
+        current_task = get_current_task()
+        if current_task is not None:
+            record.task_id = current_task.task_uid
+        else:
+            record.task_id = "(no task)"
+        return True
 
-    def set_task(self, task: Task) -> None:
-        self.task = task
 
-    def emit(self, record: logging.LogRecord) -> None:
+class LogLineFormatterMixin:
+    format: Callable[[logging.LogRecord], str]
+
+    def prepare_log_line(self, record: logging.LogRecord) -> Dict[str, Any]:
         ignore_fields = [
             "args",
             "asctime",
@@ -54,11 +56,27 @@ class KartonLogHandler(logging.Handler):
         log_line["type"] = "log"
         log_line["message"] = self.format(record)
 
-        if self.task is not None:
-            log_line["task"] = self.task.serialize()
+        current_task = get_current_task()
+        if current_task is not None:
+            log_line["task"] = current_task.serialize()
 
         log_line["hostname"] = HOSTNAME
+        return log_line
+
+
+class KartonLogHandler(logging.Handler, LogLineFormatterMixin):
+    """
+    logging.Handler that passes logs to the Karton backend.
+    """
+
+    def __init__(self, backend: KartonBackend, channel: str) -> None:
+        logging.Handler.__init__(self)
+        self.backend = backend
+        self.is_consumer_active: bool = True
+        self.channel: str = channel
 
+    def emit(self, record: logging.LogRecord) -> None:
+        log_line = self.prepare_log_line(record)
         log_consumed = self.backend.produce_log(
             log_line, logger_name=self.channel, level=record.levelname
         )

karton/core/main.py

@@ -144,10 +144,23 @@ def configuration_wizard(config_filename: str) -> None:
     log.info("Saved the new configuration file in %s", os.path.abspath(config_filename))
 
 
-def print_bind_list(config: Config) -> None:
+def print_bind_list(config: Config, output_format: str) -> None:
     backend = KartonBackend(config=config)
-    for bind in backend.get_binds():
-        print(bind)
+
+    if output_format == "table":
+        # Print a human-readable table-like version
+        print(f"{'karton name':50} {'version':10} {'karton':10}")
+        print("-" * 72)
+        for bind in backend.get_binds():
+            print(
+                f"{bind.identity:50} {bind.service_version or "-":10} {bind.version:10}"
+            )
+    elif output_format == "json":
+        # Use JSONL, each line is a JSON representing next bind
+        for bind in backend.get_binds():
+            print(backend.serialize_bind(bind))
+    else:
+        raise RuntimeError(f"Invalid output format: {output_format}")
 
 
 def delete_bind(config: Config, karton_name: str) -> None:
@@ -180,7 +193,7 @@ def delete_bind(config: Config, karton_name: str) -> None:
 
 def main() -> None:
 
-    parser = argparse.ArgumentParser(description="Your red pill to the karton-verse")
+    parser = argparse.ArgumentParser(description="Karton-core management utility")
     parser.add_argument("--version", action="version", version=__version__)
     parser.add_argument("-c", "--config-file", help="Alternative configuration path")
     parser.add_argument(
@@ -189,7 +202,14 @@ def main() -> None:
 
     subparsers = parser.add_subparsers(dest="command", help="sub-command help")
 
-    subparsers.add_parser("list", help="List active karton binds")
+    list_parser = subparsers.add_parser("list", help="List active karton binds")
+    list_parser.add_argument(
+        "-o",
+        "--output",
+        help="Short, human readable output, with names and versions only.",
+        default="table",
+        choices=("table", "json"),
+    )
 
     logs_parser = subparsers.add_parser("logs", help="Start streaming logs")
     logs_parser.add_argument(
@@ -253,7 +273,7 @@ def main() -> None:
         return
 
     if args.command == "list":
-        print_bind_list(config)
+        print_bind_list(config, args.output)
     elif args.command == "delete":
         karton_name = args.identity
         print(

karton/core/resource.py

@@ -150,34 +150,7 @@ class ResourceBase(object):
         }
 
 
-class LocalResource(ResourceBase):
-    """
-    Represents local resource with arbitrary binary data e.g. file contents.
-
-    Local resources will be uploaded to object hub (S3) during
-    task dispatching.
-
-    .. code-block:: python
-
-        # Creating resource from bytes
-        sample = Resource("original_name.exe", content=b"X5O!P%@AP[4\\
-        PZX54(P^)7CC)7}$EICAR-STANDARD-ANT...")
-
-        # Creating resource from path
-        sample = Resource("original_name.exe", path="sample/original_name.exe")
-
-    :param name: Name of the resource (e.g. name of file)
-    :param content: Resource content
-    :param path: Path of file with resource content
-    :param bucket: Alternative S3 bucket for resource
-    :param metadata: Resource metadata
-    :param uid: Alternative S3 resource id
-    :param sha256: Resource sha256 hash
-    :param fd: Seekable file descriptor
-    :param _flags: Resource flags
-    :param _close_fd: Close file descriptor after upload (default: False)
-    """
-
+class LocalResourceBase(ResourceBase):
     def __init__(
         self,
         name: str,
@@ -194,7 +167,7 @@ class LocalResource(ResourceBase):
         if len(list(filter(None, [path, content, fd]))) != 1:
             raise ValueError("You must exclusively provide a path, content or fd")
 
-        super(LocalResource, self).__init__(
+        super().__init__(
             name,
             content=content,
             path=path,
@@ -247,7 +220,7 @@ class LocalResource(ResourceBase):
         bucket: Optional[str] = None,
         metadata: Optional[Dict[str, Any]] = None,
         uid: Optional[str] = None,
-    ) -> "LocalResource":
+    ) -> "LocalResourceBase":
         """
         Resource extension, allowing to pass whole directory as a zipped resource.
 
@@ -305,6 +278,35 @@ class LocalResource(ResourceBase):
                 _close_fd=True,
             )
 
+
+class LocalResource(LocalResourceBase):
+    """
+    Represents local resource with arbitrary binary data e.g. file contents.
+
+    Local resources will be uploaded to object hub (S3) during
+    task dispatching.
+
+    .. code-block:: python
+
+        # Creating resource from bytes
+        sample = Resource("original_name.exe", content=b"X5O!P%@AP[4\\
+        PZX54(P^)7CC)7}$EICAR-STANDARD-ANT...")
+
+        # Creating resource from path
+        sample = Resource("original_name.exe", path="sample/original_name.exe")
+
+    :param name: Name of the resource (e.g. name of file)
+    :param content: Resource content
+    :param path: Path of file with resource content
+    :param bucket: Alternative S3 bucket for resource
+    :param metadata: Resource metadata
+    :param uid: Alternative S3 resource id
+    :param sha256: Resource sha256 hash
+    :param fd: Seekable file descriptor
+    :param _flags: Resource flags
+    :param _close_fd: Close file descriptor after upload (default: False)
+    """
+
     def _upload(self, backend: "KartonBackend") -> None:
         """Internal function for uploading resources