ServiceX_DID_Finder_lib 3.0.1__tar.gz → 3.2.0__tar.gz

{servicex_did_finder_lib-3.0.1 → servicex_did_finder_lib-3.2.0}/PKG-INFO

@@ -1,16 +1,18 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: ServiceX_DID_Finder_lib
-Version: 3.0.1
+Version: 3.2.0
 Summary: ServiceX DID Library Routines
+License-File: LICENSE
 Author: Gordon Watts
 Author-email: gwatts@uw.edu
-Requires-Python: >=3.8.1,<4.0.0
+Requires-Python: >=3.10,<4.0
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Requires-Dist: Celery (>=5.4,<6.0)
 Requires-Dist: make-it-sync (>=1.0.0,<2.0.0)
+Requires-Dist: python-logstash (>=0.4.8,<0.5.0)
 Requires-Dist: requests (>=2.25.0,<3.0.0)

{servicex_did_finder_lib-3.0.1 → servicex_did_finder_lib-3.2.0}/pyproject.toml

@@ -1,14 +1,16 @@
 [tool.poetry]
 name = "ServiceX_DID_Finder_lib"
-version = "3.0.1"
+version = "3.2.0"
 description = "ServiceX DID Library Routines"
 authors = ["Gordon Watts <gwatts@uw.edu>"]
 
 [tool.poetry.dependencies]
-python = "^3.8.1"
+python = "^3.10"
 make-it-sync = "^1.0.0"
 requests = "^2.25.0"
 Celery= "^5.4"
+python-logstash = "^0.4.8"
+
 
 
 [tool.poetry.group.dev]

{servicex_did_finder_lib-3.0.1 → servicex_did_finder_lib-3.2.0}/src/servicex_did_finder_lib/communication.py

@@ -1,9 +1,37 @@
+# Copyright (c) 2021-2025, IRIS-HEP
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 import argparse
 from datetime import datetime
 import json
 import logging
 import time
-from typing import Any, AsyncGenerator, Callable, Dict, List, Optional
+from typing import Any, AsyncGenerator, Callable, Dict, List, Optional, Union
 import sys
 
 import pika
@@ -15,7 +43,8 @@ from servicex_did_finder_lib.util_uri import parse_did_uri
 from .servicex_adaptor import ServiceXAdapter
 
 # The type for the callback method to handle DID's, supplied by the user.
-UserDIDHandler = Callable[[str, Dict[str, Any]], AsyncGenerator[Dict[str, Any], None]]
+UserDIDHandler = Callable[[str, Dict[str, Any]], AsyncGenerator[Union[Dict[str, Any],
+                                                                      List[Dict[str, Any]]], None]]
 
 # Given name, build the RabbitMQ queue name by appending this.
 # This is backed into how ServiceX works - do not change unless it

{servicex_did_finder_lib-3.0.1 → servicex_did_finder_lib-3.2.0}/src/servicex_did_finder_lib/did_finder_app.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022, IRIS-HEP
+# Copyright (c) 2022-2025, IRIS-HEP
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -36,6 +36,7 @@ from servicex_did_finder_lib.did_logging import initialize_root_logger
 from servicex_did_finder_lib.did_summary import DIDSummary
 from servicex_did_finder_lib.servicex_adaptor import ServiceXAdapter
 from servicex_did_finder_lib.util_uri import parse_did_uri
+from servicex_did_finder_lib import exceptions
 
 # The type for the callback method to handle DID's, supplied by the user.
 # Arguments are:
@@ -102,14 +103,7 @@ class DIDFinderTask(Task):
 
             if did_info.file_count > 0:  # otherwise wait until all files arrive then limit results
                 acc.send_on(did_info.file_count)
-        except Exception:
-            # noinspection PyTypeChecker
-            self.logger.error(
-                f"Error processing DID {did}",
-                extra={"dataset_id": dataset_id},
-                exc_info=1
-            )
-        finally:
+
             elapsed_time = int((datetime.now() - start_time).total_seconds())
             servicex.put_fileset_complete(
                 {
@@ -120,6 +114,23 @@ class DIDFinderTask(Task):
                     "elapsed-time": elapsed_time,
                 }
             )
+        except Exception as e:
+            # noinspection PyTypeChecker
+            self.logger.error(
+                f"Error processing DID {did}",
+                extra={"dataset_id": dataset_id},
+                exc_info=1
+            )
+            elapsed_time = int((datetime.now() - start_time).total_seconds())
+            error_dict: dict[str, Any] = {"elapsed-time": elapsed_time,
+                                          "message": str(e)}
+            if isinstance(e, exceptions.BaseDIDFinderException):
+                error_dict["error-type"] = e.error_type
+            else:
+                error_dict["error-type"] = "internal_failure"
+            servicex.put_fileset_error(
+                error_dict
+            )
 
 
 class DIDFinderApp(Celery):

servicex_did_finder_lib-3.2.0/src/servicex_did_finder_lib/exceptions.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2025, IRIS-HEP
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# Define some exceptions to indicate various problems in DID finding
+
+class BaseDIDFinderException(Exception):
+    """ Base exception type """
+    transient: bool = True
+    error_type: str = ""
+
+
+class NoSuchDatasetException(BaseDIDFinderException):
+    """ The dataset does not exist in the catalog """
+    transient = True
+    error_type = "does_not_exist"
+
+
+class BadDatasetNameException(BaseDIDFinderException):
+    """ The specified dataset name is invalid """
+    transient = False
+    error_type = "bad_name"
+
+
+class LookupFailureException(BaseDIDFinderException):
+    """ There was a failure when looking up the dataset in the catalog """
+    transient = True
+    error_type = "internal_failure"

servicex_did_finder_lib-3.2.0/src/servicex_did_finder_lib/logstash_logging.py

@@ -0,0 +1,147 @@
+# Copyright (c) 2019-25, IRIS-HEP
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import logging
+import os
+
+import logstash
+import functools
+
+instance = os.environ.get("INSTANCE_NAME", "Unknown")
+
+
+class LogstashFormatter(logstash.formatter.LogstashFormatterBase):
+    def __init__(self, component_name=None, message_type='Logstash', tags=None, fqdn=False):
+        super().__init__(message_type, tags, fqdn)
+        self.component_name = component_name
+
+    def format(self, record):
+        message = {
+            "@timestamp": self.format_timestamp(record.created),
+            "@version": "1",
+            "message": record.getMessage(),
+            "path": record.pathname,
+            "tags": self.tags,
+            "type": self.message_type,
+            "instance": instance,
+            "component": self.component_name,
+            # Extra Fields
+            "level": record.levelname,
+        }
+
+        # Add extra fields
+        message.update(self.get_extra_fields(record))
+
+        # If exception, add debug info
+        if record.exc_info:
+            message.update(self.get_debug_fields(record))
+
+        return self.serialize(message)
+
+
+class StreamFormatter(logging.Formatter):
+    """
+    A custom formatter that adds extras.
+    Normally log messages are "level instance component msg extra: {}"
+    """
+
+    def_keys = [
+        "name",
+        "msg",
+        "args",
+        "levelname",
+        "levelno",
+        "pathname",
+        "filename",
+        "module",
+        "exc_info",
+        "exc_text",
+        "stack_info",
+        "lineno",
+        "funcName",
+        "created",
+        "msecs",
+        "relativeCreated",
+        "thread",
+        "threadName",
+        "processName",
+        "process",
+        "message",
+        "taskName",
+    ]
+
+    def format(self, record: logging.LogRecord) -> str:
+        """
+        :param record: LogRecord
+        :return: formatted log message
+        """
+
+        string = super().format(record)
+        extra = {k: v for k, v in record.__dict__.items() if k not in self.def_keys}
+        if len(extra) > 0:
+            string += " extra: " + str(extra)
+        return string
+
+
+@functools.lru_cache
+def initialize_logging(log=None, component_name: str = None, **kwargs):
+    """
+    Get a logger and initialize it so that it outputs the correct format
+    :param component_name: Name of the component to insert into log messages
+    :param log: optional logger to initialize
+    :return: logger with the correct formatting that outputs to the console
+    """
+
+    if log is None:
+        log = logging.getLogger("servicex_did_finder_lib")
+
+    log.setLevel(logging.INFO)
+    log.propagate = False  # keep our records out of the root logger Celery hijacks
+
+    stream_handler = logging.StreamHandler()
+    stream_formatter = StreamFormatter(
+        "%(levelname)s " + f"{instance} {component_name} " + "%(message)s"
+    )
+    stream_handler.setFormatter(stream_formatter)
+    stream_handler.setLevel(log.level)
+    log.addHandler(stream_handler)
+
+    logstash_host = os.environ.get("LOGSTASH_HOST")
+
+    if logstash_host:
+        logstash_port = int(os.environ.get("LOGSTASH_PORT", 5959))
+        logstash_handler = logstash.TCPLogstashHandler(
+            logstash_host, logstash_port, version=1
+        )
+        logstash_formatter = LogstashFormatter(component_name=component_name)
+        logstash_handler.setFormatter(logstash_formatter)
+        logstash_handler.setLevel(log.level)
+        log.addHandler(logstash_handler)
+
+    log.info("Initialized logging")
+
+    return log

{servicex_did_finder_lib-3.0.1 → servicex_did_finder_lib-3.2.0}/src/servicex_did_finder_lib/servicex_adaptor.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019, IRIS-HEP
+# Copyright (c) 2019-2025, IRIS-HEP
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -74,6 +74,10 @@ class ServiceXAdapter:
                 self.logger.error(f'After {attempts} tries, failed to send ServiceX App '
                                   f'a put_file_bulk message: {mesg} - Ignoring error.')
 
+    def put_file_add(self, file):
+        # add one file
+        self.put_file_add_bulk([file])
+
     def put_fileset_complete(self, summary):
         success = False
         attempts = 0
@@ -88,3 +92,18 @@ class ServiceXAdapter:
         if not success:
             self.logger.error(f'After {attempts} tries, failed to send ServiceX App a put_file '
                               f'message: {str(summary)} - Ignoring error.')
+
+    def put_fileset_error(self, summary):
+        success = False
+        attempts = 0
+        while not success and attempts < MAX_RETRIES:
+            try:
+                requests.put(f"{self.endpoint}{self.dataset_id}/error", json=summary)
+                success = True
+            except requests.exceptions.ConnectionError:
+                self.logger.exception(f'Connection error to ServiceX App. Will retry '
+                                      f'(try {attempts} out of {MAX_RETRIES}')
+                attempts += 1
+        if not success:
+            self.logger.error(f'After {attempts} tries, failed to send ServiceX App a put_file '
+                              f'message: {str(summary)} - Ignoring error.')