logtail-python 0.4.0__py2.py3-none-any.whl → 0.4.1__py2.py3-none-any.whl

logtail/__init__.py

@@ -5,6 +5,6 @@ from .handler import LogtailHandler
 from .helpers import LogtailContext, DEFAULT_CONTEXT
 from .formatter import LogtailFormatter
 
-__version__ = '0.4.0'
+__version__ = '0.4.1'
 
 context = DEFAULT_CONTEXT

logtail/flusher.py

@@ -9,6 +9,14 @@ from .compat import queue
 RETRY_SCHEDULE = (1, 10, 60)  # seconds
 
 
+class TransportFrame(dict):
+    """A record logged by a flush worker itself, i.e. urllib3 describing our own upload.
+
+    It rides along with the next batch of regular records but never triggers an upload
+    on its own, since that upload would log again and the queue would never drain.
+    """
+
+
 class FlushWorker(threading.Thread):
     def __init__(self, upload, pipe, buffer_capacity, flush_interval, check_interval):
         threading.Thread.__init__(self)
@@ -71,7 +79,7 @@ class FlushWorker(threading.Thread):
         # count) and sends them to the Better Stack endpoint all at once. If the
         # request fails in a way that can be retried, it is retried with an
         # exponential backoff in between attempts.
-        if frame:
+        if frame and not all(isinstance(entry, TransportFrame) for entry in frame):
             response = None
             for delay in RETRY_SCHEDULE + (None, ):
                 response = self.upload(frame)
@@ -80,7 +88,12 @@ class FlushWorker(threading.Thread):
                 if delay is not None:
                     time.sleep(delay)
 
-            if response.status_code == 500 and getattr(response, "exception") != None:
+            # A network error surfaces as a Fake500 that carries an `exception`
+            # attribute; a genuine server HTTP 500 is a real `requests.Response`
+            # that has no such attribute. Use a default so the latter doesn't
+            # raise AttributeError here and kill the flush thread (dropping all
+            # subsequent logs for the life of the process).
+            if response.status_code == 500 and getattr(response, "exception", None) is not None:
                 print('Failed to send logs to Better Stack after {} retries: {}'.format(len(RETRY_SCHEDULE), response.exception))
 
         self._clean = True
@@ -93,6 +106,10 @@ class FlushWorker(threading.Thread):
             time.sleep(self.check_interval)
         self._flushing = False
 
+def in_flush_worker():
+    return isinstance(threading.current_thread(), FlushWorker)
+
+
 def _initial_time_remaining(flush_interval):
     return flush_interval
 

logtail/handler.py

@@ -2,10 +2,12 @@
 from __future__ import print_function, unicode_literals
 import logging
 import json
+import os
+import weakref
 
 from .compat import queue
 from .helpers import DEFAULT_CONTEXT
-from .flusher import FlushWorker
+from .flusher import FlushWorker, TransportFrame, in_flush_worker
 from .uploader import Uploader
 from .frame import create_frame
 
@@ -18,6 +20,8 @@ DEFAULT_DROP_EXTRA_EVENTS = True
 DEFAULT_INCLUDE_EXTRA_ATTRIBUTES = True
 DEFAULT_TIMEOUT = 30
 
+_handlers = weakref.WeakSet()
+
 
 class LogtailHandler(logging.Handler):
     def __init__(self,
@@ -50,6 +54,7 @@ class LogtailHandler(logging.Handler):
         self.dropcount = 0
         # Do not initialize the flush thread yet because it causes issues on Render.
         self.flush_thread = None
+        _handlers.add(self)
 
     def ensure_flush_thread_alive(self):
         if self.flush_thread and self.flush_thread.is_alive():
@@ -71,8 +76,12 @@ class LogtailHandler(logging.Handler):
             message = self.format(record)
             frame = create_frame(record, message, self.context, include_extra_attributes=self.include_extra_attributes)
             serializable_frame = json.loads(json.dumps(frame, default=str))
+            transport = in_flush_worker()
+            if transport:
+                serializable_frame = TransportFrame(serializable_frame)
             try:
-                self.pipe.put(serializable_frame, block=(not self.drop_extra_events))
+                # The flush worker must never block on its own queue, so its records are dropped when full.
+                self.pipe.put(serializable_frame, block=(not self.drop_extra_events) and not transport)
             except queue.Full:
                 # Only raised when not blocking, which means that extra events
                 # should be dropped.
@@ -84,3 +93,20 @@ class LogtailHandler(logging.Handler):
     def flush(self):
         if self.flush_thread and self.flush_thread.is_alive():
              self.flush_thread.flush()
+
+    def _reset_after_fork(self):
+        # A forked child inherits the parent's queue with whatever was still buffered in it,
+        # a flush thread object whose thread does not exist in the child, and an HTTP session
+        # whose socket it shares with the parent, so it starts over with fresh ones.
+        self.pipe = queue.Queue(maxsize=self.buffer_capacity)
+        self.flush_thread = None
+        self.uploader = Uploader(self.source_token, self.host, self.uploader.timeout)
+
+
+def _reset_handlers_after_fork():
+    for handler in _handlers:
+        handler._reset_after_fork()
+
+
+if hasattr(os, 'register_at_fork'):
+    os.register_at_fork(after_in_child=_reset_handlers_after_fork)

logtail/rq.py

@@ -0,0 +1,29 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+import logging
+
+import rq
+
+from .handler import LogtailHandler
+
+
+class Worker(rq.Worker):
+    """An RQ worker that sends buffered log records after every job.
+
+    RQ runs each job in a forked work horse and ends it with os._exit(), which skips the
+    interpreter shutdown that would otherwise let LogtailHandler flush. Use it with
+    `rq worker --worker-class logtail.rq.Worker` or `RQ = {"WORKER_CLASS": "logtail.rq.Worker"}`
+    in django-rq.
+    """
+
+    def perform_job(self, job, queue):
+        try:
+            return super(Worker, self).perform_job(job, queue)
+        finally:
+            for handler in _logtail_handlers():
+                handler.flush()
+
+
+def _logtail_handlers():
+    loggers = [logging.getLogger()] + [logger for logger in logging.Logger.manager.loggerDict.values() if isinstance(logger, logging.Logger)]
+    return {handler for logger in loggers for handler in logger.handlers if isinstance(handler, LogtailHandler)}

{logtail_python-0.4.0.dist-info → logtail_python-0.4.1.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: logtail-python
-Version: 0.4.0
+Version: 0.4.1
 Summary: Better Stack client library
 Home-page: https://github.com/logtail/logtail-python
-Download-URL: https://github.com/logtail/logtail-python/tarball/0.4.0
+Download-URL: https://github.com/logtail/logtail-python/tarball/0.4.1
 Author: Logtail
 Author-email: hello@logtail.com
 License: ISC

logtail_python-0.4.1.dist-info/RECORD

@@ -0,0 +1,14 @@
+logtail/__init__.py,sha256=9PQmXLJchEiOwtJ6WfZuo9AT1aL3rl6ZPD7TxLKwuz0,252
+logtail/compat.py,sha256=KbZmcmDpj_vbMotp16pADPwJ1FkEHGDNjKbOfCy-c10,141
+logtail/flusher.py,sha256=EjLbAgooBjYs0KKCMVqxRddDpMD8qnVuhXSEMZTISQw,5017
+logtail/formatter.py,sha256=nk4W1G95P19ijylPBJ8D9FlO_qbh3tfbv7xzCSfpDN8,968
+logtail/frame.py,sha256=p7vVYcajZvjH4FJDt4ADLfYljVbQXKTGKk3kFec5JUs,3335
+logtail/handler.py,sha256=CruppN_XD0W-Ifsl1iu99T0yWw66utkl7_un6V6OO1Y,4265
+logtail/helpers.py,sha256=ezDTLsTBAtCtIh_XptSYOiBrZFa-E9wo2SWTnjKgOOU,1156
+logtail/rq.py,sha256=SEjVwjp8lRLab2F0StdssBQ0-vFK1CekEgYwkSICBeo,1023
+logtail/uploader.py,sha256=Ilhh8wuAGlz1YJcQpzXtFqwD-2RYmuamKv3dFNoTWUg,868
+logtail_python-0.4.1.dist-info/licenses/LICENSE.md,sha256=aCBMqxfm7QNyFR9JiJ2GSMvhAURXN2E0yTcYNQloqI0,737
+logtail_python-0.4.1.dist-info/METADATA,sha256=F29t1-2304x3Ofa8b1cASyECF4V85ef2Ndo5yovJzmI,2429
+logtail_python-0.4.1.dist-info/WHEEL,sha256=4YBfCYNH4wlLpv3pzq1hbEuIlXA4WJabKLFurZ7eTL0,109
+logtail_python-0.4.1.dist-info/top_level.txt,sha256=QZIbQogN1yFBgjiJ-8NlYArVCL00gBoYbEUT_Ew8l-Q,8
+logtail_python-0.4.1.dist-info/RECORD,,

{logtail_python-0.4.0.dist-info → logtail_python-0.4.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (82.0.1)
+Generator: setuptools (84.0.0)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any

logtail_python-0.4.0.dist-info/RECORD

@@ -1,13 +0,0 @@
-logtail/__init__.py,sha256=lz_ouCpEkpsK8Lq9fCQ7Sqj8xbowot6JzNZGFeEfOrI,252
-logtail/compat.py,sha256=KbZmcmDpj_vbMotp16pADPwJ1FkEHGDNjKbOfCy-c10,141
-logtail/flusher.py,sha256=KKb4IEuPC_XGKaNBrbMEiwc-X4MKsYURPjY7PqR3v4k,4176
-logtail/formatter.py,sha256=nk4W1G95P19ijylPBJ8D9FlO_qbh3tfbv7xzCSfpDN8,968
-logtail/frame.py,sha256=p7vVYcajZvjH4FJDt4ADLfYljVbQXKTGKk3kFec5JUs,3335
-logtail/handler.py,sha256=9PNJPSc2S3W5uQADKE4iDFH16A7BZW90D-m7aR66Mo4,3188
-logtail/helpers.py,sha256=ezDTLsTBAtCtIh_XptSYOiBrZFa-E9wo2SWTnjKgOOU,1156
-logtail/uploader.py,sha256=Ilhh8wuAGlz1YJcQpzXtFqwD-2RYmuamKv3dFNoTWUg,868
-logtail_python-0.4.0.dist-info/licenses/LICENSE.md,sha256=aCBMqxfm7QNyFR9JiJ2GSMvhAURXN2E0yTcYNQloqI0,737
-logtail_python-0.4.0.dist-info/METADATA,sha256=Bwsctru3-I0ZkrvKuukF6j1jDB7P1nz5OYyBmBsw79M,2429
-logtail_python-0.4.0.dist-info/WHEEL,sha256=TdQ5LtNwLuxTCjgxN51AgdU5w-KkB9ttmLbzjTH02pg,109
-logtail_python-0.4.0.dist-info/top_level.txt,sha256=QZIbQogN1yFBgjiJ-8NlYArVCL00gBoYbEUT_Ew8l-Q,8
-logtail_python-0.4.0.dist-info/RECORD,,