liblogging 0.1.16__tar.gz → 0.2.3__tar.gz

{liblogging-0.1.16 → liblogging-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: liblogging
-Version: 0.1.16
+Version: 0.2.3
 Summary: Utilities for logging and sending logs.
 Home-page: https://github.com/XoriieInpottn/liblogging
 Author: xi

liblogging-0.2.3/liblogging/collector/__init__.py

@@ -0,0 +1 @@
+#!/usr/bin/env python3

liblogging-0.2.3/liblogging/collector/mongo.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+
+__author__ = "xi"
+
+import json
+import sys
+from datetime import datetime
+from queue import Empty
+from queue import Queue
+from threading import Thread
+from time import time
+from typing import Any
+
+from libdata import LazyMongoClient
+from libdata.url import URL
+from libentry import ArgumentParser
+from pydantic import BaseModel
+from pydantic import Field
+
+from ..util import split_trace_id
+
+
+class CollectorConfig(BaseModel):
+    mongo_url: str | URL
+    max_queue_size: int = Field(default=100_000, ge=1, le=100_000_000)
+    batch_size: int = Field(default=100, ge=1, le=10000)
+    wait_time: float = 0.2
+    max_wait_time: float = 0.5
+
+    def model_post_init(self, context: Any, /) -> None:
+        self.mongo_url = URL.ensure_url(self.mongo_url)
+        db, coll = self.mongo_url.get_database_and_table()
+        if db is None or coll is None:
+            raise ValueError("Both database and collection name should be given in the URL.")
+
+
+def sending_loop(queue: Queue[dict], config: CollectorConfig):
+    client = LazyMongoClient(config.mongo_url)
+
+    batch_size = config.batch_size
+    wait_time = config.wait_time
+    max_wait_time = config.max_wait_time
+
+    while True:
+        buffer = [queue.get()]
+
+        t = time()
+        for _ in range(batch_size):
+            try:
+                buffer.append(queue.get(timeout=wait_time))
+            except Empty:
+                break
+            if time() - t > max_wait_time:
+                break
+
+        if buffer[-1] is not None:
+            client.insert_many(buffer)
+        else:
+            client.insert_many(buffer[:-1])
+            break
+
+
+def process_message(line):
+    message = json.loads(line)
+    trace_id = message.get("trace_id", "")
+    if trace_id:
+        _, chat_dict = split_trace_id(trace_id)
+        message.update({
+            "uid": chat_dict.get("uid", ""),
+            "session_id": chat_dict.get("session_id", ""),
+            "turn": chat_dict.get("turn", 0)
+        })
+    create_time = message.get("create_time")
+    create_date = datetime.strptime(create_time, "%Y-%m-%d %H:%M:%S.%f").strftime("%Y-%m-%d")
+    message.update({"create_date": create_date})
+    return trace_id, message
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_schema("config", CollectorConfig)
+    config: CollectorConfig = parser.parse_args().config
+
+    queue = Queue(config.max_queue_size)
+
+    sending_thread = Thread(
+        target=sending_loop,
+        kwargs=dict(queue=queue, config=config)
+    )
+    sending_thread.start()
+
+    while True:
+        try:
+            line = sys.stdin.readline()
+            if not line:
+                break
+            line = line.strip()
+            try:
+                trace_id, message = process_message(line)
+                queue.put(message)
+                print(message)
+            except ValueError:
+                print(line)
+        except EOFError:
+            print("End of input reached. Exiting...", file=sys.stderr)
+            break
+        except Exception as e:
+            print(f"Unexpected error: {e}", file=sys.stderr)
+            continue
+
+    queue.put(None)
+    sending_thread.join()
+    print("Mongo collector exited.", file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

{liblogging-0.1.16 → liblogging-0.2.3}/liblogging/logger.py

@@ -167,7 +167,7 @@ class ContextJSONFormatter(Formatter):
         except TypeError:
             log_data["message"] = str(log_data["message"])
             output_log = json.dumps(log_data, ensure_ascii=False)
-        return output_log
+        return f"\n{output_log}\n"
 
     def formatTime(self, record):
         """to match mysql 'datetime(3)' format"""

{liblogging-0.1.16 → liblogging-0.2.3}/liblogging/sending/kafka_service.py

@@ -18,6 +18,8 @@ class KafkaService:
     def __init__(self, config):
         bootstrap_servers = config["bootstrap_servers"].split(",")
         print(f"kafka config: {config}")
+        self.default_send_timeout = config.get("send_timeout_seconds", 2)
+        self.send_timeout_increment = config.get("send_timeout_increment", 1)
         self.producer = KafkaProducer(
             bootstrap_servers=bootstrap_servers,
             key_serializer=lambda k: json.dumps(k).encode(),
@@ -44,22 +46,27 @@ class KafkaService:
         message: Dict,
         source: str,
         key: str = None,
-        topic: str = None
+        topic: str = None,
+        timeout: float = None
     ):
         if topic is None:
             topic = self.topic
         message["source"] = source
-        for _ in range(3):
+        current_timeout = timeout or self.default_send_timeout
+        for attempt in range(1, 4):
             try:
                 future = self.producer.send(
                     topic,
                     value=message,
                     key=key
                 )
-                future.get(timeout=1)
+                future.get(timeout=current_timeout)
+                print(f"Kafka send succeeded on attempt {attempt} (timeout={current_timeout}s)")
                 return True
             except Exception as e:
                 print(traceback.format_exc())
+                current_timeout += self.send_timeout_increment
+        print(f"Kafka send failed after retries (final timeout={current_timeout}s)")
         return False
 
 

{liblogging-0.1.16 → liblogging-0.2.3}/liblogging.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: liblogging
-Version: 0.1.16
+Version: 0.2.3
 Summary: Utilities for logging and sending logs.
 Home-page: https://github.com/XoriieInpottn/liblogging
 Author: xi

{liblogging-0.1.16 → liblogging-0.2.3}/liblogging.egg-info/SOURCES.txt

@@ -11,6 +11,8 @@ liblogging.egg-info/entry_points.txt
 liblogging.egg-info/requires.txt
 liblogging.egg-info/top_level.txt
 liblogging.egg-info/zip-safe
+liblogging/collector/__init__.py
+liblogging/collector/mongo.py
 liblogging/sending/__init__.py
 liblogging/sending/kafka_service.py
 liblogging/sending/log_collector.py

{liblogging-0.1.16 → liblogging-0.2.3}/liblogging.egg-info/entry_points.txt

@@ -1,2 +1,3 @@
 [console_scripts]
 liblogging_collector = liblogging.sending.log_collector:main
+liblogging_mongo_collector = liblogging.collector.mongo:main

{liblogging-0.1.16 → liblogging-0.2.3}/setup.py

@@ -12,13 +12,15 @@ if __name__ == '__main__':
         packages=[
             'liblogging',
             'liblogging.sending',
+            'liblogging.collector',
         ],
         entry_points={
             'console_scripts': [
-                'liblogging_collector = liblogging.sending.log_collector:main'
+                'liblogging_collector = liblogging.sending.log_collector:main',
+                "liblogging_mongo_collector = liblogging.collector.mongo:main"
             ]
         },
-        version='0.1.16',
+        version='0.2.3',
         description='Utilities for logging and sending logs.',
         long_description_content_type='text/markdown',
         long_description=long_description,