liblogging 0.1.7__tar.gz → 0.1.8__tar.gz

{liblogging-0.1.7 → liblogging-0.1.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: liblogging
-Version: 0.1.7
+Version: 0.1.8
 Summary: Utilities for logging and sending logs.
 Home-page: https://github.com/XoriieInpottn/liblogging
 Author: xi

liblogging-0.1.8/liblogging/sending/kafka_service.py

@@ -0,0 +1,75 @@
+import json
+import traceback
+from typing import Dict
+
+try:
+    from kafka import KafkaProducer
+except ImportError:
+    # 如果导入失败，提示用户安装 kafka-python 包
+    print("错误：未找到 'kafka' 模块。")
+    print("请运行以下命令安装所需的依赖：")
+    print("pip install kafka-python")
+    import sys
+    sys.exit(1)
+
+
+class KafkaService:
+
+    def __init__(self, config):
+        bootstrap_servers = config["bootstrap_servers"].split(",")
+        self.producer = KafkaProducer(
+            bootstrap_servers=bootstrap_servers,
+            key_serializer=lambda k: json.dumps(k).encode(),
+            value_serializer=lambda v: json.dumps(v).encode(),
+            api_version=tuple(config.get("api_version")) if config.get("api_version") else (2, 7, 0),
+            acks=config.get("acks", "all"),
+            compression_type=config.get("compression_type", "gzip"),
+            retries=config.get("retries", 10),
+            batch_size=config.get("batch_size", 163840),
+            linger_ms=config.get("linger_ms", 1),
+            max_block_ms=config.get("max_block_ms", 2000),
+            buffer_memory=config.get("buffer_memory", 335544320),
+            request_timeout_ms=config.get("request_timeout_ms", 600000),
+            security_protocol=config.get("security_protocol", "SASL_SSL"),
+            sasl_mechanism=config.get("sasl_mechanism", "SCRAM-SHA-512"),
+            sasl_plain_username=config["username"],
+            sasl_plain_password=config["password"],
+            ssl_cafile=config.get("ssl_cafile")
+        )
+        self.topic = config["topic"]
+
+    def send(
+        self,
+        message: Dict,
+        source: str,
+        key: str = None,
+        topic: str = None
+    ):
+        if topic is None:
+            topic = self.topic
+        message["source"] = source
+        for _ in range(3):
+            try:
+                future = self.producer.send(
+                    topic,
+                    value=message,
+                    key=key
+                )
+                future.get(timeout=1)
+                return True
+            except Exception as e:
+                print(traceback.format_exc())
+        return False
+
+
+class KafkaServiceFactory:
+
+    @staticmethod
+    def create_kafka_service(
+        config_path: str, cluster_name: str, env: str, ssl_cafile: str = None
+    ):
+        with open(config_path, 'r') as f:
+            config = json.load(f)
+        if ssl_cafile:
+            config["ssl_cafile"] = ssl_cafile
+        return KafkaService(config[cluster_name][env])

liblogging-0.1.8/liblogging/sending/log_collector.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+
+__author__ = "yubin"
+
+
+import argparse
+import os
+import sys
+import json
+from datetime import datetime
+import time
+
+from .kafka_service import KafkaServiceFactory
+from ..util import split_trace_id, thread_pool_manager
+
+
+def get_date_from_time(
+    time, time_format: str = "%Y-%m-%d %H:%M:%S.%f", date_format: str = "%Y-%m-%d"
+) -> str:
+    date_string = datetime.strptime(time, time_format).strftime(date_format)
+    return date_string
+
+
+def process_message(line):
+    message = json.loads(line)
+    trace_id = message.get("trace_id", "")
+    if trace_id:
+        _, chat_dict = split_trace_id(trace_id)
+        message.update({
+            "uid": chat_dict.get("uid", ""),
+            "session_id": chat_dict.get("session_id", ""),
+            "turn": chat_dict.get("turn", 0)
+        })
+    create_time = message.get("create_time")
+    message.update({"create_date": get_date_from_time(create_time)})
+    return trace_id, message
+
+
+class LogCollector:
+
+    def __init__(self, args):
+        start_time = time.time()
+        self.kafka_service = KafkaServiceFactory.create_kafka_service(
+                config_path=args.config_path,
+                cluster_name=args.cluster_name,
+                env=args.env,
+                ssl_cafile=args.ssl_cafile
+            )
+        print(f"Init kafka service time: {time.time() - start_time}")
+
+    def collect(
+        self, send_kafka: bool = False, chat_env: str = "dev", use_default_process: bool = True
+    ):
+        start_time = time.time()
+        while True:
+            try:
+                line = sys.stdin.readline().strip()
+                if not line:
+                    continue
+                print(line)
+                print(round(time.time() - start_time, 3))
+                start_time = time.time()
+                try:
+                    if use_default_process:
+                        trace_id, message = process_message(line)
+                    else:
+                        message = json.loads(line)
+                        trace_id = message.get("trace_id", "")
+
+                    if send_kafka and trace_id:
+                        print(f"Sending kafka message. env:{chat_env}")
+                        thread_pool_manager.submit(
+                            self.kafka_service.send,
+                            message={
+                                "log_history": json.dumps(message),
+                                "current_env": chat_env
+                            },
+                            key=trace_id,
+                            source=message.get("message_source")
+                        )
+                except ValueError as e:
+                    pass
+
+            except EOFError:
+                print("End of input reached. Exiting...", file=sys.stderr)
+                break
+            except Exception as e:
+                print(f"Unexpected error: {e}", file=sys.stderr)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--send-kafka", action='store_true', default=False)
+    parser.add_argument("--config-path", type=str, required=True, help="Path to Kafka config json file")
+    parser.add_argument("--cluster-name", type=str, default="cluster_2", help="Kafka cluster name")
+    parser.add_argument(
+        "--env", type=str, default=os.environ.get("CHAT_ENV", "dev"), help="Environment (e.g., dev, test, online)"
+    )
+    parser.add_argument("--ssl-cafile", type=str, required=True, help="ssl_cafile path")
+    parser.add_argument(
+        "--use-default-process",
+        type=bool,
+        default=True,
+        help="whether user default process. you can also use another function to process message by redirecting")
+
+    args = parser.parse_args()
+
+    log_collector = LogCollector(args)
+    log_collector.collect(
+        send_kafka=args.send_kafka, chat_env=args.env, use_default_process=args.use_default_process
+    )
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

{liblogging-0.1.7 → liblogging-0.1.8}/liblogging.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: liblogging
-Version: 0.1.7
+Version: 0.1.8
 Summary: Utilities for logging and sending logs.
 Home-page: https://github.com/XoriieInpottn/liblogging
 Author: xi

{liblogging-0.1.7 → liblogging-0.1.8}/liblogging.egg-info/SOURCES.txt

@@ -7,5 +7,9 @@ liblogging/util.py
 liblogging.egg-info/PKG-INFO
 liblogging.egg-info/SOURCES.txt
 liblogging.egg-info/dependency_links.txt
+liblogging.egg-info/entry_points.txt
 liblogging.egg-info/top_level.txt
-liblogging.egg-info/zip-safe
+liblogging.egg-info/zip-safe
+liblogging/sending/__init__.py
+liblogging/sending/kafka_service.py
+liblogging/sending/log_collector.py

liblogging-0.1.8/liblogging.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+liblogging_collector = liblogging.sending.log_collector:main

{liblogging-0.1.7 → liblogging-0.1.8}/setup.py

@@ -11,8 +11,14 @@ if __name__ == '__main__':
         name='liblogging',
         packages=[
             'liblogging',
+            'liblogging.sending',
         ],
-        version='0.1.7',
+        entry_points={
+            'console_scripts': [
+                'liblogging_collector = liblogging.sending.log_collector:main'
+            ]
+        },
+        version='0.1.8',
         description='Utilities for logging and sending logs.',
         long_description_content_type='text/markdown',
         long_description=long_description,