fs-pyutils 1.1.0__tar.gz

fs_pyutils-1.1.0/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.3
+Name: fs-pyutils
+Version: 1.1.0
+Summary: Python utils for fseasy scope.
+Author: fseasy
+Author-email: fseasy <xuwei.fs@outlook.com>
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+
+## FS pyutils
+
+Python Utils for fseasy scope.

fs_pyutils-1.1.0/README.md

@@ -0,0 +1,3 @@
+## FS pyutils
+
+Python Utils for fseasy scope.

fs_pyutils-1.1.0/pyproject.toml

@@ -0,0 +1,29 @@
+[project]
+name = "fs-pyutils"
+version = "1.1.0"
+description = "Python utils for fseasy scope."
+readme = "README.md"
+authors = [{ name = "fseasy", email = "xuwei.fs@outlook.com" }]
+requires-python = ">=3.12"
+dependencies = []
+
+[build-system]
+requires = ["uv_build>=0.9.25,<0.10.0"]
+build-backend = "uv_build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/fs_pyutils"]
+
+[tool.ruff]
+line-length = 120
+indent-width = 2
+exclude = []
+
+[tool.ruff.lint]
+# 确保 "I" (isort) 规则在启用列表中
+select = ["E", "F", "B", "UP", "I"]
+
+[tool.ruff.lint.isort]
+# 这里可以像 isort 一样自定义配置
+force-single-line = false # 是否强制单行导入
+combine-as-imports = true # 是否合并 import as

fs_pyutils-1.1.0/src/fs_pyutils/audio.py

@@ -0,0 +1,45 @@
+import shutil
+import subprocess
+
+
+def audio_to_mp3_bytes(
+    audio_bytes: bytes,
+    bitrate: str = "128k",  #
+    sample_rate: int | None = None,
+    channels: int | None = None,
+    ffmpeg_bin_path: str | None = None,
+) -> bytes:
+    """Any audio bytes to mp3 bytes
+    Args:
+    - bitrate: like  64k 96k 128k 192k
+    - sample_rate: like 16000 22050 24000 44100
+    - channels: like 1, 2
+    - ffmpeg_bin_path: default = "ffmpeg"
+    """
+    ffmpeg_bin_path = ffmpeg_bin_path or "ffmpeg"
+    if not shutil.which(ffmpeg_bin_path):
+        raise RuntimeError("No ffmpeg found in env")
+
+    cmd = [ffmpeg_bin_path, "-i", "pipe:0", "-codec:a", "libmp3lame", "-b:a", bitrate]
+
+    if sample_rate:
+        cmd += ["-ar", str(sample_rate)]
+
+    if channels:
+        cmd += ["-ac", str(channels)]
+
+    cmd += ["-f", "mp3", "pipe:1"]
+
+    proc = subprocess.Popen(
+        cmd,
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+
+    out, err = proc.communicate(audio_bytes)
+
+    if proc.returncode != 0:
+        raise RuntimeError(err.decode())
+
+    return out

fs_pyutils-1.1.0/src/fs_pyutils/gunicorn_logger.py

@@ -0,0 +1,73 @@
+import logging
+import os
+from typing import Any
+
+#! note1: this module must be used in python env where contains the gunicorn (it's reasonable)
+from gunicorn.config import Config
+from gunicorn.glogging import Logger as GLogger
+
+from .log_builder import JsonSyslogFormatter, NginxAlignedSyslogHandler
+
+#! note2: you must set `SYSLOG_ADDRESS` in ENV, in format `ip:port`, like: "127.0.0.1:5140"
+g_syslog_addr_str = os.environ["SYSLOG_ADDRESS"]
+#! it's better also set the hostname.
+g_hostname = os.getenv("HOSTNAME", "gunicorn-default")
+
+
+class GunicornSyslogLogger(GLogger):
+  def setup(self, cfg: Config) -> None:
+    super().setup(cfg)
+
+    ip, port = g_syslog_addr_str.split(":")
+    syslog_address = (ip, int(port))
+
+    # 继承 gunicorn 的日志级别 => cfg.loglevel 是 string, 必须得改成 int
+    level = self.LOG_LEVELS.get(cfg.loglevel.lower(), logging.INFO)
+
+    # --- 准备你的 Handler 和 Formatter ---
+    syslog_handler = NginxAlignedSyslogHandler(
+      address=syslog_address,
+      hostname=g_hostname,
+      facility=23,  # Local7
+    )
+
+    # 使用你现有的 JsonSyslogFormatter
+    json_fmt = JsonSyslogFormatter(g_hostname)
+    syslog_handler.setFormatter(json_fmt)
+    syslog_handler.setLevel(level)
+
+    # --- 将 Handler 绑定到 Gunicorn 的两个核心 Logger ---
+    # 1. 错误日志 (gunicorn.error) (we dont' remove the default error log handler, just append)
+    self.error_log.addHandler(syslog_handler)
+    self.error_log.setLevel(level)
+
+    # 2. 访问日志 (gunicorn.access)
+    # 移除默认的 access_log handler (防止重复打印到 stdout)
+    self.access_log.handlers = []
+    self.access_log.addHandler(syslog_handler)
+    self.access_log.setLevel(level)
+    self.access_log.propagate = False
+
+  def access(self, resp, req, environ, request_time) -> None:  # type: ignore
+    """
+    这个方法覆盖了 Gunicorn 默认逻辑。
+    它把本该塞进 args 的字典，转而塞进 extra。
+    这样就能被你现有的 _get_extra_kv 自动识别并转为 JSON 字段。
+    """
+    atoms = self.atoms(resp, req, environ, request_time)
+    # 消息内容可以设为 "access" 或者根据 access_log_format 渲染
+    self.access_log.info("access", extra=atoms)
+
+
+class GunicornJsonFormatter(JsonSyslogFormatter):
+  def _build_log_dict(self, record: logging.LogRecord) -> dict[str, Any]:
+    log_data = super()._build_log_dict(record)
+
+    # mapping key for readability
+    mapping = {"h": "remote_ip", "m": "method", "U": "path", "s": "status", "M": "duration_ms", "a": "user_agent"}
+
+    for short_key, long_key in mapping.items():
+      if short_key in log_data:
+        log_data[long_key] = log_data.pop(short_key)
+
+    return log_data

fs_pyutils-1.1.0/src/fs_pyutils/import.py

@@ -0,0 +1,56 @@
+import sys
+from importlib.util import module_from_spec, spec_from_file_location
+from pathlib import Path
+from types import ModuleType
+from typing import Union
+
+
+def import_module_from_path(
+  module_name: str,
+  file_path: Union[str, Path],
+  *,
+  register: bool = True,
+) -> ModuleType:
+  """
+  Dynamically import a Python module from a file path.
+
+  Parameters
+  ----------
+  module_name:
+      Name to assign to the module (used in sys.modules).
+
+  file_path:
+      Path to the .py file.
+
+  register:
+      Whether to register the module in sys.modules.
+
+  Returns
+  -------
+  ModuleType
+      The imported module.
+
+  Raises
+  ------
+  FileNotFoundError
+  ImportError
+  """
+
+  path = Path(file_path)
+
+  if not path.exists():
+    raise FileNotFoundError(path)
+
+  spec = spec_from_file_location(module_name, path)
+
+  if spec is None or spec.loader is None:
+    raise ImportError(f"Cannot load module from {path}")
+
+  module = module_from_spec(spec)
+
+  if register:
+    sys.modules[module_name] = module
+
+  spec.loader.exec_module(module)
+
+  return module

fs_pyutils-1.1.0/src/fs_pyutils/log_builder.py

@@ -0,0 +1,183 @@
+import json
+import logging
+import socket
+import sys
+from datetime import datetime
+from logging.handlers import SysLogHandler
+
+# import traceback
+from typing import Any
+from urllib.parse import urlparse
+
+
+def build_logger(
+  name: str, level: int, syslog_address: tuple[str, int] | None = None, domain: str | None = None
+) -> logging.Logger:
+  """
+  Args:
+    syslog_address: used for syslog (you can setup a grafana alloy), will send a json log
+    domain: used in the json logger
+  """
+  logger = logging.getLogger(name)
+
+  streamHandler = logging.StreamHandler(stream=sys.stderr)
+  fmt = SingleLineFormatter("%(asctime)s/%(name)s/%(levelname)s/%(filename)s:%(lineno)d> %(message)s")
+  streamHandler.setFormatter(fmt)
+  streamHandler.setLevel(level)
+
+  logger.addHandler(streamHandler)
+  logger.setLevel(level)
+
+  # syslog
+  if syslog_address:
+    try:
+      syslog_handler = NginxAlignedSyslogHandler(
+        address=syslog_address,
+        hostname=_domain2hostname(domain),
+        facility=SysLogHandler.LOG_LOCAL7,
+      )
+      json_fmt = JsonSyslogFormatter(_domain2hostname(domain))
+      syslog_handler.setFormatter(json_fmt)
+      syslog_handler.setLevel(level)
+      logger.addHandler(syslog_handler)
+    except Exception as e:
+      logger.warning(f"Failed to add syslog, err={e}")
+  return logger
+
+
+class SingleLineFormatter(logging.Formatter):
+  def format(self, record: logging.LogRecord) -> str:
+    fmt_line = super().format(record)
+    single_line = fmt_line.replace("\n", " ↵ ")
+    # naively append extra fields
+    extra_kv = _get_extra_kv(record)
+    if extra_kv:
+      extra_line = json.dumps(extra_kv, ensure_ascii=False, default=str)
+      single_line = f"{single_line} extra={extra_line}"
+    return single_line
+
+
+class JsonSyslogFormatter(logging.Formatter):
+  """A formatter for syslog in json format"""
+
+  def __init__(self, host: str):
+    """init with a hostname"""
+    super().__init__(fmt="%(message)s")
+    self._host = host
+
+  def _build_log_dict(self, record: logging.LogRecord) -> dict[str, Any]:
+    iso_time = datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="seconds")
+
+    log_data = {
+      "host": self._host,
+      "time": iso_time,  # to align the loki receiver
+      "level": record.levelname,
+      "logger": record.name,
+      "file": f"{record.filename}:{record.lineno}",
+      "msg": record.getMessage(),
+    }
+    if record.exc_info:
+      log_data["traceback"] = self.formatException(record.exc_info)
+
+    # add info from extra={...}
+    log_data.update(_get_extra_kv(record))
+    return log_data
+
+  def format(self, record: logging.LogRecord) -> str:
+    log_data = self._build_log_dict(record)
+    return json.dumps(log_data, default=str)
+
+
+class NginxAlignedSyslogHandler(logging.Handler):
+  """
+  严格遵守 RFC 3164，生成与 Nginx 完全一致的 Syslog UDP 报文。
+  解决了 Python SysLogHandler 日期填充错误、格式错位的问题。
+  """
+
+  def __init__(self, address: tuple[str, int], hostname: str, facility: int = 23):
+    super().__init__()
+    self.address = address
+    # 强制替换非法字符，保持与 Nginx (site_docgate) 类似的纯净 TAG
+    self.app_name = f"{hostname.replace('.', '_').replace('-', '_')}_fastapi"
+    self.hostname = hostname
+    self.facility = facility  # Nginx 默认多用 Local7 (23)
+    self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+
+  def emit(self, record: logging.LogRecord) -> None:
+    try:
+      # 1. 提取 JSON message
+      msg = self.format(record).lstrip()  # remove the potential leading space to keep the format
+
+      # 2. 计算 PRI (Facility * 8 + Severity)
+      severity_map = {
+        logging.DEBUG: 7,
+        logging.INFO: 6,
+        logging.WARNING: 4,
+        logging.ERROR: 3,
+        logging.CRITICAL: 2,
+      }
+      severity = severity_map.get(record.levelno, 6)
+      pri = (self.facility * 8) + severity
+
+      # 3. 严格生成 RFC 3164 时间戳: "Mmm dd hh:mm:ss"
+      # 注意：个位数日期必须用空格补齐，比如 "Feb  3" 不能是 "Feb 03"
+      now = datetime.now()
+      month = now.strftime("%b")
+      day = now.day
+      day_str = f"{day:>2}"  # 右对齐，不足补空格。这一步完美解决解析失败问题！
+      time_str = now.strftime("%H:%M:%S")
+      timestamp = f"{month} {day_str} {time_str}"
+
+      # 4. 严格拼接，完全复刻 Nginx 格式: <PRI>TIMESTAMP HOSTNAME TAG: MSG
+      # 注意 self.app_name 后面紧跟冒号和空格 ": "
+      syslog_msg = f"<{pri}>{timestamp} {self.hostname} {self.app_name}: {msg}\n"
+
+      # 5. 发送 UDP 包
+      self.sock.sendto(syslog_msg.encode("utf-8"), self.address)
+    except Exception:
+      self.handleError(record)
+
+
+def _get_extra_kv(record: logging.LogRecord) -> dict[str, Any]:
+  RESERVED_ATTRS = set(
+    (
+      "args",
+      "asctime",
+      "created",
+      "exc_info",
+      "exc_text",
+      "filename",
+      "funcName",
+      "levelname",
+      "levelno",
+      "lineno",
+      "message",
+      "module",
+      "msecs",
+      "msg",
+      "name",
+      "pathname",
+      "process",
+      "processName",
+      "relativeCreated",
+      "stack_info",
+      "thread",
+      "threadName",
+      "taskName",
+    )
+  )
+  extra_data: dict[str, Any] = {}
+  for key, value in record.__dict__.items():
+    if key not in RESERVED_ATTRS:
+      extra_data[key] = value
+  return extra_data
+
+
+def _domain2hostname(domain: str | None) -> str:
+  _DEFAULT_HOST = "default_app"
+  if not domain:
+    return _DEFAULT_HOST
+  host = urlparse(domain if "://" in domain else "https://" + domain).hostname
+  if not host:
+    host = _DEFAULT_HOST
+  return host

fs_pyutils-1.1.0/src/fs_pyutils/systemd_notifier.py

@@ -0,0 +1,104 @@
+"""Systemd notifier, should be used with fastapi.
+1. intercept the gunicorn default `READY` signal (by pop env var: NOTIFY_SOCKET)
+2. export a lifespan manager with following features:
+   - notify READY/STOPPING
+   - db watchdog for systemd.
+
+To enable systemd watchdog:
+1. in systemd service. set:
+   Type=notify
+
+   WatchdogSec=45(or any other value)
+   Restart=always
+"""
+
+import asyncio
+import logging
+import os
+import socket
+from collections.abc import AsyncGenerator
+from contextlib import asynccontextmanager
+from typing import Any
+
+from fastapi import FastAPI
+from sqlalchemy.ext.asyncio import AsyncEngine
+
+_CUSTOM_NOTIFY_SOCKET_VAR_NAME = "CUSTOM_NOTIFY_SOCKET"
+
+
+def intercept_server_ready_signal() -> None:
+  """pop NOTIFY_SOCKET from env and set it to our custom var"""
+  # 拦截 Systemd 的通知 Socket
+  # Gunicorn 内部逻辑是：如果检测到环境变量 NOTIFY_SOCKET，就会在启动后发送 READY=1。
+  # 我们在 Gunicorn 初始化前把它移走，存入自定义变量中。
+  _real_notify_socket = os.environ.pop("NOTIFY_SOCKET", None)
+  if _real_notify_socket:
+    # 存入一个 Gunicorn 不认识，但我们代码能找到的变量名
+    os.environ[_CUSTOM_NOTIFY_SOCKET_VAR_NAME] = _real_notify_socket
+
+
+@asynccontextmanager
+async def systemd_notifier_lifespan(
+  app: FastAPI, async_db_engine: AsyncEngine, logger: logging.Logger | None = None
+) -> AsyncGenerator[Any, None]:
+  """Please call this after you do anything else! because it'll send `READY/STOPPING` signal"""
+  del app
+  # 启动看门狗后台任务
+  task = asyncio.create_task(_db_watchdog_task(async_db_engine, logger=logger))
+  # 发送启动成功
+  _send_sd_notify("READY=1", logger=logger)
+  try:
+    yield
+  finally:
+    _send_sd_notify("STOPPING=1", logger=logger)
+
+    try:
+      task.cancel()
+    except asyncio.CancelledError:
+      pass  # 预期的取消异常
+    if logger:
+      logger.info("systemd: db watchdog stopped.")
+    _send_sd_notify("STATUS=db watchdog stopped", logger=logger)
+
+
+async def _db_watchdog_task(async_db_engine: AsyncEngine, logger: logging.Logger | None = None) -> None:
+  """后台监控任务：定期执行 SQL 任务，确保 SQL 运行正常，并喂狗"""
+  interval_usec = os.getenv("WATCHDOG_USEC")
+  if not interval_usec:
+    if logger:
+      logger.warning("db watchdog isn't set in systemd. skip watchdog task")
+    return
+  watch_sleep_sec = round(int(interval_usec) / 1_000_000 / 3)  # try to report 3 times in one watch duration
+  _send_sd_notify(f"STATUS=prepared to create systemd watchdog task, report interval={watch_sleep_sec}s", logger=logger)
+  while True:
+    try:
+      async with async_db_engine.begin() as conn:
+        await conn.exec_driver_sql("SELECT 1")
+
+      # 2. 如果成功执行，说明事件循环没死，DB 没卡死，通知 Systemd
+      _send_sd_notify("WATCHDOG=1", logger=logger)
+      _send_sd_notify("STATUS=db watchdog send", logger=logger)  # test
+
+    except Exception as e:
+      # 如果连不上 DB 报错，故意不喂狗，让 Systemd 几秒后重启我们
+      if logger:
+        logger.exception(f"db watchdog health check failed: {e}")
+      _send_sd_notify(f"STATUS=db watchdog health check failed, e={e}", logger=logger)
+
+    await asyncio.sleep(watch_sleep_sec)
+
+
+def _send_sd_notify(data: str, logger: logging.Logger | None = None):
+  # 读取拦截后的 Socket 地址
+  addr = os.getenv(_CUSTOM_NOTIFY_SOCKET_VAR_NAME)
+  if not addr:
+    return
+  if addr.startswith("@"):
+    addr = "\0" + addr[1:]
+  try:
+    with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
+      sock.connect(addr)
+      sock.sendall(data.encode())
+  except Exception as e:
+    if logger:
+      logger.warning(f"Failed to send watchdog signal to systemd: {e}")