filepulse 0.1.0__py3-none-any.whl

filepulse/__init__.py

@@ -0,0 +1,3 @@
+"""filepulse — 文件监控与归档同步工具."""
+
+__version__ = "0.1.0"

filepulse/__main__.py

@@ -0,0 +1,5 @@
+"""Allow running as ``python -m filepulse``."""
+
+from .cli import main
+
+main()

filepulse/archive.py

@@ -0,0 +1,215 @@
+"""Archive monitor: detects new zip files, extracts and merges to targets."""
+
+import glob as glob_mod
+import logging
+import shutil
+import zipfile
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, List, Tuple
+
+from .base import BaseMonitor
+from .hooks import run_hook
+from .utils import merge_folder
+
+logger = logging.getLogger(__name__)
+
+
+class ArchiveMonitor(BaseMonitor):
+    """Scan for zip archives matching glob patterns, extract, and merge."""
+
+    state_filename = "archive_monitor_state.json"
+
+    # ---- discovery ------------------------------------------------------
+
+    def find_new_zips(self, task: Dict) -> List[Path]:
+        """Return unprocessed zip paths for *task* sorted by name."""
+        base_dir = Path(self.config['archive_monitor'].get('base_dir', '.'))
+        pattern = task.get('source_pattern', '')
+        full_pattern = str(base_dir / pattern)
+
+        matches = glob_mod.glob(full_pattern, recursive=True)
+        task_state = self.state.get(task['name'], {})
+        new: list[Path] = []
+
+        for zip_path_str in sorted(matches):
+            zp = Path(zip_path_str)
+            if not zp.exists():
+                continue
+            key = str(zp)
+            mtime = zp.stat().st_mtime
+            if key in task_state and task_state[key].get('mtime') == mtime:
+                continue
+            new.append(zp)
+        return new
+
+    # ---- process one archive --------------------------------------------
+
+    def process_archive(self, task: Dict, zip_path: Path) -> bool:
+        """Extract *zip_path* and merge into configured target directories.
+
+        Supports ``flatten_targets`` (copy all files flat) and the legacy
+        ``target_dir`` / ``flatten`` config keys.
+        """
+        name: str = task['name']
+        archive_dir = Path(task['archive_dir'])
+        extract_temp = Path(self.config['archive_monitor']['extract_temp'])
+        rename_to: str = task.get('rename_to', '')
+
+        # Resolve target directories
+        target_dirs: List[str] = task.get('target_dirs')
+        if target_dirs is None:
+            td = task.get('target_dir', '')
+            target_dirs = [td] if td else []
+        if not target_dirs:
+            logger.error("任务 %s 未配置 target_dir 或 target_dirs", name)
+            return False
+
+        # Flatten set
+        flatten_set: set[str] = set()
+        flat_targets = task.get('flatten_targets', [])
+        if isinstance(flat_targets, list):
+            for p in flat_targets:
+                flatten_set.add(str(Path(p).resolve()))
+        if task.get('flatten', False):
+            flatten_set = {str(Path(td).resolve()) for td in target_dirs}
+
+        zip_mtime = zip_path.stat().st_mtime
+
+        # ---- before hook ----
+        hook_before = task.get('on_before_process', '')
+        if hook_before:
+            env = {
+                'MONITOR_TYPE': 'archive',
+                'MONITOR_EVENT': 'before_process',
+                'MONITOR_SOURCE': str(zip_path),
+                'MONITOR_TARGET': target_dirs[0],
+                'MONITOR_TASK': name,
+            }
+            if not run_hook(
+                hook_before, env,
+                retries=self.config['archive_monitor'].get('hook_retries', 3),
+                delay=self.config['archive_monitor'].get('hook_retry_delay', 1),
+            ):
+                logger.warning("前置钩子失败，跳过处理: %s", zip_path)
+                return False
+
+        # ---- extract ----
+        extract_temp.mkdir(parents=True, exist_ok=True)
+        extract_root = extract_temp / f"{name}_{zip_path.stem}"
+        if extract_root.exists():
+            shutil.rmtree(extract_root)
+        extract_root.mkdir(exist_ok=True)
+
+        try:
+            with zipfile.ZipFile(zip_path, 'r') as zf:
+                zf.extractall(extract_root)
+        except zipfile.BadZipFile:
+            logger.exception("无效的压缩包 %s", zip_path)
+            shutil.rmtree(extract_root, ignore_errors=True)
+            return False
+
+        top_items = list(extract_root.iterdir())
+        if not top_items:
+            logger.warning("压缩包为空: %s", zip_path)
+            shutil.rmtree(extract_root, ignore_errors=True)
+            return False
+
+        # ---- merge to each target ----
+        for td in target_dirs:
+            target = Path(td).resolve()
+            target.mkdir(parents=True, exist_ok=True)
+
+            if str(target) in flatten_set:
+                logger.info("目标目录 %s 启用扁平化", target)
+                files_to_copy = [p for p in extract_root.rglob('*') if p.is_file()]
+                for src_file in files_to_copy:
+                    dest = target / src_file.name
+                    if dest.exists():
+                        stem, suffix = dest.stem, dest.suffix
+                        counter = 1
+                        while dest.exists():
+                            dest = target / f"{stem}_{counter}{suffix}"
+                            counter += 1
+                    shutil.copy2(str(src_file), str(dest))
+                logger.info("扁平化复制 %d 个文件到: %s", len(files_to_copy), target)
+            else:
+                for item in top_items:
+                    item_name = rename_to if rename_to else item.name
+                    dest = target / item_name
+                    dest.parent.mkdir(parents=True, exist_ok=True)
+                    if item.is_dir():
+                        shutil.copytree(str(item), str(dest), dirs_exist_ok=True)
+                    else:
+                        shutil.copy2(str(item), str(dest))
+                logger.info("已复制到: %s", target)
+
+        shutil.rmtree(extract_root, ignore_errors=True)
+
+        # ---- archive the zip ----
+        archive_dir.mkdir(parents=True, exist_ok=True)
+        archive_dest = archive_dir / zip_path.name
+        if archive_dest.exists():
+            ts = datetime.fromtimestamp(zip_mtime).strftime('%Y%m%d_%H%M%S')
+            archive_dest = archive_dir / f"{zip_path.stem}.{ts}{zip_path.suffix}"
+        shutil.move(str(zip_path), str(archive_dest))
+        logger.info("已归档zip: %s -> %s", zip_path, archive_dest)
+
+        # ---- persist state ----
+        self.state.setdefault(name, {})[str(zip_path)] = {
+            'mtime': zip_mtime,
+            'processed_at': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
+            'archive_path': str(archive_dest),
+        }
+        self.save_state()
+
+        # ---- after hook ----
+        hook_after = task.get('on_after_process', '')
+        if hook_after:
+            env = {
+                'MONITOR_TYPE': 'archive',
+                'MONITOR_EVENT': 'after_process',
+                'MONITOR_SOURCE': str(zip_path),
+                'MONITOR_TARGET': target_dirs[0],
+                'MONITOR_TASK': name,
+                'MONITOR_EXTRACTED': target_dirs[0],
+            }
+            run_hook(
+                hook_after, env,
+                retries=self.config['archive_monitor'].get('hook_retries', 3),
+                delay=self.config['archive_monitor'].get('hook_retry_delay', 1),
+            )
+
+        return True
+
+    # ---- run once / continuous ------------------------------------------
+
+    def run_once(self) -> Tuple[int, int]:
+        tasks = self.config.get('archives', [])
+        if not tasks:
+            logger.warning("配置文件中未定义归档监控任务 ([[archives]])")
+            return 0, 0
+
+        processed = 0
+        errors = 0
+        for task in tasks:
+            logger.info("检查归档任务: %s", task['name'])
+            new_zips = self.find_new_zips(task)
+            if new_zips:
+                logger.info("发现 %d 个新压缩包", len(new_zips))
+            for zp in new_zips:
+                try:
+                    logger.info("处理压缩包: %s", zp)
+                    if self.process_archive(task, zp):
+                        processed += 1
+                    else:
+                        errors += 1
+                except Exception:
+                    logger.exception("处理压缩包失败 %s", zp)
+                    errors += 1
+        return processed, errors
+
+    def run_continuous(self) -> None:
+        interval = self.config['archive_monitor'].get('interval', 10.0)
+        logger.info("监控 %d 个归档任务", len(self.config.get('archives', [])))
+        super().run_continuous(interval)

filepulse/base.py

@@ -0,0 +1,71 @@
+"""Shared base class for FileMonitor and ArchiveMonitor."""
+
+import logging
+import signal
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Dict, Tuple
+
+from . import paths
+from .config import load_config
+from .state import StateManager
+from .utils import setup_logging
+
+logger = logging.getLogger(__name__)
+
+
+class BaseMonitor(ABC):
+    """Common scaffolding: config loading, state persistence, signal
+    handling, and the continuous polling loop."""
+
+    # Override in subclass to pick the right state file name.
+    state_filename: str = "monitor_state.json"
+
+    def __init__(self, config_path: str):
+        self.config_path = Path(config_path)
+        self.config = load_config(config_path)
+        self.logger = setup_logging()
+        self.running = True
+
+        state_path = paths.get_data_dir() / self.state_filename
+        self.state_mgr = StateManager(state_path)
+        self.state: Dict = self.state_mgr.load()
+
+        signal.signal(signal.SIGINT, self._signal_handler)
+        signal.signal(signal.SIGTERM, self._signal_handler)
+
+    # ---- abstract -------------------------------------------------------
+
+    @abstractmethod
+    def run_once(self) -> Tuple[int, int]:
+        """Perform one polling cycle. Returns (success_count, error_count)."""
+        ...
+
+    # ---- continuous loop ------------------------------------------------
+
+    def run_continuous(self, interval: float) -> None:
+        """Block and poll at *interval* seconds until stopped."""
+        self.logger.info("启动监控服务，检查间隔: %.1f 秒", interval)
+        while self.running:
+            try:
+                self.run_once()
+            except KeyboardInterrupt:
+                self.logger.info("收到中断信号，停止监控")
+                break
+            except Exception:
+                self.logger.exception("监控循环错误")
+            if self.running:
+                import time
+                time.sleep(interval)
+        self.logger.info("监控服务已停止")
+
+    # ---- signal ---------------------------------------------------------
+
+    def _signal_handler(self, signum, frame):
+        self.logger.info("收到信号 %d，准备停止...", signum)
+        self.running = False
+
+    # ---- state persistence ----------------------------------------------
+
+    def save_state(self) -> None:
+        self.state_mgr.save(self.state)

filepulse/cli.py

@@ -0,0 +1,329 @@
+"""filepulse CLI — file sync and archive monitor."""
+
+import hashlib
+import os
+import sys
+import time
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+import click
+
+from . import __version__
+from . import paths
+from .archive import ArchiveMonitor
+from .monitor import FileMonitor
+from .utils import get_file_info, setup_logging
+
+
+# ======================================================================
+# PID helpers
+# ======================================================================
+
+def _pid_path(name: str) -> Path:
+    """Resolve a PID file under the data directory."""
+    return paths.get_data_dir() / f"{name}.pid"
+
+
+def _read_pid(name: str) -> Optional[int]:
+    try:
+        return int(_pid_path(name).read_text().strip())
+    except Exception:
+        return None
+
+
+def _pid_running(pid: int) -> bool:
+    try:
+        os.kill(pid, 0)
+        return True
+    except OSError:
+        return False
+
+
+def _write_pid(name: str) -> None:
+    _pid_path(name).write_text(str(os.getpid()))
+
+
+def _cleanup_pid(name: str) -> None:
+    _pid_path(name).unlink(missing_ok=True)
+
+
+_CONTEXT_SETTINGS = {'help_option_names': ['-h', '--help']}
+
+
+# ======================================================================
+# CLI entry point
+# ======================================================================
+
+@click.group(context_settings=_CONTEXT_SETTINGS)
+@click.version_option(__version__, '-V', '--version')
+def main():
+    """filepulse — 文件监控与归档同步工具
+
+    轮询检测源文件变化并自动同步到目标位置，支持 zip 归档自动解压处理。
+    """
+
+
+# ======================================================================
+# sync — file monitoring
+# ======================================================================
+
+@main.group()
+def sync():
+    """文件同步 — 监控源文件变化并复制到目标."""
+
+
+@sync.command('run')
+@click.argument('config', type=click.Path(exists=True))
+@click.option('--daemon', '-d', is_flag=True, help='后台运行')
+def sync_run(config: str, daemon: bool):
+    """持续监控文件变化（前台阻塞运行）."""
+    logger = setup_logging(daemon)
+    monitor = FileMonitor(config)
+
+    if daemon:
+        _write_pid('monitor')
+        logger.info("后台进程 PID: %d", os.getpid())
+
+    try:
+        monitor.run_continuous()
+    finally:
+        if daemon:
+            _cleanup_pid('monitor')
+
+
+@sync.command('once')
+@click.argument('config', type=click.Path(exists=True))
+def sync_once(config: str):
+    """一次性同步所有已变更的文件."""
+    setup_logging()
+    monitor = FileMonitor(config)
+    synced, errors = monitor.run_once()
+    click.echo(f"同步完成: {synced} 个文件已更新，{errors} 个错误")
+
+
+@sync.command('check')
+@click.argument('file', type=click.Path())
+@click.argument('config', type=click.Path(exists=True))
+def sync_check(file: str, config: str):
+    """检查单个文件是否有变更."""
+    setup_logging()
+    monitor = FileMonitor(config)
+    source = Path(file)
+    file_id = hashlib.md5(str(source).encode()).hexdigest()[:8]
+    changed = monitor.has_file_changed(source, file_id)
+
+    click.echo(f"文件: {source}")
+    click.echo(f"状态: {'已更改' if changed else '未更改'}")
+    if source.exists():
+        info = get_file_info(source)
+        if info:
+            click.echo(f"大小: {info['size']} 字节")
+            click.echo(f"修改时间: {datetime.fromtimestamp(info['mtime'])}")
+
+
+@sync.command('stats')
+@click.argument('config', type=click.Path(exists=True))
+def sync_stats(config: str):
+    """列出已配置的文件映射."""
+    monitor = FileMonitor(config)
+    files = monitor.config.get('files', {})
+    click.echo(f"监控文件数量: {len(files)}")
+    click.echo()
+    for file_id, mapping in files.items():
+        if isinstance(mapping, str):
+            click.echo(f"  {mapping}")
+        elif isinstance(mapping, dict):
+            click.echo(f"  {mapping.get('source', '')} -> {mapping.get('target', '')}")
+
+
+# ======================================================================
+# archive — zip monitoring
+# ======================================================================
+
+@main.group()
+def archive():
+    """归档处理 — 监控 zip 压缩包并自动解压合并."""
+
+
+@archive.command('run')
+@click.argument('config', type=click.Path(exists=True))
+@click.option('--daemon', '-d', is_flag=True, help='后台运行')
+def archive_run(config: str, daemon: bool):
+    """持续监控新 zip 文件（前台阻塞运行）."""
+    logger = setup_logging(daemon)
+    monitor = ArchiveMonitor(config)
+
+    if daemon:
+        _write_pid('archive_monitor')
+        logger.info("后台进程 PID: %d", os.getpid())
+
+    try:
+        monitor.run_continuous()
+    finally:
+        if daemon:
+            _cleanup_pid('archive_monitor')
+
+
+@archive.command('once')
+@click.argument('config', type=click.Path(exists=True))
+def archive_once(config: str):
+    """一次性处理所有待处理的 zip."""
+    setup_logging()
+    monitor = ArchiveMonitor(config)
+    processed, errors = monitor.run_once()
+    click.echo(f"归档处理完成: {processed} 个已处理，{errors} 个错误")
+
+
+@archive.command('check')
+@click.argument('config', type=click.Path(exists=True))
+def archive_check(config: str):
+    """列出待处理 zip 文件（干跑，不实际处理）."""
+    monitor = ArchiveMonitor(config)
+    tasks = monitor.config.get('archives', [])
+    if not tasks:
+        click.echo("未配置归档监控任务")
+        return
+    for task in tasks:
+        click.echo(f"\n任务: {task['name']}")
+        click.echo(f"  模式: {task.get('source_pattern', '')}")
+        click.echo(f"  目标: {task.get('target_dir', task.get('target_dirs', ''))}")
+        new_zips = monitor.find_new_zips(task)
+        if new_zips:
+            click.echo(f"  待处理 ({len(new_zips)}):")
+            for zp in new_zips:
+                click.echo(f"    - {zp}")
+        else:
+            click.echo("  待处理: 无")
+
+
+# ======================================================================
+# daemon — process management
+# ======================================================================
+
+@main.group()
+def daemon():
+    """守护进程管理 — 查看/停止后台监控."""
+
+
+@daemon.command('status')
+def daemon_status():
+    """查看监控守护进程状态."""
+    # File sync
+    pid = _read_pid('monitor')
+    if pid and _pid_running(pid):
+        click.echo(f"✓ 文件同步监控运行中 (PID: {pid})")
+    else:
+        click.echo("✗ 文件同步监控未运行")
+        _cleanup_pid('monitor')
+
+    # Archive
+    pid = _read_pid('archive_monitor')
+    if pid and _pid_running(pid):
+        click.echo(f"✓ 归档监控运行中 (PID: {pid})")
+    else:
+        click.echo("✗ 归档监控未运行")
+        _cleanup_pid('archive_monitor')
+
+
+@daemon.command('stop')
+@click.option('--sync/--no-sync', 'stop_sync', default=True, help='停止文件同步守护进程')
+@click.option('--archive/--no-archive', 'stop_archive', default=True, help='停止归档守护进程')
+def daemon_stop(stop_sync: bool, stop_archive: bool):
+    """停止后台监控进程."""
+    stopped = 0
+    for label, pid_name in [('文件同步', 'monitor'), ('归档', 'archive_monitor')]:
+        if label == '文件同步' and not stop_sync:
+            continue
+        if label == '归档' and not stop_archive:
+            continue
+        pid = _read_pid(pid_name)
+        if pid and _pid_running(pid):
+            os.kill(pid, 15)  # SIGTERM
+            _cleanup_pid(pid_name)
+            click.echo(f"✓ {label}监控已停止 (PID: {pid})")
+            stopped += 1
+        else:
+            _cleanup_pid(pid_name)
+    if stopped == 0:
+        click.echo("无运行中的监控进程")
+
+
+@daemon.command('logs')
+@click.argument('num', type=int, default=50)
+def daemon_logs(num: int):
+    """查看最近 N 行日志."""
+    logfile = paths.get_cache_dir() / 'monitor.log'
+    if not logfile.exists():
+        click.echo("暂无日志文件")
+        return
+    lines = logfile.read_text().splitlines()
+    click.echo('\n'.join(lines[-num:]))
+
+
+@daemon.command('reload')
+def daemon_reload():
+    """重载配置（停止后重新后台启动）."""
+    click.echo("停止监控...")
+    ctx = click.get_current_context()
+    # Invoke stop first
+    ctx.invoke(daemon_stop, stop_sync=True, stop_archive=True)
+    time.sleep(1)
+    click.echo("请手动运行 'filepulse sync run --daemon <config>' 或 "
+               "'filepulse archive run --daemon <config>' 重新启动")
+
+
+# ======================================================================
+# utility commands
+# ======================================================================
+
+@main.command('clean')
+@click.option('--logs-days', type=int, default=7, help='删除多少天前的日志')
+@click.option('--backup-days', type=int, default=30, help='删除多少天前的备份')
+def clean_cmd(logs_days: int, backup_days: int):
+    """清理旧日志和备份文件."""
+    import subprocess
+
+    log_dir = paths.get_cache_dir()
+    if log_dir.exists():
+        result = subprocess.run(
+            ['find', str(log_dir), '-name', '*.log', '-mtime', f'+{logs_days}', '-delete'],
+            capture_output=True, text=True,
+        )
+        click.echo(f"✓ 已清理 {logs_days} 天前的日志")
+
+    # Backup files are stored next to targets, find them
+    result = subprocess.run(
+        ['find', str(Path.cwd()), '-name', '*.bak.*', '-mtime', f'+{backup_days}', '-delete'],
+        capture_output=True, text=True,
+    )
+    click.echo(f"✓ 已清理 {backup_days} 天前的备份")
+    click.echo("✓ 清理完成")
+
+
+@main.command('treasury-run')
+@click.option('--send-only', is_flag=True, help='仅发送邮件，跳过 ETL 和渲染')
+def treasury_run(send_only: bool):
+    """运行司库日报流程（调用 hooks/treasury_daily.sh）."""
+    script = Path(__file__).resolve().parent.parent.parent / 'hooks' / 'treasury_daily.sh'
+    if not script.exists():
+        click.echo(f"✗ 脚本不存在: {script}")
+        raise SystemExit(1)
+
+    import subprocess
+    args = ['bash', str(script)]
+    if send_only:
+        args.append('--send-only')
+    click.echo("司库日报流程 (前台运行)")
+    subprocess.run(args, check=False)
+
+
+# ======================================================================
+# entry point
+# ======================================================================
+# entry point
+# ======================================================================
+
+if __name__ == '__main__':
+    main()

filepulse/config.py

@@ -0,0 +1,55 @@
+"""Configuration loading and schema defaults."""
+
+from pathlib import Path
+from typing import Dict
+
+import tomli
+
+
+# Default values merged into user config
+MONITOR_DEFAULTS: Dict = {
+    'interval': 2.0,
+    'check_size': True,
+    'check_mtime': True,
+    'check_hash': False,
+    'create_backup': True,
+    'backup_dir': 'backup',
+    'hook_retries': 3,
+    'hook_retry_delay': 1,
+}
+
+ARCHIVE_MONITOR_DEFAULTS: Dict = {
+    'interval': 10.0,
+    'extract_temp': '/tmp/sync_extract',
+    'base_dir': '.',
+    'hook_retries': 3,
+    'hook_retry_delay': 1,
+}
+
+
+def _merge_defaults(config: Dict, section: str, defaults: Dict) -> Dict:
+    """Merge default values into a config section without overwriting user values."""
+    if section not in config:
+        config[section] = {}
+    for key, value in defaults.items():
+        if key not in config[section]:
+            config[section][key] = value
+    return config
+
+
+def load_config(config_path: str) -> Dict:
+    """Load a TOML configuration file and merge in defaults.
+
+    Raises SystemExit if the file is unreadable or invalid TOML.
+    """
+    path = Path(config_path)
+    try:
+        with open(path, 'rb') as f:
+            config = tomli.load(f)
+    except Exception as e:
+        raise SystemExit(f"错误: 无法加载配置文件 {config_path}: {e}")
+
+    _merge_defaults(config, 'monitor', MONITOR_DEFAULTS)
+    _merge_defaults(config, 'archive_monitor', ARCHIVE_MONITOR_DEFAULTS)
+
+    return config