flova 0.1.0__py3-none-any.whl

flova/__main__.py

@@ -0,0 +1,3 @@
+from flova.entrypoints.cli import main
+
+raise SystemExit(main())

flova/application/deploy.py

@@ -0,0 +1,46 @@
+import hashlib
+import hmac
+import json
+import logging
+import threading
+
+from flova.config import Webhook
+from flova.core.ports import Repository, SyncHook
+
+
+class DeploymentService:
+    def __init__(self, config: Webhook, repository: Repository, hook: SyncHook, lock=None):
+        self.config, self.repository, self.hook = config, repository, hook
+        self.lock = lock if lock is not None else threading.Lock()
+
+    def handle(self, payload: bytes, signature: str, event: str):
+        expected = (
+            "sha256=" + hmac.new(self.config.secret.encode(), payload, hashlib.sha256).hexdigest()
+        )
+        if not signature or not hmac.compare_digest(expected, signature):
+            return 401, "Invalid signature"
+        if event != "push":
+            return 200, "Ignored non-push event"
+        try:
+            data = json.loads(payload)
+        except (ValueError, UnicodeDecodeError):
+            return 400, "Invalid JSON"
+        if not isinstance(data, dict):
+            return 400, "Invalid JSON"
+        if data.get("ref") != "refs/heads/" + self.config.branch:
+            return 200, "Ignored branch"
+        if not self.lock.acquire(blocking=False):
+            return 409, "Deployment in progress"
+        try:
+            if not self.repository.sync(self.config.repository, self.config.branch):
+                return 500, "Deploy failed"
+            try:
+                self.hook.trigger()
+            except Exception as error:
+                logging.warning("Post-sync hook failed: %s", type(error).__name__)
+            return 200, "Deploy success"
+        except Exception as error:
+            logging.error("Deployment failed: %s", type(error).__name__)
+            return 500, "Deploy failed"
+        finally:
+            self.lock.release()

flova/application/finance.py

@@ -0,0 +1,206 @@
+import hashlib
+import json
+import os
+import subprocess
+from datetime import datetime
+from pathlib import Path
+from zoneinfo import ZoneInfo
+
+from flova.core.errors import StageError
+from flova.core.ports import CommandRunner
+
+
+def as_json(text, stage):
+    try:
+        return json.loads(text)
+    except (ValueError, TypeError):
+        raise StageError(stage, "invalid-json") from None
+
+
+def prepare(args, run):
+    day = args.date or datetime.now(ZoneInfo("Asia/Shanghai")).date().isoformat()
+    datetime.strptime(day, "%Y-%m-%d")
+    config = json.loads((args.flow / "config/download.json").read_text())
+    errors = []
+    ready = 0
+    for provider, settings in config.items():
+        stage = "download"
+        try:
+            if provider not in {"alipay", "wechat"}:
+                raise StageError("download", "unsupported-provider")
+            if not args.offline:
+                result = as_json(
+                    run([args.dflow, "download", "-p", provider], stage, args.flow, timeout=300),
+                    stage,
+                )
+                if result.get("status") != "success":
+                    raise StageError(stage, result.get("status", "unknown"))
+                stage = "unzip"
+                target = Path(settings["target"])
+                output = target / f"{day}.{settings['suffix']}"
+                # A stale output must not make a failed unzip look successful.
+                import tempfile
+
+                with tempfile.TemporaryDirectory(prefix="n8n-unzip-", dir=target) as directory:
+                    run(
+                        [
+                            args.oub,
+                            "unzip",
+                            "--path",
+                            str(target / f"{day}.zip"),
+                            "--output",
+                            directory,
+                            "--name",
+                            output.name,
+                        ],
+                        stage,
+                        args.flow,
+                    )
+                    extracted = Path(directory) / output.name
+                    if not extracted.is_file() or extracted.stat().st_size == 0:
+                        raise StageError(stage, "missing-output")
+                    os.replace(extracted, output)
+            ready += 1
+        except (StageError, OSError, subprocess.TimeoutExpired) as error:
+            errors.append(
+                {
+                    "ok": False,
+                    "stage": stage,
+                    "provider": provider,
+                    "message": str(error)
+                    if isinstance(error, StageError)
+                    else f"{stage} failed; check server configuration",
+                }
+            )
+    if not ready:
+        return errors
+    try:
+        flags = [] if args.offline else ["--write"]
+        report = as_json(
+            run(
+                [
+                    args.fa,
+                    "--config",
+                    str(args.config),
+                    "bill",
+                    "sync",
+                    "daily",
+                    "--date",
+                    day,
+                    "--json",
+                    *flags,
+                ],
+                "import",
+                args.flow,
+            ),
+            "import",
+        )
+        if not args.offline:
+            # Preserve old markers so restoring the old workflow cannot immediately
+            # append the same successfully imported files again.
+            for source in report.get("sources", []):
+                if source.get("status") not in {"changed", "unchanged"} or not source.get("path"):
+                    continue
+                path = Path(source["path"])
+                raw = path.read_bytes()
+                if hashlib.sha256(raw).hexdigest() != source.get("sha256"):
+                    raise StageError("import", "source-changed-after-sync")
+                marker = Path(str(path) + ".md5")
+                temporary = Path(str(marker) + ".n8n-tmp")
+                temporary.write_text(hashlib.md5(raw).hexdigest() + "\n")
+                os.replace(temporary, marker)
+        payload = as_json(
+            run(
+                [
+                    args.fa,
+                    "--config",
+                    str(args.config),
+                    "classify",
+                    "extract",
+                    "--root",
+                    str(args.ledger.parent),
+                ],
+                "extract",
+                args.flow,
+            ),
+            "extract",
+        )
+        if payload.get("schema_version") != 1 or not isinstance(payload.get("transactions"), list):
+            raise StageError("extract", "invalid-payload")
+        return errors + [
+            {
+                "ok": True,
+                "payload": payload,
+                "import": report,
+                "has_fixme": bool(payload["transactions"]),
+            }
+        ]
+    except StageError as error:
+        return errors + [{"ok": False, "stage": error.stage, "message": str(error)}]
+
+
+def finish(args, run):
+    results = {}
+    flags = ["--write"] if args.write else []
+    if args.decisions_base64:
+        results["classification"] = as_json(
+            run(
+                [
+                    args.fa,
+                    "--config",
+                    str(args.config),
+                    "classify",
+                    "apply",
+                    "--root",
+                    str(args.ledger.parent),
+                    "--input-base64",
+                    args.decisions_base64,
+                    "--validator",
+                    "docker exec fava bean-check /bean/main.bean",
+                    *flags,
+                ],
+                "classify",
+                args.flow,
+            ),
+            "classify",
+        )
+    if args.subscriptions:
+        results["subscriptions"] = as_json(
+            run(
+                [
+                    args.fa,
+                    "--config",
+                    str(args.config),
+                    "subscriptions",
+                    "generate",
+                    "--ledger",
+                    str(args.ledger),
+                    "--subscriptions",
+                    str(args.subscriptions),
+                    "--json",
+                    *flags,
+                ],
+                "subscriptions",
+                args.flow,
+            ),
+            "subscriptions",
+        )
+    elif args.write:
+        # Keep the existing subscription operation until its actual plan is known.
+        # A missing Makefile target is an error, never an implicit empty plan.
+        run(["make", "subscriptions"], "subscriptions", args.ledger.parent)
+        results["subscriptions"] = {"legacy_make": True}
+    else:
+        results["subscriptions"] = {"status": "plan-path-required"}
+    return {"ok": True, **results}
+
+
+class FinanceService:
+    def __init__(self, runner: CommandRunner):
+        self.runner = runner
+
+    def prepare(self, args):
+        return prepare(args, self.runner.run)
+
+    def finish(self, args):
+        return finish(args, self.runner.run)

flova/application/ledger.py

@@ -0,0 +1,24 @@
+from datetime import date
+
+from flova.core.ports import QueryGateway
+
+
+class LedgerService:
+    def __init__(self, gateway: QueryGateway):
+        self.gateway = gateway
+
+    def execute(self, payload):
+        if not isinstance(payload, dict):
+            raise ValueError("request body must be a JSON object")
+        start, end = payload.get("start_date"), payload.get("end_date")
+        if not isinstance(start, str) or not isinstance(end, str):
+            raise ValueError("start_date and end_date must be strings")
+        start, end = date.fromisoformat(start), date.fromisoformat(end)
+        if end < start:
+            raise ValueError("end_date must not be earlier than start_date")
+        if (end - start).days >= 366:
+            raise ValueError("date range must not exceed 366 days")
+        limit = payload.get("max_transactions", 500)
+        if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= 500:
+            raise ValueError("max_transactions must be between 1 and 500")
+        return self.gateway.query(start, end, limit)

flova/application/services.py

@@ -0,0 +1,15 @@
+from flova.core.ports import Runtime
+
+
+class ServiceManager:
+    def __init__(self, runtime: Runtime):
+        self.runtime = runtime
+
+    def execute(self, action, output=None, *, keep_stopped=False):
+        if keep_stopped and action != "pack":
+            raise ValueError("--keep-stopped is only valid for service pack")
+        if action == "pack":
+            return self.runtime.pack(output, keep_stopped=keep_stopped)
+        if action in {"start", "restore", "stop", "check", "status"}:
+            return getattr(self.runtime, action)()
+        raise ValueError("Unknown service action")

flova/bootstrap.py

@@ -0,0 +1,35 @@
+from flova.application.deploy import DeploymentService
+from flova.application.finance import FinanceService
+from flova.application.services import ServiceManager
+from flova.infrastructure.git import GitRepository
+from flova.infrastructure.hook import HttpSyncHook
+from flova.infrastructure.process import SubprocessRunner
+from flova.infrastructure.runtime import DockerSystemRuntime
+
+
+def finance():
+    return FinanceService(SubprocessRunner())
+
+
+def services(config):
+    return ServiceManager(DockerSystemRuntime(config.root))
+
+
+def deployments(config):
+    import threading
+
+    locks = {}
+    result = {}
+    for hook in config.webhooks.values():
+        lock = locks.setdefault(hook.repository.resolve(), threading.Lock())
+        result[hook.path] = DeploymentService(
+            hook, GitRepository(), HttpSyncHook(hook.after_sync_url, hook.after_sync_token), lock
+        )
+    return result
+
+
+def ledger(config):
+    from flova.application.ledger import LedgerService
+    from flova.infrastructure.beancount import BeancountQuery
+
+    return LedgerService(BeancountQuery(config.ledger))

flova/config.py

@@ -0,0 +1,108 @@
+import os
+import re
+import tomllib
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from flova.core.errors import ConfigError
+
+
+@dataclass(frozen=True)
+class Webhook:
+    path: str
+    repository: Path
+    branch: str
+    secret: str = field(repr=False)
+    after_sync_url: str = ""
+    after_sync_token: str = field(default="", repr=False)
+
+
+@dataclass(frozen=True)
+class Config:
+    root: Path
+    webhook_host: str = "0.0.0.0"
+    webhook_port: int = 58129
+    ledger_host: str = "0.0.0.0"
+    ledger_port: int = 8080
+    ledger: Path = Path("/bills/main.bean")
+    webhooks: dict[str, Webhook] = field(default_factory=dict)
+
+
+def _expand(value):
+    if isinstance(value, str):
+
+        def replace(match):
+            key = match.group(1)
+            if key not in os.environ:
+                raise ConfigError("Missing environment variable: " + key)
+            return os.environ[key]
+
+        return re.sub(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", replace, value)
+    if isinstance(value, dict):
+        return {k: _expand(v) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_expand(v) for v in value]
+    return value
+
+
+def load_config(path: Path, root: Path | None = None) -> Config:
+    try:
+        with path.open("rb") as f:
+            raw = _expand(tomllib.load(f))
+    except (OSError, tomllib.TOMLDecodeError) as error:
+        raise ConfigError("Unable to read configuration: " + type(error).__name__) from None
+    allowed = {"root", "webhook", "ledger", "webhooks"}
+    if set(raw) - allowed:
+        raise ConfigError("Unknown configuration sections")
+    root = (root or Path(raw.get("root", "/root/.flow"))).absolute()
+
+    def resolve(value):
+        p = Path(value)
+        return p if p.is_absolute() else root / p
+
+    webhook, ledger = raw.get("webhook", {}), raw.get("ledger", {})
+    for section, fields in [(webhook, {"host", "port"}), (ledger, {"file", "host", "port"})]:
+        if set(section) - fields:
+            raise ConfigError("Unknown server configuration fields")
+    hooks = {}
+    for name, data in raw.get("webhooks", {}).items():
+        if set(data) - {
+            "path",
+            "repository",
+            "branch",
+            "secret",
+            "after_sync_url",
+            "after_sync_token",
+        }:
+            raise ConfigError("Unknown webhook fields: " + name)
+        for key in ["path", "repository", "branch", "secret"]:
+            if not isinstance(data.get(key), str) or not data[key]:
+                raise ConfigError("Missing webhook field: " + name + "." + key)
+        if not data["path"].startswith("/"):
+            raise ConfigError("Webhook path must start with /")
+        if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/-]*", data["branch"]):
+            raise ConfigError("Invalid Git branch")
+        hook = Webhook(
+            data["path"],
+            resolve(data["repository"]),
+            data["branch"],
+            data["secret"],
+            data.get("after_sync_url", ""),
+            data.get("after_sync_token", ""),
+        )
+        if hook.path in [h.path for h in hooks.values()]:
+            raise ConfigError("Duplicate webhook path")
+        hooks[name] = hook
+    for section in [webhook, ledger]:
+        port = section.get("port", 58129 if section is webhook else 8080)
+        if isinstance(port, bool) or not isinstance(port, int) or not 1 <= port <= 65535:
+            raise ConfigError("Invalid server port")
+    return Config(
+        root,
+        webhook.get("host", "0.0.0.0"),
+        webhook.get("port", 58129),
+        ledger.get("host", "0.0.0.0"),
+        ledger.get("port", 8080),
+        resolve(ledger.get("file", "data/account/main.bean")),
+        hooks,
+    )

flova/core/errors.py

@@ -0,0 +1,8 @@
+class ConfigError(ValueError):
+    """Invalid local configuration; messages never contain credential values."""
+
+
+class StageError(RuntimeError):
+    def __init__(self, stage, code):
+        self.stage, self.code = stage, code
+        super().__init__(f"{stage} failed (exit {code}); inspect server logs")

flova/core/ports.py

@@ -0,0 +1,27 @@
+from pathlib import Path
+from typing import Any, Protocol
+
+
+class CommandRunner(Protocol):
+    def run(self, command: list[str], stage: str, cwd: Path, timeout: int = 180) -> str: ...
+
+
+class Repository(Protocol):
+    def sync(self, path: Path, branch: str) -> bool: ...
+
+
+class QueryGateway(Protocol):
+    def query(self, start, end, limit: int) -> dict[str, Any]: ...
+
+
+class SyncHook(Protocol):
+    def trigger(self) -> None: ...
+
+
+class Runtime(Protocol):
+    def start(self) -> None: ...
+    def restore(self) -> None: ...
+    def stop(self) -> None: ...
+    def status(self) -> None: ...
+    def check(self) -> None: ...
+    def pack(self, output: Path, *, keep_stopped: bool = False) -> None: ...

flova/entrypoints/cli.py

@@ -0,0 +1,147 @@
+import argparse
+import fcntl
+import json
+import logging
+import os
+import subprocess
+import sys
+import time
+from importlib.metadata import version
+from pathlib import Path
+
+from flova import bootstrap
+from flova.config import load_config
+from flova.core.errors import ConfigError, StageError
+from flova.entrypoints.http import create_server, serve
+
+
+def parser():
+    p = argparse.ArgumentParser(prog="flova")
+    p.add_argument("--version", action="version", version="flova " + version("flova"))
+    p.add_argument("--root", type=Path, default=None)
+    p.add_argument("--config", type=Path)
+    commands = p.add_subparsers(dest="group", required=True)
+    commands.add_parser("config").add_argument("action", choices=["check"])
+    service = commands.add_parser("service")
+    service.add_argument("action", choices=["start", "restore", "stop", "check", "status", "pack"])
+    service.add_argument("--output", type=Path)
+    service.add_argument(
+        "--keep-stopped",
+        action="store_true",
+        help="leave source stopped after a successful migration pack",
+    )
+    download = commands.add_parser("download")
+    download.add_argument("--provider", "-p", default="alipay", choices=["alipay", "wechat"])
+    download.add_argument("--download-config", type=Path)
+    commands.add_parser("webhook").add_argument("action", choices=["serve"])
+    ledger = commands.add_parser("ledger")
+    ledger.add_argument("action", choices=["serve", "query"])
+    ledger.add_argument("--start")
+    ledger.add_argument("--end")
+    ledger.add_argument("--max-transactions", type=int, default=500)
+    f = commands.add_parser("finance")
+    f.add_argument("action", choices=["prepare", "finish"])
+    f.add_argument("--flow", type=Path)
+    f.add_argument("--date")
+    f.add_argument("--offline", action="store_true")
+    f.add_argument("--write", action="store_true")
+    f.add_argument("--decisions-base64", default="")
+    f.add_argument("--subscriptions", type=Path)
+    f.add_argument("--ledger", type=Path)
+    f.add_argument("--fa")
+    f.add_argument("--dflow")
+    f.add_argument("--oub")
+    return p
+
+
+def main(argv=None):
+    os.umask(0o077)
+    args = parser().parse_args(argv)
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
+    root = args.root or Path(os.environ.get("FLOVA_ROOT", "/root/.flow"))
+    path = args.config or Path(os.environ.get("FLOVA_CONFIG", str(root / "config/flova.toml")))
+    try:
+        config = load_config(path, args.root)
+        if args.group == "config":
+            print("configuration is valid")
+            return 0
+        if args.group == "service":
+            if os.geteuid() != 0:
+                raise ConfigError("Service management requires root")
+            output = args.output or config.root / "backups" / (
+                "flow-transfer-" + time.strftime("%Y%m%d-%H%M%S") + ".tar.gz"
+            )
+            bootstrap.services(config).execute(args.action, output, keep_stopped=args.keep_stopped)
+            return 0
+        if args.group == "download":
+            from flova.infrastructure.download import BillDownloader
+
+            result = BillDownloader().download(
+                args.provider, args.download_config or config.root / "config/download.json"
+            )
+            print(json.dumps(result))
+            return 0 if result["status"] == "success" else 1
+        if args.group == "webhook":
+            if not config.webhooks:
+                raise ConfigError("No webhooks configured")
+            serve(
+                create_server(
+                    config.webhook_host,
+                    config.webhook_port,
+                    deployments=bootstrap.deployments(config),
+                )
+            )
+            return 0
+        if args.group == "ledger":
+            query = bootstrap.ledger(config)
+            if args.action == "serve":
+                serve(create_server(config.ledger_host, config.ledger_port, query=query))
+                return 0
+            print(
+                json.dumps(
+                    query.execute(
+                        {
+                            "start_date": args.start,
+                            "end_date": args.end,
+                            "max_transactions": args.max_transactions,
+                        }
+                    )
+                )
+            )
+            return 0
+        args.flow = args.flow or config.root
+        args.config = args.flow / "config/fane.yaml"
+        args.ledger = args.ledger or args.flow / "data/account/main.bean"
+        for name in ["fa", "dflow", "oub"]:
+            executable = "flova-download" if name == "dflow" else name
+            setattr(args, name, getattr(args, name) or str(args.flow / "runtime/bin" / executable))
+        lock = args.flow / "state/locks/n8n-finance.lock"
+        lock.parent.mkdir(parents=True, exist_ok=True)
+        with lock.open("a") as handle:
+            fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            job = bootstrap.finance()
+            result = job.prepare(args) if args.action == "prepare" else job.finish(args)
+        print(json.dumps(result, ensure_ascii=False))
+        return 0
+    except (
+        ConfigError,
+        StageError,
+        RuntimeError,
+        OSError,
+        ValueError,
+        KeyError,
+        subprocess.SubprocessError,
+    ) as error:
+        payload = {
+            "ok": False,
+            "stage": getattr(error, "stage", args.group),
+            "message": str(error)
+            if isinstance(error, (ConfigError, StageError, RuntimeError))
+            else type(error).__name__,
+        }
+        print(json.dumps(payload))
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

flova/entrypoints/download.py

@@ -0,0 +1,24 @@
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from flova.infrastructure.download import BillDownloader
+
+
+def main(argv=None):
+    p = argparse.ArgumentParser(prog="dflow")
+    p.add_argument("action", choices=["download"])
+    p.add_argument("--provider", "-p", default="alipay", choices=["alipay", "wechat"])
+    p.add_argument("--config", type=Path, default=Path("/root/.flow/config/download.json"))
+    args = p.parse_args(argv)
+    try:
+        result = BillDownloader().download(args.provider, args.config)
+    except Exception as error:
+        result = {"status": "error", "provider": args.provider, "message": type(error).__name__}
+    print(json.dumps(result))
+    return 0 if result["status"] == "success" else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())