knowflow 0.1.0__py3-none-any.whl

knowflow/cli/__init__.py

@@ -0,0 +1,265 @@
+#!/usr/bin/env python3
+"""
+KnowFlow CLI — Agent Knowledge Pipeline
+
+Usage:
+    knowflow init <template> [--dir DIR] [--storage git|local] [--kb obsidian|local]
+    knowflow run [--plugin NAME]
+    knowflow status
+    knowflow commit [MESSAGE]
+    knowflow archive [--dry-run]
+    knowflow list
+    knowflow version
+"""
+import sys
+import os
+from pathlib import Path
+
+def main():
+    args = sys.argv[1:]
+    if not args or args[0] in ("-h", "--help"):
+        print(__doc__.strip())
+        return
+    
+    cmd = args[0]
+    
+    if cmd == "version":
+        from knowflow import __version__
+        print(f"KnowFlow v{__version__}")
+    
+    elif cmd == "list":
+        from knowflow.cli.template_manager import list_templates
+        templates = list_templates()
+        print(f"\n📦 Available templates ({len(templates)}):\n")
+        for t in templates:
+            print(f"  {t['display']:25s}  {t['description']}")
+        print()
+    
+    elif cmd == "init":
+        _init(args[1:])
+    
+    elif cmd == "run":
+        _run(args[1:])
+    
+    elif cmd == "status":
+        _status()
+    
+    elif cmd == "commit":
+        msg = " ".join(args[1:]) if len(args) > 1 else ""
+        _commit(msg)
+    
+    elif cmd == "archive":
+        dry_run = "--dry-run" in args
+        _archive(dry_run)
+    
+    elif cmd == "dashboard":
+        _dashboard(args[1:])
+    
+    else:
+        print(f"Unknown command: {cmd}")
+        print(__doc__.strip())
+
+
+def _init(args):
+    from knowflow.cli.template_manager import init_project, list_templates
+    
+    template = args[0] if args else None
+    output_dir = "."
+    storage = "git"
+    kb = "local"
+    
+    for i, a in enumerate(args):
+        if a == "--dir" and i + 1 < len(args): output_dir = args[i + 1]
+        if a == "--storage" and i + 1 < len(args): storage = args[i + 1]
+        if a == "--kb" and i + 1 < len(args): kb = args[i + 1]
+    
+    if not template:
+        print("❌ Please specify a template. Available:")
+        for t in list_templates():
+            print(f"  {t['name']:25s}  {t['description']}")
+        return
+    
+    try:
+        files = init_project(template, output_dir, storage, kb)
+        print(f"\n✅ Created project '{template}' in {output_dir}/")
+        for f in files:
+            print(f"   📄 {f}")
+        print(f"\n   Next: cd {output_dir} && knowflow run")
+    except ValueError as e:
+        print(f"❌ {e}")
+
+
+def _run(args):
+    from knowflow.core.pipeline import PipelineRunner
+    from knowflow.transforms import MarkdownExport, ProfileIndex
+    from knowflow.adapters.storage import create_storage
+    
+    # Detect project root
+    root = Path.cwd()
+    config_path = root / "knowflow.toml"
+    if not config_path.exists():
+        print("❌ No knowflow.toml found. Run 'knowflow init' first.")
+        return
+    
+    import yaml
+    with open(config_path) as f:
+        config = yaml.safe_load(f)
+    
+    # Parse args
+    plugin_name = None
+    dry_run = "--dry-run" in args
+    verbose = "--verbose" in args
+    for i, a in enumerate(args):
+        if a == "--plugin" and i + 1 < len(args):
+            plugin_name = args[i + 1]
+    
+    # Create storage
+    storage_cfg = config.get("storage", {"backend": "git", "path": str(root)})
+    store = create_storage(storage_cfg["backend"], config=storage_cfg)
+    
+    # Setup pipeline
+    runner = PipelineRunner(config)
+    runner.register(MarkdownExport())
+    runner.register(ProfileIndex())
+    
+    data_dir = root / "data" if (root / "data").exists() else root
+    output_dir = root / "output"
+    output_dir.mkdir(exist_ok=True)
+    
+    if plugin_name:
+        results = [runner.run_plugin(plugin_name, store, data_dir, output_dir, dry_run, verbose)]
+    else:
+        plugins = [p.stem for p in sorted(data_dir.iterdir()) if p.is_dir() and p.name not in ("output", ".git", ".github")]
+        if not plugins:
+            # Fall back to the data directory itself as a single plugin
+            plugins = [root.name]
+        results = runner.run_all(plugins, store, data_dir, output_dir, dry_run, verbose)
+    
+    for r in results:
+        icon = "✅" if r["success"] else "❌"
+        print(f"  {icon} {r['plugin']}")
+        for sname, sres in r["stages"].items():
+            status = sres.get("status", "?")
+            if status == "ok":
+                gen = sres.get("generated", 0)
+                print(f"     ├─ {sname}: {gen} files generated" if gen else f"     ├─ {sname}: ✅")
+            elif status == "skipped":
+                print(f"     ├─ {sname}: ⏭️  {sres.get('reason', '')}")
+
+
+def _status():
+    from knowflow.adapters.storage import create_storage
+    
+    root = Path.cwd()
+    config_path = root / "knowflow.toml"
+    backend = "git"
+    if config_path.exists():
+        import yaml
+        with open(config_path) as f:
+            cfg = yaml.safe_load(f)
+            backend = cfg.get("storage", {}).get("backend", "git")
+    
+    try:
+        store = create_storage(backend, config={"path": str(root)})
+        items = store.walk() if hasattr(store, 'walk') else []
+        commits = store.log(5) if hasattr(store, 'log') else []
+        
+        print(f"\n📊 KnowFlow Status")
+        print(f"   Root: {root}")
+        print(f"   Backend: {backend}")
+        print(f"   Files: {len(items)}")
+        if commits:
+            print(f"   Last commits:")
+            for c in commits:
+                short = c['id'][:8]
+                msg = c['message'][:50]
+                print(f"     {short}  {msg}")
+        else:
+            print(f"   (no version history)")
+        print()
+    except Exception as e:
+        print(f"❌ Status error: {e}")
+
+
+def _commit(message: str = ""):
+    from knowflow.adapters.storage import create_storage
+    root = Path.cwd()
+    store = create_storage("git", config={"path": str(root)})
+    rev = store.commit(message)
+    if rev:
+        print(f"✅ Committed: {rev[:8]} — {message or 'auto'}")
+    else:
+        print("   (nothing to commit)")
+
+
+def _archive(dry_run: bool = False):
+    from knowflow.transforms.archive import ArchiveStage
+    from knowflow.core.pipeline import PipelineContext
+    
+    root = Path.cwd()
+    data_dir = root / "data"
+    if not data_dir.exists():
+        print("❌ No data/ directory found")
+        return
+    
+    stage = ArchiveStage()
+    for plugin_dir in sorted(data_dir.iterdir()):
+        if plugin_dir.is_dir():
+            ctx = PipelineContext(
+                storage=None, plugin_name=plugin_dir.name,
+                plugin_config={}, data_dir=plugin_dir,
+                output_dir=root / "output" / plugin_dir.name,
+                dry_run=dry_run,
+            )
+            result = stage.process(ctx)
+            if dry_run:
+                print(f"  {plugin_dir}: {result.get('candidates', 0)} files to archive")
+            else:
+                print(f"  {plugin_dir}: {result.get('archives', 0)} archives, {result.get('freed_kb', 0)}KB freed")
+
+
+def _dashboard(args):
+    """Generate web dashboard from knowledge data."""
+    from knowflow.dashboard import generate
+    
+    root = Path.cwd()
+    knowledge_dir = root
+    auto_detect = True
+    password = None
+    
+    for i, a in enumerate(args):
+        if a == "--dir" and i + 1 < len(args):
+            knowledge_dir = Path(args[i + 1])
+            auto_detect = False
+        if a == "--password" and i + 1 < len(args):
+            password = args[i + 1]
+    
+    if auto_detect and not (knowledge_dir / "MANIFEST.yaml").exists():
+        home_kb = Path.home() / "knowledge"
+        if home_kb.exists():
+            knowledge_dir = home_kb
+        else:
+            print("❌ No knowledge directory found. Specify: knowflow dashboard --dir /path/to/knowledge")
+            return
+    
+    output = knowledge_dir / "dashboard"
+    for i, a in enumerate(args):
+        if a == "--output" and i + 1 < len(args):
+            output = Path(args[i + 1])
+    
+    if not password:
+        import getpass
+        password = getpass.getpass("设置仪表盘密码: ")
+    
+    try:
+        path = generate(knowledge_dir, output, password=password)
+        print(f"   Open: file://{path.absolute()}")
+        print(f"   Host with: cd {output} && python3 -m http.server 8080")
+    except Exception as e:
+        print(f"❌ Dashboard generation failed: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    main()

knowflow/cli/template_manager.py

@@ -0,0 +1,100 @@
+"""
+knowflow/cli/template_manager.py — Template system
+====================================================
+Init new projects from built-in templates.
+
+Usage:
+    knowflow init football     # Create football prediction project
+    knowflow list              # List available templates
+"""
+from pathlib import Path
+import shutil
+import yaml
+
+TEMPLATES_DIR = Path(__file__).resolve().parent.parent / "templates"
+
+
+def list_templates() -> list[dict]:
+    """List all available templates with descriptions"""
+    templates = []
+    for tdir in sorted(TEMPLATES_DIR.iterdir()):
+        if tdir.is_dir():
+            cfg_path = tdir / "plugin.yaml"
+            if cfg_path.exists():
+                with open(cfg_path) as f:
+                    cfg = yaml.safe_load(f) or {}
+                templates.append({
+                    "name": tdir.name,
+                    "display": cfg.get("display_name", tdir.name),
+                    "description": cfg.get("description", ""),
+                })
+    return templates
+
+
+def init_project(template_name: str, output_dir: str, storage_backend: str = "git",
+                 kb_backend: str = "local") -> list[str]:
+    """
+    Initialize a new project from a template.
+    
+    Args:
+        template_name: Name of template (e.g. "football-prediction")
+        output_dir: Where to create the project
+        storage_backend: Storage type (git, local, s3, etc.)
+        kb_backend: Knowledge base type (obsidian, local, notion, etc.)
+    
+    Returns:
+        List of created files (relative to output_dir)
+    """
+    src = TEMPLATES_DIR / template_name
+    if not src.exists():
+        raise ValueError(f"Template '{template_name}' not found. "
+                         f"Available: {[t['name'] for t in list_templates()]}")
+    
+    dst = Path(output_dir).expanduser().resolve()
+    dst.mkdir(parents=True, exist_ok=True)
+    
+    created = []
+    
+    # Copy plugin config
+    if (src / "plugin.yaml").exists():
+        shutil.copy2(src / "plugin.yaml", dst / "plugin.yaml")
+        created.append("plugin.yaml")
+    
+    # Create data directory
+    plugin_name = template_name.split("-")[0]
+    (dst / plugin_name).mkdir(parents=True, exist_ok=True)
+    created.append(f"{plugin_name}/")
+    
+    # Create knowflow.toml config
+    config = {
+        "project": {"name": template_name, "version": "0.1.0"},
+        "storage": {"backend": storage_backend, "path": str(dst)},
+        "knowledge": {"backend": kb_backend, "path": str(dst / "output")},
+        "pipeline": ["markdown_export", "profile_index"],
+    }
+    config_path = dst / "knowflow.toml"
+    config_path.write_text(yaml.dump(config, default_flow_style=False), encoding="utf-8")
+    created.append("knowflow.toml")
+    
+    # Create README
+    readme = f"""# {template_name}
+
+Generated from KnowFlow template: {template_name}
+
+## Usage
+
+```bash
+# Add data
+echo '{{"key": "value"}}' > {plugin_name}/data/my-data.json
+
+# Run pipeline
+knowflow run
+
+# View output
+ls {plugin_name}/output/
+```
+"""
+    (dst / "README.md").write_text(readme, encoding="utf-8")
+    created.append("README.md")
+    
+    return created

knowflow/core/__init__.py

@@ -0,0 +1,10 @@
+"""knowflow/core/__init__.py"""
+from .storage import StorageAdapter, StorageItem, StorageError, StorageNotFoundError
+from .knowledge import KnowledgeAdapter, KnowledgePage, KnowledgeError
+from .pipeline import TransformStage, PipelineRunner, PipelineContext, PipelineError
+
+__all__ = [
+    'StorageAdapter', 'StorageItem', 'StorageError', 'StorageNotFoundError',
+    'KnowledgeAdapter', 'KnowledgePage', 'KnowledgeError',
+    'TransformStage', 'PipelineRunner', 'PipelineContext', 'PipelineError',
+]

knowflow/core/knowledge.py

@@ -0,0 +1,85 @@
+"""
+knowflow/core/knowledge.py — Knowledge Base Adapter Interface
+==============================================================
+Knowledge Base = HOW humans read and interact with the data.
+Different from Storage (where data lives).
+
+Built-in adapters:
+  - local:    Plain markdown directory
+  - obsidian: Obsidian vault (markdown + .obsidian config)
+  - notion:   Notion database (via API)
+  - ima:      腾讯IMA知识库 (via OpenAPI)
+  - mkdocs:   MkDocs static site
+  - gitbook:  GitBook-style documentation
+
+Usage:
+    from knowflow.adapters.knowledge import create_knowledge, create_kb  # create_kb 为别名
+    kb = create_knowledge("obsidian", path="~/my-vault")
+    kb.sync([KnowledgePage(title="周报", content="# 周报\\n…", path="weekly/2026-10-02.md")])
+    kb.search("阅读习惯", limit=5)
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Optional
+
+
+class KnowledgeError(Exception): pass
+
+
+@dataclass
+class KnowledgePage:
+    """A page/document in the knowledge base"""
+    title: str
+    content: str           # Markdown content
+    path: str              # Relative path within KB
+    tags: list[str] = None
+    metadata: dict = None
+
+
+class KnowledgeAdapter(ABC):
+    """
+    Knowledge Base Adapter Interface
+    
+    Transforms structured data from Storage into human-readable
+    documents in the target knowledge base system.
+    
+    A single data source can sync to multiple KBs simultaneously
+    (e.g., Push to Obsidian + Notion + IMA at the same time).
+    """
+    
+    def __init__(self, config: dict = None):
+        self.config = config or {}
+    
+    @abstractmethod
+    def name(self) -> str:
+        """Human-readable name of this KB backend"""
+        ...
+    
+    @abstractmethod
+    def sync(self, pages: list[KnowledgePage]) -> dict:
+        """
+        Sync pages to the knowledge base.
+        
+        Args:
+            pages: List of KnowledgePage objects to sync
+        
+        Returns:
+            {"synced": 5, "skipped": 2, "errors": [...]}
+        """
+        ...
+    
+    @abstractmethod
+    def search(self, query: str, limit: int = 10) -> list[KnowledgePage]:
+        """
+        Search the knowledge base.
+        Returns matching pages.
+        """
+        ...
+    
+    def validate_config(self) -> list[str]:
+        """
+        Check if configuration is valid.
+        Returns list of error messages (empty = valid).
+        """
+        return []

knowflow/core/pipeline.py

@@ -0,0 +1,119 @@
+"""
+knowflow/core/pipeline.py — Data Pipeline Framework
+====================================================
+Transforms raw data (JSON) → formatted output (MD/docs).
+Each transform is a pluggable stage.
+
+Built-in stages:
+  - markdown_export:  JSON → Markdown
+  - profile_index:    Auto-generate PROFILE.md
+  - archive:          Hot → Warm → Cold lifecycle
+  - token_compress:   TokenJuice-style compression
+  - validate_schema:  Validate against plugin schema
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from pathlib import Path
+import importlib
+import inspect
+import yaml
+
+
+@dataclass
+class PipelineContext:
+    """Context passed through each pipeline stage"""
+    storage: object           # StorageAdapter instance
+    plugin_name: str          # e.g. "football"
+    plugin_config: dict       # Parsed plugin.yaml
+    data_dir: Path            # Where data files live
+    output_dir: Path          # Where output goes (obsidian/)
+    dry_run: bool = False
+    verbose: bool = False
+
+
+class PipelineError(Exception): pass
+
+
+class TransformStage(ABC):
+    """
+    A single stage in the data pipeline.
+    
+    Stages are loaded from knowflow.toml [pipeline] config.
+    To add a custom stage: subclass TransformStage and register.
+    """
+    
+    def __init__(self, name: str):
+        self.name = name
+    
+    @abstractmethod
+    def process(self, ctx: PipelineContext) -> dict:
+        """
+        Execute this stage.
+        Returns: {"status": "ok"|"skipped"|"error", "detail": "..."}
+        """
+        ...
+    
+    def should_run(self, ctx: PipelineContext) -> bool:
+        """Override to add conditional execution."""
+        return True
+
+
+class PipelineRunner:
+    """Orchestrates pipeline execution"""
+    
+    def __init__(self, config: dict):
+        self.config = config
+        self.stages: dict[str, TransformStage] = {}
+    
+    def register(self, stage: TransformStage):
+        """Register a pipeline stage"""
+        self.stages[stage.name] = stage
+    
+    def load_plugin_config(self, plugin_dir: Path) -> dict:
+        """Load plugin.yaml from a plugin directory"""
+        path = plugin_dir / "plugin.yaml"
+        if path.exists():
+            with open(path) as f:
+                return yaml.safe_load(f) or {}
+        return {}
+    
+    def run_plugin(self, plugin_name: str, storage, data_dir: Path, output_dir: Path,
+                   dry_run: bool = False, verbose: bool = False) -> dict:
+        """Run pipeline for one plugin"""
+        result = {"plugin": plugin_name, "stages": {}, "success": True}
+        
+        ctx = PipelineContext(
+            storage=storage,
+            plugin_name=plugin_name,
+            plugin_config={},
+            data_dir=data_dir / plugin_name,
+            output_dir=output_dir / plugin_name,
+            dry_run=dry_run,
+            verbose=verbose,
+        )
+        
+        for stage_name, stage in self.stages.items():
+            if not stage.should_run(ctx):
+                result["stages"][stage_name] = {"status": "skipped"}
+                continue
+            
+            try:
+                stage_result = stage.process(ctx)
+                result["stages"][stage_name] = stage_result
+                if stage_result.get("status") == "error":
+                    result["success"] = False
+            except Exception as e:
+                result["stages"][stage_name] = {"status": "error", "detail": str(e)}
+                result["success"] = False
+        
+        return result
+    
+    def run_all(self, plugins: list, storage, data_dir: Path, output_dir: Path,
+                dry_run: bool = False, verbose: bool = False) -> list[dict]:
+        """Run pipeline for all plugins"""
+        results = []
+        for plugin in plugins:
+            result = self.run_plugin(plugin, storage, data_dir, output_dir, dry_run, verbose)
+            results.append(result)
+        return results