notegen 0.1.0__tar.gz

notegen-0.1.0/.gitignore

@@ -0,0 +1,11 @@
+.venv/
+__pycache__/
+*.pyc
+*.pyo
+.pytest_cache/
+.ruff_cache/
+*.egg-info/
+dist/
+build/
+.coverage
+htmlcov/

notegen-0.1.0/PKG-INFO

@@ -0,0 +1,91 @@
+Metadata-Version: 2.4
+Name: notegen
+Version: 0.1.0
+Summary: Convert YouTube videos, playlists, and web pages into Obsidian markdown notes using LLMs
+Project-URL: Homepage, https://github.com/sidhunair280/notegen
+Project-URL: Bug Tracker, https://github.com/sidhunair280/notegen/issues
+Author-email: Sidhu Nair <sidhunair280@gmail.com>
+License: MIT
+Keywords: knowledge-management,llm,markdown,notes,obsidian,youtube
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: End Users/Desktop
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Text Processing :: Markup :: Markdown
+Classifier: Topic :: Utilities
+Requires-Python: >=3.11
+Requires-Dist: anyio>=4
+Requires-Dist: beautifulsoup4>=4.12
+Requires-Dist: httpx>=0.27
+Requires-Dist: litellm>=1.40
+Requires-Dist: pyyaml>=6
+Requires-Dist: rich>=13
+Requires-Dist: tiktoken>=0.7
+Requires-Dist: trafilatura>=1.12
+Requires-Dist: typer>=0.12
+Requires-Dist: youtube-transcript-api>=0.6
+Requires-Dist: yt-dlp>=2024.1
+Description-Content-Type: text/markdown
+
+# notegen
+
+Convert YouTube videos, playlists, and web pages into structured Obsidian-flavored markdown notes using LLMs.
+
+## Install
+
+```bash
+pip install notegen
+```
+
+## Usage
+
+```bash
+# Auto-detect source type
+notegen https://youtube.com/watch?v=...
+notegen https://example.com/article
+notegen transcript.txt
+
+# Explicit commands
+notegen video <youtube-url>
+notegen playlist <playlist-url> [--force]
+notegen web <url>
+notegen text <file-or-stdin>
+
+# Config
+notegen config init   # create ~/.config/notes-gen/config.yaml
+notegen config show   # print resolved config
+```
+
+## Options
+
+| Flag | Description |
+|---|---|
+| `-o / --output-dir` | Override output directory |
+| `-m / --model` | LiteLLM model string (e.g. `openai/gpt-4o`) |
+| `--no-mermaid` | Disable mermaid diagram generation |
+| `--force` | Skip playlist videos without captions instead of aborting |
+
+## Config file (`~/.config/notes-gen/config.yaml`)
+
+```yaml
+output_dir: ~/notes
+model: anthropic/claude-sonnet-4-6
+mermaid: true
+max_concurrent: 5
+web_max_pages: 50
+web_max_depth: 3
+```
+
+API keys are read from environment variables (e.g. `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`).
+
+## Output format
+
+Obsidian-flavored markdown with YAML frontmatter, `## / ###` headings, `> [!TIP]` / `> [!WARNING]` callouts, mermaid diagrams, and `[[wikilinks]]`.
+
+## Requirements
+
+- Python ≥ 3.11
+- An API key for your chosen LLM provider

notegen-0.1.0/README.md

@@ -0,0 +1,59 @@
+# notegen
+
+Convert YouTube videos, playlists, and web pages into structured Obsidian-flavored markdown notes using LLMs.
+
+## Install
+
+```bash
+pip install notegen
+```
+
+## Usage
+
+```bash
+# Auto-detect source type
+notegen https://youtube.com/watch?v=...
+notegen https://example.com/article
+notegen transcript.txt
+
+# Explicit commands
+notegen video <youtube-url>
+notegen playlist <playlist-url> [--force]
+notegen web <url>
+notegen text <file-or-stdin>
+
+# Config
+notegen config init   # create ~/.config/notes-gen/config.yaml
+notegen config show   # print resolved config
+```
+
+## Options
+
+| Flag | Description |
+|---|---|
+| `-o / --output-dir` | Override output directory |
+| `-m / --model` | LiteLLM model string (e.g. `openai/gpt-4o`) |
+| `--no-mermaid` | Disable mermaid diagram generation |
+| `--force` | Skip playlist videos without captions instead of aborting |
+
+## Config file (`~/.config/notes-gen/config.yaml`)
+
+```yaml
+output_dir: ~/notes
+model: anthropic/claude-sonnet-4-6
+mermaid: true
+max_concurrent: 5
+web_max_pages: 50
+web_max_depth: 3
+```
+
+API keys are read from environment variables (e.g. `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`).
+
+## Output format
+
+Obsidian-flavored markdown with YAML frontmatter, `## / ###` headings, `> [!TIP]` / `> [!WARNING]` callouts, mermaid diagrams, and `[[wikilinks]]`.
+
+## Requirements
+
+- Python ≥ 3.11
+- An API key for your chosen LLM provider

notegen-0.1.0/notes_gen/cli.py

@@ -0,0 +1,166 @@
+from pathlib import Path
+from typing import Optional
+
+import typer
+import yaml
+
+from notes_gen.config import DEFAULT_CONFIG_PATH, Config, load_config, merge_cli_overrides
+
+app = typer.Typer(no_args_is_help=True, help="Convert YouTube/web content to Obsidian notes.")
+config_app = typer.Typer(help="Manage configuration.")
+app.add_typer(config_app, name="config")
+
+_KNOWN_SUBCOMMANDS = {"video", "playlist", "web", "text", "config", "auto"}
+
+
+def _run_auto(source: str, cfg: Config, force: bool = False) -> None:
+    if "youtube.com/playlist" in source or ("list=" in source and "youtube.com" in source):
+        from notes_gen.sources.youtube import run_playlist_pipeline
+
+        index_path = run_playlist_pipeline(source, cfg, force=force)
+        typer.echo(f"Playlist notes written to {index_path.parent}")
+    elif "youtube.com/watch" in source or "youtu.be/" in source:
+        from notes_gen.sources.youtube import run_video_pipeline
+
+        output_path = run_video_pipeline(source, cfg)
+        typer.echo(f"Notes written to {output_path}")
+    elif source.startswith("http://") or source.startswith("https://"):
+        from notes_gen.sources.web import run_web_crawl_pipeline
+
+        output_path = run_web_crawl_pipeline(source, cfg)
+        typer.echo(f"Notes written to {output_path}")
+    else:
+        from notes_gen.sources.text import run_text_pipeline
+
+        output_path = run_text_pipeline(source, cfg)
+        typer.echo(f"Notes written to {output_path}")
+
+
+@config_app.command("init")
+def config_init() -> None:
+    """Create default config at ~/.config/notes-gen/config.yaml."""
+    if DEFAULT_CONFIG_PATH.exists():
+        typer.echo(f"Config already exists: {DEFAULT_CONFIG_PATH}")
+        raise typer.Exit(1)
+    DEFAULT_CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
+    defaults = Config()
+    data = {
+        "output_dir": str(defaults.output_dir),
+        "model": defaults.model,
+        "mermaid": defaults.mermaid,
+        "max_concurrent": defaults.max_concurrent,
+        "web_max_pages": defaults.web_max_pages,
+        "web_max_depth": defaults.web_max_depth,
+    }
+    DEFAULT_CONFIG_PATH.write_text(yaml.dump(data, default_flow_style=False))
+    typer.echo(f"Config written to {DEFAULT_CONFIG_PATH}")
+
+
+@config_app.command("show")
+def config_show() -> None:
+    """Print resolved config as YAML."""
+    cfg = load_config()
+    data = {
+        "output_dir": str(cfg.output_dir),
+        "model": cfg.model,
+        "mermaid": cfg.mermaid,
+        "max_concurrent": cfg.max_concurrent,
+        "web_max_pages": cfg.web_max_pages,
+        "web_max_depth": cfg.web_max_depth,
+    }
+    typer.echo(yaml.dump(data, default_flow_style=False), nl=False)
+
+
+@app.command()
+def video(
+    url: str = typer.Argument(..., help="YouTube video URL"),
+    output_dir: Optional[Path] = typer.Option(None, "--output-dir", "-o"),
+    model: Optional[str] = typer.Option(None, "--model", "-m"),
+    no_mermaid: bool = typer.Option(False, "--no-mermaid"),
+    verbose: bool = typer.Option(False, "--verbose", "-v"),
+    force: bool = typer.Option(False, "--force"),
+) -> None:
+    """Generate notes from a YouTube video."""
+    from notes_gen.sources.youtube import run_video_pipeline
+
+    cfg = load_config()
+    cfg = merge_cli_overrides(cfg, output_dir=output_dir, model=model, mermaid=not no_mermaid)
+    output_path = run_video_pipeline(url, cfg)
+    typer.echo(f"Notes written to {output_path}")
+
+
+@app.command()
+def playlist(
+    url: str = typer.Argument(..., help="YouTube playlist URL"),
+    output_dir: Optional[Path] = typer.Option(None, "--output-dir", "-o"),
+    model: Optional[str] = typer.Option(None, "--model", "-m"),
+    no_mermaid: bool = typer.Option(False, "--no-mermaid"),
+    verbose: bool = typer.Option(False, "--verbose", "-v"),
+    force: bool = typer.Option(False, "--force"),
+) -> None:
+    """Generate notes from a YouTube playlist."""
+    from notes_gen.sources.youtube import run_playlist_pipeline
+
+    cfg = load_config()
+    cfg = merge_cli_overrides(cfg, output_dir=output_dir, model=model, mermaid=not no_mermaid)
+    index_path = run_playlist_pipeline(url, cfg, force=force)
+    typer.echo(f"Playlist notes written to {index_path.parent}")
+
+
+@app.command()
+def web(
+    url: str = typer.Argument(..., help="Web URL to fetch"),
+    output_dir: Optional[Path] = typer.Option(None, "--output-dir", "-o"),
+    model: Optional[str] = typer.Option(None, "--model", "-m"),
+    no_mermaid: bool = typer.Option(False, "--no-mermaid"),
+    verbose: bool = typer.Option(False, "--verbose", "-v"),
+) -> None:
+    """Generate notes from a web page (crawls same-domain links)."""
+    from notes_gen.sources.web import run_web_crawl_pipeline
+
+    cfg = load_config()
+    cfg = merge_cli_overrides(cfg, output_dir=output_dir, model=model, mermaid=not no_mermaid)
+    output_path = run_web_crawl_pipeline(url, cfg)
+    typer.echo(f"Notes written to {output_path}")
+
+
+@app.command()
+def text(
+    source: str = typer.Argument(..., help="File path or '-' for stdin"),
+    output_dir: Optional[Path] = typer.Option(None, "--output-dir", "-o"),
+    model: Optional[str] = typer.Option(None, "--model", "-m"),
+    no_mermaid: bool = typer.Option(False, "--no-mermaid"),
+    verbose: bool = typer.Option(False, "--verbose", "-v"),
+) -> None:
+    """Generate notes from a text file or stdin."""
+    from notes_gen.sources.text import run_text_pipeline
+
+    cfg = load_config()
+    cfg = merge_cli_overrides(cfg, output_dir=output_dir, model=model, mermaid=not no_mermaid)
+    output_path = run_text_pipeline(source, cfg)
+    typer.echo(f"Notes written to {output_path}")
+
+
+@app.command()
+def auto(
+    source: str = typer.Argument(..., help="YouTube URL, web URL, or file path"),
+    output_dir: Optional[Path] = typer.Option(None, "--output-dir", "-o"),
+    model: Optional[str] = typer.Option(None, "--model", "-m"),
+    no_mermaid: bool = typer.Option(False, "--no-mermaid"),
+    verbose: bool = typer.Option(False, "--verbose", "-v"),
+    force: bool = typer.Option(False, "--force"),
+) -> None:
+    """Auto-detect source type and generate notes."""
+    cfg = load_config()
+    cfg = merge_cli_overrides(cfg, output_dir=output_dir, model=model, mermaid=not no_mermaid)
+    _run_auto(source, cfg, force=force)
+
+
+def main() -> None:
+    """Entry point: inject 'auto' subcommand when bare URL/file is passed."""
+    import sys
+
+    args = sys.argv[1:]
+    if args and not args[0].startswith("-") and args[0] not in _KNOWN_SUBCOMMANDS:
+        sys.argv.insert(1, "auto")
+    app()

notegen-0.1.0/notes_gen/config.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, replace
+from pathlib import Path
+from typing import Optional
+
+import yaml
+
+DEFAULT_CONFIG_PATH = Path.home() / ".config" / "notes-gen" / "config.yaml"
+
+DEFAULT_MODEL = "anthropic/claude-sonnet-4-6"
+
+
+@dataclass
+class Config:
+    output_dir: Path = None  # type: ignore[assignment]
+    model: str = DEFAULT_MODEL
+    mermaid: bool = True
+    max_concurrent: int = 5
+    web_max_pages: int = 50
+    web_max_depth: int = 3
+
+    def __post_init__(self) -> None:
+        if self.output_dir is None:
+            self.output_dir = Path.home() / "notes"
+        self.output_dir = Path(self.output_dir)
+
+
+def load_config(path: Path = DEFAULT_CONFIG_PATH) -> Config:
+    if not path.exists():
+        return Config()
+    raw = yaml.safe_load(path.read_text()) or {}
+    kwargs: dict = {}
+    if "output_dir" in raw:
+        kwargs["output_dir"] = Path(raw["output_dir"])
+    for field in ("model", "mermaid", "max_concurrent", "web_max_pages", "web_max_depth"):
+        if field in raw:
+            kwargs[field] = raw[field]
+    return Config(**kwargs)
+
+
+def merge_cli_overrides(
+    cfg: Config,
+    *,
+    output_dir: Optional[Path] = None,
+    model: Optional[str] = None,
+    mermaid: Optional[bool] = None,
+) -> Config:
+    overrides: dict = {}
+    if output_dir is not None:
+        overrides["output_dir"] = output_dir
+    if model is not None:
+        overrides["model"] = model
+    if mermaid is not None:
+        overrides["mermaid"] = mermaid
+    return replace(cfg, **overrides)

notegen-0.1.0/notes_gen/output/formatter.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+import re
+import unicodedata
+from datetime import date
+
+import yaml
+
+
+def slugify(text: str) -> str:
+    if not text:
+        return ""
+    text = unicodedata.normalize("NFKD", text)
+    text = text.encode("ascii", "ignore").decode("ascii")
+    text = text.lower()
+    text = re.sub(r"[^\w\s-]", "", text)
+    text = re.sub(r"[\s_]+", "-", text)
+    text = text.strip("-")
+    return text
+
+
+def build_frontmatter(
+    title: str,
+    source: str,
+    type: str,
+    tags: list[str],
+    date: date,
+) -> str:
+    data = {
+        "title": title,
+        "source": source,
+        "type": type,
+        "tags": tags,
+        "date": date.isoformat(),
+    }
+    return f"---\n{yaml.dump(data, default_flow_style=False, allow_unicode=True)}---\n"

notegen-0.1.0/notes_gen/output/writer.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def write_note(
+    path: Path,
+    content: str,
+    overwrite_policy: str = "overwrite",
+) -> Path:
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    if path.exists():
+        if overwrite_policy == "skip":
+            return path
+        if overwrite_policy == "rename":
+            stem = path.stem
+            suffix = path.suffix
+            counter = 1
+            new_path = path.parent / f"{stem}-{counter}{suffix}"
+            while new_path.exists():
+                counter += 1
+                new_path = path.parent / f"{stem}-{counter}{suffix}"
+            new_path.write_text(content, encoding="utf-8")
+            return new_path
+
+    path.write_text(content, encoding="utf-8")
+    return path
+
+
+def write_index(playlist_dir: Path, video_slugs: list[str]) -> Path:
+    playlist_dir.mkdir(parents=True, exist_ok=True)
+    title = playlist_dir.name.replace("-", " ").title()
+    lines = [f"# {title}\n"]
+    for slug in video_slugs:
+        lines.append(f"- [[{slug}]]")
+    content = "\n".join(lines) + "\n"
+    index_path = playlist_dir / "index.md"
+    index_path.write_text(content, encoding="utf-8")
+    return index_path

notegen-0.1.0/notes_gen/processing/chunker.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import tiktoken
+
+_ENCODING = tiktoken.get_encoding("cl100k_base")
+
+
+def count_tokens(text: str) -> int:
+    return len(_ENCODING.encode(text))
+
+
+def chunk_text(text: str, max_tokens: int = 12000, overlap: int = 200) -> list[str]:
+    if not text.strip():
+        return []
+
+    tokens = _ENCODING.encode(text)
+    total = len(tokens)
+
+    if total <= max_tokens:
+        return [text]
+
+    chunks: list[str] = []
+    start = 0
+    while start < total:
+        end = min(start + max_tokens, total)
+        chunk_tokens = tokens[start:end]
+        chunks.append(_ENCODING.decode(chunk_tokens))
+        if end >= total:
+            break
+        start = end - overlap
+
+    return chunks

notegen-0.1.0/notes_gen/processing/filter.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import re
+
+_META_PATTERNS = [
+    r"(?i)\b(sponsored?|sponsorship)\b.{0,120}",
+    r"(?i)\bsmash\s+that\s+(like|subscribe)\b.{0,80}",
+    r"(?i)\blike\s+(and\s+)?subscribe\b.{0,80}",
+    r"(?i)\bsubscribe\s+(to\s+)?(my|the|this)\s+(channel|newsletter|podcast)\b.{0,80}",
+    r"(?i)\bfollow\s+(me\s+)?on\s+(twitter|instagram|tiktok|linkedin|facebook)\b.{0,80}",
+    r"(?i)\bcheck\s+out\s+my\s+merch\b.{0,100}",
+    r"(?i)\bsupport\s+(me\s+)?on\s+patreon\b.{0,100}",
+    r"(?i)\bjoin\s+(my\s+)?discord\b.{0,80}",
+    r"(?i)\bget\s+\d+%\s+off\b.{0,120}",
+    r"(?i)\buse\s+(code|promo)\s+\w+\s+(for|to\s+get)\b.{0,120}",
+    r"(?i)\bnordvpn\b.{0,150}",
+    r"(?i)\bteespring\b.{0,100}",
+]
+
+_COMPILED = [re.compile(p) for p in _META_PATTERNS]
+
+
+def remove_meta(text: str) -> str:
+    if not text:
+        return text
+    lines = text.split("\n")
+    cleaned: list[str] = []
+    for line in lines:
+        if any(pat.search(line) for pat in _COMPILED):
+            continue
+        cleaned.append(line)
+    return "\n".join(cleaned)

notegen-0.1.0/notes_gen/processing/llm.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import litellm
+
+from notes_gen.config import Config
+
+_SYSTEM_PROMPT = """\
+You are an expert note-taker converting transcripts and articles into structured Obsidian notes.
+
+Rules:
+- Use Obsidian-flavored markdown: YAML frontmatter (omit here — added externally),
+  ## and ### headings only
+- Use `> [!TIP]` and `> [!WARNING]` callouts for important insights
+- Use mermaid diagrams for flows and architectures when appropriate
+- Use [[wikilinks]] for cross-references to related concepts
+- Be comprehensive — never truncate to hit a length limit
+- Auto-infer relevant tags from content
+- Write for a technical audience learning the subject
+"""
+
+_USER_PROMPT_TEMPLATE = """\
+Convert the following content into structured Obsidian notes:
+
+<content>
+{chunk}
+</content>
+
+Produce well-organized markdown notes with clear headings, key concepts,
+code examples where relevant, and callouts for important points.
+"""
+
+
+def generate_notes(chunks: list[str], cfg: Config) -> str:
+    results: list[str] = []
+    for chunk in chunks:
+        messages = [
+            {"role": "system", "content": _SYSTEM_PROMPT},
+            {"role": "user", "content": _USER_PROMPT_TEMPLATE.format(chunk=chunk)},
+        ]
+        response = litellm.completion(
+            model=cfg.model,
+            messages=messages,
+            temperature=0.3,
+        )
+        results.append(response.choices[0].message.content)
+    return "\n\n".join(results)

notegen-0.1.0/notes_gen/processing/merger.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+import re
+
+_HEADER_RE = re.compile(r"^(#{1,3})\s+(.+)$", re.MULTILINE)
+
+
+def merge(notes: list[str]) -> str:
+    if not notes:
+        return ""
+
+    seen_headers: set[str] = set()
+    output_sections: list[str] = []
+
+    for note in notes:
+        if not note.strip():
+            continue
+        sections = _split_into_sections(note)
+        for header, body in sections:
+            key = header.strip().lower() if header else None
+            if key and key in seen_headers:
+                continue
+            if key:
+                seen_headers.add(key)
+            output_sections.append((header, body))
+
+    if not output_sections:
+        return ""
+
+    parts: list[str] = []
+    for header, body in output_sections:
+        if header:
+            parts.append(f"{header}\n\n{body.strip()}")
+        else:
+            parts.append(body.strip())
+
+    return "\n\n".join(p for p in parts if p)
+
+
+def _split_into_sections(text: str) -> list[tuple[str, str]]:
+    """Split text into (header, body) pairs. First section may have no header."""
+    positions = [(m.start(), m.group()) for m in _HEADER_RE.finditer(text)]
+    if not positions:
+        return [("", text)]
+
+    sections: list[tuple[str, str]] = []
+    if positions[0][0] > 0:
+        sections.append(("", text[: positions[0][0]]))
+
+    for i, (pos, header) in enumerate(positions):
+        header_end = pos + len(header)
+        next_pos = positions[i + 1][0] if i + 1 < len(positions) else len(text)
+        body = text[header_end:next_pos].lstrip("\n")
+        sections.append((header, body))
+
+    return sections