longecho 0.3.0__py3-none-any.whl

longecho/__init__.py

@@ -0,0 +1,29 @@
+"""longecho -- compliance validator, discovery, and site builder for durable personal archives."""
+
+__version__ = "0.3.0"
+
+from .build import BuildResult, build_site
+from .checker import (
+    DURABLE_FORMAT_CATEGORIES,
+    ComplianceResult,
+    EchoSource,
+    Readme,
+    check_compliance,
+    parse_readme,
+)
+from .discovery import discover_sources, matches_query, search_sources
+
+__all__ = [
+    "BuildResult",
+    "ComplianceResult",
+    "DURABLE_FORMAT_CATEGORIES",
+    "EchoSource",
+    "Readme",
+    "__version__",
+    "build_site",
+    "check_compliance",
+    "discover_sources",
+    "matches_query",
+    "parse_readme",
+    "search_sources",
+]

longecho/__main__.py

@@ -0,0 +1,5 @@
+"""Allow running longecho as a module: python -m longecho."""
+
+from .cli import app
+
+app()

longecho/build.py

@@ -0,0 +1,250 @@
+"""longecho site builder -- generates a single-file application from a longecho archive."""
+
+import re
+from dataclasses import dataclass
+from datetime import date, datetime
+from pathlib import Path
+from typing import Optional
+
+from jinja2 import Environment, PackageLoader, select_autoescape
+
+from . import __version__
+from .checker import EchoSource, check_compliance
+from .discovery import should_skip_directory
+
+
+@dataclass
+class BuildResult:
+    """Result of a site build operation."""
+
+    success: bool
+    output_path: Optional[Path] = None
+    sources_count: int = 0
+    error: Optional[str] = None
+
+
+def get_jinja_env() -> Environment:
+    """Create Jinja2 environment with templates."""
+    return Environment(
+        loader=PackageLoader("longecho", "templates"),
+        autoescape=select_autoescape(["html", "xml"]),
+    )
+
+
+def _sanitize_html(html: str) -> str:
+    """Strip dangerous HTML elements from rendered markdown."""
+    # Remove dangerous tags with content
+    html = re.sub(
+        r"<(script|style|iframe|object|embed)\b[^>]*>.*?</\1>",
+        "", html, flags=re.DOTALL | re.IGNORECASE,
+    )
+    # Remove self-closing dangerous tags
+    html = re.sub(
+        r"<(script|style|iframe|object|embed|base|meta|link)\b[^>]*/?>",
+        "", html, flags=re.IGNORECASE,
+    )
+    # Remove opening-only dangerous tags (base, meta, link are void elements)
+    html = re.sub(
+        r"<(base|meta|link)\b[^>]*>",
+        "", html, flags=re.IGNORECASE,
+    )
+    # Remove form tags
+    html = re.sub(r"</?form\b[^>]*>", "", html, flags=re.IGNORECASE)
+    # Remove on* event handlers (quoted)
+    html = re.sub(r"\s+on\w+\s*=\s*[\"'][^\"']*[\"']", "", html, flags=re.IGNORECASE)
+    # Remove on* event handlers (unquoted, stop at > or whitespace)
+    html = re.sub(r"\s+on\w+\s*=\s*[^\s>]+", "", html, flags=re.IGNORECASE)
+    # Remove javascript: URIs in href/src/action attributes
+    html = re.sub(
+        r'(href|src|action)\s*=\s*["\']?\s*javascript:[^"\'>\s]*["\']?',
+        "", html, flags=re.IGNORECASE,
+    )
+    return html
+
+
+def markdown_to_html(content: str) -> str:
+    """Convert markdown to sanitized HTML."""
+    import markdown
+
+    html: str = markdown.markdown(content, extensions=["fenced_code", "tables"])
+    return _sanitize_html(html)
+
+
+def _get_data_files(source: EchoSource, output_path: Path) -> list[dict]:
+    """Get data files in a source directory, with relative paths from output_path."""
+    files = []
+    for fmt in source.durable_formats:
+        for f in source.path.glob(f"*{fmt}"):
+            if f.is_file():
+                try:
+                    rel = Path(f.relative_to(output_path.parent))
+                except ValueError:
+                    # output_path is outside the archive — use absolute file:// URI
+                    rel = Path(f.as_uri()) if hasattr(f, "as_uri") else f
+                files.append({"name": f.name, "path": str(rel)})
+    return sorted(files, key=lambda x: x["name"])
+
+
+def discover_sub_sources(source: EchoSource) -> list[EchoSource]:
+    """Discover sub-sources using contents field or auto-discovery."""
+    path = source.path
+    sources: list[EchoSource] = []
+
+    if source.contents:
+        # Curated: only listed paths, in order
+        for entry in source.contents:
+            entry_path = entry.get("path", "")
+            sub_path = (path / entry_path).resolve()
+            # Prevent path traversal
+            if not sub_path.is_relative_to(path):
+                continue
+            if not sub_path.is_dir():
+                continue
+
+            result = check_compliance(sub_path)
+            if result.compliant and result.source:
+                sources.append(result.source)
+    else:
+        # Auto-discover: all compliant subdirectories, alphabetical
+        for item in sorted(path.iterdir()):
+            if not item.is_dir():
+                continue
+            if item.name == "site" or should_skip_directory(item.name):
+                continue
+
+            result = check_compliance(item)
+            if result.compliant and result.source:
+                sources.append(result.source)
+
+    return sources
+
+
+def make_json_safe(obj: object) -> object:
+    """Recursively convert non-JSON-serializable types (e.g. datetime.date) to strings."""
+    if isinstance(obj, dict):
+        return {k: make_json_safe(v) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [make_json_safe(v) for v in obj]
+    if isinstance(obj, (date, datetime)):
+        return obj.isoformat()
+    return obj
+
+
+def _source_to_json(source: EchoSource, output_path: Path) -> dict:
+    """Convert an EchoSource to a JSON-serializable dict for the SFA, recursively."""
+    readme_html = ""
+    try:
+        content = source.readme_path.read_text(encoding="utf-8")
+        readme_html = markdown_to_html(content)
+    except (OSError, UnicodeDecodeError):
+        pass
+
+    frontmatter: dict = make_json_safe(source.frontmatter or {})  # type: ignore[assignment]
+
+    # Recursively discover and convert children
+    child_sources = discover_sub_sources(source)
+    children = [_source_to_json(c, output_path) for c in child_sources]
+
+    # Compute relative path to source's site/index.html if it exists
+    site_url = None
+    if source.has_site and source.site_path:
+        try:
+            rel = source.site_path / "index.html"
+            site_url = str(rel.relative_to(output_path.parent))
+        except ValueError:
+            site_url = (source.site_path / "index.html").as_uri()
+
+    return {
+        "name": source.name,
+        "description": source.description,
+        "formats": source.durable_formats,
+        "frontmatter": frontmatter,
+        "readme_html": readme_html,
+        "data_files": _get_data_files(source, output_path),
+        "site_url": site_url,
+        "children": children,
+    }
+
+
+def _generate_site_readme(
+    name: str,
+    sources: list[EchoSource],
+    output_path: Path,
+) -> None:
+    """Generate a longecho-compliant README.md for the site/ directory."""
+    import yaml
+
+    source_names = ", ".join(s.name for s in sources)
+    now = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
+
+    frontmatter = {
+        "name": f"{name} Site",
+        "description": "Static site generated from longecho archive",
+        "generator": f"longecho build v{__version__}",
+        "datetime": now,
+        "contents": [
+            {"path": "index.html", "description": "Single-file browsable archive"},
+        ],
+    }
+    fm_yaml = yaml.dump(frontmatter, default_flow_style=False, sort_keys=False)
+
+    content = f"---\n{fm_yaml}---\n\nGenerated by longecho from {len(sources)} source(s): {source_names}.\nOpen index.html in any browser to explore.\n"
+    readme_path = output_path / "README.md"
+    readme_path.write_text(content, encoding="utf-8")
+
+
+def _count_sources(data: list[dict]) -> int:
+    """Count all sources including nested children."""
+    total = len(data)
+    for s in data:
+        total += _count_sources(s.get("children", []))
+    return total
+
+
+def build_site(
+    path: Path,
+    output: Optional[Path] = None,
+) -> BuildResult:
+    """Build a single-file application for a longecho archive."""
+    path = Path(path).resolve()
+
+    if not path.exists():
+        return BuildResult(success=False, error=f"Path does not exist: {path}")
+
+    if not path.is_dir():
+        return BuildResult(success=False, error=f"Path is not a directory: {path}")
+
+    result = check_compliance(path)
+    if not result.compliant or not result.source:
+        return BuildResult(
+            success=False,
+            error=f"Not a longecho archive: {result.reason}"
+        )
+
+    root_source = result.source
+    sources = discover_sub_sources(root_source)
+
+    output_path = Path(output).resolve() if output else path / "site"
+    output_path.mkdir(parents=True, exist_ok=True)
+
+    sources_data = [_source_to_json(s, output_path) for s in sources]
+
+    env = get_jinja_env()
+    template = env.get_template("sfa.html")
+    html = template.render(
+        name=root_source.name,
+        description=root_source.description,
+        sources_data=sources_data,
+        generated_at=datetime.now().strftime("%Y-%m-%d %H:%M"),
+    )
+
+    index_path = output_path / "index.html"
+    index_path.write_text(html, encoding="utf-8")
+
+    _generate_site_readme(root_source.name, sources, output_path)
+
+    return BuildResult(
+        success=True,
+        output_path=output_path,
+        sources_count=_count_sources(sources_data),
+    )

longecho/checker.py

@@ -0,0 +1,253 @@
+"""longecho compliance checker."""
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+
+import yaml
+
+# Single source of truth for durable format categories.
+# All display, spec, and compliance logic derives from this structure.
+DURABLE_FORMAT_CATEGORIES: dict[str, list[str]] = {
+    "Structured data": [".db", ".sqlite", ".sqlite3", ".json", ".jsonl"],
+    "Documents": [".md", ".markdown", ".txt", ".text", ".rst", ".html", ".htm"],
+    "Archives": [".zip"],
+    "Images": [".jpg", ".jpeg", ".png", ".webp", ".gif"],
+    "Tabular / data": [".csv", ".tsv", ".xml", ".yaml", ".yml"],
+}
+
+# Derived flat set for O(1) membership testing
+DURABLE_EXTENSIONS: set[str] = {
+    ext for exts in DURABLE_FORMAT_CATEGORIES.values() for ext in exts
+}
+
+EXCLUDE_PATTERNS: set[str] = {
+    "README.md", "README.txt",
+    "CLAUDE.md", "CHANGELOG.md",
+    ".gitignore", ".gitattributes",
+    "pyproject.toml", "setup.py", "setup.cfg",
+    "requirements.txt",
+}
+
+DEFAULT_FORMAT_SCAN_DEPTH: int = 2
+MAX_README_SUMMARY_LENGTH: int = 500
+
+
+@dataclass
+class Readme:
+    """A README parsed into its constituent parts."""
+
+    frontmatter: Optional[dict]
+    body: str
+    title: Optional[str]
+    summary: Optional[str]
+
+
+def _split_frontmatter(content: str) -> tuple[Optional[dict], str]:
+    """Split YAML frontmatter from markdown content.
+
+    Returns (frontmatter dict or None, remaining body).
+    """
+    if not content.startswith("---"):
+        return None, content
+
+    lines = content.split("\n")
+    # Find closing ---
+    for i, line in enumerate(lines[1:], start=1):
+        if line.strip() == "---":
+            yaml_content = "\n".join(lines[1:i])
+            body = "\n".join(lines[i + 1 :])
+            try:
+                parsed = yaml.safe_load(yaml_content)
+                if isinstance(parsed, dict):
+                    return parsed, body
+                return None, body
+            except yaml.YAMLError:
+                return None, body
+
+    # No closing delimiter found
+    return None, content
+
+
+def parse_readme(readme_path: Path) -> Optional[Readme]:
+    """Parse a README into frontmatter, title, and summary. Returns None if unreadable."""
+    try:
+        content = readme_path.read_text(encoding="utf-8")
+    except (OSError, UnicodeDecodeError):
+        return None
+
+    frontmatter, body = _split_frontmatter(content)
+
+    title = None
+    summary_lines: list[str] = []
+    in_paragraph = False
+
+    for line in body.split("\n"):
+        stripped = line.strip()
+
+        if stripped.startswith("# ") and title is None:
+            title = stripped[2:].strip()
+            continue
+
+        if stripped.startswith("#"):
+            if summary_lines:
+                break
+            continue
+
+        if not stripped:
+            if in_paragraph:
+                break
+            continue
+
+        in_paragraph = True
+        summary_lines.append(stripped)
+
+    summary = " ".join(summary_lines)[:MAX_README_SUMMARY_LENGTH] if summary_lines else None
+    return Readme(frontmatter=frontmatter, body=body, title=title, summary=summary)
+
+
+@dataclass
+class EchoSource:
+    """A longecho-compliant data source."""
+
+    path: Path
+    readme_path: Path
+    name: str
+    description: str
+    durable_formats: list[str] = field(default_factory=list)
+    has_site: bool = False
+    site_path: Optional[Path] = None
+    frontmatter: Optional[dict] = None
+    contents: Optional[list[dict]] = None
+
+    def __str__(self) -> str:
+        desc = self.description or "No description"
+        if len(desc) > 60:
+            desc = desc[:57] + "..."
+        return f"{self.path}: {desc}"
+
+
+@dataclass
+class ComplianceResult:
+    """Result of a longecho compliance check."""
+
+    compliant: bool
+    path: Path
+    reason: Optional[str] = None
+    source: Optional[EchoSource] = None  # populated when compliant
+
+    def __str__(self) -> str:
+        if self.compliant:
+            return f"longecho-compliant: {self.path}"
+        return f"Not longecho-compliant: {self.path} ({self.reason})"
+
+
+def find_readme(path: Path) -> Optional[Path]:
+    """Find README file at the root of a directory."""
+    for name in ["README.md", "README.txt", "readme.md", "readme.txt"]:
+        readme = path / name
+        if readme.is_file():
+            return readme
+    return None
+
+
+def detect_durable_formats(path: Path, max_depth: int = DEFAULT_FORMAT_SCAN_DEPTH) -> list[str]:
+    """Detect durable file formats in a directory up to max_depth."""
+    found: set[str] = set()
+
+    def scan_directory(dir_path: Path, depth: int):
+        if depth > max_depth:
+            return
+
+        try:
+            for item in dir_path.iterdir():
+                if item.name.startswith("."):
+                    continue
+
+                if item.is_file():
+                    if item.name not in EXCLUDE_PATTERNS:
+                        suffix = item.suffix.lower()
+                        if suffix in DURABLE_EXTENSIONS:
+                            found.add(suffix)
+                elif item.is_dir():
+                    scan_directory(item, depth + 1)
+        except PermissionError:
+            pass
+
+    scan_directory(path, 0)
+    return sorted(found)
+
+
+def is_durable_format(extension: str) -> bool:
+    """Check if a file extension is a durable format (with or without leading dot)."""
+    ext = extension.lower()
+    if not ext.startswith("."):
+        ext = "." + ext
+    return ext in DURABLE_EXTENSIONS
+
+
+def _parse_contents(frontmatter: dict) -> Optional[list[dict]]:
+    """Parse the contents field from frontmatter into normalized dicts."""
+    raw = frontmatter.get("contents")
+    if not isinstance(raw, list):
+        return None
+
+    entries = []
+    for item in raw:
+        if isinstance(item, dict) and "path" in item:
+            entries.append(item)
+    return entries if entries else None
+
+
+def check_compliance(path: Path) -> ComplianceResult:
+    """Check if a directory is longecho-compliant (has README + durable formats)."""
+    path = Path(path).resolve()
+
+    if not path.exists():
+        return ComplianceResult(compliant=False, path=path, reason="Path does not exist")
+    if not path.is_dir():
+        return ComplianceResult(compliant=False, path=path, reason="Path is not a directory")
+
+    readme_file = find_readme(path)
+    if not readme_file:
+        return ComplianceResult(
+            compliant=False, path=path, reason="No README.md or README.txt found"
+        )
+
+    durable = detect_durable_formats(path)
+
+    if not durable:
+        return ComplianceResult(
+            compliant=False, path=path, reason="No durable data formats found"
+        )
+
+    readme = parse_readme(readme_file)
+
+    # Name cascade: frontmatter name > # Heading > dirname
+    name = (readme.title if readme else None) or path.name
+    description = (readme.summary if readme else None) or ""
+    frontmatter = readme.frontmatter if readme else None
+    contents = None
+
+    if frontmatter:
+        name = frontmatter.get("name", name)
+        description = frontmatter.get("description", description)
+        contents = _parse_contents(frontmatter)
+
+    site_dir = path / "site"
+    has_site = site_dir.exists() and (site_dir / "index.html").exists()
+    site_path = site_dir if has_site else None
+
+    source = EchoSource(
+        path=path,
+        readme_path=readme_file,
+        name=name,
+        description=description,
+        durable_formats=durable,
+        has_site=has_site,
+        site_path=site_path,
+        frontmatter=frontmatter,
+        contents=contents,
+    )
+
+    return ComplianceResult(compliant=True, path=path, source=source)