markdownizer 0.4.0__tar.gz → 0.4.2__tar.gz

{markdownizer-0.4.0 → markdownizer-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: markdownizer
-Version: 0.4.0
+Version: 0.4.2
 Summary: Extract documentation from Python projects into Markdown without rewriting it
 Author-email: Mohammad Hasan Khoddami <mohammadh.khoddami@gmail.com>
 License: MIT License

{markdownizer-0.4.0 → markdownizer-0.4.2}/markdownizer/__init__.py

@@ -8,7 +8,7 @@ from markdownizer.extractor import extract_project
 from markdownizer.ir import IR_VERSION, ProjectIR, build_project_ir
 from markdownizer.optimizer import OptimizedContext, optimize_context
 
-__version__ = "0.4.0"
+__version__ = "0.4.2"
 __all__ = [
     "extract_project",
     "build_project_ir",

{markdownizer-0.4.0 → markdownizer-0.4.2}/markdownizer/cli.py

@@ -83,7 +83,9 @@ def build_parser() -> argparse.ArgumentParser:
         description=(
             "Extract existing documentation from a Python project into "
             "Markdown files. Use 'markdownizer build <project>' for format "
-            "selection; the legacy form below is kept as a compatibility alias."
+            "selection; the legacy form below is kept as a compatibility "
+            "alias. If your project directory is named 'build', 'context', "
+            "or 'stats', prefix it with './'."
         ),
     )
     parser.set_defaults(command="build")

{markdownizer-0.4.0 → markdownizer-0.4.2}/markdownizer/imports.py

@@ -67,11 +67,17 @@ def _resolve_relative(
 
 
 def _collect_statement_imports(
-    node: ast.AST, module_map: dict[str, str], from_rel_path: str
+    module: ast.Module, module_map: dict[str, str], from_rel_path: str
 ) -> list[ImportEdge]:
+    """Collect module-level import statements.
+
+    Only top-level statements count: imports inside function bodies, class
+    bodies, or ``if __name__ == "__main__":`` blocks are not structural
+    module dependencies. ``from __future__`` imports are filtered out.
+    """
     edges: list[ImportEdge] = []
 
-    for child in ast.walk(node):
+    for child in module.body:
         if isinstance(child, ast.Import):
             for alias in child.names:
                 target = _resolve_absolute(alias.name, module_map)
@@ -87,6 +93,8 @@ def _collect_statement_imports(
                     )
                 )
         elif isinstance(child, ast.ImportFrom):
+            if child.module == "__future__":
+                continue
             if child.module is None:
                 # ``from . import x`` — the imported names are resolved
                 # relative to the current package.

{markdownizer-0.4.0 → markdownizer-0.4.2}/markdownizer/ir.py

@@ -27,7 +27,7 @@ from typing import Any
 
 from markdownizer.classifier import classify
 from markdownizer.imports import ImportEdge, collect_import_edges
-from markdownizer.parser import DocObject, parse_file
+from markdownizer.parser import DocObject, parse_file, read_python_source
 from markdownizer.scanner import scan_python_files
 
 IR_VERSION = 1
@@ -336,7 +336,7 @@ def build_project_ir(project_root: Path, exclude: list[str] | None = None) -> Pr
     for py_file in files:
         rel = py_file.relative_to(project_root).as_posix()
         dotted = _module_dotted_name(rel)
-        source = py_file.read_text(encoding="utf-8")
+        source = read_python_source(py_file)
         module_obj, objects = parse_file(str(py_file), source=source)
 
         module_symbols = [_symbol_from_docobject(o, rel) for o in objects]

{markdownizer-0.4.0 → markdownizer-0.4.2}/markdownizer/optimizer/rank.py

@@ -11,6 +11,10 @@ Methods:
   (fixed iteration count for full determinism)
 * ``fanout`` — in-degree of each module in the import graph
 * ``simple`` — uniform file scores (no graph)
+
+Rank values are deterministic for a given platform (IEEE-754 addition
+order); the IR hash excludes ranks, so ``ir.hash`` is stable across
+platforms even if floating-point tie-breaking ever differs.
 """
 
 from __future__ import annotations

{markdownizer-0.4.0 → markdownizer-0.4.2}/markdownizer/optimizer/slice.py

@@ -4,12 +4,14 @@ Turns a ranked ProjectIR into an AI-ready context artifact within a token
 budget, using deterministic layered emission:
 
 * layer 0 — project index (packages, modules, docstrings)
-* layer 1 — signatures + docstrings for public symbols
-* layer 2 — full source for ranked symbols
-* layer 3 — remaining symbol docstrings/comments when budget allows
-
-High-ranked symbols are placed at the start and end of the artifact
-(primacy/recency bias); lower-ranked content sits in the middle.
+* layer 1 — per-symbol blocks: full source for the highest-ranked symbols,
+  signature-only for the rest (or docstrings only for ``include_source``
+  profiles without source)
+
+Each symbol is emitted exactly once. High-ranked symbols are placed at the
+start and end of the artifact (primacy/recency bias); lower-ranked content
+sits in the middle. The reported token estimate counts the complete
+artifact, header included.
 """
 
 from __future__ import annotations
@@ -102,6 +104,55 @@ def _block_tokens(text: str, prefer_tiktoken: bool) -> float:
     return count_tokens(text, prefer_tiktoken=prefer_tiktoken)
 
 
+def _symbol_block(
+    symbol: Symbol,
+    profile: Profile,
+    prefer_tiktoken: bool,
+    budget: float,
+) -> tuple[str, float] | None:
+    """Render one symbol exactly once; ``None`` when nothing fits the budget."""
+    comments = profile.include_comments
+    if profile.include_source is True:
+        src_text = render_object(
+            symbol,
+            None,
+            include_source=True,
+            include_comments=comments,
+        )
+        src_cost = _block_tokens(src_text, prefer_tiktoken)
+        if src_cost <= budget:
+            return src_text, src_cost
+        sig_text = render_object(
+            symbol,
+            None,
+            include_source="signature",
+            include_comments=comments,
+        )
+        sig_cost = _block_tokens(sig_text, prefer_tiktoken)
+        return (sig_text, sig_cost) if sig_cost <= budget else None
+
+    if profile.include_source == "signature":
+        sig_text = render_object(
+            symbol,
+            None,
+            include_source="signature",
+            include_comments=comments,
+        )
+        sig_cost = _block_tokens(sig_text, prefer_tiktoken)
+        return (sig_text, sig_cost) if sig_cost <= budget else None
+
+    if symbol.docstring:
+        doc_text = render_object(
+            symbol,
+            None,
+            include_source=False,
+            include_comments=comments,
+        )
+        doc_cost = _block_tokens(doc_text, prefer_tiktoken)
+        return (doc_text, doc_cost) if doc_cost <= budget else None
+    return None
+
+
 def optimize_context(
     ir: ProjectIR,
     max_tokens: float,
@@ -114,19 +165,20 @@ def optimize_context(
 
     Args:
         ir: The project IR (ranked in place by this function).
-        max_tokens: Target token budget (soft limit, ±10% slop).
+        max_tokens: Target token budget (soft limit, ±10% slop). Must be > 0.
         profile: Profile preset name (see :mod:`markdownizer.optimizer.profiles`).
         query: Optional space-separated keyword prefilter on symbol names,
             frameworks, and docstrings (deterministic substring match).
         rank_method: ``pagerank`` (default), ``fanout``, or ``simple``.
         prefer_tiktoken: Use ``tiktoken`` when installed for token estimates.
+
+    Raises:
+        ValueError: If ``max_tokens`` is not positive.
     """
+    if float(max_tokens) <= 0:
+        raise ValueError(f"max_tokens must be positive, got {max_tokens!r}")
     rank_ir(ir, method=rank_method)
     profile_obj = get_profile(profile)
-    budget = float(max_tokens) * _BUDGET_SLOP
-    parts: list[str] = []
-    total: float = 0.0
-    included: int = 0
     total_symbols = len(ir.symbols)
 
     header = (
@@ -134,90 +186,45 @@ def optimize_context(
         f"(profile={profile_obj.name}, rank={rank_method}, "
         f"max_tokens={int(max_tokens)})"
     )
+    header_cost = _block_tokens(header, prefer_tiktoken)
+    budget = float(max_tokens) * _BUDGET_SLOP - header_cost
+
+    parts: list[str] = []
+    total: float = 0.0
 
     index = _render_index(ir)
-    if _block_tokens(index, prefer_tiktoken) <= budget:
+    index_cost = _block_tokens(index, prefer_tiktoken)
+    if index_cost <= budget:
         parts.append(index)
-        total += _block_tokens(index, prefer_tiktoken)
+        total += index_cost
 
     module_index = _render_module_index(ir)
-    if _block_tokens(module_index, prefer_tiktoken) <= budget:
+    module_cost = _block_tokens(module_index, prefer_tiktoken)
+    if module_cost <= budget:
         parts.append(module_index)
-        total += _block_tokens(module_index, prefer_tiktoken)
+        total += module_cost
 
     symbols = _select_symbols(ir, profile_obj, query)
     ordered = _edge_place(symbols)
 
-    signatures: list[tuple[Symbol, str, float]] = []
-    sources: list[tuple[Symbol, str, float]] = []
-    remainder: list[tuple[Symbol, str, float]] = []
-
+    included = 0
     for symbol in ordered:
-        if profile_obj.include_source is not False:
-            sig_text = render_object(
-                symbol,
-                None,
-                include_source="signature",
-                include_comments=profile_obj.include_comments,
-            )
-            sig_cost = _block_tokens(sig_text, prefer_tiktoken)
-            if sig_cost <= budget:
-                signatures.append((symbol, sig_text, sig_cost))
-
-        if profile_obj.include_source is True:
-            src_text = render_object(
-                symbol,
-                None,
-                include_source=True,
-                include_comments=profile_obj.include_comments,
-            )
-            src_cost = _block_tokens(src_text, prefer_tiktoken)
-            sources.append((symbol, src_text, src_cost))
-
-        if profile_obj.include_source is False and symbol.docstring:
-            doc_text = render_object(
-                symbol,
-                None,
-                include_source=False,
-                include_comments=profile_obj.include_comments,
-            )
-            doc_cost = _block_tokens(doc_text, prefer_tiktoken)
-            remainder.append((symbol, doc_text, doc_cost))
-
-    if profile_obj.include_source is not False:
-        for _, sig_text, sig_cost in signatures:
-            if total + sig_cost > budget:
-                break
-            parts.append("---")
-            parts.append("")
-            parts.append(sig_text)
-            total += sig_cost
-            included += 1
-
-    if profile_obj.include_source is True:
-        for _, src_text, src_cost in sources:
-            if total + src_cost > budget:
-                break
-            parts.append("---")
-            parts.append("")
-            parts.append(src_text)
-            total += src_cost
-            included += 1
-
-    if profile_obj.include_source is False:
-        for _, doc_text, doc_cost in remainder:
-            if total + doc_cost > budget:
-                break
-            parts.append("---")
-            parts.append("")
-            parts.append(doc_text)
-            total += doc_cost
-            included += 1
+        block = _symbol_block(symbol, profile_obj, prefer_tiktoken, budget)
+        if block is None:
+            continue
+        text, cost = block
+        if total + cost > budget:
+            break
+        parts.append("---")
+        parts.append("")
+        parts.append(text)
+        total += cost
+        included += 1
 
     text = "\n".join([header, "", *parts]).rstrip() + "\n"
     return OptimizedContext(
         text=text,
-        estimated_tokens=total,
+        estimated_tokens=_block_tokens(text, prefer_tiktoken),
         max_tokens=float(max_tokens),
         profile=profile_obj.name,
         rank_method=rank_method,

{markdownizer-0.4.0 → markdownizer-0.4.2}/markdownizer/optimizer/tokens.py

@@ -18,7 +18,7 @@ try:
 
     _ENCODER = _tiktoken.get_encoding("cl100k_base")
     _HAS_TIKTOKEN = True
-except Exception:  # pragma: no cover - depends on optional dependency
+except (ImportError, OSError):  # pragma: no cover - depends on optional dependency
     _ENCODER = None
     _HAS_TIKTOKEN = False
 

{markdownizer-0.4.0 → markdownizer-0.4.2}/markdownizer/parser.py

@@ -6,6 +6,7 @@ import ast
 import io
 import tokenize
 from dataclasses import dataclass, field
+from pathlib import Path
 from typing import Protocol
 
 
@@ -72,6 +73,19 @@ def _rightmost_name(name: str) -> str:
     return name.rsplit(".", 1)[-1] if name else ""
 
 
+def read_python_source(path: Path) -> str:
+    """Read a Python source file, tolerating non-UTF-8 encodings.
+
+    Uses ``utf-8-sig`` (which also strips a UTF-8 BOM) and falls back to
+    latin-1, which can decode every byte sequence without raising. A single
+    legacy-encoded file must never abort an entire project extraction.
+    """
+    try:
+        return path.read_text(encoding="utf-8-sig")
+    except UnicodeDecodeError:
+        return path.read_text(encoding="latin-1")
+
+
 def _unparse_or_empty(node: ast.AST | None) -> str:
     """Render an AST node back to source text, or ``""`` if unavailable."""
     if node is None:
@@ -337,8 +351,7 @@ def parse_file(file_path: str, source: str | None = None) -> tuple[DocObject, li
     ``source`` may be passed in to avoid re-reading the file from disk.
     """
     if source is None:
-        with open(file_path, encoding="utf-8") as fh:
-            source = fh.read()
+        source = read_python_source(Path(file_path))
 
     source_lines = source.splitlines()
     all_comments = _collect_comments(source)

{markdownizer-0.4.0 → markdownizer-0.4.2}/markdownizer.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: markdownizer
-Version: 0.4.0
+Version: 0.4.2
 Summary: Extract documentation from Python projects into Markdown without rewriting it
 Author-email: Mohammad Hasan Khoddami <mohammadh.khoddami@gmail.com>
 License: MIT License

{markdownizer-0.4.0 → markdownizer-0.4.2}/tests/test_cli.py

@@ -222,6 +222,30 @@ def test_stats_invalid_rank_exits(tmp_path):
         main(["stats", str(tmp_path), "--rank", "magic"])
 
 
+def test_stats_empty_project(tmp_path, capsys):
+    assert main(["stats", str(tmp_path)]) == 0
+    captured = capsys.readouterr()
+    assert "Symbols: 0" in captured.out
+
+
+def test_context_query_flag(tmp_path, capsys):
+    (tmp_path / "auth.py").write_text('"""Auth."""\ndef login():\n    """Login."""\n    pass\n')
+    (tmp_path / "other.py").write_text(
+        '"""Other."""\ndef unrelated():\n    """Nope."""\n    pass\n'
+    )
+    out = tmp_path / "out"
+    assert main(["context", str(tmp_path), "-o", str(out), "--query", "login"]) == 0
+    text = (out / "context.md").read_text(encoding="utf-8")
+    assert "login" in text
+    assert "unrelated" not in text
+
+
+def test_context_negative_max_tokens_returns_1(tmp_path, capsys):
+    (tmp_path / "mod.py").write_text('"""docs"""\n')
+    assert main(["context", str(tmp_path), "--max-tokens", "-10"]) == 1
+    assert "max_tokens must be positive" in capsys.readouterr().err
+
+
 def _assert_exits_with(argv: list[str], expected: str, capsys: pytest.CaptureFixture[str]) -> None:
     with pytest.raises(SystemExit):
         main(argv)

{markdownizer-0.4.0 → markdownizer-0.4.2}/tests/test_extractor.py

@@ -119,3 +119,15 @@ def test_extract_error_on_unwritable_output(sample_project, tmp_path):
         pytest.skip("directory is writable; cannot simulate")
     with pytest.raises(OSError):
         extract_project(sample_project, out / "sub" / "nested")
+
+
+def test_extract_survives_non_utf8_file(tmp_path):
+    """End-to-end: a legacy-encoded file does not abort extraction."""
+    (tmp_path / "good.py").write_text('"""Fine."""\ndef ok():\n    pass\n')
+    (tmp_path / "legacy.py").write_bytes(b'"""caf\xe9"""\ndef old():\n    pass\n')
+    out = tmp_path / "out"
+    written = extract_project(tmp_path, out)
+    assert sorted(p.name for p in written) == ["_root.md"]
+    md = (out / "_root.md").read_text(encoding="utf-8")
+    assert "# Function: old" in md
+    assert "café" in md

{markdownizer-0.4.0 → markdownizer-0.4.2}/tests/test_imports.py

@@ -144,3 +144,42 @@ def test_dotted_import_resolution_to_init():
 
 def test_empty_source():
     assert _edges("") == []
+
+
+def test_nested_function_import_not_collected():
+    source = "def f():\n    import os\n    return os.path\n"
+    assert _edges(source) == []
+
+
+def test_main_guard_import_not_collected():
+    source = "import pkg.models\n\nif __name__ == '__main__':\n    import sys\n"
+    edges = _edges(source)
+    assert [e.imported_module for e in edges] == ["pkg.models"]
+
+
+def test_class_body_import_not_collected():
+    source = "class C:\n    def m(self):\n        import os\n"
+    assert _edges(source) == []
+
+
+def test_future_import_filtered():
+    source = "from __future__ import annotations\nimport pkg.models\n"
+    edges = _edges(source)
+    assert [e.imported_module for e in edges] == ["pkg.models"]
+
+
+def test_try_wrapped_import_not_collected():
+    """try/except-wrapped imports are conservatively treated as external."""
+    source = "try:\n    import pkg.models\nexcept ImportError:\n    pass\n"
+    assert _edges(source) == []
+
+
+def test_latin1_source_decodes():
+    from markdownizer.imports import collect_import_edges
+
+    edges = collect_import_edges(
+        '"""caf\xe9"""\nimport os\n'.encode("latin-1").decode("latin-1"),
+        "mod.py",
+        {"mod.py": "mod.py"},
+    )
+    assert [e.imported_module for e in edges] == ["os"]

{markdownizer-0.4.0 → markdownizer-0.4.2}/tests/test_ir.py

@@ -245,3 +245,34 @@ def test_unsorted_input_still_sorted(tmp_path):
     ir = build_project_ir(tmp_path)
     assert [m.path for m in ir.modules] == ["a/y.py", "b/z.py"]
     assert [s.qualified_name for s in ir.symbols] == ["y", "z"]
+
+
+def test_build_survives_non_utf8_file(tmp_path):
+    """A single latin-1 encoded file must not abort the whole build."""
+    from tests.conftest import write_py
+
+    write_py(tmp_path / "good.py", '"""Fine."""\ndef ok():\n    pass\n')
+    (tmp_path / "legacy.py").write_bytes(b'"""caf\xe9 latin-1"""\ndef old():\n    pass\n')
+    ir = build_project_ir(tmp_path)
+    assert [m.path for m in ir.modules] == ["good.py", "legacy.py"]
+    legacy = next(m for m in ir.modules if m.path == "legacy.py")
+    assert legacy.docstring == "café latin-1"
+    assert ir.stats.symbol_count == 2
+
+
+def test_build_survives_utf8_bom_file(tmp_path):
+    from tests.conftest import write_py
+
+    write_py(tmp_path / "a.py", "def f():\n    pass\n")
+    (tmp_path / "bom.py").write_bytes(b'\xef\xbb\xbf"""BOM doc."""\ndef g():\n    pass\n')
+    ir = build_project_ir(tmp_path)
+    assert [m.path for m in ir.modules] == ["a.py", "bom.py"]
+    bom = next(m for m in ir.modules if m.path == "bom.py")
+    assert bom.docstring == "BOM doc."
+
+
+def test_non_utf8_hash_stable(tmp_path):
+    (tmp_path / "legacy.py").write_bytes(b'"""caf\xe9"""\ndef old():\n    pass\n')
+    ir1 = build_project_ir(tmp_path)
+    ir2 = build_project_ir(tmp_path)
+    assert ir1.hash == ir2.hash

{markdownizer-0.4.0 → markdownizer-0.4.2}/tests/test_optimizer.py

@@ -166,3 +166,76 @@ def test_tiny_budget_only_header(tmp_path):
     ctx = optimize_context(ir, max_tokens=1)
     assert ctx.included_symbols == 0
     assert ctx.text.startswith("# Context:")
+
+
+def test_negative_budget_raises(tmp_path):
+    ir = _build(tmp_path)
+    with pytest.raises(ValueError):
+        optimize_context(ir, max_tokens=0)
+    with pytest.raises(ValueError):
+        optimize_context(ir, max_tokens=-50)
+
+
+def test_debugging_emits_each_symbol_once(tmp_path):
+    """M1: full source contains the signature — never duplicate it."""
+    ir = _build(tmp_path)
+    ctx = optimize_context(ir, max_tokens=10000, profile="debugging")
+    assert ctx.included_symbols == ctx.total_symbols
+    assert "## Signature" not in ctx.text
+    assert ctx.text.count("## Source Code") == ctx.included_symbols
+
+
+def test_debugging_falls_back_to_signature_when_source_does_not_fit(tmp_path):
+    """A symbol whose full source exceeds the budget still gets a signature."""
+    from tests.conftest import write_py
+
+    body = "\n".join(f"    x{i} = {i} * 7" for i in range(200))
+    write_py(
+        tmp_path / "big.py",
+        f'"""Big."""\ndef huge():\n    """Huge."""\n{body}\n    return 1\n',
+    )
+    ir = __import__("markdownizer").build_project_ir(tmp_path)
+    ctx = optimize_context(ir, max_tokens=150, profile="debugging")
+    assert "## Signature" in ctx.text
+    assert "## Source Code" not in ctx.text
+    assert ctx.included_symbols == 1
+
+
+def test_included_symbols_counts_unique_symbols(tmp_path):
+    """M2: each symbol counts once regardless of emission depth."""
+    ir = _build(tmp_path)
+    ctx = optimize_context(ir, max_tokens=10000, profile="debugging")
+    assert ctx.included_symbols == ctx.total_symbols
+
+
+def test_estimated_tokens_matches_artifact(tmp_path):
+    """M3: the estimate counts the complete artifact, header included."""
+    from markdownizer.optimizer.tokens import count_tokens
+
+    ir = _build(tmp_path)
+    for profile in ("architecture", "api", "debugging", "onboarding"):
+        ctx = optimize_context(ir, max_tokens=10000, profile=profile)
+        assert ctx.estimated_tokens == count_tokens(ctx.text), profile
+
+
+def test_edge_placement_orders_top_symbols_first(tmp_path):
+    """The highest-ranked symbol leads the artifact; the second leads the tail."""
+    ir = _build(tmp_path)
+    ctx = optimize_context(ir, max_tokens=10000, profile="api")
+    ranked = sorted(ir.symbols, key=lambda s: -s.rank)
+    top, second = ranked[0], ranked[1]
+    body = ctx.text.split("# Context:")[1]
+    top_index = body.find(f"# {top.framework}: {top.name}")
+    second_index = body.find(f"# {second.framework}: {second.name}")
+    assert top_index < second_index
+
+
+def test_context_query_flag(tmp_path):
+    from tests.conftest import write_py
+
+    write_py(tmp_path / "auth.py", '"""Auth."""\ndef login():\n    """Login."""\n    pass\n')
+    write_py(tmp_path / "other.py", '"""Other."""\ndef unrelated():\n    """Nope."""\n    pass\n')
+    ir = __import__("markdownizer").build_project_ir(tmp_path)
+    ctx = optimize_context(ir, max_tokens=5000, profile="api", query="login")
+    assert "login" in ctx.text
+    assert "unrelated" not in ctx.text

{markdownizer-0.4.0 → markdownizer-0.4.2}/tests/test_parser.py

@@ -232,3 +232,27 @@ def test_file_path_populated(write_file):
     path = write_file("mod.py", "def f():\n    pass\n")
     _, objects = parse_file(str(path))
     assert Path(objects[0].file_path) == path
+
+
+def test_latin1_file_parses(tmp_path):
+    from markdownizer.parser import read_python_source
+
+    path = tmp_path / "legacy.py"
+    path.write_bytes(b'"""caf\xe9"""\ndef old():\n    pass\n')
+    source = read_python_source(path)
+    assert "café" in source
+    module, objects = parse_file(str(path))
+    assert module.docstring == "café"
+    assert objects[0].name == "old"
+
+
+def test_utf8_bom_file_parses(tmp_path):
+    from markdownizer.parser import read_python_source
+
+    path = tmp_path / "bom.py"
+    path.write_bytes(b'\xef\xbb\xbf"""BOM."""\ndef g():\n    pass\n')
+    source = read_python_source(path)
+    assert not source.startswith("\ufeff")
+    module, objects = parse_file(str(path))
+    assert module.docstring == "BOM."
+    assert objects[0].name == "g"