markdownizer 0.4.1__tar.gz → 0.4.2__tar.gz

{markdownizer-0.4.1 → markdownizer-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: markdownizer
-Version: 0.4.1
+Version: 0.4.2
 Summary: Extract documentation from Python projects into Markdown without rewriting it
 Author-email: Mohammad Hasan Khoddami <mohammadh.khoddami@gmail.com>
 License: MIT License

{markdownizer-0.4.1 → markdownizer-0.4.2}/markdownizer/__init__.py

@@ -8,7 +8,7 @@ from markdownizer.extractor import extract_project
 from markdownizer.ir import IR_VERSION, ProjectIR, build_project_ir
 from markdownizer.optimizer import OptimizedContext, optimize_context
 
-__version__ = "0.4.1"
+__version__ = "0.4.2"
 __all__ = [
     "extract_project",
     "build_project_ir",

{markdownizer-0.4.1 → markdownizer-0.4.2}/markdownizer/imports.py

@@ -67,11 +67,17 @@ def _resolve_relative(
 
 
 def _collect_statement_imports(
-    node: ast.AST, module_map: dict[str, str], from_rel_path: str
+    module: ast.Module, module_map: dict[str, str], from_rel_path: str
 ) -> list[ImportEdge]:
+    """Collect module-level import statements.
+
+    Only top-level statements count: imports inside function bodies, class
+    bodies, or ``if __name__ == "__main__":`` blocks are not structural
+    module dependencies. ``from __future__`` imports are filtered out.
+    """
     edges: list[ImportEdge] = []
 
-    for child in ast.walk(node):
+    for child in module.body:
         if isinstance(child, ast.Import):
             for alias in child.names:
                 target = _resolve_absolute(alias.name, module_map)
@@ -87,6 +93,8 @@ def _collect_statement_imports(
                     )
                 )
         elif isinstance(child, ast.ImportFrom):
+            if child.module == "__future__":
+                continue
             if child.module is None:
                 # ``from . import x`` — the imported names are resolved
                 # relative to the current package.

{markdownizer-0.4.1 → markdownizer-0.4.2}/markdownizer/ir.py

@@ -27,7 +27,7 @@ from typing import Any
 
 from markdownizer.classifier import classify
 from markdownizer.imports import ImportEdge, collect_import_edges
-from markdownizer.parser import DocObject, parse_file
+from markdownizer.parser import DocObject, parse_file, read_python_source
 from markdownizer.scanner import scan_python_files
 
 IR_VERSION = 1
@@ -336,7 +336,7 @@ def build_project_ir(project_root: Path, exclude: list[str] | None = None) -> Pr
     for py_file in files:
         rel = py_file.relative_to(project_root).as_posix()
         dotted = _module_dotted_name(rel)
-        source = py_file.read_text(encoding="utf-8")
+        source = read_python_source(py_file)
         module_obj, objects = parse_file(str(py_file), source=source)
 
         module_symbols = [_symbol_from_docobject(o, rel) for o in objects]

{markdownizer-0.4.1 → markdownizer-0.4.2}/markdownizer/parser.py

@@ -6,6 +6,7 @@ import ast
 import io
 import tokenize
 from dataclasses import dataclass, field
+from pathlib import Path
 from typing import Protocol
 
 
@@ -72,6 +73,19 @@ def _rightmost_name(name: str) -> str:
     return name.rsplit(".", 1)[-1] if name else ""
 
 
+def read_python_source(path: Path) -> str:
+    """Read a Python source file, tolerating non-UTF-8 encodings.
+
+    Uses ``utf-8-sig`` (which also strips a UTF-8 BOM) and falls back to
+    latin-1, which can decode every byte sequence without raising. A single
+    legacy-encoded file must never abort an entire project extraction.
+    """
+    try:
+        return path.read_text(encoding="utf-8-sig")
+    except UnicodeDecodeError:
+        return path.read_text(encoding="latin-1")
+
+
 def _unparse_or_empty(node: ast.AST | None) -> str:
     """Render an AST node back to source text, or ``""`` if unavailable."""
     if node is None:
@@ -337,8 +351,7 @@ def parse_file(file_path: str, source: str | None = None) -> tuple[DocObject, li
     ``source`` may be passed in to avoid re-reading the file from disk.
     """
     if source is None:
-        with open(file_path, encoding="utf-8") as fh:
-            source = fh.read()
+        source = read_python_source(Path(file_path))
 
     source_lines = source.splitlines()
     all_comments = _collect_comments(source)

{markdownizer-0.4.1 → markdownizer-0.4.2}/markdownizer.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: markdownizer
-Version: 0.4.1
+Version: 0.4.2
 Summary: Extract documentation from Python projects into Markdown without rewriting it
 Author-email: Mohammad Hasan Khoddami <mohammadh.khoddami@gmail.com>
 License: MIT License

{markdownizer-0.4.1 → markdownizer-0.4.2}/tests/test_extractor.py

@@ -119,3 +119,15 @@ def test_extract_error_on_unwritable_output(sample_project, tmp_path):
         pytest.skip("directory is writable; cannot simulate")
     with pytest.raises(OSError):
         extract_project(sample_project, out / "sub" / "nested")
+
+
+def test_extract_survives_non_utf8_file(tmp_path):
+    """End-to-end: a legacy-encoded file does not abort extraction."""
+    (tmp_path / "good.py").write_text('"""Fine."""\ndef ok():\n    pass\n')
+    (tmp_path / "legacy.py").write_bytes(b'"""caf\xe9"""\ndef old():\n    pass\n')
+    out = tmp_path / "out"
+    written = extract_project(tmp_path, out)
+    assert sorted(p.name for p in written) == ["_root.md"]
+    md = (out / "_root.md").read_text(encoding="utf-8")
+    assert "# Function: old" in md
+    assert "café" in md

{markdownizer-0.4.1 → markdownizer-0.4.2}/tests/test_imports.py

@@ -144,3 +144,42 @@ def test_dotted_import_resolution_to_init():
 
 def test_empty_source():
     assert _edges("") == []
+
+
+def test_nested_function_import_not_collected():
+    source = "def f():\n    import os\n    return os.path\n"
+    assert _edges(source) == []
+
+
+def test_main_guard_import_not_collected():
+    source = "import pkg.models\n\nif __name__ == '__main__':\n    import sys\n"
+    edges = _edges(source)
+    assert [e.imported_module for e in edges] == ["pkg.models"]
+
+
+def test_class_body_import_not_collected():
+    source = "class C:\n    def m(self):\n        import os\n"
+    assert _edges(source) == []
+
+
+def test_future_import_filtered():
+    source = "from __future__ import annotations\nimport pkg.models\n"
+    edges = _edges(source)
+    assert [e.imported_module for e in edges] == ["pkg.models"]
+
+
+def test_try_wrapped_import_not_collected():
+    """try/except-wrapped imports are conservatively treated as external."""
+    source = "try:\n    import pkg.models\nexcept ImportError:\n    pass\n"
+    assert _edges(source) == []
+
+
+def test_latin1_source_decodes():
+    from markdownizer.imports import collect_import_edges
+
+    edges = collect_import_edges(
+        '"""caf\xe9"""\nimport os\n'.encode("latin-1").decode("latin-1"),
+        "mod.py",
+        {"mod.py": "mod.py"},
+    )
+    assert [e.imported_module for e in edges] == ["os"]

{markdownizer-0.4.1 → markdownizer-0.4.2}/tests/test_ir.py

@@ -245,3 +245,34 @@ def test_unsorted_input_still_sorted(tmp_path):
     ir = build_project_ir(tmp_path)
     assert [m.path for m in ir.modules] == ["a/y.py", "b/z.py"]
     assert [s.qualified_name for s in ir.symbols] == ["y", "z"]
+
+
+def test_build_survives_non_utf8_file(tmp_path):
+    """A single latin-1 encoded file must not abort the whole build."""
+    from tests.conftest import write_py
+
+    write_py(tmp_path / "good.py", '"""Fine."""\ndef ok():\n    pass\n')
+    (tmp_path / "legacy.py").write_bytes(b'"""caf\xe9 latin-1"""\ndef old():\n    pass\n')
+    ir = build_project_ir(tmp_path)
+    assert [m.path for m in ir.modules] == ["good.py", "legacy.py"]
+    legacy = next(m for m in ir.modules if m.path == "legacy.py")
+    assert legacy.docstring == "café latin-1"
+    assert ir.stats.symbol_count == 2
+
+
+def test_build_survives_utf8_bom_file(tmp_path):
+    from tests.conftest import write_py
+
+    write_py(tmp_path / "a.py", "def f():\n    pass\n")
+    (tmp_path / "bom.py").write_bytes(b'\xef\xbb\xbf"""BOM doc."""\ndef g():\n    pass\n')
+    ir = build_project_ir(tmp_path)
+    assert [m.path for m in ir.modules] == ["a.py", "bom.py"]
+    bom = next(m for m in ir.modules if m.path == "bom.py")
+    assert bom.docstring == "BOM doc."
+
+
+def test_non_utf8_hash_stable(tmp_path):
+    (tmp_path / "legacy.py").write_bytes(b'"""caf\xe9"""\ndef old():\n    pass\n')
+    ir1 = build_project_ir(tmp_path)
+    ir2 = build_project_ir(tmp_path)
+    assert ir1.hash == ir2.hash

{markdownizer-0.4.1 → markdownizer-0.4.2}/tests/test_parser.py

@@ -232,3 +232,27 @@ def test_file_path_populated(write_file):
     path = write_file("mod.py", "def f():\n    pass\n")
     _, objects = parse_file(str(path))
     assert Path(objects[0].file_path) == path
+
+
+def test_latin1_file_parses(tmp_path):
+    from markdownizer.parser import read_python_source
+
+    path = tmp_path / "legacy.py"
+    path.write_bytes(b'"""caf\xe9"""\ndef old():\n    pass\n')
+    source = read_python_source(path)
+    assert "café" in source
+    module, objects = parse_file(str(path))
+    assert module.docstring == "café"
+    assert objects[0].name == "old"
+
+
+def test_utf8_bom_file_parses(tmp_path):
+    from markdownizer.parser import read_python_source
+
+    path = tmp_path / "bom.py"
+    path.write_bytes(b'\xef\xbb\xbf"""BOM."""\ndef g():\n    pass\n')
+    source = read_python_source(path)
+    assert not source.startswith("\ufeff")
+    module, objects = parse_file(str(path))
+    assert module.docstring == "BOM."
+    assert objects[0].name == "g"