openscad-parser 2.5.2__tar.gz → 2.6.1__tar.gz

{openscad_parser-2.5.2 → openscad_parser-2.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openscad_parser
-Version: 2.5.2
+Version: 2.6.1
 Summary: A PEG parser to read OpenSCAD language source code, with optional AST tree generation.
 Keywords: openscad,openscad parser,parser
 Author: Revar Desmera

{openscad_parser-2.5.2 → openscad_parser-2.6.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "openscad_parser"
-version = "2.5.2"
+version = "2.6.1"
 description = "A PEG parser to read OpenSCAD language source code, with optional AST tree generation."
 readme = "README.rst"
 authors = [

{openscad_parser-2.5.2 → openscad_parser-2.6.1}/src/openscad_parser/ast/__init__.py

@@ -1,4 +1,7 @@
+import hashlib
+import json
 import os
+import pickle
 import platform
 from typing import Optional
 from arpeggio import NoMatch
@@ -253,22 +256,240 @@ def getASTfromString(code: str, include_comments: bool = False, origin: str = "<
     return ast
 
 
-# Module-level cache for AST trees
+# Module-level in-memory cache for per-file AST trees (no includes resolved)
+# Key: tuple of (absolute file path (str), include_comments (bool))
+# Value: tuple of (AST nodes, modification timestamp)
+_ast_cache: dict[tuple[str, bool], tuple[list[ASTNode] | None, float]] = {}
+
+# Resolved (includes-expanded) cache
 # Key: tuple of (absolute file path (str), include_comments (bool), process_includes (bool))
 # Value: tuple of (AST nodes, modification timestamp)
-_ast_cache: dict[tuple[str, bool, bool], tuple[list[ASTNode] | None, float]] = {}
+_resolved_cache: dict[tuple[str, bool, bool], tuple[list[ASTNode] | None, float]] = {}
+
+
+def _get_disk_cache_dir() -> Optional[str]:
+    """Get the disk cache directory, creating it if needed."""
+    cache_dir = os.environ.get('OPENSCAD_PARSER_CACHE_DIR')
+    if not cache_dir:
+        home = os.path.expanduser('~')
+        if platform.system() == 'Darwin':
+            cache_dir = os.path.join(home, 'Library', 'Caches', 'openscad_parser')
+        elif platform.system() == 'Windows':  # pragma: no cover
+            cache_dir = os.path.join(os.environ.get('LOCALAPPDATA', home), 'openscad_parser', 'cache')
+        else:
+            cache_dir = os.path.join(home, '.cache', 'openscad_parser')
+    try:
+        os.makedirs(cache_dir, exist_ok=True)
+        return cache_dir
+    except OSError:  # pragma: no cover
+        return None
+
+
+def _disk_cache_path(file_path: str, include_comments: bool) -> Optional[str]:
+    """Get the disk cache file path for a given source file."""
+    cache_dir = _get_disk_cache_dir()
+    if not cache_dir:
+        return None  # pragma: no cover
+    key = f"{file_path}:{include_comments}"
+    h = hashlib.sha256(key.encode()).hexdigest()[:16]
+    return os.path.join(cache_dir, f"{h}.pickle")
+
+
+def _load_from_disk_cache(file_path: str, include_comments: bool, current_mtime: float) -> Optional[list[ASTNode]]:
+    """Try to load a file's AST from disk cache."""
+    cache_path = _disk_cache_path(file_path, include_comments)
+    if not cache_path or not os.path.exists(cache_path):
+        return None
+    try:
+        with open(cache_path, 'rb') as f:
+            cached_mtime, ast = pickle.load(f)
+        if cached_mtime == current_mtime:
+            return ast
+    except (OSError, pickle.UnpicklingError, ValueError, EOFError):
+        pass
+    return None
+
+
+def _save_to_disk_cache(file_path: str, include_comments: bool, mtime: float, ast: list[ASTNode] | None):
+    """Save a file's AST to disk cache and update the manifest."""
+    cache_path = _disk_cache_path(file_path, include_comments)
+    if not cache_path:
+        return  # pragma: no cover
+    try:
+        with open(cache_path, 'wb') as f:
+            pickle.dump((mtime, ast), f, protocol=pickle.HIGHEST_PROTOCOL)
+    except OSError:  # pragma: no cover
+        return
+    cache_fname = os.path.basename(cache_path)
+    _manifest_update(cache_fname, file_path)
+    _evict_stale_cache()
+
+
+def _manifest_path() -> Optional[str]:
+    """Get the path to the cache manifest file."""
+    cache_dir = _get_disk_cache_dir()
+    if not cache_dir:
+        return None  # pragma: no cover
+    return os.path.join(cache_dir, "manifest.json")
+
+
+def _manifest_load() -> dict[str, str]:
+    """Load the manifest: {cache_filename: source_file_path}."""
+    path = _manifest_path()
+    if not path or not os.path.exists(path):
+        return {}
+    try:
+        with open(path, 'r') as f:
+            return json.load(f)
+    except (OSError, json.JSONDecodeError):
+        return {}
+
+
+def _manifest_save(manifest: dict[str, str]):
+    """Save the manifest to disk."""
+    path = _manifest_path()
+    if not path:
+        return  # pragma: no cover
+    try:
+        with open(path, 'w') as f:
+            json.dump(manifest, f)
+    except OSError:  # pragma: no cover
+        pass
+
+
+def _manifest_update(cache_fname: str, source_path: str):
+    """Add or update an entry in the manifest."""
+    manifest = _manifest_load()
+    manifest[cache_fname] = source_path
+    _manifest_save(manifest)
+
+
+def _evict_stale_cache():
+    """Remove cache entries whose source files no longer exist."""
+    cache_dir = _get_disk_cache_dir()
+    if not cache_dir:
+        return  # pragma: no cover
+    manifest = _manifest_load()
+    if not manifest:
+        return
+    stale_keys = [
+        fname for fname, source_path in manifest.items()
+        if not os.path.exists(source_path)
+    ]
+    if not stale_keys:
+        return
+    for fname in stale_keys:
+        cache_file = os.path.join(cache_dir, fname)
+        try:
+            os.remove(cache_file)
+        except OSError:
+            pass
+        del manifest[fname]
+    _manifest_save(manifest)
 
 
 def clear_ast_cache():
     """Clear the in-memory AST cache.
-    
+
     This function removes all cached AST trees, forcing all subsequent
     calls to getASTfromFile() to re-parse files.
-    
+
     Example:
         clear_ast_cache()  # Clear all cached ASTs
     """
     _ast_cache.clear()
+    _resolved_cache.clear()
+
+
+def clear_disk_cache():
+    """Clear the on-disk AST cache.
+
+    This function removes all cached AST files from disk, forcing all subsequent
+    calls to re-parse files from scratch.
+
+    Example:
+        clear_disk_cache()  # Remove all disk-cached ASTs
+    """
+    cache_dir = _get_disk_cache_dir()
+    if cache_dir and os.path.isdir(cache_dir):
+        for fname in os.listdir(cache_dir):
+            if fname.endswith('.pickle') or fname == 'manifest.json':
+                try:
+                    os.remove(os.path.join(cache_dir, fname))
+                except OSError:  # pragma: no cover
+                    pass
+
+
+def _parse_single_file(file_path: str, include_comments: bool = False) -> list[ASTNode] | None:
+    """Parse a single file without resolving includes. Uses memory and disk cache.
+
+    Returns the AST with IncludeStatement nodes intact (not expanded).
+    """
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(f"File {file_path} not found")
+
+    current_mtime = os.path.getmtime(file_path)
+    cache_key = (file_path, include_comments)
+
+    # Check in-memory cache
+    if cache_key in _ast_cache:
+        cached_ast, cached_mtime = _ast_cache[cache_key]
+        if cached_mtime == current_mtime:
+            return cached_ast
+
+    # Check disk cache
+    disk_result = _load_from_disk_cache(file_path, include_comments, current_mtime)
+    if disk_result is not None:
+        _ast_cache[cache_key] = (disk_result, current_mtime)
+        return disk_result
+
+    # Parse the file
+    with open(file_path, 'r', encoding='utf-8') as f:
+        code = f.read()
+
+    source_map = SourceMap()
+    source_map.add_origin(file_path, code)
+
+    parser = getOpenSCADParser(reduce_tree=False, include_comments=include_comments)
+    ast = parse_ast(parser, code, source_map=source_map)
+
+    # Cache in memory and on disk
+    _ast_cache[cache_key] = (ast, current_mtime)
+    _save_to_disk_cache(file_path, include_comments, current_mtime, ast)
+
+    return ast
+
+
+def _resolve_includes(ast_nodes: list[ASTNode] | None, current_file: str,
+                      include_comments: bool = False,
+                      visited: set | None = None) -> list[ASTNode] | None:
+    """Resolve IncludeStatement nodes by parsing and inlining referenced files."""
+    if ast_nodes is None:
+        return None
+    if visited is None:
+        visited = set()
+
+    result = []
+    for node in ast_nodes:
+        if isinstance(node, IncludeStatement):
+            filename = node.filepath.val
+            lib_file = findLibraryFile(current_file, filename)
+            if lib_file is None:
+                raise FileNotFoundError(
+                    f"Included file '{filename}' not found. "
+                    f"Searched relative to: {current_file if current_file else 'current directory'}"
+                )
+            lib_file = os.path.abspath(lib_file)
+            if lib_file in visited:
+                continue
+            visited.add(lib_file)
+            included_ast = _parse_single_file(lib_file, include_comments)
+            included_ast = _resolve_includes(included_ast, lib_file, include_comments, visited)
+            if included_ast:
+                result.extend(included_ast)
+        else:
+            result.append(node)
+    return result
 
 
 def getASTfromFile(file: str, include_comments: bool = False, process_includes: bool = True) -> list[ASTNode] | None:
@@ -278,20 +499,21 @@ def getASTfromFile(file: str, include_comments: bool = False, process_includes:
     This function reads the contents of the provided OpenSCAD file, processes include statements,
     parses it using the OpenSCAD parser, and returns the resulting AST (or list of AST nodes).
 
-    The function caches AST trees in memory. Cache entries are automatically invalidated
-    if the file's modification timestamp changes, ensuring that updated files are re-parsed.
+    The function caches AST trees both in memory and on disk. Cache entries are automatically
+    invalidated if a file's modification timestamp changes, ensuring updated files are re-parsed.
+    Each included file is parsed independently and cached separately, so only changed files
+    need re-parsing.
 
     Important: The `process_includes` parameter affects the AST structure:
-    
-    - When `process_includes=True` (default): Include statements are processed before parsing,
-      meaning the included file contents are inserted into the source code, and the AST will
-      NOT contain `IncludeStatement` nodes. The AST represents the code as if all includes
-      have been expanded.
-    
+
+    - When `process_includes=True` (default): Include statements are processed and resolved,
+      meaning the included file's AST nodes are inlined, and the AST will NOT contain
+      `IncludeStatement` nodes. The AST represents the code as if all includes have been expanded.
+
     - When `process_includes=False`: Include statements are NOT processed, and the AST will
       contain `IncludeStatement` nodes wherever `include <file>` statements appear in the
       source code.
-    
+
     Note: Unlike `include` statements, `use <file>` statements are ALWAYS parsed into
     `UseStatement` AST nodes, regardless of the `process_includes` setting. This is because
     `use` statements only affect module and function lookup at runtime, not source inclusion.
@@ -316,54 +538,34 @@ def getASTfromFile(file: str, include_comments: bool = False, process_includes:
         # Get AST with IncludeStatement nodes instead of processing includes
         ast_with_include_nodes = getASTfromFile("my_model.scad", process_includes=False)
     """
-    # Get absolute path for consistent cache keys
     file_path = os.path.abspath(file)
-    
-    # Check if file exists and get its modification time
+
     if not os.path.exists(file_path):
         raise FileNotFoundError(f"File {file} not found")
-    
+
     current_mtime = os.path.getmtime(file_path)
-    
-    # Cache key includes file path, include_comments flag, and process_includes flag
-    cache_key = (file_path, include_comments, process_includes)
-    
-    # Check cache
-    if cache_key in _ast_cache:
-        cached_ast, cached_mtime = _ast_cache[cache_key]
-        # If file hasn't been modified, return cached AST
+
+    # For process_includes=False, just parse the single file
+    if not process_includes:
+        return _parse_single_file(file_path, include_comments)
+
+    # Check resolved cache (in-memory only since resolved ASTs depend on multiple files)
+    resolved_key = (file_path, include_comments, True)
+    if resolved_key in _resolved_cache:
+        cached_ast, cached_mtime = _resolved_cache[resolved_key]
         if cached_mtime == current_mtime:
             return cached_ast
-        # Otherwise, invalidate the cache entry
-        del _ast_cache[cache_key]
-    
-    # Read the file
-    with open(file_path, 'r', encoding='utf-8') as f:
-        code = f.read()
-    
-    # Create source map and process includes if requested
-    source_map = SourceMap()
-    source_map.add_origin(file_path, code)
-    
-    if process_includes:
-        try:
-            source_map = process_includes_func(source_map, file_path)
-        except FileNotFoundError as e:
-            # Re-raise file not found errors as-is
-            raise
-        except Exception as e:  # pragma: no cover
-            raise Exception(f"Error processing includes: {e}")
-    
-    # Get the combined string for parsing
-    combined_code = source_map.get_combined_string()
-    
-    # Parse
-    parser = getOpenSCADParser(reduce_tree=False, include_comments=include_comments)
-    ast = parse_ast(parser, combined_code, source_map=source_map)
-    
-    # Cache the result with current modification time
-    _ast_cache[cache_key] = (ast, current_mtime)
-    
+
+    # Parse the file independently (uses per-file cache)
+    ast = _parse_single_file(file_path, include_comments)
+
+    # Resolve all include statements recursively
+    visited = {file_path}
+    ast = _resolve_includes(ast, file_path, include_comments, visited)
+
+    # Cache the resolved result
+    _resolved_cache[resolved_key] = (ast, current_mtime)
+
     return ast
 
 

{openscad_parser-2.5.2 → openscad_parser-2.6.1}/src/openscad_parser/ast/builder.py

@@ -80,8 +80,13 @@ class SemanticChildren(list):
         self._rule_map = rule_map
 
     def __getattr__(self, name):
+        if name == '_rule_map':
+            raise AttributeError(name)
         return self._rule_map.get(name, [])
 
+    def __reduce__(self):
+        return (list, (list(self),))
+
     def get_rule(self, rule_name, index=0):
         """Return the index-th result for rule_name, or [] if absent/out of range."""
         results = self._rule_map.get(rule_name, [])
@@ -95,7 +100,7 @@ class ASTBuilderVisitor(PTNodeVisitor):
     
     def __init__(self, parser, source_map=None, file=""):
         """Initialize the visitor with the parser and optional source map or file path.
-        
+
         Args:
             parser: The Arpeggio parser instance (needed to access input for position conversion)
             source_map: Optional SourceMap for tracking positions across multiple origins
@@ -105,10 +110,12 @@ class ASTBuilderVisitor(PTNodeVisitor):
         self.parser = parser
         if source_map is not None:
             self.source_map = source_map
+            self._has_source_map = bool(source_map._segments)
             self.file = ""  # Not used when source_map is provided
         else:
             # Backward compatibility: create a simple source map from file
             self.source_map = SourceMap()
+            self._has_source_map = False
             if file:
                 # We can't get the content here, but we'll handle it in _get_node_position
                 self.file = file
@@ -194,17 +201,9 @@ class ASTBuilderVisitor(PTNodeVisitor):
         """Return the combined-string offset one past the last character of node."""
         if node is None:
             return 0
-        try:
-            # NonTerminal: recurse to find the furthest end among children
-            end = getattr(node, 'position', 0)
-            for child in node:
-                child_end = self._get_node_end_position(child)
-                if child_end > end:
-                    end = child_end
+        end = getattr(node, 'position_end', None)
+        if end is not None:
             return end
-        except TypeError:
-            pass
-        # Terminal: start + length of matched value
         pos = getattr(node, 'position', 0)
         val = getattr(node, 'value', '')
         return pos + len(str(val))
@@ -225,7 +224,7 @@ class ASTBuilderVisitor(PTNodeVisitor):
         end_pos = self._get_node_end_position(node)
 
         # Use SourceMap if available to map position back to original origin
-        if hasattr(self, 'source_map') and self.source_map.get_segments():
+        if self._has_source_map:
             return self.source_map.get_location(char_pos, end_pos)
         else:
             # Fallback: calculate line/column from character position

{openscad_parser-2.5.2 → openscad_parser-2.6.1}/src/openscad_parser/ast/source_map.py

@@ -172,11 +172,15 @@ class SourceMap:
                     # Calculate the actual line/column for the after segment
                     # Count lines in the part that was removed + before
                     removed_and_before = segment.content[:replace_end_in_segment]
-                    line_count = removed_and_before.count('\n') - line_count_adjustment
+                    line_count = removed_and_before.count('\n') + line_count_adjustment
                     if line_count > 0:
-                        last_newline = removed_and_before.rfind('\n')
-                        after_segment.start_line = segment.start_line + line_count
-                        after_segment.start_column = len(removed_and_before) - last_newline
+                        if line_count_adjustment and removed_and_before.count('\n') == 0:
+                            after_segment.start_line = segment.start_line + line_count
+                            after_segment.start_column = 1
+                        else:
+                            last_newline = removed_and_before.rfind('\n')
+                            after_segment.start_line = segment.start_line + line_count
+                            after_segment.start_column = len(removed_and_before) - last_newline
                     else:
                         after_segment.start_line = segment.start_line
                         after_segment.start_column = segment.start_column + len(removed_and_before)