rolfedh-doc-utils 0.1.4__py3-none-any.whl → 0.1.41__py3-none-any.whl

doc_utils/unused_adoc.py

@@ -5,49 +5,177 @@ import re
 from .file_utils import collect_files, write_manifest_and_archive
 from .topic_map_parser import detect_repo_type, get_all_topic_map_references
 
-def find_unused_adoc(scan_dirs, archive_dir, archive=False, exclude_dirs=None, exclude_files=None):
+def find_scan_directories(base_path='.', exclude_dirs=None):
+    """
+    Automatically find all 'modules' and 'assemblies' directories in the repository.
+    
+    Returns a list of paths to scan.
+    """
+    scan_dirs = []
+    exclude_dirs = exclude_dirs or []
+    
+    for root, dirs, files in os.walk(base_path):
+        # Skip symbolic links to prevent issues
+        dirs[:] = [d for d in dirs if not os.path.islink(os.path.join(root, d))]
+        
+        # Skip excluded directories
+        for exclude_dir in exclude_dirs:
+            abs_exclude = os.path.abspath(exclude_dir)
+            if os.path.abspath(root).startswith(abs_exclude):
+                dirs[:] = []  # Don't descend into excluded directories
+                break
+        
+        # Skip hidden directories and common non-content directories
+        dirs[:] = [d for d in dirs if not d.startswith('.') and d not in ['node_modules', 'build', 'dist', 'target']]
+        
+        # Look for modules and assemblies directories
+        for d in dirs:
+            if d in ['modules', 'assemblies']:
+                dir_path = os.path.join(root, d)
+                # Check if this directory or any subdirectory contains .adoc files
+                has_adoc = False
+                for subroot, subdirs, subfiles in os.walk(dir_path):
+                    # Skip symbolic links
+                    subdirs[:] = [sd for sd in subdirs if not os.path.islink(os.path.join(subroot, sd))]
+                    if any(f.endswith('.adoc') for f in subfiles):
+                        has_adoc = True
+                        break
+                if has_adoc:
+                    scan_dirs.append(dir_path)
+    
+    # Also check for modules/rn pattern if modules exists
+    modules_dirs = [d for d in scan_dirs if os.path.basename(d) == 'modules']
+    for modules_dir in modules_dirs:
+        rn_dir = os.path.join(modules_dir, 'rn')
+        if os.path.isdir(rn_dir):
+            # Check if rn directory or subdirectories contain .adoc files
+            has_adoc = False
+            for subroot, subdirs, subfiles in os.walk(rn_dir):
+                subdirs[:] = [sd for sd in subdirs if not os.path.islink(os.path.join(subroot, sd))]
+                if any(f.endswith('.adoc') for f in subfiles):
+                    has_adoc = True
+                    break
+            if has_adoc:
+                scan_dirs.append(rn_dir)
+    
+    return scan_dirs
+
+def find_unused_adoc(scan_dirs=None, archive_dir='./archive', archive=False, exclude_dirs=None, exclude_files=None, include_commented=False):
     # Print safety warning
     print("\n⚠️  SAFETY: Work in a git branch! Run without --archive first to preview.\n")
-    
+
+    # If no scan_dirs provided, auto-discover them
+    if not scan_dirs:
+        scan_dirs = find_scan_directories(exclude_dirs=exclude_dirs)
+        if scan_dirs:
+            print(f"Auto-discovered directories to scan:")
+            for dir_path in sorted(scan_dirs):
+                print(f"  - {dir_path}")
+        else:
+            print("No 'modules' or 'assemblies' directories found containing .adoc files.")
+            print("Please run this tool from your documentation repository root.")
+            return
+
     # Detect repository type
     repo_type = detect_repo_type()
     print(f"Detected repository type: {repo_type}")
-    
+
     # Collect all .adoc files in scan directories
     asciidoc_files = collect_files(scan_dirs, {'.adoc'}, exclude_dirs, exclude_files)
-    
-    # Track which files are referenced
-    referenced_files = set()
-    
+
+    # Track which files are referenced (uncommented and commented separately)
+    referenced_files = set()  # Files in uncommented includes
+    commented_only_files = {}  # Files referenced ONLY in commented lines: {basename: [(file, line_num, line_text)]}
+
     if repo_type == 'topic_map':
         # For OpenShift-docs style repos, get references from topic maps
         topic_references = get_all_topic_map_references()
         # Convert to basenames for comparison
         referenced_files.update(os.path.basename(ref) for ref in topic_references)
-    
-    # Always scan for include:: directives in all .adoc files
+
+    # Patterns for finding includes (both commented and uncommented)
     include_pattern = re.compile(r'include::(.+?)\[')
+    commented_include_pattern = re.compile(r'^\s*//.*include::(.+?)\[')
+
     adoc_files = collect_files(['.'], {'.adoc'}, exclude_dirs, exclude_files)
-    
+
     for file_path in adoc_files:
         try:
             with open(file_path, 'r', encoding='utf-8') as f:
-                content = f.read()
-                includes = include_pattern.findall(content)
-                # Extract just the filename from the include path
-                for include in includes:
-                    # Handle both relative and absolute includes
-                    include_basename = os.path.basename(include)
-                    referenced_files.add(include_basename)
+                lines = f.readlines()
+
+                for line_num, line in enumerate(lines, 1):
+                    # Check if this is a commented include
+                    commented_match = commented_include_pattern.search(line)
+                    if commented_match:
+                        include_basename = os.path.basename(commented_match.group(1))
+                        # Track location of commented reference
+                        if include_basename not in commented_only_files:
+                            commented_only_files[include_basename] = []
+                        commented_only_files[include_basename].append((file_path, line_num, line.strip()))
+                    else:
+                        # Check for uncommented includes
+                        uncommented_match = include_pattern.search(line)
+                        if uncommented_match:
+                            include_basename = os.path.basename(uncommented_match.group(1))
+                            referenced_files.add(include_basename)
+                            # If we found an uncommented reference, remove from commented_only tracking
+                            if include_basename in commented_only_files:
+                                del commented_only_files[include_basename]
         except Exception as e:
             print(f"Warning: could not read {file_path}: {e}")
-    
-    # Find unused files by comparing basenames
-    unused_files = [f for f in asciidoc_files if os.path.basename(f) not in referenced_files]
+
+    # Determine which files are unused based on the include_commented flag
+    if include_commented:
+        # When --commented is used: treat files with commented-only references as unused
+        # Only files with uncommented references are considered "used"
+        unused_files = [f for f in asciidoc_files if os.path.basename(f) not in referenced_files]
+        commented_only_unused = []
+    else:
+        # Default behavior: files referenced only in commented lines are considered "used"
+        # They should NOT be in the unused list, but we track them for reporting
+        all_referenced = referenced_files.union(set(commented_only_files.keys()))
+        unused_files = [f for f in asciidoc_files if os.path.basename(f) not in all_referenced]
+
+        # Generate list of files referenced only in comments for the report
+        commented_only_unused = []
+        for basename, references in commented_only_files.items():
+            # Find the full path for this basename in asciidoc_files
+            matching_files = [f for f in asciidoc_files if os.path.basename(f) == basename]
+            for f in matching_files:
+                commented_only_unused.append((f, references))
+
     unused_files = list(dict.fromkeys(unused_files))  # Remove duplicates
-    
+
+    # Print summary
     print(f"Found {len(unused_files)} unused files out of {len(asciidoc_files)} total files in scan directories")
-    
+
+    # Generate detailed report for commented-only references
+    if commented_only_unused and not include_commented:
+        report_path = os.path.join(archive_dir, 'commented-references-report.txt')
+        os.makedirs(archive_dir, exist_ok=True)
+
+        with open(report_path, 'w', encoding='utf-8') as report:
+            report.write("Files Referenced Only in Commented Lines\n")
+            report.write("=" * 70 + "\n\n")
+            report.write(f"Found {len(commented_only_unused)} files that are referenced only in commented-out includes.\n")
+            report.write("These files are considered 'used' by default and will NOT be archived.\n\n")
+            report.write("To archive these files along with other unused files, use the --commented flag.\n\n")
+            report.write("-" * 70 + "\n\n")
+
+            for file_path, references in sorted(commented_only_unused):
+                report.write(f"File: {file_path}\n")
+                report.write(f"Referenced in {len(references)} commented line(s):\n")
+                for ref_file, line_num, line_text in references:
+                    report.write(f"  {ref_file}:{line_num}\n")
+                    report.write(f"    {line_text}\n")
+                report.write("\n")
+
+        print(f"\n📋 Found {len(commented_only_unused)} files referenced only in commented lines.")
+        print(f"   Detailed report saved to: {report_path}")
+        print(f"   These files are considered 'used' and will NOT be archived by default.")
+        print(f"   To include them in the archive operation, use the --commented flag.\n")
+
     return write_manifest_and_archive(
         unused_files, archive_dir, 'to-archive', 'to-archive', archive=archive
     )

doc_utils/unused_attributes.py

@@ -6,19 +6,62 @@ Functions:
 - find_adoc_files: Recursively find all .adoc files in a directory (ignoring symlinks).
 - scan_for_attribute_usage: Find which attributes are used in a set of .adoc files.
 - find_unused_attributes: Main function to return unused attributes.
+- find_attributes_files: Find all potential attributes files in the repository.
 """
 
 import os
 import re
-from typing import Set, List
+from pathlib import Path
+from typing import Set, List, Optional
 
 def parse_attributes_file(attr_file: str) -> Set[str]:
+    # AsciiDoc configuration attributes that control the processor itself
+    # These should be ignored as they won't appear in content
+    IGNORED_ATTRIBUTES = {
+        'data-uri',
+        'doctype',
+        'experimental',
+        'idprefix',
+        'imagesdir',
+        'includes',
+        'sectanchors',
+        'sectlinks',
+        'source-highlighter',
+        'linkattrs',
+        'toclevels',
+        'idseparator',
+        'icons',
+        'iconsdir',
+        'generated-dir',
+        'code-examples',
+        'doc-guides',
+        'doc-examples',
+    }
+
     attributes = set()
-    with open(attr_file, 'r', encoding='utf-8') as f:
-        for line in f:
-            match = re.match(r'^:([\w-]+):', line.strip())
-            if match:
-                attributes.add(match.group(1))
+
+    # Check if file exists
+    if not os.path.exists(attr_file):
+        raise FileNotFoundError(f"Attributes file not found: {attr_file}")
+
+    # Check if it's a file (not a directory)
+    if not os.path.isfile(attr_file):
+        raise ValueError(f"Path is not a file: {attr_file}")
+
+    try:
+        with open(attr_file, 'r', encoding='utf-8') as f:
+            for line in f:
+                match = re.match(r'^:([\w-]+):', line.strip())
+                if match:
+                    attr_name = match.group(1)
+                    # Skip ignored configuration attributes
+                    if attr_name not in IGNORED_ATTRIBUTES:
+                        attributes.add(attr_name)
+    except PermissionError:
+        raise PermissionError(f"Permission denied reading file: {attr_file}")
+    except UnicodeDecodeError as e:
+        raise ValueError(f"Unable to read file (encoding issue): {attr_file}\n{str(e)}")
+
     return attributes
 
 def find_adoc_files(root_dir: str) -> List[str]:
@@ -33,18 +76,187 @@ def find_adoc_files(root_dir: str) -> List[str]:
 
 def scan_for_attribute_usage(adoc_files: List[str], attributes: Set[str]) -> Set[str]:
     used = set()
+    # Pattern for attribute references: {attribute-name}
     attr_pattern = re.compile(r'\{([\w-]+)\}')
+    # Patterns for conditional directives: ifdef::attr[], ifndef::attr[], endif::attr[]
+    conditional_pattern = re.compile(r'(?:ifdef|ifndef|endif)::([\w-]+)\[')
+
     for file in adoc_files:
         with open(file, 'r', encoding='utf-8') as f:
             for line in f:
+                # Check for {attribute} references
                 for match in attr_pattern.findall(line):
                     if match in attributes:
                         used.add(match)
+                # Check for ifdef::attribute[], ifndef::attribute[], endif::attribute[]
+                for match in conditional_pattern.findall(line):
+                    if match in attributes:
+                        used.add(match)
     return used
 
+def find_attributes_files(root_dir: str = '.') -> List[str]:
+    """Find all attributes.adoc files in the repository."""
+    attributes_files = []
+    root_path = Path(root_dir)
+
+    # Common attribute file patterns
+    patterns = ['**/attributes.adoc', '**/attributes*.adoc', '**/*attributes.adoc', '**/*-attributes.adoc']
+
+    for pattern in patterns:
+        for path in root_path.glob(pattern):
+            # Skip hidden directories and common build directories
+            parts = path.parts
+            if any(part.startswith('.') or part in ['target', 'build', 'node_modules', '.archive'] for part in parts):
+                continue
+            # Convert to string and avoid duplicates
+            str_path = str(path)
+            if str_path not in attributes_files:
+                attributes_files.append(str_path)
+
+    # Sort for consistent ordering
+    attributes_files.sort()
+    return attributes_files
+
+
+def select_attributes_file(attributes_files: List[str]) -> Optional[str]:
+    """Interactive selection of attributes file from a list."""
+    if not attributes_files:
+        return None
+
+    if len(attributes_files) == 1:
+        print(f"Found attributes file: {attributes_files[0]}")
+        response = input("Use this file? (y/n): ").strip().lower()
+        if response == 'y':
+            return attributes_files[0]
+        else:
+            response = input("Enter the path to your attributes file: ").strip()
+            if os.path.exists(response) and os.path.isfile(response):
+                return response
+            else:
+                print(f"Error: File not found: {response}")
+                return None
+
+    # Multiple files found
+    print("\nFound multiple attributes files:")
+    for i, file_path in enumerate(attributes_files, 1):
+        print(f"  {i}. {file_path}")
+    print(f"  {len(attributes_files) + 1}. Enter custom path")
+
+    while True:
+        response = input(f"\nSelect option (1-{len(attributes_files) + 1}) or 'q' to quit: ").strip()
+        if response.lower() == 'q':
+            return None
+
+        try:
+            choice = int(response)
+            if 1 <= choice <= len(attributes_files):
+                return attributes_files[choice - 1]
+            elif choice == len(attributes_files) + 1:
+                response = input("Enter the path to your attributes file: ").strip()
+                if os.path.exists(response) and os.path.isfile(response):
+                    return response
+                else:
+                    print(f"Error: File not found: {response}")
+            else:
+                print(f"Invalid choice. Please enter a number between 1 and {len(attributes_files) + 1}")
+        except ValueError:
+            print("Invalid input. Please enter a number.")
+
+    return None
+
+
 def find_unused_attributes(attr_file: str, adoc_root: str = '.') -> List[str]:
     attributes = parse_attributes_file(attr_file)
     adoc_files = find_adoc_files(adoc_root)
     used = scan_for_attribute_usage(adoc_files, attributes)
     unused = sorted(attributes - used)
     return unused
+
+
+def comment_out_unused_attributes(attr_file: str, unused_attrs: List[str]) -> int:
+    """
+    Comment out unused attributes in the attributes file.
+
+    Args:
+        attr_file: Path to the attributes file
+        unused_attrs: List of unused attribute names
+
+    Returns:
+        Number of attributes commented out
+    """
+    if not unused_attrs:
+        return 0
+
+    # Read the file
+    with open(attr_file, 'r', encoding='utf-8') as f:
+        lines = f.readlines()
+
+    # Create a set for faster lookup
+    unused_set = set(unused_attrs)
+    commented_count = 0
+
+    # Process each line
+    new_lines = []
+    for line in lines:
+        # Check if this line defines an attribute
+        match = re.match(r'^:([\w-]+):', line)
+        if match and match.group(1) in unused_set:
+            # Comment out this line
+            new_lines.append(f'// Unused {line}')
+            commented_count += 1
+        else:
+            new_lines.append(line)
+
+    # Write back to the file
+    with open(attr_file, 'w', encoding='utf-8') as f:
+        f.writelines(new_lines)
+
+    return commented_count
+
+
+def remove_unused_attributes(attr_file: str, unused_attrs: List[str] = None) -> int:
+    """
+    Remove unused attributes from the attributes file.
+
+    This removes lines that either:
+    - Define an attribute in the unused_attrs list, or
+    - Are already marked with "// Unused" prefix
+
+    Args:
+        attr_file: Path to the attributes file
+        unused_attrs: Optional list of unused attribute names. If None, only
+                      removes lines already marked with "// Unused".
+
+    Returns:
+        Number of lines removed
+    """
+    # Read the file
+    with open(attr_file, 'r', encoding='utf-8') as f:
+        lines = f.readlines()
+
+    # Create a set for faster lookup
+    unused_set = set(unused_attrs) if unused_attrs else set()
+    removed_count = 0
+
+    # Process each line
+    new_lines = []
+    for line in lines:
+        # Check if line is already marked as unused
+        if line.startswith('// Unused '):
+            removed_count += 1
+            continue
+
+        # Check if this line defines an unused attribute
+        if unused_attrs:
+            match = re.match(r'^:([\w-]+):', line)
+            if match and match.group(1) in unused_set:
+                removed_count += 1
+                continue
+
+        new_lines.append(line)
+
+    # Write back to the file
+    with open(attr_file, 'w', encoding='utf-8') as f:
+        f.writelines(new_lines)
+
+    return removed_count

doc_utils/unused_images.py

@@ -6,26 +6,98 @@ from .file_utils import collect_files, write_manifest_and_archive
 
 IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.svg'}
 
-def find_unused_images(scan_dirs, archive_dir, archive=False, exclude_dirs=None, exclude_files=None):
+def find_unused_images(scan_dirs, archive_dir, archive=False, exclude_dirs=None, exclude_files=None, include_commented=False):
     # Print safety warning
     print("\n⚠️  SAFETY: Work in a git branch! Run without --archive first to preview.\n")
-    
+
     image_files = collect_files(scan_dirs, IMAGE_EXTENSIONS, exclude_dirs, exclude_files)
     adoc_files = collect_files(['.'], {'.adoc'}, exclude_dirs, exclude_files)
-    referenced_images = set()
+
+    # Track which images are referenced (uncommented and commented separately)
+    referenced_images = set()  # Images in uncommented references
+    commented_only_images = {}  # Images referenced ONLY in commented lines: {basename: [(file, line_num, line_text)]}
+
+    # Patterns for finding image references (both commented and uncommented)
     image_ref_pattern = re.compile(r'(?i)image::([^\[]+)[\[]|image:([^\[]+)[\[]|"([^"\s]+\.(?:png|jpg|jpeg|gif|svg))"')
+    commented_line_pattern = re.compile(r'^\s*//')
+
     for adoc_file in adoc_files:
         try:
             with open(adoc_file, 'r', encoding='utf-8') as f:
-                content = f.read()
-                for match in image_ref_pattern.findall(content):
-                    for group in match:
-                        if group:
-                            referenced_images.add(os.path.basename(group))
+                lines = f.readlines()
+
+                for line_num, line in enumerate(lines, 1):
+                    # Check if this line is commented
+                    is_commented = commented_line_pattern.match(line)
+
+                    # Find all image references in the line
+                    for match in image_ref_pattern.findall(line):
+                        for group in match:
+                            if group:
+                                image_basename = os.path.basename(group)
+
+                                if is_commented:
+                                    # Track location of commented reference
+                                    if image_basename not in commented_only_images:
+                                        commented_only_images[image_basename] = []
+                                    commented_only_images[image_basename].append((adoc_file, line_num, line.strip()))
+                                else:
+                                    # Add to uncommented references
+                                    referenced_images.add(image_basename)
+                                    # If we found an uncommented reference, remove from commented_only tracking
+                                    if image_basename in commented_only_images:
+                                        del commented_only_images[image_basename]
         except Exception as e:
             print(f"Warning: could not read {adoc_file}: {e}")
-    unused_images = [f for f in image_files if os.path.basename(f) not in referenced_images]
+
+    # Determine which images are unused based on the include_commented flag
+    if include_commented:
+        # When --commented is used: treat images with commented-only references as unused
+        # Only images with uncommented references are considered "used"
+        unused_images = [f for f in image_files if os.path.basename(f) not in referenced_images]
+        commented_only_unused = []
+    else:
+        # Default behavior: images referenced only in commented lines are considered "used"
+        # They should NOT be in the unused list, but we track them for reporting
+        all_referenced = referenced_images.union(set(commented_only_images.keys()))
+        unused_images = [f for f in image_files if os.path.basename(f) not in all_referenced]
+
+        # Generate list of images referenced only in comments for the report
+        commented_only_unused = []
+        for basename, references in commented_only_images.items():
+            # Find the full path for this basename in image_files
+            matching_files = [f for f in image_files if os.path.basename(f) == basename]
+            for f in matching_files:
+                commented_only_unused.append((f, references))
+
     unused_images = list(dict.fromkeys(unused_images))
+
+    # Generate detailed report for commented-only references
+    if commented_only_unused and not include_commented:
+        report_path = os.path.join(archive_dir, 'commented-image-references-report.txt')
+        os.makedirs(archive_dir, exist_ok=True)
+
+        with open(report_path, 'w', encoding='utf-8') as report:
+            report.write("Images Referenced Only in Commented Lines\n")
+            report.write("=" * 70 + "\n\n")
+            report.write(f"Found {len(commented_only_unused)} images that are referenced only in commented-out lines.\n")
+            report.write("These images are considered 'used' by default and will NOT be archived.\n\n")
+            report.write("To archive these images along with other unused images, use the --commented flag.\n\n")
+            report.write("-" * 70 + "\n\n")
+
+            for file_path, references in sorted(commented_only_unused):
+                report.write(f"Image: {file_path}\n")
+                report.write(f"Referenced in {len(references)} commented line(s):\n")
+                for ref_file, line_num, line_text in references:
+                    report.write(f"  {ref_file}:{line_num}\n")
+                    report.write(f"    {line_text}\n")
+                report.write("\n")
+
+        print(f"\n📋 Found {len(commented_only_unused)} images referenced only in commented lines.")
+        print(f"   Detailed report saved to: {report_path}")
+        print(f"   These images are considered 'used' and will NOT be archived by default.")
+        print(f"   To include them in the archive operation, use the --commented flag.\n")
+
     return write_manifest_and_archive(
         unused_images, archive_dir, 'unused-images', 'unused-images', archive=archive
     )