claude-mpm 4.3.20__py3-none-any.whl → 4.3.22__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. claude_mpm/VERSION +1 -1
  2. claude_mpm/agents/agent_loader.py +2 -2
  3. claude_mpm/agents/agent_loader_integration.py +2 -2
  4. claude_mpm/agents/async_agent_loader.py +2 -2
  5. claude_mpm/agents/base_agent_loader.py +2 -2
  6. claude_mpm/agents/frontmatter_validator.py +2 -2
  7. claude_mpm/agents/system_agent_config.py +2 -2
  8. claude_mpm/agents/templates/data_engineer.json +1 -2
  9. claude_mpm/cli/commands/doctor.py +2 -2
  10. claude_mpm/cli/commands/mpm_init.py +560 -47
  11. claude_mpm/cli/commands/mpm_init_handler.py +6 -0
  12. claude_mpm/cli/parsers/mpm_init_parser.py +39 -1
  13. claude_mpm/cli/startup_logging.py +11 -9
  14. claude_mpm/commands/mpm-init.md +76 -12
  15. claude_mpm/config/agent_config.py +2 -2
  16. claude_mpm/config/paths.py +2 -2
  17. claude_mpm/core/agent_name_normalizer.py +2 -2
  18. claude_mpm/core/config.py +2 -1
  19. claude_mpm/core/config_aliases.py +2 -2
  20. claude_mpm/core/file_utils.py +1 -0
  21. claude_mpm/core/log_manager.py +2 -2
  22. claude_mpm/core/tool_access_control.py +2 -2
  23. claude_mpm/core/unified_agent_registry.py +2 -2
  24. claude_mpm/core/unified_paths.py +2 -2
  25. claude_mpm/experimental/cli_enhancements.py +3 -2
  26. claude_mpm/hooks/base_hook.py +2 -2
  27. claude_mpm/hooks/instruction_reinforcement.py +2 -2
  28. claude_mpm/hooks/validation_hooks.py +2 -2
  29. claude_mpm/scripts/mpm_doctor.py +2 -2
  30. claude_mpm/services/agents/loading/agent_profile_loader.py +2 -2
  31. claude_mpm/services/agents/loading/base_agent_manager.py +2 -2
  32. claude_mpm/services/agents/loading/framework_agent_loader.py +2 -2
  33. claude_mpm/services/agents/management/agent_capabilities_generator.py +2 -2
  34. claude_mpm/services/agents/management/agent_management_service.py +2 -2
  35. claude_mpm/services/agents/memory/memory_categorization_service.py +5 -2
  36. claude_mpm/services/agents/memory/memory_file_service.py +27 -6
  37. claude_mpm/services/agents/memory/memory_format_service.py +5 -2
  38. claude_mpm/services/agents/memory/memory_limits_service.py +3 -2
  39. claude_mpm/services/agents/registry/deployed_agent_discovery.py +2 -2
  40. claude_mpm/services/agents/registry/modification_tracker.py +4 -4
  41. claude_mpm/services/async_session_logger.py +2 -1
  42. claude_mpm/services/claude_session_logger.py +2 -2
  43. claude_mpm/services/core/path_resolver.py +3 -2
  44. claude_mpm/services/diagnostics/diagnostic_runner.py +4 -3
  45. claude_mpm/services/event_bus/direct_relay.py +2 -1
  46. claude_mpm/services/event_bus/event_bus.py +2 -1
  47. claude_mpm/services/event_bus/relay.py +2 -2
  48. claude_mpm/services/framework_claude_md_generator/content_assembler.py +2 -2
  49. claude_mpm/services/infrastructure/daemon_manager.py +2 -2
  50. claude_mpm/services/memory/cache/simple_cache.py +2 -2
  51. claude_mpm/services/project/archive_manager.py +981 -0
  52. claude_mpm/services/project/documentation_manager.py +536 -0
  53. claude_mpm/services/project/enhanced_analyzer.py +491 -0
  54. claude_mpm/services/project/project_organizer.py +904 -0
  55. claude_mpm/services/response_tracker.py +2 -2
  56. claude_mpm/services/socketio/handlers/connection.py +14 -33
  57. claude_mpm/services/socketio/server/eventbus_integration.py +2 -2
  58. claude_mpm/services/version_control/version_parser.py +5 -4
  59. claude_mpm/storage/state_storage.py +2 -2
  60. claude_mpm/utils/agent_dependency_loader.py +49 -0
  61. claude_mpm/utils/common.py +542 -0
  62. claude_mpm/utils/database_connector.py +298 -0
  63. claude_mpm/utils/error_handler.py +2 -1
  64. claude_mpm/utils/log_cleanup.py +2 -2
  65. claude_mpm/utils/path_operations.py +2 -2
  66. claude_mpm/utils/robust_installer.py +56 -0
  67. claude_mpm/utils/session_logging.py +2 -2
  68. claude_mpm/utils/subprocess_utils.py +2 -2
  69. claude_mpm/validation/agent_validator.py +2 -2
  70. {claude_mpm-4.3.20.dist-info → claude_mpm-4.3.22.dist-info}/METADATA +1 -1
  71. {claude_mpm-4.3.20.dist-info → claude_mpm-4.3.22.dist-info}/RECORD +75 -69
  72. {claude_mpm-4.3.20.dist-info → claude_mpm-4.3.22.dist-info}/WHEEL +0 -0
  73. {claude_mpm-4.3.20.dist-info → claude_mpm-4.3.22.dist-info}/entry_points.txt +0 -0
  74. {claude_mpm-4.3.20.dist-info → claude_mpm-4.3.22.dist-info}/licenses/LICENSE +0 -0
  75. {claude_mpm-4.3.20.dist-info → claude_mpm-4.3.22.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,904 @@
1
+ """
2
+ Project Organizer Service for Claude MPM Project Initialization
3
+ ===============================================================
4
+
5
+ This service verifies and creates standard project directory structures
6
+ for optimal Claude Code and Claude MPM usage.
7
+
8
+ Key Features:
9
+ - Standard directory structure verification
10
+ - Missing directory creation with proper permissions
11
+ - .gitignore configuration and updates
12
+ - Temporary directory management
13
+ - Project structure documentation generation
14
+
15
+ Author: Claude MPM Development Team
16
+ Created: 2025-01-26
17
+ """
18
+
19
+ import os
20
+ import stat
21
+ from pathlib import Path
22
+ from typing import Dict, List, Optional, Set, Tuple
23
+
24
+ from rich.console import Console
25
+
26
+ from claude_mpm.core.logging_utils import get_logger
27
+ logger = get_logger(__name__)
28
+ console = Console()
29
+
30
+
31
+ class ProjectOrganizer:
32
+ """Manages project directory structure and organization."""
33
+
34
+ # Standard directory structure for Claude MPM projects
35
+ STANDARD_DIRECTORIES = {
36
+ "tmp": "Temporary files, test outputs, and experiments",
37
+ "scripts": "Project scripts and automation tools",
38
+ "docs": "Project documentation",
39
+ "docs/_archive": "Archived documentation versions",
40
+ "tests": "Test suites and test fixtures",
41
+ ".claude-mpm": "Claude MPM configuration and data",
42
+ ".claude-mpm/memories": "Agent memory storage",
43
+ ".claude/agents": "Deployed agent configurations",
44
+ "src": "Source code (for libraries/packages)",
45
+ }
46
+
47
+ # Comprehensive gitignore patterns for Claude MPM projects
48
+ GITIGNORE_DIRS = {
49
+ # Temporary and cache directories
50
+ "tmp/",
51
+ "temp/",
52
+ "*.tmp",
53
+ "*.temp",
54
+ "*.cache",
55
+ ".claude-mpm/cache/",
56
+ ".claude-mpm/logs/",
57
+ ".claude/cache/",
58
+
59
+ # Python artifacts
60
+ "__pycache__/",
61
+ "*.py[cod]",
62
+ "*$py.class",
63
+ "*.so",
64
+ ".Python",
65
+ "*.egg-info/",
66
+ "*.egg",
67
+ "dist/",
68
+ "build/",
69
+ "develop-eggs/",
70
+ ".eggs/",
71
+ "wheels/",
72
+ "pip-wheel-metadata/",
73
+ "*.manifest",
74
+ "*.spec",
75
+
76
+ # Testing and coverage
77
+ ".pytest_cache/",
78
+ ".coverage",
79
+ ".coverage.*",
80
+ "htmlcov/",
81
+ ".tox/",
82
+ ".nox/",
83
+ "*.cover",
84
+ ".hypothesis/",
85
+ ".pytype/",
86
+ "coverage.xml",
87
+ "*.pytest_cache",
88
+
89
+ # Virtual environments
90
+ ".env",
91
+ ".venv",
92
+ "env/",
93
+ "venv/",
94
+ "ENV/",
95
+ "env.bak/",
96
+ "venv.bak/",
97
+ "virtualenv/",
98
+ ".conda/",
99
+ "conda-env/",
100
+
101
+ # IDE and editor files
102
+ ".vscode/",
103
+ ".idea/",
104
+ "*.swp",
105
+ "*.swo",
106
+ "*~",
107
+ ".project",
108
+ ".pydevproject",
109
+ ".settings/",
110
+ "*.sublime-workspace",
111
+ "*.sublime-project",
112
+
113
+ # OS-specific files
114
+ ".DS_Store",
115
+ "Thumbs.db",
116
+ "ehthumbs.db",
117
+ "Desktop.ini",
118
+ "$RECYCLE.BIN/",
119
+ "*.cab",
120
+ "*.msi",
121
+ "*.msm",
122
+ "*.msp",
123
+ "*.lnk",
124
+
125
+ # Logs and databases
126
+ "*.log",
127
+ "*.sql",
128
+ "*.sqlite",
129
+ "*.sqlite3",
130
+ "*.db",
131
+ "logs/",
132
+
133
+ # Node/JavaScript
134
+ "node_modules/",
135
+ "npm-debug.log*",
136
+ "yarn-debug.log*",
137
+ "yarn-error.log*",
138
+ ".npm",
139
+ ".yarn/",
140
+
141
+ # Documentation builds
142
+ "_build/",
143
+ "site/",
144
+ "docs/_build/",
145
+
146
+ # Security and credentials
147
+ ".env.*",
148
+ "*.pem",
149
+ "*.key",
150
+ "*.cert",
151
+ "*.crt",
152
+ ".secrets/",
153
+ "credentials/",
154
+
155
+ # Claude MPM specific
156
+ ".claude-mpm/*.log",
157
+ ".claude-mpm/sessions/",
158
+ ".claude-mpm/tmp/",
159
+ ".claude/sessions/",
160
+ "*.mpm.tmp",
161
+
162
+ # Backup files
163
+ "*.bak",
164
+ "*.backup",
165
+ "*.old",
166
+ "backup/",
167
+ "backups/",
168
+ }
169
+
170
+ # Project type specific structures
171
+ PROJECT_STRUCTURES = {
172
+ "web": ["public", "src/components", "src/pages", "src/styles"],
173
+ "api": ["src/routes", "src/models", "src/middleware", "src/services"],
174
+ "cli": ["src/commands", "src/utils", "src/config"],
175
+ "library": ["src", "examples", "benchmarks"],
176
+ "mobile": ["src/screens", "src/components", "src/services", "assets"],
177
+ "fullstack": ["frontend", "backend", "shared", "infrastructure"],
178
+ }
179
+
180
+ def __init__(self, project_path: Path):
181
+ """Initialize the project organizer."""
182
+ self.project_path = project_path
183
+ self.gitignore_path = project_path / ".gitignore"
184
+ self.structure_report = {}
185
+
186
+ def verify_structure(self, project_type: Optional[str] = None) -> Dict:
187
+ """Verify project structure and identify missing components."""
188
+ report = {
189
+ "project_path": str(self.project_path),
190
+ "exists": [],
191
+ "missing": [],
192
+ "issues": [],
193
+ "recommendations": [],
194
+ }
195
+
196
+ # Check standard directories
197
+ for dir_name, description in self.STANDARD_DIRECTORIES.items():
198
+ dir_path = self.project_path / dir_name
199
+ if dir_path.exists():
200
+ report["exists"].append({
201
+ "path": dir_name,
202
+ "description": description,
203
+ "is_directory": dir_path.is_dir(),
204
+ })
205
+ else:
206
+ report["missing"].append({
207
+ "path": dir_name,
208
+ "description": description,
209
+ "required": self._is_required_directory(dir_name, project_type),
210
+ })
211
+
212
+ # Check project-type specific directories
213
+ if project_type and project_type in self.PROJECT_STRUCTURES:
214
+ for dir_name in self.PROJECT_STRUCTURES[project_type]:
215
+ dir_path = self.project_path / dir_name
216
+ if not dir_path.exists():
217
+ report["missing"].append({
218
+ "path": dir_name,
219
+ "description": f"{project_type} specific directory",
220
+ "required": False,
221
+ })
222
+
223
+ # Check for common issues
224
+ report["issues"] = self._check_common_issues()
225
+
226
+ # Generate recommendations
227
+ report["recommendations"] = self._generate_recommendations(report)
228
+
229
+ self.structure_report = report
230
+ return report
231
+
232
+ def _is_required_directory(self, dir_name: str, project_type: Optional[str]) -> bool:
233
+ """Determine if a directory is required for the project."""
234
+ # Always required directories
235
+ always_required = {"tmp", "scripts", "docs"}
236
+ if dir_name in always_required:
237
+ return True
238
+
239
+ # Type-specific requirements
240
+ if project_type:
241
+ if project_type in ["library", "package"] and dir_name == "src":
242
+ return True
243
+ if project_type in ["web", "api", "fullstack"] and dir_name == "tests":
244
+ return True
245
+
246
+ return False
247
+
248
+ def _check_common_issues(self) -> List[Dict]:
249
+ """Check for common project organization issues."""
250
+ issues = []
251
+
252
+ # Check for files in wrong locations
253
+ root_files = list(self.project_path.glob("*.py"))
254
+ test_files_in_root = [f for f in root_files if "test" in f.name.lower()]
255
+ if test_files_in_root:
256
+ issues.append({
257
+ "type": "misplaced_tests",
258
+ "description": "Test files found in project root",
259
+ "files": [str(f.name) for f in test_files_in_root],
260
+ "recommendation": "Move test files to tests/ directory",
261
+ })
262
+
263
+ script_files_in_root = [
264
+ f for f in root_files
265
+ if f.name.lower().endswith((".sh", ".bash", ".py"))
266
+ and not f.name.startswith(".")
267
+ and f.name not in ["setup.py", "pyproject.toml"]
268
+ ]
269
+ if script_files_in_root:
270
+ issues.append({
271
+ "type": "misplaced_scripts",
272
+ "description": "Script files found in project root",
273
+ "files": [str(f.name) for f in script_files_in_root[:5]], # Limit to 5
274
+ "recommendation": "Move scripts to scripts/ directory",
275
+ })
276
+
277
+ # Check for missing .gitignore
278
+ if not self.gitignore_path.exists():
279
+ issues.append({
280
+ "type": "missing_gitignore",
281
+ "description": "No .gitignore file found",
282
+ "recommendation": "Create .gitignore with standard patterns",
283
+ })
284
+ else:
285
+ # Check .gitignore completeness
286
+ gitignore_content = self.gitignore_path.read_text()
287
+ missing_patterns = []
288
+ for pattern in ["tmp/", "__pycache__", ".env", "*.log"]:
289
+ if pattern not in gitignore_content:
290
+ missing_patterns.append(pattern)
291
+
292
+ if missing_patterns:
293
+ issues.append({
294
+ "type": "incomplete_gitignore",
295
+ "description": "Common patterns missing from .gitignore",
296
+ "patterns": missing_patterns,
297
+ "recommendation": "Update .gitignore with missing patterns",
298
+ })
299
+
300
+ # Check for large files that should be in tmp
301
+ large_files = []
302
+ for file in self.project_path.rglob("*"):
303
+ if file.is_file() and not any(part.startswith(".") for part in file.parts):
304
+ try:
305
+ size_mb = file.stat().st_size / (1024 * 1024)
306
+ if size_mb > 10: # Files larger than 10MB
307
+ if "tmp" not in str(file) and "node_modules" not in str(file):
308
+ large_files.append({
309
+ "path": str(file.relative_to(self.project_path)),
310
+ "size_mb": round(size_mb, 2),
311
+ })
312
+ except (OSError, PermissionError):
313
+ continue
314
+
315
+ if large_files:
316
+ issues.append({
317
+ "type": "large_files",
318
+ "description": "Large files outside tmp/ directory",
319
+ "files": large_files[:5], # Limit to 5
320
+ "recommendation": "Consider moving large files to tmp/ or adding to .gitignore",
321
+ })
322
+
323
+ return issues
324
+
325
+ def _generate_recommendations(self, report: Dict) -> List[str]:
326
+ """Generate recommendations based on verification report."""
327
+ recommendations = []
328
+
329
+ # Recommend creating missing required directories
330
+ required_missing = [d for d in report["missing"] if d.get("required")]
331
+ if required_missing:
332
+ recommendations.append(
333
+ f"Create {len(required_missing)} required directories: "
334
+ + ", ".join(d["path"] for d in required_missing)
335
+ )
336
+
337
+ # Recommend fixing issues
338
+ if report["issues"]:
339
+ for issue in report["issues"]:
340
+ recommendations.append(issue["recommendation"])
341
+
342
+ # Recommend documentation
343
+ if "docs" not in [d["path"] for d in report["exists"]]:
344
+ recommendations.append("Create docs/ directory for project documentation")
345
+
346
+ return recommendations
347
+
348
+ def create_missing_directories(self, force: bool = False) -> Dict:
349
+ """Create missing standard directories."""
350
+ created = []
351
+ skipped = []
352
+ errors = []
353
+
354
+ report = self.verify_structure()
355
+
356
+ for dir_info in report["missing"]:
357
+ dir_path = self.project_path / dir_info["path"]
358
+
359
+ # Skip non-required unless force
360
+ if not dir_info.get("required") and not force:
361
+ skipped.append(dir_info["path"])
362
+ continue
363
+
364
+ try:
365
+ dir_path.mkdir(parents=True, exist_ok=True)
366
+ created.append(dir_info["path"])
367
+ logger.info(f"Created directory: {dir_path}")
368
+
369
+ # Add README for certain directories
370
+ self._add_directory_readme(dir_path, dir_info["description"])
371
+
372
+ except Exception as e:
373
+ errors.append({"path": dir_info["path"], "error": str(e)})
374
+ logger.error(f"Failed to create {dir_path}: {e}")
375
+
376
+ return {
377
+ "created": created,
378
+ "skipped": skipped,
379
+ "errors": errors,
380
+ }
381
+
382
+ def _add_directory_readme(self, dir_path: Path, description: str) -> None:
383
+ """Add README file to newly created directory."""
384
+ readme_path = dir_path / "README.md"
385
+
386
+ # Only add README for certain directories
387
+ readme_dirs = ["tmp", "scripts", "docs/_archive", ".claude-mpm/memories"]
388
+ if any(str(dir_path).endswith(d) for d in readme_dirs):
389
+ if not readme_path.exists():
390
+ content = f"""# {dir_path.name}
391
+
392
+ {description}
393
+
394
+ ## Purpose
395
+
396
+ This directory is used for {description.lower()}.
397
+
398
+ ## Usage Guidelines
399
+
400
+ """
401
+ if "tmp" in str(dir_path):
402
+ content += """- Place all temporary files and test outputs here
403
+ - This directory is gitignored - contents will not be committed
404
+ - Clean up old files periodically to save disk space
405
+ - Use subdirectories to organize different types of temporary files
406
+ """
407
+ elif "scripts" in str(dir_path):
408
+ content += """- All project scripts should be placed here
409
+ - Use descriptive names for scripts
410
+ - Include comments and usage instructions in scripts
411
+ - Make scripts executable with `chmod +x script_name.sh`
412
+ """
413
+ elif "_archive" in str(dir_path):
414
+ content += """- Archived documentation versions are stored here
415
+ - Files are timestamped when archived
416
+ - Preserve important historical documentation
417
+ - Review and clean up old archives periodically
418
+ """
419
+ elif "memories" in str(dir_path):
420
+ content += """- Agent memory files are stored here
421
+ - Each agent can have its own memory file
422
+ - Memories persist between sessions
423
+ - Update memories when project knowledge changes
424
+ """
425
+
426
+ readme_path.write_text(content)
427
+ logger.debug(f"Created README in {dir_path}")
428
+
429
+ def update_gitignore(self, additional_patterns: Optional[List[str]] = None) -> bool:
430
+ """Update or create .gitignore file with standard patterns."""
431
+ try:
432
+ # Read existing content
433
+ existing_patterns = set()
434
+ if self.gitignore_path.exists():
435
+ content = self.gitignore_path.read_text()
436
+ existing_patterns = set(line.strip() for line in content.splitlines() if line.strip() and not line.startswith("#"))
437
+ else:
438
+ content = ""
439
+
440
+ # Combine all patterns
441
+ all_patterns = self.GITIGNORE_DIRS.copy()
442
+ if additional_patterns:
443
+ all_patterns.update(additional_patterns)
444
+
445
+ # Find missing patterns
446
+ missing = all_patterns - existing_patterns
447
+
448
+ if missing:
449
+ # Add section for new patterns
450
+ new_section = "\n# Added by Claude MPM /mpm-init\n"
451
+ for pattern in sorted(missing):
452
+ new_section += f"{pattern}\n"
453
+
454
+ # Append to file
455
+ if content and not content.endswith("\n"):
456
+ content += "\n"
457
+ content += new_section
458
+
459
+ # Write updated content
460
+ self.gitignore_path.write_text(content)
461
+ logger.info(f"Updated .gitignore with {len(missing)} patterns")
462
+ return True
463
+ else:
464
+ logger.info(".gitignore already contains all standard patterns")
465
+ return False
466
+
467
+ except Exception as e:
468
+ logger.error(f"Failed to update .gitignore: {e}")
469
+ return False
470
+
471
+ def organize_misplaced_files(self, dry_run: bool = True, auto_safe: bool = True) -> Dict:
472
+ """Organize misplaced files into proper directories.
473
+
474
+ Args:
475
+ dry_run: If True, only report what would be moved without moving
476
+ auto_safe: If True, only move files that are clearly safe to move
477
+ """
478
+ moves = []
479
+ skipped = []
480
+ errors = []
481
+
482
+ # Files that should never be moved from root
483
+ protected_root_files = {
484
+ "setup.py", "pyproject.toml", "package.json", "package-lock.json",
485
+ "requirements.txt", "Pipfile", "Pipfile.lock", "poetry.lock",
486
+ "Makefile", "makefile", "Dockerfile", "docker-compose.yml",
487
+ ".gitignore", ".gitattributes", "LICENSE", "README.md", "README.rst",
488
+ "CHANGELOG.md", "CONTRIBUTING.md", "CODE_OF_CONDUCT.md",
489
+ "CLAUDE.md", "CODE.md", "DEVELOPER.md", "STRUCTURE.md", "OPS.md",
490
+ ".env.example", ".env.sample", "VERSION", "BUILD_NUMBER"
491
+ }
492
+
493
+ # Scan root directory for misplaced files
494
+ root_files = list(self.project_path.glob("*"))
495
+ for file in root_files:
496
+ if file.is_file() and file.name not in protected_root_files:
497
+ target_dir = None
498
+ confidence = "low" # low, medium, high
499
+ reason = ""
500
+
501
+ # Determine target directory based on file patterns
502
+ file_lower = file.name.lower()
503
+
504
+ # Test files (high confidence)
505
+ if "test" in file_lower and file.suffix == ".py":
506
+ if file_lower.startswith("test_") or file_lower.endswith("_test.py"):
507
+ target_dir = "tests"
508
+ confidence = "high"
509
+ reason = "Test file pattern detected"
510
+
511
+ # Script files (medium-high confidence)
512
+ elif file.suffix in [".sh", ".bash"]:
513
+ target_dir = "scripts"
514
+ confidence = "high"
515
+ reason = "Shell script file"
516
+ elif file.suffix == ".py" and any(pattern in file_lower for pattern in ["script", "run", "cli", "tool"]):
517
+ target_dir = "scripts"
518
+ confidence = "medium"
519
+ reason = "Python script pattern detected"
520
+
521
+ # Log and temporary files (high confidence)
522
+ elif file.suffix in [".log", ".tmp", ".temp", ".cache"]:
523
+ target_dir = "tmp"
524
+ confidence = "high"
525
+ reason = "Temporary/log file"
526
+ elif file_lower.startswith(("tmp_", "temp_", "test_output", "debug_")):
527
+ target_dir = "tmp"
528
+ confidence = "high"
529
+ reason = "Temporary file pattern"
530
+
531
+ # Documentation files (medium confidence)
532
+ elif file.suffix in [".md", ".rst", ".txt"] and file.name not in protected_root_files:
533
+ if any(pattern in file_lower for pattern in ["notes", "draft", "todo", "spec", "design"]):
534
+ target_dir = "docs"
535
+ confidence = "medium"
536
+ reason = "Documentation file pattern"
537
+
538
+ # Data files (medium confidence)
539
+ elif file.suffix in [".csv", ".json", ".xml", ".yaml", ".yml"]:
540
+ if file.suffix in [".yaml", ".yml"] and any(pattern in file_lower for pattern in ["config", "settings"]):
541
+ # Config files might belong in root
542
+ confidence = "low"
543
+ else:
544
+ target_dir = "data"
545
+ confidence = "medium"
546
+ reason = "Data file"
547
+
548
+ # Build artifacts (high confidence)
549
+ elif file.suffix in [".whl", ".tar.gz", ".zip"] and "dist" not in str(file.parent):
550
+ target_dir = "dist"
551
+ confidence = "high"
552
+ reason = "Build artifact"
553
+
554
+ # Example files (medium confidence)
555
+ elif "example" in file_lower or "sample" in file_lower:
556
+ target_dir = "examples"
557
+ confidence = "medium"
558
+ reason = "Example file pattern"
559
+
560
+ if target_dir:
561
+ # Check if we should move based on confidence and auto_safe setting
562
+ should_move = (confidence == "high") if auto_safe else (confidence in ["high", "medium"])
563
+
564
+ if should_move:
565
+ target_path = self.project_path / target_dir / file.name
566
+ moves.append({
567
+ "source": str(file.relative_to(self.project_path)),
568
+ "target": str(target_path.relative_to(self.project_path)),
569
+ "reason": reason,
570
+ "confidence": confidence,
571
+ })
572
+
573
+ if not dry_run:
574
+ try:
575
+ # Create target directory if needed
576
+ target_path.parent.mkdir(parents=True, exist_ok=True)
577
+ # Move file
578
+ file.rename(target_path)
579
+ logger.info(f"Moved {file.name} to {target_dir}/ ({reason})")
580
+ except Exception as e:
581
+ errors.append({
582
+ "file": str(file.name),
583
+ "error": str(e)
584
+ })
585
+ logger.error(f"Failed to move {file.name}: {e}")
586
+ else:
587
+ skipped.append({
588
+ "file": str(file.name),
589
+ "suggested_dir": target_dir,
590
+ "confidence": confidence,
591
+ "reason": f"Low confidence move - skipped (use --no-auto-safe to include)"
592
+ })
593
+
594
+ # Also check for deeply nested test files that should be in tests/
595
+ if not auto_safe: # Only in non-safe mode
596
+ for test_file in self.project_path.rglob("*test*.py"):
597
+ # Skip if already in tests directory
598
+ if "tests" in test_file.parts or "test" in test_file.parts:
599
+ continue
600
+ # Skip if in node_modules or venv
601
+ if any(part in test_file.parts for part in ["node_modules", "venv", ".venv", "site-packages"]):
602
+ continue
603
+
604
+ target_path = self.project_path / "tests" / test_file.name
605
+ moves.append({
606
+ "source": str(test_file.relative_to(self.project_path)),
607
+ "target": str(target_path.relative_to(self.project_path)),
608
+ "reason": "Test file found outside tests directory",
609
+ "confidence": "medium",
610
+ })
611
+
612
+ if not dry_run:
613
+ try:
614
+ target_path.parent.mkdir(parents=True, exist_ok=True)
615
+ test_file.rename(target_path)
616
+ logger.info(f"Moved {test_file.name} to tests/")
617
+ except Exception as e:
618
+ errors.append({
619
+ "file": str(test_file.relative_to(self.project_path)),
620
+ "error": str(e)
621
+ })
622
+
623
+ return {
624
+ "dry_run": dry_run,
625
+ "auto_safe": auto_safe,
626
+ "proposed_moves": moves if dry_run else [],
627
+ "completed_moves": [] if dry_run else moves,
628
+ "skipped": skipped,
629
+ "errors": errors,
630
+ "total": len(moves),
631
+ "total_skipped": len(skipped),
632
+ "total_errors": len(errors),
633
+ }
634
+
635
+ def generate_structure_documentation(self) -> str:
636
+ """Generate markdown documentation of project structure."""
637
+ doc = """# Project Structure Report
638
+
639
+ ## Structure Validation
640
+
641
+ """
642
+ # Add validation summary
643
+ validation = self.validate_structure()
644
+ doc += f"**Overall Grade:** {validation.get('grade', 'Not evaluated')}\n"
645
+ doc += f"**Score:** {validation.get('score', 0)}/100\n\n"
646
+
647
+ if validation["errors"]:
648
+ doc += "### 🔴 Critical Issues\n"
649
+ for error in validation["errors"]:
650
+ doc += f"- {error}\n"
651
+ doc += "\n"
652
+
653
+ if validation["warnings"]:
654
+ doc += "### ⚠️ Warnings\n"
655
+ for warning in validation["warnings"]:
656
+ doc += f"- {warning}\n"
657
+ doc += "\n"
658
+
659
+ doc += "## Directory Organization\n\n"
660
+
661
+ # Document existing structure
662
+ for dir_name, description in self.STANDARD_DIRECTORIES.items():
663
+ dir_path = self.project_path / dir_name
664
+ if dir_path.exists():
665
+ doc += f"### ✅ `{dir_name}/`\n{description}\n\n"
666
+
667
+ # List some contents
668
+ try:
669
+ contents = list(dir_path.iterdir())[:5]
670
+ if contents:
671
+ doc += "**Contents:**\n"
672
+ for item in contents:
673
+ if item.is_dir():
674
+ doc += f"- {item.name}/ (directory)\n"
675
+ else:
676
+ doc += f"- {item.name}\n"
677
+ if len(list(dir_path.iterdir())) > 5:
678
+ doc += f"- ... and {len(list(dir_path.iterdir())) - 5} more items\n"
679
+ doc += "\n"
680
+ except PermissionError:
681
+ doc += "*Permission denied to list contents*\n\n"
682
+ else:
683
+ doc += f"### ❌ `{dir_name}/` (Missing)\n{description}\n\n"
684
+
685
+ # Document misplaced files
686
+ organize_result = self.organize_misplaced_files(dry_run=True, auto_safe=True)
687
+ if organize_result["proposed_moves"]:
688
+ doc += "## 📦 Files to Reorganize\n\n"
689
+ doc += "The following files could be better organized:\n\n"
690
+ for move in organize_result["proposed_moves"][:10]: # Limit to 10
691
+ doc += f"- `{move['source']}` → `{move['target']}`\n"
692
+ doc += f" - Reason: {move['reason']}\n"
693
+ doc += f" - Confidence: {move['confidence']}\n"
694
+ if len(organize_result["proposed_moves"]) > 10:
695
+ doc += f"\n... and {len(organize_result['proposed_moves']) - 10} more files\n"
696
+ doc += "\n"
697
+
698
+ # Document gitignore status
699
+ doc += "## .gitignore Configuration\n\n"
700
+ if self.gitignore_path.exists():
701
+ gitignore_content = self.gitignore_path.read_text()
702
+ critical_patterns = ["tmp/", "__pycache__", ".env", "*.log", ".claude-mpm/cache/"]
703
+ doc += "### Critical Patterns Status:\n"
704
+ for pattern in critical_patterns:
705
+ status = "✅" if pattern in gitignore_content else "❌"
706
+ doc += f"- {status} `{pattern}`\n"
707
+ doc += "\n"
708
+ else:
709
+ doc += "❌ No .gitignore file found\n\n"
710
+
711
+ # Add recommendations
712
+ if self.structure_report and self.structure_report.get("recommendations"):
713
+ doc += "## 💡 Recommendations\n\n"
714
+ for rec in self.structure_report["recommendations"]:
715
+ doc += f"- {rec}\n"
716
+ doc += "\n"
717
+
718
+ # Add quick fix commands
719
+ doc += "## 🛠️ Quick Fix Commands\n\n"
720
+ doc += "```bash\n"
721
+ doc += "# Run complete initialization\n"
722
+ doc += "claude-mpm mpm-init --organize\n\n"
723
+ doc += "# Review without changes\n"
724
+ doc += "claude-mpm mpm-init --review\n\n"
725
+ doc += "# Update existing documentation\n"
726
+ doc += "claude-mpm mpm-init --update\n"
727
+ doc += "```\n"
728
+
729
+ return doc
730
+
731
+ def generate_structure_report_json(self) -> Dict:
732
+ """Generate a comprehensive JSON structure report."""
733
+ validation = self.validate_structure()
734
+ organize_result = self.organize_misplaced_files(dry_run=True, auto_safe=True)
735
+
736
+ report = {
737
+ "timestamp": str(Path.cwd()),
738
+ "project_path": str(self.project_path),
739
+ "validation": validation,
740
+ "directories": {},
741
+ "misplaced_files": organize_result,
742
+ "gitignore": {
743
+ "exists": self.gitignore_path.exists(),
744
+ "patterns_status": {}
745
+ },
746
+ "statistics": {
747
+ "total_directories": 0,
748
+ "total_files": 0,
749
+ "misplaced_files": len(organize_result.get("proposed_moves", [])),
750
+ "structure_score": validation.get("score", 0)
751
+ }
752
+ }
753
+
754
+ # Check directory status
755
+ for dir_name, description in self.STANDARD_DIRECTORIES.items():
756
+ dir_path = self.project_path / dir_name
757
+ report["directories"][dir_name] = {
758
+ "exists": dir_path.exists(),
759
+ "description": description,
760
+ "file_count": len(list(dir_path.glob("*"))) if dir_path.exists() else 0,
761
+ "is_directory": dir_path.is_dir() if dir_path.exists() else None
762
+ }
763
+ if dir_path.exists():
764
+ report["statistics"]["total_directories"] += 1
765
+
766
+ # Check gitignore patterns
767
+ if self.gitignore_path.exists():
768
+ gitignore_content = self.gitignore_path.read_text()
769
+ critical_patterns = ["tmp/", "__pycache__", ".env", "*.log", ".claude-mpm/cache/"]
770
+ for pattern in critical_patterns:
771
+ report["gitignore"]["patterns_status"][pattern] = pattern in gitignore_content
772
+
773
+ # Count total files
774
+ for item in self.project_path.rglob("*"):
775
+ if item.is_file():
776
+ report["statistics"]["total_files"] += 1
777
+
778
+ return report
779
+
780
+ def ensure_project_ready(self, auto_organize: bool = False, safe_mode: bool = True) -> Tuple[bool, List[str]]:
781
+ """Ensure project is ready for Claude MPM usage.
782
+
783
+ Args:
784
+ auto_organize: Automatically organize misplaced files
785
+ safe_mode: Only perform safe operations
786
+ """
787
+ actions_taken = []
788
+ issues_found = []
789
+
790
+ # Verify structure first
791
+ self.verify_structure()
792
+
793
+ # Create required directories
794
+ result = self.create_missing_directories(force=False)
795
+ if result["created"]:
796
+ actions_taken.append(f"Created {len(result['created'])} directories")
797
+
798
+ # Create tmp directory with proper README if it doesn't exist
799
+ tmp_dir = self.project_path / "tmp"
800
+ if not tmp_dir.exists():
801
+ tmp_dir.mkdir(parents=True, exist_ok=True)
802
+ self._add_directory_readme(tmp_dir, "Temporary files, test outputs, and experiments")
803
+ actions_taken.append("Created tmp/ directory with README")
804
+
805
+ # Update .gitignore with comprehensive patterns
806
+ if self.update_gitignore():
807
+ actions_taken.append("Updated .gitignore with comprehensive patterns")
808
+
809
+ # Check if organization is needed
810
+ organize_result = self.organize_misplaced_files(dry_run=True, auto_safe=safe_mode)
811
+ if organize_result["proposed_moves"]:
812
+ if auto_organize:
813
+ # Perform the organization
814
+ actual_result = self.organize_misplaced_files(dry_run=False, auto_safe=safe_mode)
815
+ if actual_result["completed_moves"]:
816
+ actions_taken.append(
817
+ f"Organized {len(actual_result['completed_moves'])} files into proper directories"
818
+ )
819
+ if actual_result["errors"]:
820
+ issues_found.append(
821
+ f"Failed to move {len(actual_result['errors'])} files"
822
+ )
823
+ else:
824
+ actions_taken.append(
825
+ f"Identified {len(organize_result['proposed_moves'])} files to reorganize (use --organize to move)"
826
+ )
827
+ if organize_result["skipped"]:
828
+ actions_taken.append(
829
+ f"Skipped {len(organize_result['skipped'])} low-confidence moves"
830
+ )
831
+
832
+ # Check for remaining issues
833
+ if self.structure_report.get("issues"):
834
+ for issue in self.structure_report["issues"]:
835
+ if issue["type"] not in ["misplaced_scripts", "misplaced_tests"]: # These may be handled
836
+ issues_found.append(issue["description"])
837
+
838
+ # Generate structure validation report
839
+ validation_report = self.validate_structure()
840
+ if not validation_report["is_valid"]:
841
+ issues_found.extend(validation_report["errors"])
842
+
843
+ ready = len(issues_found) == 0
844
+ return ready, actions_taken
845
+
846
+ def validate_structure(self) -> Dict:
847
+ """Validate the project structure meets Claude MPM standards."""
848
+ validation = {
849
+ "is_valid": True,
850
+ "errors": [],
851
+ "warnings": [],
852
+ "score": 100,
853
+ }
854
+
855
+ # Check critical directories exist
856
+ critical_dirs = ["tmp", "scripts", "docs"]
857
+ for dir_name in critical_dirs:
858
+ if not (self.project_path / dir_name).exists():
859
+ validation["is_valid"] = False
860
+ validation["errors"].append(f"Missing critical directory: {dir_name}/")
861
+ validation["score"] -= 10
862
+
863
+ # Check .gitignore completeness
864
+ if not self.gitignore_path.exists():
865
+ validation["is_valid"] = False
866
+ validation["errors"].append("No .gitignore file found")
867
+ validation["score"] -= 15
868
+ else:
869
+ gitignore_content = self.gitignore_path.read_text()
870
+ critical_patterns = ["tmp/", "__pycache__", ".env", "*.log"]
871
+ for pattern in critical_patterns:
872
+ if pattern not in gitignore_content:
873
+ validation["warnings"].append(f"Missing gitignore pattern: {pattern}")
874
+ validation["score"] -= 2
875
+
876
+ # Check for files in wrong locations
877
+ root_files = list(self.project_path.glob("*"))
878
+ misplaced_count = 0
879
+ for file in root_files:
880
+ if file.is_file():
881
+ if "test" in file.name.lower() and file.suffix == ".py":
882
+ misplaced_count += 1
883
+ elif file.suffix in [".sh", ".bash"] and file.name not in ["Makefile"]:
884
+ misplaced_count += 1
885
+ elif file.suffix in [".log", ".tmp", ".cache"]:
886
+ misplaced_count += 1
887
+
888
+ if misplaced_count > 0:
889
+ validation["warnings"].append(f"{misplaced_count} files potentially misplaced in root")
890
+ validation["score"] -= min(misplaced_count * 2, 20)
891
+
892
+ # Score interpretation
893
+ if validation["score"] >= 90:
894
+ validation["grade"] = "A - Excellent structure"
895
+ elif validation["score"] >= 80:
896
+ validation["grade"] = "B - Good structure"
897
+ elif validation["score"] >= 70:
898
+ validation["grade"] = "C - Acceptable structure"
899
+ elif validation["score"] >= 60:
900
+ validation["grade"] = "D - Needs improvement"
901
+ else:
902
+ validation["grade"] = "F - Poor structure"
903
+
904
+ return validation