ctx-packer 0.1.4__tar.gz

ctx_packer-0.1.4/.context-pack.yaml.example

@@ -0,0 +1,286 @@
+# Context Packer Configuration Example
+# Copy this file to .context-pack.yaml in your repository root and customize as needed
+
+# ==============================================================================
+# OUTPUT SETTINGS
+# ==============================================================================
+
+# Output format for generated context packs
+# Options: "xml" | "zip"
+# - xml: Single file with Repomix-style structure (for paste workflows)
+# - zip: Archive with preserved directory structure + manifest (for upload workflows)
+format: zip
+
+# Maximum token budget for LLM context window
+# The system reserves 20% for metadata, using 80% for actual file content
+# Common values:
+#   - Claude 3.5 Sonnet: 200000
+#   - GPT-4 Turbo: 128000
+#   - GPT-4: 8192
+#   - Claude 2: 100000
+token_budget: 100000
+
+# Output directory for generated context packs
+# Relative paths are resolved from repository root
+output_path: ./output
+
+# ==============================================================================
+# SCORING WEIGHTS
+# ==============================================================================
+
+# Weight for semantic similarity score (0.0 to 1.0)
+# Higher values prioritize files matching the query semantically
+# Recommended: 0.6 for query-driven workflows
+semantic_weight: 0.6
+
+# Weight for PageRank structural importance score (0.0 to 1.0)
+# Higher values prioritize core dependencies and central files
+# Recommended: 0.4 for balanced ranking
+# Note: semantic_weight + pagerank_weight should equal 1.0
+pagerank_weight: 0.4
+
+# ==============================================================================
+# FILE FILTERING
+# ==============================================================================
+
+# Include test files in the context pack
+# Set to true if you want test files included in the output
+include_tests: false
+
+# Glob patterns for files to include
+# Uses Python glob syntax with ** for recursive matching
+include_patterns:
+  - "**/*.py"          # Python files
+  - "**/*.js"          # JavaScript files
+  - "**/*.ts"          # TypeScript files
+  - "**/*.tsx"         # TypeScript React files
+  - "**/*.jsx"         # JavaScript React files
+  - "**/*.java"        # Java files
+  - "**/*.go"          # Go files
+  - "**/*.rs"          # Rust files
+  - "**/*.cpp"         # C++ files
+  - "**/*.c"           # C files
+  - "**/*.h"           # Header files
+  - "**/*.rb"          # Ruby files
+  - "**/*.php"         # PHP files
+  - "**/*.swift"       # Swift files
+  - "**/*.kt"          # Kotlin files
+  - "**/*.scala"       # Scala files
+
+# Glob patterns for files to exclude
+# These patterns override include_patterns
+exclude_patterns:
+  - "*.min.js"         # Minified JavaScript
+  - "*.min.css"        # Minified CSS
+  - "node_modules/**"  # Node.js dependencies
+  - "__pycache__/**"   # Python bytecode cache
+  - ".git/**"          # Git repository data
+  - ".svn/**"          # SVN repository data
+  - "venv/**"          # Python virtual environment
+  - "env/**"           # Python virtual environment
+  - ".venv/**"         # Python virtual environment
+  - "dist/**"          # Build output
+  - "build/**"         # Build output
+  - "target/**"        # Build output (Rust, Java)
+  - "*.pyc"            # Python bytecode
+  - "*.pyo"            # Python optimized bytecode
+  - "*.so"             # Shared objects
+  - "*.dylib"          # Dynamic libraries
+  - "*.dll"            # Windows DLLs
+  - ".DS_Store"        # macOS metadata
+  - "Thumbs.db"        # Windows thumbnails
+
+# ==============================================================================
+# BACKEND SELECTION
+# ==============================================================================
+
+backends:
+  # Vector index backend for semantic search
+  # Options: "auto" | "leann" | "faiss"
+  # - auto: Try LEANN first, fall back to FAISS if unavailable
+  # - leann: Use LEANN only (97% storage savings, graph-based)
+  # - faiss: Use FAISS only (battle-tested HNSW index)
+  vector_index: auto
+
+  # Graph backend for PageRank computation
+  # Options: "auto" | "igraph" | "networkx"
+  # - auto: Try igraph first, fall back to NetworkX if unavailable
+  # - igraph: Use igraph only (C++ backend, <1s for 10k files)
+  # - networkx: Use NetworkX only (pure Python, <10s for 10k files)
+  graph: auto
+
+  # Embeddings backend for vector generation
+  # Options: "auto" | "local" | "api"
+  # - auto: Try local first, fall back to API on OOM
+  # - local: Use sentence-transformers only (requires GPU or sufficient RAM)
+  # - api: Use OpenAI API only (requires OPENAI_API_KEY env var)
+  embeddings: auto
+
+# ==============================================================================
+# EMBEDDINGS CONFIGURATION
+# ==============================================================================
+
+embeddings:
+  # Sentence-transformers model for local embeddings
+  # Popular options:
+  #   - all-MiniLM-L6-v2: Fast, 384 dimensions (default)
+  #   - all-mpnet-base-v2: Better quality, 768 dimensions
+  #   - multi-qa-mpnet-base-dot-v1: Optimized for Q&A
+  model: all-MiniLM-L6-v2
+
+  # Device for local embeddings computation
+  # Options: "cpu" | "cuda" | "mps"
+  # - cpu: Use CPU (slower but works everywhere)
+  # - cuda: Use NVIDIA GPU (requires CUDA)
+  # - mps: Use Apple Silicon GPU (macOS only)
+  device: cpu
+
+  # Batch size for embedding generation
+  # Larger values are faster but use more memory
+  # Reduce if you encounter OOM errors
+  # Recommended values:
+  #   - CPU: 16-32
+  #   - GPU: 64-128
+  batch_size: 32
+
+  # API provider for fallback embeddings
+  # Options: "openai"
+  # Future: "cohere", "anthropic", "huggingface"
+  api_provider: openai
+
+  # Environment variable name containing API key
+  # Set this environment variable before running context-pack
+  # Example: export OPENAI_API_KEY="sk-..."
+  api_key_env: OPENAI_API_KEY
+
+# ==============================================================================
+# PERFORMANCE TUNING
+# ==============================================================================
+
+performance:
+  # Maximum number of worker threads for parallel processing
+  # Set to 0 to use all available CPU cores
+  # Recommended: 4 for most systems
+  max_workers: 4
+
+  # Cache embeddings to disk for faster subsequent runs
+  # Embeddings are stored in .context-pack/embeddings.cache
+  # Set to false to disable caching (saves disk space)
+  cache_embeddings: true
+
+  # Enable incremental indexing
+  # Only re-index files that have changed since last run
+  # Set to false to force full re-indexing every time
+  incremental_index: true
+
+# ==============================================================================
+# LOGGING CONFIGURATION
+# ==============================================================================
+
+logging:
+  # Log level for console output
+  # Options: "DEBUG" | "INFO" | "WARNING" | "ERROR"
+  # - DEBUG: All messages including detailed timing
+  # - INFO: Progress messages and warnings
+  # - WARNING: Warnings and errors only
+  # - ERROR: Errors only
+  console_level: INFO
+
+  # Log level for file output
+  # File logs are always written to .context-pack/logs/
+  # Recommended: DEBUG for troubleshooting
+  file_level: DEBUG
+
+  # Enable verbose mode for detailed timing information
+  # Logs execution time for each phase
+  verbose: false
+
+# ==============================================================================
+# ADVANCED OPTIONS
+# ==============================================================================
+
+advanced:
+  # PageRank damping factor (0.0 to 1.0)
+  # Higher values give more weight to incoming links
+  # Standard value: 0.85
+  pagerank_damping: 0.85
+
+  # PageRank boost factor for changed files (1.0 to 10.0)
+  # Multiplier applied to PageRank scores of changed files
+  # Higher values prioritize changed files more strongly
+  # Recommended: 2.0 for PR reviews
+  changed_file_boost: 2.0
+
+  # Maximum number of iterations for PageRank convergence
+  # Higher values improve accuracy but increase computation time
+  pagerank_max_iterations: 100
+
+  # Convergence tolerance for PageRank
+  # Stop iterating when score changes are below this threshold
+  pagerank_tolerance: 0.0001
+
+  # Minimum file size in bytes to include
+  # Files smaller than this are excluded
+  # Set to 0 to include all files
+  min_file_size: 0
+
+  # Maximum file size in bytes to include
+  # Files larger than this are excluded
+  # Set to 0 for no limit
+  # Recommended: 1000000 (1 MB) to exclude large generated files
+  max_file_size: 1000000
+
+  # Enable AST parsing round-trip validation
+  # Verifies that parse → print → parse produces equivalent structure
+  # Useful for debugging parser issues
+  # Warning: Significantly slows down indexing
+  validate_roundtrip: false
+
+# ==============================================================================
+# EXAMPLE CONFIGURATIONS
+# ==============================================================================
+
+# Example 1: PR Review Workflow
+# Optimize for reviewing changed files and their dependencies
+# 
+# semantic_weight: 0.4
+# pagerank_weight: 0.6
+# changed_file_boost: 3.0
+# format: zip
+
+# Example 2: Bug Investigation
+# Optimize for semantic search with natural language queries
+# 
+# semantic_weight: 0.8
+# pagerank_weight: 0.2
+# format: xml
+# token_budget: 50000
+
+# Example 3: Documentation Generation
+# Optimize for core API files and public interfaces
+# 
+# semantic_weight: 0.5
+# pagerank_weight: 0.5
+# include_patterns:
+#   - "**/api/**/*.py"
+#   - "**/models/**/*.py"
+#   - "**/schemas/**/*.py"
+
+# Example 4: Minimal Dependencies (No Optional Backends)
+# Use only core dependencies with fallback algorithms
+# 
+# backends:
+#   vector_index: faiss
+#   graph: networkx
+#   embeddings: api
+
+# Example 5: Maximum Performance (All Primary Backends)
+# Use all primary backends for optimal speed
+# 
+# backends:
+#   vector_index: leann
+#   graph: igraph
+#   embeddings: local
+# embeddings:
+#   device: cuda
+#   batch_size: 128

ctx_packer-0.1.4/AUTHORS.md

@@ -0,0 +1,30 @@
+# Authors
+
+This file lists the contributors to Context Packer.
+
+## Creator and Lead Maintainer
+
+- **zamery** <zaob.ogn@gmail.com>
+  - Project creator and lead maintainer
+  - Initial implementation and design
+
+## Contributors
+
+<!-- Contributors will be added here as they contribute -->
+
+We welcome contributions from everyone! See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.
+
+## Acknowledgments
+
+Special thanks to:
+
+- The open source community for inspiration and tools
+- Tree-sitter project for AST parsing capabilities
+- FAISS team for vector indexing
+- igraph and NetworkX teams for graph analysis
+- Sentence Transformers for embedding models
+- All users who report bugs and suggest improvements
+
+---
+
+To add yourself to this list, please include your name and email in your pull request.

ctx_packer-0.1.4/CHANGELOG.md

@@ -0,0 +1,138 @@
+# Changelog
+## [0.1.4] - 2026-03-25
+
+### Added
+- `context-pack status` - Show index size, file count, backend info
+- `context-pack vacuum` - Optimize SQLite database
+- `context-pack reindex-domain` - Only rebuild domain_map.db (fast)
+- SQLite DomainMapDB backend for large repositories (>10K files)
+- Phase 1-4: Parallel Write → Shadow Read → SQLite Primary → Cleanup
+
+### Changed
+- Tuned retrieval weights: domain_boost 0.4 → 0.25
+- Improved query classification for better ranking
+- Fixed DomainMapDB.drop-in compatibility with RetrievalEngine
+
+### Fixed
+- Missing `directories_for()` method in DomainMapDB
+- Broken try block structure in query.py
+
+
+## [0.1.3] - 2026-03-24
+
+### Added
+- `context-pack status` - Show index size, file count, backend info
+- `context-pack vacuum` - Optimize SQLite database
+- `context-pack reindex-domain` - Only rebuild domain_map.db (fast)
+- SQLite DomainMapDB backend for large repositories (>10K files)
+- Phase 1-4: Parallel Write → Shadow Read → SQLite Primary → Cleanup
+
+### Changed
+- Tuned retrieval weights: domain_boost 0.4 → 0.25
+- Improved query classification for better ranking
+- Fixed DomainMapDB.drop-in compatibility with RetrievalEngine
+
+### Fixed
+- Missing `directories_for()` method in DomainMapDB
+- Broken try block structure in query.py
+
+
+## [0.1.2] - 2026-03-24
+
+### Added
+- `context-pack status` - Show index size, file count, backend info
+- `context-pack vacuum` - Optimize SQLite database
+- `context-pack reindex-domain` - Only rebuild domain_map.db (fast)
+- SQLite DomainMapDB backend for large repositories (>10K files)
+- Phase 1-4: Parallel Write → Shadow Read → SQLite Primary → Cleanup
+
+### Changed
+- Tuned retrieval weights: domain_boost 0.4 → 0.25
+- Improved query classification for better ranking
+- Fixed DomainMapDB.drop-in compatibility with RetrievalEngine
+
+### Fixed
+- Missing `directories_for()` method in DomainMapDB
+- Broken try block structure in query.py
+
+
+## [0.1.1] - 2026-03-24
+
+### Added
+- `context-pack status` - Show index size, file count, backend info
+- `context-pack vacuum` - Optimize SQLite database
+- `context-pack reindex-domain` - Only rebuild domain_map.db (fast)
+- SQLite DomainMapDB backend for large repositories (>10K files)
+- Phase 1-4: Parallel Write → Shadow Read → SQLite Primary → Cleanup
+
+### Changed
+- Tuned retrieval weights: domain_boost 0.4 → 0.25
+- Improved query classification for better ranking
+- Fixed DomainMapDB.drop-in compatibility with RetrievalEngine
+
+### Fixed
+- Missing `directories_for()` method in DomainMapDB
+- Broken try block structure in query.py
+
+
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- Initial project structure with comprehensive documentation
+- README.md with installation instructions and usage examples
+- CONTRIBUTING.md with development guidelines
+- CODE_OF_CONDUCT.md based on Contributor Covenant 2.1
+- Example configuration file (.context-pack.yaml.example)
+- GPL-3.0-or-later license
+
+## [0.1.0] - 2025-01-XX
+
+### Added
+- AST-based code parsing with Tree-sitter (primary) and regex fallback
+- Vector indexing with LEANN (primary) and FAISS (fallback)
+- Dependency graph analysis with igraph (primary) and NetworkX (fallback)
+- Hybrid ranking combining semantic search and PageRank
+- Token budget management with greedy knapsack algorithm
+- Dual output formats: XML (Repomix-style) and ZIP (with manifest)
+- Incremental indexing with staleness detection
+- Comprehensive fallback strategies for all components
+- CLI interface with index, query, and pack commands
+- Configuration management via YAML files
+- Structured logging with dual output (console + file)
+- Property-based testing with Hypothesis
+- Integration tests for full workflow
+- Performance benchmarks for all components
+
+### Features
+- Support for Python, JavaScript, and TypeScript parsing
+- Local embeddings with sentence-transformers
+- OpenAI API fallback for embeddings
+- Changed file boosting for PR reviews
+- Configurable semantic and PageRank weights
+- File filtering with include/exclude patterns
+- Automatic backend selection and fallback
+- Round-trip validation for AST parsing
+- Token counting with ±2% accuracy
+
+### Performance
+- Indexing: <5 minutes for 10,000 files (primary backends)
+- Query: <10 seconds for 10,000 files (primary backends)
+- Parsing: <5 seconds per 1,000 lines of code
+- LEANN: 97% storage savings vs traditional vector indexes
+
+### Documentation
+- Comprehensive README with quick start guide
+- Detailed configuration documentation
+- API documentation with Google-style docstrings
+- Contributing guidelines
+- Code of Conduct
+- Example workflows for common use cases
+
+[Unreleased]: https://github.com/maemreyo/zmr-ctx-paker/compare/v0.1.0...HEAD
+[0.1.0]: https://github.com/maemreyo/zmr-ctx-paker/releases/tag/v0.1.0

ctx_packer-0.1.4/CODE_OF_CONDUCT.md

@@ -0,0 +1,77 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at **zaob.ogn@gmail.com**. All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org), version 2.1, available at https://www.contributor-covenant.org/version/2/1/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity).
+
+For answers to common questions about this code of conduct, see the FAQ at https://www.contributor-covenant.org/faq. Translations are available at https://www.contributor-covenant.org/translations.