folderfuse 0.0.1__tar.gz → 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. folderfuse-0.1.0/.gitignore +87 -0
  2. {folderfuse-0.0.1 → folderfuse-0.1.0}/LICENSE +1 -1
  3. folderfuse-0.1.0/PKG-INFO +276 -0
  4. folderfuse-0.1.0/README.md +218 -0
  5. folderfuse-0.1.0/folderfuse/__init__.py +6 -0
  6. folderfuse-0.1.0/folderfuse/__main__.py +10 -0
  7. folderfuse-0.1.0/folderfuse/api/__init__.py +15 -0
  8. folderfuse-0.1.0/folderfuse/api/launcher.py +19 -0
  9. folderfuse-0.1.0/folderfuse/api/port_hunter.py +31 -0
  10. folderfuse-0.1.0/folderfuse/api/routes.py +377 -0
  11. folderfuse-0.1.0/folderfuse/api/schemas.py +129 -0
  12. folderfuse-0.1.0/folderfuse/api/server.py +94 -0
  13. folderfuse-0.1.0/folderfuse/api/tree_builder.py +97 -0
  14. folderfuse-0.1.0/folderfuse/cli/__init__.py +7 -0
  15. folderfuse-0.1.0/folderfuse/cli/app.py +381 -0
  16. folderfuse-0.1.0/folderfuse/cli/config/__init__.py +27 -0
  17. folderfuse-0.1.0/folderfuse/cli/config/loader.py +196 -0
  18. folderfuse-0.1.0/folderfuse/cli/config/models.py +39 -0
  19. folderfuse-0.1.0/folderfuse/cli/terminal/__init__.py +17 -0
  20. folderfuse-0.1.0/folderfuse/cli/terminal/clipboard.py +29 -0
  21. folderfuse-0.1.0/folderfuse/cli/terminal/console.py +118 -0
  22. folderfuse-0.1.0/folderfuse/cli/terminal/meta.py +28 -0
  23. folderfuse-0.1.0/folderfuse/core/__init__.py +41 -0
  24. folderfuse-0.1.0/folderfuse/core/crawler/__init__.py +25 -0
  25. folderfuse-0.1.0/folderfuse/core/crawler/crawler.py +174 -0
  26. folderfuse-0.1.0/folderfuse/core/crawler/detector.py +70 -0
  27. folderfuse-0.1.0/folderfuse/core/crawler/ignore.py +93 -0
  28. folderfuse-0.1.0/folderfuse/core/crawler/models.py +61 -0
  29. folderfuse-0.1.0/folderfuse/core/extractors/__init__.py +24 -0
  30. folderfuse-0.1.0/folderfuse/core/extractors/base.py +32 -0
  31. folderfuse-0.1.0/folderfuse/core/extractors/docx.py +61 -0
  32. folderfuse-0.1.0/folderfuse/core/extractors/models.py +27 -0
  33. folderfuse-0.1.0/folderfuse/core/extractors/notebook.py +68 -0
  34. folderfuse-0.1.0/folderfuse/core/extractors/pdf.py +61 -0
  35. folderfuse-0.1.0/folderfuse/core/extractors/profiler.py +100 -0
  36. folderfuse-0.1.0/folderfuse/core/extractors/registry.py +50 -0
  37. folderfuse-0.1.0/folderfuse/core/extractors/spreadsheet.py +158 -0
  38. folderfuse-0.1.0/folderfuse/core/layout/__init__.py +37 -0
  39. folderfuse-0.1.0/folderfuse/core/layout/engine.py +81 -0
  40. folderfuse-0.1.0/folderfuse/core/layout/formatters.py +171 -0
  41. folderfuse-0.1.0/folderfuse/core/layout/models.py +52 -0
  42. folderfuse-0.1.0/folderfuse/core/layout/ordering.py +52 -0
  43. folderfuse-0.1.0/folderfuse/core/layout/tree_generator.py +53 -0
  44. folderfuse-0.1.0/folderfuse/core/layout/working_files.py +57 -0
  45. folderfuse-0.1.0/folderfuse/core/pipeline.py +268 -0
  46. folderfuse-0.1.0/folderfuse/core/security/__init__.py +27 -0
  47. folderfuse-0.1.0/folderfuse/core/security/models.py +41 -0
  48. folderfuse-0.1.0/folderfuse/core/security/patterns.py +77 -0
  49. folderfuse-0.1.0/folderfuse/core/security/scrubber.py +113 -0
  50. folderfuse-0.1.0/folderfuse/core/tokenizers/__init__.py +23 -0
  51. folderfuse-0.1.0/folderfuse/core/tokenizers/base.py +45 -0
  52. folderfuse-0.1.0/folderfuse/core/tokenizers/claude.py +58 -0
  53. folderfuse-0.1.0/folderfuse/core/tokenizers/deepseek.py +49 -0
  54. folderfuse-0.1.0/folderfuse/core/tokenizers/gemini.py +55 -0
  55. folderfuse-0.1.0/folderfuse/core/tokenizers/models.py +38 -0
  56. folderfuse-0.1.0/folderfuse/core/tokenizers/openai.py +58 -0
  57. folderfuse-0.1.0/folderfuse/core/tokenizers/registry.py +113 -0
  58. folderfuse-0.1.0/folderfuse/core/transform/__init__.py +35 -0
  59. folderfuse-0.1.0/folderfuse/core/transform/formatters.py +27 -0
  60. folderfuse-0.1.0/folderfuse/core/transform/languages.py +112 -0
  61. folderfuse-0.1.0/folderfuse/core/transform/models.py +52 -0
  62. folderfuse-0.1.0/folderfuse/core/transform/selector.py +25 -0
  63. folderfuse-0.1.0/folderfuse/core/transform/skeletonizer.py +243 -0
  64. folderfuse-0.1.0/folderfuse/py.typed +0 -0
  65. folderfuse-0.1.0/folderfuse/web/dist/assets/index-B_uycAtT.js +231 -0
  66. folderfuse-0.1.0/folderfuse/web/dist/assets/index-BfFYCGUS.css +1 -0
  67. folderfuse-0.1.0/folderfuse/web/dist/index.html +16 -0
  68. folderfuse-0.1.0/pyproject.toml +192 -0
  69. folderfuse-0.1.0/tests/__init__.py +0 -0
  70. folderfuse-0.1.0/tests/benchmarks/__init__.py +0 -0
  71. folderfuse-0.1.0/tests/benchmarks/test_performance.py +112 -0
  72. folderfuse-0.1.0/tests/conftest.py +26 -0
  73. folderfuse-0.1.0/tests/e2e/__init__.py +0 -0
  74. folderfuse-0.1.0/tests/e2e/test_golden_e2e.py +225 -0
  75. folderfuse-0.1.0/tests/security/__init__.py +0 -0
  76. folderfuse-0.1.0/tests/security/test_penetration.py +178 -0
  77. folderfuse-0.1.0/tests/test_api.py +61 -0
  78. folderfuse-0.1.0/tests/test_api_bundle.py +75 -0
  79. folderfuse-0.1.0/tests/test_api_routes.py +76 -0
  80. folderfuse-0.1.0/tests/test_api_server.py +65 -0
  81. folderfuse-0.1.0/tests/test_cli.py +101 -0
  82. folderfuse-0.1.0/tests/test_cli_config.py +106 -0
  83. folderfuse-0.1.0/tests/test_cli_terminal.py +105 -0
  84. folderfuse-0.1.0/tests/test_core.py +27 -0
  85. folderfuse-0.1.0/tests/test_crawler.py +165 -0
  86. folderfuse-0.1.0/tests/test_cross_platform.py +124 -0
  87. folderfuse-0.1.0/tests/test_extractors.py +151 -0
  88. folderfuse-0.1.0/tests/test_golden.py +108 -0
  89. folderfuse-0.1.0/tests/test_layout.py +162 -0
  90. folderfuse-0.1.0/tests/test_pipeline.py +141 -0
  91. folderfuse-0.1.0/tests/test_security.py +92 -0
  92. folderfuse-0.1.0/tests/test_tokenizers.py +118 -0
  93. folderfuse-0.1.0/tests/test_transform.py +130 -0
  94. folderfuse-0.0.1/PKG-INFO +0 -20
  95. folderfuse-0.0.1/README.md +0 -5
  96. folderfuse-0.0.1/pyproject.toml +0 -26
  97. folderfuse-0.0.1/src/folderfuse/__init__.py +0 -3
  98. folderfuse-0.0.1/src/folderfuse/__main__.py +0 -5
@@ -0,0 +1,87 @@
1
+ # Python artifacts
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+ *.so
6
+ .Python
7
+ build/
8
+ develop-eggs/
9
+ dist/
10
+ downloads/
11
+ eggs/
12
+ .eggs/
13
+ # lib/
14
+ lib64/
15
+ parts/
16
+ sdist/
17
+ var/
18
+ wheels/
19
+ share/python-wheels/
20
+ *.egg-info/
21
+ .installed.cfg
22
+ *.egg
23
+ MANIFEST
24
+
25
+
26
+
27
+ # Virtual Environments
28
+ .env
29
+ .venv/
30
+ env/
31
+ venv/
32
+ ENV/
33
+ env.bak/
34
+ venv.bak/
35
+
36
+ # UV Cache
37
+ .uv/
38
+ uv.lock
39
+
40
+ # Testing & Coverage
41
+ .pytest_cache/
42
+ .coverage
43
+ .coverage.*
44
+ htmlcov/
45
+ coverage.xml
46
+ *.cover
47
+ *.py,cover
48
+ .hypothesis/
49
+
50
+ # Type Checking
51
+ .mypy_cache/
52
+ .dmypy.json
53
+ dmypy.json
54
+
55
+ # Node & Frontend (Source dependencies)
56
+ node_modules/
57
+ frontend/dist/
58
+ frontend/.vite/
59
+ npm-debug.log*
60
+ yarn-debug.log*
61
+ yarn-error.log*
62
+ pnpm-debug.log*
63
+ .pnpm-store/
64
+
65
+ # Compiled static bundle in wheel package (built during release)
66
+ folderfuse/web/dist/*
67
+ !folderfuse/web/dist/.gitkeep
68
+
69
+ # IDEs & System Files
70
+ .idea/
71
+ .vscode/
72
+ *.swp
73
+ *.swo
74
+ *~
75
+ .DS_Store
76
+ Thumbs.db
77
+
78
+ # Secrets & Local Configurations
79
+ *.env
80
+ *.env.*
81
+ !.env.example
82
+ *.pem
83
+ *.key
84
+ folderfuse.yaml
85
+ .folderfuse.json
86
+
87
+ .secrets
@@ -1,6 +1,6 @@
1
1
  MIT License
2
2
 
3
- Copyright (c) 2026 FolderFuse Authors
3
+ Copyright (c) 2026 Latent Oxygen Studios
4
4
 
5
5
  Permission is hereby granted, free of charge, to any person obtaining a copy
6
6
  of this software and associated documentation files (the "Software"), to deal
@@ -0,0 +1,276 @@
1
+ Metadata-Version: 2.5
2
+ Name: folderfuse
3
+ Version: 0.1.0
4
+ Summary: High-performance codebase context packager, AST skeletonizer, and multi-model token optimizer for LLMs.
5
+ Project-URL: Homepage, https://github.com/Samm-G/folder-fuse
6
+ Project-URL: Repository, https://github.com/Samm-G/folder-fuse
7
+ Project-URL: Issues, https://github.com/Samm-G/folder-fuse/issues
8
+ Author: Latent Oxygen Studios
9
+ License: MIT
10
+ License-File: LICENSE
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Operating System :: OS Independent
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Topic :: Software Development :: Code Generators
21
+ Classifier: Topic :: Utilities
22
+ Classifier: Typing :: Typed
23
+ Requires-Python: >=3.10
24
+ Requires-Dist: fastapi>=0.110.0
25
+ Requires-Dist: nbformat>=5.9.0
26
+ Requires-Dist: openpyxl>=3.1.2
27
+ Requires-Dist: pathspec>=0.12.1
28
+ Requires-Dist: pypdf>=4.0.0
29
+ Requires-Dist: pyperclip>=1.8.2
30
+ Requires-Dist: python-docx>=1.1.0
31
+ Requires-Dist: pyyaml>=6.0.1
32
+ Requires-Dist: rich>=13.7.0
33
+ Requires-Dist: tiktoken>=0.7.0
34
+ Requires-Dist: tokenizers>=0.19.0
35
+ Requires-Dist: tree-sitter-c-sharp>=0.21.0
36
+ Requires-Dist: tree-sitter-c>=0.21.0
37
+ Requires-Dist: tree-sitter-cpp>=0.21.0
38
+ Requires-Dist: tree-sitter-go>=0.21.0
39
+ Requires-Dist: tree-sitter-java>=0.21.0
40
+ Requires-Dist: tree-sitter-javascript>=0.21.0
41
+ Requires-Dist: tree-sitter-python>=0.21.0
42
+ Requires-Dist: tree-sitter-rust>=0.21.0
43
+ Requires-Dist: tree-sitter-typescript>=0.21.0
44
+ Requires-Dist: tree-sitter>=0.22.0
45
+ Requires-Dist: typer>=0.12.0
46
+ Requires-Dist: uvicorn[standard]>=0.28.0
47
+ Provides-Extra: dev
48
+ Requires-Dist: httpx>=0.27.0; extra == 'dev'
49
+ Requires-Dist: mypy>=1.9.0; extra == 'dev'
50
+ Requires-Dist: pre-commit>=3.7.0; extra == 'dev'
51
+ Requires-Dist: pytest-asyncio>=0.23.0; extra == 'dev'
52
+ Requires-Dist: pytest-cov>=5.0.0; extra == 'dev'
53
+ Requires-Dist: pytest>=8.0.0; extra == 'dev'
54
+ Requires-Dist: ruff>=0.4.0; extra == 'dev'
55
+ Requires-Dist: types-openpyxl>=3.1.5.20240307; extra == 'dev'
56
+ Requires-Dist: types-pyyaml>=6.0.12.20240311; extra == 'dev'
57
+ Description-Content-Type: text/markdown
58
+
59
+ # FolderFuse
60
+
61
+ [![PyPI Version](https://img.shields.io/pypi/v/folderfuse.svg)](https://pypi.org/project/folderfuse/)
62
+ [![Python Version](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/downloads/)
63
+ [![CI Status](https://github.com/Samm-G/folder-fuse/actions/workflows/ci.yml/badge.svg)](https://github.com/Samm-G/folder-fuse/actions)
64
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
65
+ [![Air-Gapped](https://img.shields.io/badge/telemetry-zero--egress-success.svg)](#security--privacy)
66
+
67
+ High-performance codebase context packager, AST skeletonizer, and multi-model token optimizer engineered for Large Language Models (LLMs).
68
+
69
+ ---
70
+
71
+ ## Overview
72
+
73
+ **FolderFuse** packages local codebases, documentation repositories, and mixed-format folders into model-ready, token-optimized context bundles. It operates under a strict **dual-entry architecture**: a headless Unix-pipe CLI for terminal power users and automated CI/CD pipelines, alongside an embedded local Web UI for interactive preparation.
74
+
75
+ FolderFuse is strictly an **exporter**—it never dispatches requests directly to external LLM APIs, ensuring workflows remain 100% private, zero-cost, and completely air-gapped.
76
+
77
+ ```
78
+ ┌──────────────────────────────────────────────────────────────────┐
79
+ │ FolderFuse Core │
80
+ │ │
81
+ │ ┌───────────────────────┐ ┌──────────────────────┐ │
82
+ │ │ Headless CLI │ │ FastAPI Engine │ │
83
+ │ │ (Typer / Rich) │ │ (Local 127.0.0.1) │ │
84
+ │ └──────────┬────────────┘ └──────────┬───────────┘ │
85
+ │ │ │ │
86
+ │ │ ┌───────────┴───────────┐ │
87
+ │ │ │ Embedded React UI │ │
88
+ │ │ │ (Vite + Tailwind) │ │
89
+ │ │ └───────────┬───────────┘ │
90
+ │ └───────────────────┬─────────────────┘ │
91
+ │ ▼ │
92
+ │ ┌─────────────────────────────────────────────────────────────┐ │
93
+ │ │ Processing Pipeline │ │
94
+ │ │ │ │
95
+ │ │ 1. Filesystem Crawler (.gitignore, root jail containment) │ │
96
+ │ │ 2. Document Extractors (.pdf, .docx, .xlsx, .ipynb) │ │
97
+ │ │ 3. AST Skeletonizer (Tree-Sitter multi-language parser) │ │
98
+ │ │ 4. Security Sanitizer (Zero-leak session deduplication) │ │
99
+ │ │ 5. Multi-Model Tokenizers (Claude, Gemini 3, DeepSeek V4) │ │
100
+ │ │ 6. KV-Cache Optimizer (5-step layout ordering engine) │ │
101
+ │ └──────────────────────────────┬──────────────────────────────┘ │
102
+ │ ▼ │
103
+ │ Export: Stdout / File / Clipboard │
104
+ └──────────────────────────────────────────────────────────────────┘
105
+ ```
106
+
107
+ ---
108
+
109
+ ## Core Capabilities
110
+
111
+ - **Dual-Entry Architecture:**
112
+ - **Headless POSIX CLI:** Optimized for Unix piping (`folderfuse pack . | pbcopy`). Payload directed strictly to `stdout`; progress spinners, logs, and token meters routed to `stderr`.
113
+ - **Embedded Web Dashboard:** Single-command boot (`folderfuse ui .`) launching an air-gapped React/Vite dashboard served directly from Python wheel assets on `127.0.0.1`.
114
+ - **Multi-Language AST Skeletonization (Tree-Sitter):**
115
+ Prunes function/method implementation bodies while preserving class definitions, method signatures, return types, docstrings, and top-level constants/type aliases. Supports Python, TypeScript, JavaScript, Go, Rust, Java, C, C++, and C#.
116
+ - **Prompt-Cache Optimization (Anthropic & Gemini):**
117
+ Structures prompt layout deterministically into a 5-step hierarchy to maximize KV-cache hit rates:
118
+ 1. System instructions prefix
119
+ 2. ASCII repository directory tree
120
+ 3. Reference documentation (`*.md`, `docs/**`)
121
+ 4. Stable source code
122
+ 5. Ephemeral / working target files placed at the prompt tail (Git-aware + glob overrides)
123
+ - **Multi-Model Tokenizer Engine:**
124
+ 100% offline exact token calculations for **Anthropic Claude** (Claude 3.5 Sonnet, Haiku), **Google Gemini** (Gemini 3.8 Flash, 3.7 Flash, 3.5 Flash), **DeepSeek** (DeepSeek-V4.1-Flash, DeepSeek-V4-Pro-0813), and **OpenAI** (GPT-4o, o200k/cl100k BPE).
125
+ - **Zero-Leak Secret Sanitization:**
126
+ Detects credentials (OpenAI, Anthropic, AWS, GitHub PATs, JWTs, private keys) and replaces them with deterministic synthetic tokens (`[REDACTED:OPENAI_KEY:1]`), preserving cross-file references without leaking raw secrets.
127
+ - **Rich Document Parsing & Statistical Data Profiling:**
128
+ - **Spreadsheets (`.xlsx`, `.csv`):** Markdown tables with configurable row/column caps and an automated statistical data profile summary (null percentages, unique counts, numeric stats).
129
+ - **Jupyter Notebooks (`.ipynb`):** Strips heavy base64 image plots while preserving code cells and execution output streams.
130
+ - **Word Docs (`.docx`) & PDFs (`.pdf`):** Extracts headings, bullet lists, and page-by-page text.
131
+
132
+ ---
133
+
134
+ ## Installation
135
+
136
+ Install `folderfuse` from PyPI:
137
+
138
+ ```bash
139
+ pip install folderfuse
140
+ ```
141
+
142
+ Or using `uv`:
143
+
144
+ ```bash
145
+ uv add folderfuse
146
+ ```
147
+
148
+ Or run directly without permanent installation via `pipx`:
149
+
150
+ ```bash
151
+ pipx run folderfuse ui
152
+ ```
153
+
154
+ ---
155
+
156
+ ## Quickstart
157
+
158
+ ### 1. Headless CLI (`folderfuse pack`)
159
+
160
+ Package the current repository into an Anthropic Claude XML prompt and copy it directly to your system clipboard:
161
+
162
+ ```bash
163
+ folderfuse pack . --model claude --clipboard
164
+ ```
165
+
166
+ Stream directly to standard output for Unix pipelines:
167
+
168
+ ```bash
169
+ # Pipe directly into macOS clipboard
170
+ folderfuse pack ./my-project -q | pbcopy
171
+
172
+ # Save formatted markdown context bundle to disk
173
+ folderfuse pack ./my-project --format markdown --output bundle.md
174
+ ```
175
+
176
+ Apply AST skeletonization to secondary libraries:
177
+
178
+ ```bash
179
+ folderfuse pack . --skeleton "src/utils/**,src/legacy/**" --strict
180
+ ```
181
+
182
+ ### 2. Interactive Local Web Dashboard (`folderfuse ui`)
183
+
184
+ Launch the local web server and open the dashboard in your default browser:
185
+
186
+ ```bash
187
+ folderfuse ui /path/to/project
188
+ ```
189
+
190
+ Options for UI server:
191
+
192
+ ```bash
193
+ folderfuse ui . --port 8080 --no-browser
194
+ ```
195
+
196
+ ---
197
+
198
+ ## CLI Flag Reference (`folderfuse pack`)
199
+
200
+ | Flag | Short | Default | Description |
201
+ | :----------------------- | :---- | :------------------ | :--------------------------------------------------------------------- |
202
+ | `PATH` | | `.` | Target directory path to inspect and package. |
203
+ | `--model` | `-m` | `claude-3-5-sonnet` | Tokenizer profile (`claude`, `gemini`, `openai`, `deepseek`). |
204
+ | `--format` | `-f` | `xml` | Output layout (`xml`, `markdown`, `json`, `raw`). |
205
+ | `--output` | `-o` | `None` | Destination file path to write bundle to. |
206
+ | `--clipboard` | `-c` | `False` | Copy generated context bundle to system clipboard. |
207
+ | `--skeleton` | | `None` | Glob patterns for AST skeletonization (repeatable or comma-separated). |
208
+ | `--exclude` | | `None` | Additional glob patterns to exclude from ingestion. |
209
+ | `--working` | | `None` | Globs for volatile files to place at prompt tail. |
210
+ | `--no-gitignore` | | `False` | Disable automatic `.gitignore` rule evaluation. |
211
+ | `--follow-symlinks` | | `False` | Permit traversing symlinks pointing outside root jail. |
212
+ | `--scrub-secrets` | | `True` | Toggle automated secret and credential masking. |
213
+ | `--omit-sensitive-files` | | `False` | Omit sensitive files (`.env`, `id_rsa`) instead of scrubbing. |
214
+ | `--max-size-kb` | | `500` | Skip text files exceeding threshold in KB. |
215
+ | `--instructions` | | `None` | Custom system framing instructions for prompt header. |
216
+ | `--strict` | | `False` | Exit with code `3` if total tokens exceed model context limit. |
217
+ | `--json-meta` | | `False` | Emit structured JSON diagnostic metadata to stderr. |
218
+ | `--quiet` | `-q` | `False` | Suppress all diagnostic banners, tables, and spinners. |
219
+
220
+ ### POSIX Exit Codes
221
+ - `0`: Success (Context bundle generated cleanly).
222
+ - `1`: General runtime or filesystem write error.
223
+ - `2`: Target directory not found or invalid path.
224
+ - `3`: Context window budget overflow (when `--strict` is set).
225
+
226
+ ---
227
+
228
+ ## Configuration File (`folderfuse.yaml`)
229
+
230
+ FolderFuse supports repository-level configuration files (`folderfuse.yaml` or `.folderfuse.json`) located at the target directory root:
231
+
232
+ ```yaml
233
+ version: "1"
234
+ settings:
235
+ default_model: "claude-3-5-sonnet"
236
+ default_format: "xml"
237
+ max_file_size_kb: 500
238
+ scrub_secrets: true
239
+ strict: false
240
+ include_instructions: true
241
+ instruction_text: "You are an expert systems engineer evaluating this repository."
242
+
243
+ exclusions:
244
+ - "dist/**"
245
+ - "build/**"
246
+ - "**/*.spec.ts"
247
+
248
+ skeletonize:
249
+ - "legacy/**/*.py"
250
+ - "src/vendor/**/*.ts"
251
+
252
+ working_files:
253
+ - "src/active_feature.py"
254
+
255
+ custom_redactions:
256
+ - name: "INTERNAL_CODENAME"
257
+ regex: "(?i)project-zeus"
258
+ replace: "[PROJECT_ZEUS]"
259
+ ```
260
+
261
+ *Precedence:* Built-in Defaults < User Global Config (`~/.config/folderfuse/config.yaml`) < Target Repo Config (`folderfuse.yaml`) < CLI Flags.
262
+
263
+ ---
264
+
265
+ ## Security & Privacy
266
+
267
+ - **Air-Gapped Operation:** FolderFuse makes zero network requests, contains zero analytics SDKs, and bundles all frontend and grammar assets locally inside the Python wheel.
268
+ - **Localhost-Only Binding:** The embedded FastAPI server binds strictly to loopback interfaces (`127.0.0.1`).
269
+ - **Root Jail Containment:** Symbolic links resolving outside the designated root directory are blocked by default to prevent path traversal attacks.
270
+
271
+ ---
272
+
273
+ ## License
274
+
275
+ Distributed under the terms of the [MIT License](LICENSE).
276
+ Copyright © 2026 Latent Oxygen Studios.
@@ -0,0 +1,218 @@
1
+ # FolderFuse
2
+
3
+ [![PyPI Version](https://img.shields.io/pypi/v/folderfuse.svg)](https://pypi.org/project/folderfuse/)
4
+ [![Python Version](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/downloads/)
5
+ [![CI Status](https://github.com/Samm-G/folder-fuse/actions/workflows/ci.yml/badge.svg)](https://github.com/Samm-G/folder-fuse/actions)
6
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
7
+ [![Air-Gapped](https://img.shields.io/badge/telemetry-zero--egress-success.svg)](#security--privacy)
8
+
9
+ High-performance codebase context packager, AST skeletonizer, and multi-model token optimizer engineered for Large Language Models (LLMs).
10
+
11
+ ---
12
+
13
+ ## Overview
14
+
15
+ **FolderFuse** packages local codebases, documentation repositories, and mixed-format folders into model-ready, token-optimized context bundles. It operates under a strict **dual-entry architecture**: a headless Unix-pipe CLI for terminal power users and automated CI/CD pipelines, alongside an embedded local Web UI for interactive preparation.
16
+
17
+ FolderFuse is strictly an **exporter**—it never dispatches requests directly to external LLM APIs, ensuring workflows remain 100% private, zero-cost, and completely air-gapped.
18
+
19
+ ```
20
+ ┌──────────────────────────────────────────────────────────────────┐
21
+ │ FolderFuse Core │
22
+ │ │
23
+ │ ┌───────────────────────┐ ┌──────────────────────┐ │
24
+ │ │ Headless CLI │ │ FastAPI Engine │ │
25
+ │ │ (Typer / Rich) │ │ (Local 127.0.0.1) │ │
26
+ │ └──────────┬────────────┘ └──────────┬───────────┘ │
27
+ │ │ │ │
28
+ │ │ ┌───────────┴───────────┐ │
29
+ │ │ │ Embedded React UI │ │
30
+ │ │ │ (Vite + Tailwind) │ │
31
+ │ │ └───────────┬───────────┘ │
32
+ │ └───────────────────┬─────────────────┘ │
33
+ │ ▼ │
34
+ │ ┌─────────────────────────────────────────────────────────────┐ │
35
+ │ │ Processing Pipeline │ │
36
+ │ │ │ │
37
+ │ │ 1. Filesystem Crawler (.gitignore, root jail containment) │ │
38
+ │ │ 2. Document Extractors (.pdf, .docx, .xlsx, .ipynb) │ │
39
+ │ │ 3. AST Skeletonizer (Tree-Sitter multi-language parser) │ │
40
+ │ │ 4. Security Sanitizer (Zero-leak session deduplication) │ │
41
+ │ │ 5. Multi-Model Tokenizers (Claude, Gemini 3, DeepSeek V4) │ │
42
+ │ │ 6. KV-Cache Optimizer (5-step layout ordering engine) │ │
43
+ │ └──────────────────────────────┬──────────────────────────────┘ │
44
+ │ ▼ │
45
+ │ Export: Stdout / File / Clipboard │
46
+ └──────────────────────────────────────────────────────────────────┘
47
+ ```
48
+
49
+ ---
50
+
51
+ ## Core Capabilities
52
+
53
+ - **Dual-Entry Architecture:**
54
+ - **Headless POSIX CLI:** Optimized for Unix piping (`folderfuse pack . | pbcopy`). Payload directed strictly to `stdout`; progress spinners, logs, and token meters routed to `stderr`.
55
+ - **Embedded Web Dashboard:** Single-command boot (`folderfuse ui .`) launching an air-gapped React/Vite dashboard served directly from Python wheel assets on `127.0.0.1`.
56
+ - **Multi-Language AST Skeletonization (Tree-Sitter):**
57
+ Prunes function/method implementation bodies while preserving class definitions, method signatures, return types, docstrings, and top-level constants/type aliases. Supports Python, TypeScript, JavaScript, Go, Rust, Java, C, C++, and C#.
58
+ - **Prompt-Cache Optimization (Anthropic & Gemini):**
59
+ Structures prompt layout deterministically into a 5-step hierarchy to maximize KV-cache hit rates:
60
+ 1. System instructions prefix
61
+ 2. ASCII repository directory tree
62
+ 3. Reference documentation (`*.md`, `docs/**`)
63
+ 4. Stable source code
64
+ 5. Ephemeral / working target files placed at the prompt tail (Git-aware + glob overrides)
65
+ - **Multi-Model Tokenizer Engine:**
66
+ 100% offline exact token calculations for **Anthropic Claude** (Claude 3.5 Sonnet, Haiku), **Google Gemini** (Gemini 3.8 Flash, 3.7 Flash, 3.5 Flash), **DeepSeek** (DeepSeek-V4.1-Flash, DeepSeek-V4-Pro-0813), and **OpenAI** (GPT-4o, o200k/cl100k BPE).
67
+ - **Zero-Leak Secret Sanitization:**
68
+ Detects credentials (OpenAI, Anthropic, AWS, GitHub PATs, JWTs, private keys) and replaces them with deterministic synthetic tokens (`[REDACTED:OPENAI_KEY:1]`), preserving cross-file references without leaking raw secrets.
69
+ - **Rich Document Parsing & Statistical Data Profiling:**
70
+ - **Spreadsheets (`.xlsx`, `.csv`):** Markdown tables with configurable row/column caps and an automated statistical data profile summary (null percentages, unique counts, numeric stats).
71
+ - **Jupyter Notebooks (`.ipynb`):** Strips heavy base64 image plots while preserving code cells and execution output streams.
72
+ - **Word Docs (`.docx`) & PDFs (`.pdf`):** Extracts headings, bullet lists, and page-by-page text.
73
+
74
+ ---
75
+
76
+ ## Installation
77
+
78
+ Install `folderfuse` from PyPI:
79
+
80
+ ```bash
81
+ pip install folderfuse
82
+ ```
83
+
84
+ Or using `uv`:
85
+
86
+ ```bash
87
+ uv add folderfuse
88
+ ```
89
+
90
+ Or run directly without permanent installation via `pipx`:
91
+
92
+ ```bash
93
+ pipx run folderfuse ui
94
+ ```
95
+
96
+ ---
97
+
98
+ ## Quickstart
99
+
100
+ ### 1. Headless CLI (`folderfuse pack`)
101
+
102
+ Package the current repository into an Anthropic Claude XML prompt and copy it directly to your system clipboard:
103
+
104
+ ```bash
105
+ folderfuse pack . --model claude --clipboard
106
+ ```
107
+
108
+ Stream directly to standard output for Unix pipelines:
109
+
110
+ ```bash
111
+ # Pipe directly into macOS clipboard
112
+ folderfuse pack ./my-project -q | pbcopy
113
+
114
+ # Save formatted markdown context bundle to disk
115
+ folderfuse pack ./my-project --format markdown --output bundle.md
116
+ ```
117
+
118
+ Apply AST skeletonization to secondary libraries:
119
+
120
+ ```bash
121
+ folderfuse pack . --skeleton "src/utils/**,src/legacy/**" --strict
122
+ ```
123
+
124
+ ### 2. Interactive Local Web Dashboard (`folderfuse ui`)
125
+
126
+ Launch the local web server and open the dashboard in your default browser:
127
+
128
+ ```bash
129
+ folderfuse ui /path/to/project
130
+ ```
131
+
132
+ Options for UI server:
133
+
134
+ ```bash
135
+ folderfuse ui . --port 8080 --no-browser
136
+ ```
137
+
138
+ ---
139
+
140
+ ## CLI Flag Reference (`folderfuse pack`)
141
+
142
+ | Flag | Short | Default | Description |
143
+ | :----------------------- | :---- | :------------------ | :--------------------------------------------------------------------- |
144
+ | `PATH` | | `.` | Target directory path to inspect and package. |
145
+ | `--model` | `-m` | `claude-3-5-sonnet` | Tokenizer profile (`claude`, `gemini`, `openai`, `deepseek`). |
146
+ | `--format` | `-f` | `xml` | Output layout (`xml`, `markdown`, `json`, `raw`). |
147
+ | `--output` | `-o` | `None` | Destination file path to write bundle to. |
148
+ | `--clipboard` | `-c` | `False` | Copy generated context bundle to system clipboard. |
149
+ | `--skeleton` | | `None` | Glob patterns for AST skeletonization (repeatable or comma-separated). |
150
+ | `--exclude` | | `None` | Additional glob patterns to exclude from ingestion. |
151
+ | `--working` | | `None` | Globs for volatile files to place at prompt tail. |
152
+ | `--no-gitignore` | | `False` | Disable automatic `.gitignore` rule evaluation. |
153
+ | `--follow-symlinks` | | `False` | Permit traversing symlinks pointing outside root jail. |
154
+ | `--scrub-secrets` | | `True` | Toggle automated secret and credential masking. |
155
+ | `--omit-sensitive-files` | | `False` | Omit sensitive files (`.env`, `id_rsa`) instead of scrubbing. |
156
+ | `--max-size-kb` | | `500` | Skip text files exceeding threshold in KB. |
157
+ | `--instructions` | | `None` | Custom system framing instructions for prompt header. |
158
+ | `--strict` | | `False` | Exit with code `3` if total tokens exceed model context limit. |
159
+ | `--json-meta` | | `False` | Emit structured JSON diagnostic metadata to stderr. |
160
+ | `--quiet` | `-q` | `False` | Suppress all diagnostic banners, tables, and spinners. |
161
+
162
+ ### POSIX Exit Codes
163
+ - `0`: Success (Context bundle generated cleanly).
164
+ - `1`: General runtime or filesystem write error.
165
+ - `2`: Target directory not found or invalid path.
166
+ - `3`: Context window budget overflow (when `--strict` is set).
167
+
168
+ ---
169
+
170
+ ## Configuration File (`folderfuse.yaml`)
171
+
172
+ FolderFuse supports repository-level configuration files (`folderfuse.yaml` or `.folderfuse.json`) located at the target directory root:
173
+
174
+ ```yaml
175
+ version: "1"
176
+ settings:
177
+ default_model: "claude-3-5-sonnet"
178
+ default_format: "xml"
179
+ max_file_size_kb: 500
180
+ scrub_secrets: true
181
+ strict: false
182
+ include_instructions: true
183
+ instruction_text: "You are an expert systems engineer evaluating this repository."
184
+
185
+ exclusions:
186
+ - "dist/**"
187
+ - "build/**"
188
+ - "**/*.spec.ts"
189
+
190
+ skeletonize:
191
+ - "legacy/**/*.py"
192
+ - "src/vendor/**/*.ts"
193
+
194
+ working_files:
195
+ - "src/active_feature.py"
196
+
197
+ custom_redactions:
198
+ - name: "INTERNAL_CODENAME"
199
+ regex: "(?i)project-zeus"
200
+ replace: "[PROJECT_ZEUS]"
201
+ ```
202
+
203
+ *Precedence:* Built-in Defaults < User Global Config (`~/.config/folderfuse/config.yaml`) < Target Repo Config (`folderfuse.yaml`) < CLI Flags.
204
+
205
+ ---
206
+
207
+ ## Security & Privacy
208
+
209
+ - **Air-Gapped Operation:** FolderFuse makes zero network requests, contains zero analytics SDKs, and bundles all frontend and grammar assets locally inside the Python wheel.
210
+ - **Localhost-Only Binding:** The embedded FastAPI server binds strictly to loopback interfaces (`127.0.0.1`).
211
+ - **Root Jail Containment:** Symbolic links resolving outside the designated root directory are blocked by default to prevent path traversal attacks.
212
+
213
+ ---
214
+
215
+ ## License
216
+
217
+ Distributed under the terms of the [MIT License](LICENSE).
218
+ Copyright © 2026 Latent Oxygen Studios.
@@ -0,0 +1,6 @@
1
+ """FolderFuse: High-performance codebase context packager and multi-model token optimizer."""
2
+
3
+ from __future__ import annotations
4
+
5
+ __version__ = "0.1.0"
6
+ __all__: list[str] = ["__version__"]
@@ -0,0 +1,10 @@
1
+ """Module entrypoint for python -m folderfuse."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import sys
6
+
7
+ from folderfuse.cli import app
8
+
9
+ if __name__ == "__main__":
10
+ sys.exit(app())
@@ -0,0 +1,15 @@
1
+ """Local HTTP API and static asset server."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from folderfuse.api.launcher import launch_browser
6
+ from folderfuse.api.port_hunter import find_available_port, is_port_available
7
+ from folderfuse.api.server import create_app, run_server
8
+
9
+ __all__: list[str] = [
10
+ "create_app",
11
+ "find_available_port",
12
+ "is_port_available",
13
+ "launch_browser",
14
+ "run_server",
15
+ ]
@@ -0,0 +1,19 @@
1
+ """Automated browser launching with headless fault tolerance."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import threading
6
+ import webbrowser
7
+ from contextlib import suppress
8
+
9
+
10
+ def launch_browser(url: str, delay_seconds: float = 0.5) -> None:
11
+ """Launch user default browser after a safety delay allowing server boot."""
12
+
13
+ def _open() -> None:
14
+ with suppress(Exception):
15
+ webbrowser.open(url)
16
+
17
+ timer = threading.Timer(delay_seconds, _open)
18
+ timer.daemon = True
19
+ timer.start()
@@ -0,0 +1,31 @@
1
+ """Socket-based port availability probing and dynamic port discovery."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import socket
6
+
7
+
8
+ def is_port_available(host: str, port: int) -> bool:
9
+ """Check if a specific host:port endpoint can be bound and listened on."""
10
+ try:
11
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
12
+ sock.bind((host, port))
13
+ sock.listen(1)
14
+ return True
15
+ except OSError:
16
+ return False
17
+
18
+
19
+ def find_available_port(
20
+ host: str = "127.0.0.1",
21
+ start_port: int = 8080,
22
+ max_attempts: int = 20,
23
+ ) -> int:
24
+ """Probe sequentially from start_port across max_attempts for an open socket."""
25
+ for offset in range(max_attempts):
26
+ candidate = start_port + offset
27
+ if is_port_available(host, candidate):
28
+ return candidate
29
+
30
+ end_port = start_port + max_attempts - 1
31
+ raise RuntimeError(f"No available ports found on {host} in range {start_port}..{end_port}.")