folderfuse 0.0.1__tar.gz → 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- folderfuse-0.1.0/.gitignore +87 -0
- {folderfuse-0.0.1 → folderfuse-0.1.0}/LICENSE +1 -1
- folderfuse-0.1.0/PKG-INFO +276 -0
- folderfuse-0.1.0/README.md +218 -0
- folderfuse-0.1.0/folderfuse/__init__.py +6 -0
- folderfuse-0.1.0/folderfuse/__main__.py +10 -0
- folderfuse-0.1.0/folderfuse/api/__init__.py +15 -0
- folderfuse-0.1.0/folderfuse/api/launcher.py +19 -0
- folderfuse-0.1.0/folderfuse/api/port_hunter.py +31 -0
- folderfuse-0.1.0/folderfuse/api/routes.py +377 -0
- folderfuse-0.1.0/folderfuse/api/schemas.py +129 -0
- folderfuse-0.1.0/folderfuse/api/server.py +94 -0
- folderfuse-0.1.0/folderfuse/api/tree_builder.py +97 -0
- folderfuse-0.1.0/folderfuse/cli/__init__.py +7 -0
- folderfuse-0.1.0/folderfuse/cli/app.py +381 -0
- folderfuse-0.1.0/folderfuse/cli/config/__init__.py +27 -0
- folderfuse-0.1.0/folderfuse/cli/config/loader.py +196 -0
- folderfuse-0.1.0/folderfuse/cli/config/models.py +39 -0
- folderfuse-0.1.0/folderfuse/cli/terminal/__init__.py +17 -0
- folderfuse-0.1.0/folderfuse/cli/terminal/clipboard.py +29 -0
- folderfuse-0.1.0/folderfuse/cli/terminal/console.py +118 -0
- folderfuse-0.1.0/folderfuse/cli/terminal/meta.py +28 -0
- folderfuse-0.1.0/folderfuse/core/__init__.py +41 -0
- folderfuse-0.1.0/folderfuse/core/crawler/__init__.py +25 -0
- folderfuse-0.1.0/folderfuse/core/crawler/crawler.py +174 -0
- folderfuse-0.1.0/folderfuse/core/crawler/detector.py +70 -0
- folderfuse-0.1.0/folderfuse/core/crawler/ignore.py +93 -0
- folderfuse-0.1.0/folderfuse/core/crawler/models.py +61 -0
- folderfuse-0.1.0/folderfuse/core/extractors/__init__.py +24 -0
- folderfuse-0.1.0/folderfuse/core/extractors/base.py +32 -0
- folderfuse-0.1.0/folderfuse/core/extractors/docx.py +61 -0
- folderfuse-0.1.0/folderfuse/core/extractors/models.py +27 -0
- folderfuse-0.1.0/folderfuse/core/extractors/notebook.py +68 -0
- folderfuse-0.1.0/folderfuse/core/extractors/pdf.py +61 -0
- folderfuse-0.1.0/folderfuse/core/extractors/profiler.py +100 -0
- folderfuse-0.1.0/folderfuse/core/extractors/registry.py +50 -0
- folderfuse-0.1.0/folderfuse/core/extractors/spreadsheet.py +158 -0
- folderfuse-0.1.0/folderfuse/core/layout/__init__.py +37 -0
- folderfuse-0.1.0/folderfuse/core/layout/engine.py +81 -0
- folderfuse-0.1.0/folderfuse/core/layout/formatters.py +171 -0
- folderfuse-0.1.0/folderfuse/core/layout/models.py +52 -0
- folderfuse-0.1.0/folderfuse/core/layout/ordering.py +52 -0
- folderfuse-0.1.0/folderfuse/core/layout/tree_generator.py +53 -0
- folderfuse-0.1.0/folderfuse/core/layout/working_files.py +57 -0
- folderfuse-0.1.0/folderfuse/core/pipeline.py +268 -0
- folderfuse-0.1.0/folderfuse/core/security/__init__.py +27 -0
- folderfuse-0.1.0/folderfuse/core/security/models.py +41 -0
- folderfuse-0.1.0/folderfuse/core/security/patterns.py +77 -0
- folderfuse-0.1.0/folderfuse/core/security/scrubber.py +113 -0
- folderfuse-0.1.0/folderfuse/core/tokenizers/__init__.py +23 -0
- folderfuse-0.1.0/folderfuse/core/tokenizers/base.py +45 -0
- folderfuse-0.1.0/folderfuse/core/tokenizers/claude.py +58 -0
- folderfuse-0.1.0/folderfuse/core/tokenizers/deepseek.py +49 -0
- folderfuse-0.1.0/folderfuse/core/tokenizers/gemini.py +55 -0
- folderfuse-0.1.0/folderfuse/core/tokenizers/models.py +38 -0
- folderfuse-0.1.0/folderfuse/core/tokenizers/openai.py +58 -0
- folderfuse-0.1.0/folderfuse/core/tokenizers/registry.py +113 -0
- folderfuse-0.1.0/folderfuse/core/transform/__init__.py +35 -0
- folderfuse-0.1.0/folderfuse/core/transform/formatters.py +27 -0
- folderfuse-0.1.0/folderfuse/core/transform/languages.py +112 -0
- folderfuse-0.1.0/folderfuse/core/transform/models.py +52 -0
- folderfuse-0.1.0/folderfuse/core/transform/selector.py +25 -0
- folderfuse-0.1.0/folderfuse/core/transform/skeletonizer.py +243 -0
- folderfuse-0.1.0/folderfuse/py.typed +0 -0
- folderfuse-0.1.0/folderfuse/web/dist/assets/index-B_uycAtT.js +231 -0
- folderfuse-0.1.0/folderfuse/web/dist/assets/index-BfFYCGUS.css +1 -0
- folderfuse-0.1.0/folderfuse/web/dist/index.html +16 -0
- folderfuse-0.1.0/pyproject.toml +192 -0
- folderfuse-0.1.0/tests/__init__.py +0 -0
- folderfuse-0.1.0/tests/benchmarks/__init__.py +0 -0
- folderfuse-0.1.0/tests/benchmarks/test_performance.py +112 -0
- folderfuse-0.1.0/tests/conftest.py +26 -0
- folderfuse-0.1.0/tests/e2e/__init__.py +0 -0
- folderfuse-0.1.0/tests/e2e/test_golden_e2e.py +225 -0
- folderfuse-0.1.0/tests/security/__init__.py +0 -0
- folderfuse-0.1.0/tests/security/test_penetration.py +178 -0
- folderfuse-0.1.0/tests/test_api.py +61 -0
- folderfuse-0.1.0/tests/test_api_bundle.py +75 -0
- folderfuse-0.1.0/tests/test_api_routes.py +76 -0
- folderfuse-0.1.0/tests/test_api_server.py +65 -0
- folderfuse-0.1.0/tests/test_cli.py +101 -0
- folderfuse-0.1.0/tests/test_cli_config.py +106 -0
- folderfuse-0.1.0/tests/test_cli_terminal.py +105 -0
- folderfuse-0.1.0/tests/test_core.py +27 -0
- folderfuse-0.1.0/tests/test_crawler.py +165 -0
- folderfuse-0.1.0/tests/test_cross_platform.py +124 -0
- folderfuse-0.1.0/tests/test_extractors.py +151 -0
- folderfuse-0.1.0/tests/test_golden.py +108 -0
- folderfuse-0.1.0/tests/test_layout.py +162 -0
- folderfuse-0.1.0/tests/test_pipeline.py +141 -0
- folderfuse-0.1.0/tests/test_security.py +92 -0
- folderfuse-0.1.0/tests/test_tokenizers.py +118 -0
- folderfuse-0.1.0/tests/test_transform.py +130 -0
- folderfuse-0.0.1/PKG-INFO +0 -20
- folderfuse-0.0.1/README.md +0 -5
- folderfuse-0.0.1/pyproject.toml +0 -26
- folderfuse-0.0.1/src/folderfuse/__init__.py +0 -3
- folderfuse-0.0.1/src/folderfuse/__main__.py +0 -5
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# Python artifacts
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.so
|
|
6
|
+
.Python
|
|
7
|
+
build/
|
|
8
|
+
develop-eggs/
|
|
9
|
+
dist/
|
|
10
|
+
downloads/
|
|
11
|
+
eggs/
|
|
12
|
+
.eggs/
|
|
13
|
+
# lib/
|
|
14
|
+
lib64/
|
|
15
|
+
parts/
|
|
16
|
+
sdist/
|
|
17
|
+
var/
|
|
18
|
+
wheels/
|
|
19
|
+
share/python-wheels/
|
|
20
|
+
*.egg-info/
|
|
21
|
+
.installed.cfg
|
|
22
|
+
*.egg
|
|
23
|
+
MANIFEST
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
# Virtual Environments
|
|
28
|
+
.env
|
|
29
|
+
.venv/
|
|
30
|
+
env/
|
|
31
|
+
venv/
|
|
32
|
+
ENV/
|
|
33
|
+
env.bak/
|
|
34
|
+
venv.bak/
|
|
35
|
+
|
|
36
|
+
# UV Cache
|
|
37
|
+
.uv/
|
|
38
|
+
uv.lock
|
|
39
|
+
|
|
40
|
+
# Testing & Coverage
|
|
41
|
+
.pytest_cache/
|
|
42
|
+
.coverage
|
|
43
|
+
.coverage.*
|
|
44
|
+
htmlcov/
|
|
45
|
+
coverage.xml
|
|
46
|
+
*.cover
|
|
47
|
+
*.py,cover
|
|
48
|
+
.hypothesis/
|
|
49
|
+
|
|
50
|
+
# Type Checking
|
|
51
|
+
.mypy_cache/
|
|
52
|
+
.dmypy.json
|
|
53
|
+
dmypy.json
|
|
54
|
+
|
|
55
|
+
# Node & Frontend (Source dependencies)
|
|
56
|
+
node_modules/
|
|
57
|
+
frontend/dist/
|
|
58
|
+
frontend/.vite/
|
|
59
|
+
npm-debug.log*
|
|
60
|
+
yarn-debug.log*
|
|
61
|
+
yarn-error.log*
|
|
62
|
+
pnpm-debug.log*
|
|
63
|
+
.pnpm-store/
|
|
64
|
+
|
|
65
|
+
# Compiled static bundle in wheel package (built during release)
|
|
66
|
+
folderfuse/web/dist/*
|
|
67
|
+
!folderfuse/web/dist/.gitkeep
|
|
68
|
+
|
|
69
|
+
# IDEs & System Files
|
|
70
|
+
.idea/
|
|
71
|
+
.vscode/
|
|
72
|
+
*.swp
|
|
73
|
+
*.swo
|
|
74
|
+
*~
|
|
75
|
+
.DS_Store
|
|
76
|
+
Thumbs.db
|
|
77
|
+
|
|
78
|
+
# Secrets & Local Configurations
|
|
79
|
+
*.env
|
|
80
|
+
*.env.*
|
|
81
|
+
!.env.example
|
|
82
|
+
*.pem
|
|
83
|
+
*.key
|
|
84
|
+
folderfuse.yaml
|
|
85
|
+
.folderfuse.json
|
|
86
|
+
|
|
87
|
+
.secrets
|
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: folderfuse
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: High-performance codebase context packager, AST skeletonizer, and multi-model token optimizer for LLMs.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Samm-G/folder-fuse
|
|
6
|
+
Project-URL: Repository, https://github.com/Samm-G/folder-fuse
|
|
7
|
+
Project-URL: Issues, https://github.com/Samm-G/folder-fuse/issues
|
|
8
|
+
Author: Latent Oxygen Studios
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Software Development :: Code Generators
|
|
21
|
+
Classifier: Topic :: Utilities
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Requires-Dist: fastapi>=0.110.0
|
|
25
|
+
Requires-Dist: nbformat>=5.9.0
|
|
26
|
+
Requires-Dist: openpyxl>=3.1.2
|
|
27
|
+
Requires-Dist: pathspec>=0.12.1
|
|
28
|
+
Requires-Dist: pypdf>=4.0.0
|
|
29
|
+
Requires-Dist: pyperclip>=1.8.2
|
|
30
|
+
Requires-Dist: python-docx>=1.1.0
|
|
31
|
+
Requires-Dist: pyyaml>=6.0.1
|
|
32
|
+
Requires-Dist: rich>=13.7.0
|
|
33
|
+
Requires-Dist: tiktoken>=0.7.0
|
|
34
|
+
Requires-Dist: tokenizers>=0.19.0
|
|
35
|
+
Requires-Dist: tree-sitter-c-sharp>=0.21.0
|
|
36
|
+
Requires-Dist: tree-sitter-c>=0.21.0
|
|
37
|
+
Requires-Dist: tree-sitter-cpp>=0.21.0
|
|
38
|
+
Requires-Dist: tree-sitter-go>=0.21.0
|
|
39
|
+
Requires-Dist: tree-sitter-java>=0.21.0
|
|
40
|
+
Requires-Dist: tree-sitter-javascript>=0.21.0
|
|
41
|
+
Requires-Dist: tree-sitter-python>=0.21.0
|
|
42
|
+
Requires-Dist: tree-sitter-rust>=0.21.0
|
|
43
|
+
Requires-Dist: tree-sitter-typescript>=0.21.0
|
|
44
|
+
Requires-Dist: tree-sitter>=0.22.0
|
|
45
|
+
Requires-Dist: typer>=0.12.0
|
|
46
|
+
Requires-Dist: uvicorn[standard]>=0.28.0
|
|
47
|
+
Provides-Extra: dev
|
|
48
|
+
Requires-Dist: httpx>=0.27.0; extra == 'dev'
|
|
49
|
+
Requires-Dist: mypy>=1.9.0; extra == 'dev'
|
|
50
|
+
Requires-Dist: pre-commit>=3.7.0; extra == 'dev'
|
|
51
|
+
Requires-Dist: pytest-asyncio>=0.23.0; extra == 'dev'
|
|
52
|
+
Requires-Dist: pytest-cov>=5.0.0; extra == 'dev'
|
|
53
|
+
Requires-Dist: pytest>=8.0.0; extra == 'dev'
|
|
54
|
+
Requires-Dist: ruff>=0.4.0; extra == 'dev'
|
|
55
|
+
Requires-Dist: types-openpyxl>=3.1.5.20240307; extra == 'dev'
|
|
56
|
+
Requires-Dist: types-pyyaml>=6.0.12.20240311; extra == 'dev'
|
|
57
|
+
Description-Content-Type: text/markdown
|
|
58
|
+
|
|
59
|
+
# FolderFuse
|
|
60
|
+
|
|
61
|
+
[](https://pypi.org/project/folderfuse/)
|
|
62
|
+
[](https://www.python.org/downloads/)
|
|
63
|
+
[](https://github.com/Samm-G/folder-fuse/actions)
|
|
64
|
+
[](https://opensource.org/licenses/MIT)
|
|
65
|
+
[](#security--privacy)
|
|
66
|
+
|
|
67
|
+
High-performance codebase context packager, AST skeletonizer, and multi-model token optimizer engineered for Large Language Models (LLMs).
|
|
68
|
+
|
|
69
|
+
---
|
|
70
|
+
|
|
71
|
+
## Overview
|
|
72
|
+
|
|
73
|
+
**FolderFuse** packages local codebases, documentation repositories, and mixed-format folders into model-ready, token-optimized context bundles. It operates under a strict **dual-entry architecture**: a headless Unix-pipe CLI for terminal power users and automated CI/CD pipelines, alongside an embedded local Web UI for interactive preparation.
|
|
74
|
+
|
|
75
|
+
FolderFuse is strictly an **exporter**—it never dispatches requests directly to external LLM APIs, ensuring workflows remain 100% private, zero-cost, and completely air-gapped.
|
|
76
|
+
|
|
77
|
+
```
|
|
78
|
+
┌──────────────────────────────────────────────────────────────────┐
|
|
79
|
+
│ FolderFuse Core │
|
|
80
|
+
│ │
|
|
81
|
+
│ ┌───────────────────────┐ ┌──────────────────────┐ │
|
|
82
|
+
│ │ Headless CLI │ │ FastAPI Engine │ │
|
|
83
|
+
│ │ (Typer / Rich) │ │ (Local 127.0.0.1) │ │
|
|
84
|
+
│ └──────────┬────────────┘ └──────────┬───────────┘ │
|
|
85
|
+
│ │ │ │
|
|
86
|
+
│ │ ┌───────────┴───────────┐ │
|
|
87
|
+
│ │ │ Embedded React UI │ │
|
|
88
|
+
│ │ │ (Vite + Tailwind) │ │
|
|
89
|
+
│ │ └───────────┬───────────┘ │
|
|
90
|
+
│ └───────────────────┬─────────────────┘ │
|
|
91
|
+
│ ▼ │
|
|
92
|
+
│ ┌─────────────────────────────────────────────────────────────┐ │
|
|
93
|
+
│ │ Processing Pipeline │ │
|
|
94
|
+
│ │ │ │
|
|
95
|
+
│ │ 1. Filesystem Crawler (.gitignore, root jail containment) │ │
|
|
96
|
+
│ │ 2. Document Extractors (.pdf, .docx, .xlsx, .ipynb) │ │
|
|
97
|
+
│ │ 3. AST Skeletonizer (Tree-Sitter multi-language parser) │ │
|
|
98
|
+
│ │ 4. Security Sanitizer (Zero-leak session deduplication) │ │
|
|
99
|
+
│ │ 5. Multi-Model Tokenizers (Claude, Gemini 3, DeepSeek V4) │ │
|
|
100
|
+
│ │ 6. KV-Cache Optimizer (5-step layout ordering engine) │ │
|
|
101
|
+
│ └──────────────────────────────┬──────────────────────────────┘ │
|
|
102
|
+
│ ▼ │
|
|
103
|
+
│ Export: Stdout / File / Clipboard │
|
|
104
|
+
└──────────────────────────────────────────────────────────────────┘
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
---
|
|
108
|
+
|
|
109
|
+
## Core Capabilities
|
|
110
|
+
|
|
111
|
+
- **Dual-Entry Architecture:**
|
|
112
|
+
- **Headless POSIX CLI:** Optimized for Unix piping (`folderfuse pack . | pbcopy`). Payload directed strictly to `stdout`; progress spinners, logs, and token meters routed to `stderr`.
|
|
113
|
+
- **Embedded Web Dashboard:** Single-command boot (`folderfuse ui .`) launching an air-gapped React/Vite dashboard served directly from Python wheel assets on `127.0.0.1`.
|
|
114
|
+
- **Multi-Language AST Skeletonization (Tree-Sitter):**
|
|
115
|
+
Prunes function/method implementation bodies while preserving class definitions, method signatures, return types, docstrings, and top-level constants/type aliases. Supports Python, TypeScript, JavaScript, Go, Rust, Java, C, C++, and C#.
|
|
116
|
+
- **Prompt-Cache Optimization (Anthropic & Gemini):**
|
|
117
|
+
Structures prompt layout deterministically into a 5-step hierarchy to maximize KV-cache hit rates:
|
|
118
|
+
1. System instructions prefix
|
|
119
|
+
2. ASCII repository directory tree
|
|
120
|
+
3. Reference documentation (`*.md`, `docs/**`)
|
|
121
|
+
4. Stable source code
|
|
122
|
+
5. Ephemeral / working target files placed at the prompt tail (Git-aware + glob overrides)
|
|
123
|
+
- **Multi-Model Tokenizer Engine:**
|
|
124
|
+
100% offline exact token calculations for **Anthropic Claude** (Claude 3.5 Sonnet, Haiku), **Google Gemini** (Gemini 3.8 Flash, 3.7 Flash, 3.5 Flash), **DeepSeek** (DeepSeek-V4.1-Flash, DeepSeek-V4-Pro-0813), and **OpenAI** (GPT-4o, o200k/cl100k BPE).
|
|
125
|
+
- **Zero-Leak Secret Sanitization:**
|
|
126
|
+
Detects credentials (OpenAI, Anthropic, AWS, GitHub PATs, JWTs, private keys) and replaces them with deterministic synthetic tokens (`[REDACTED:OPENAI_KEY:1]`), preserving cross-file references without leaking raw secrets.
|
|
127
|
+
- **Rich Document Parsing & Statistical Data Profiling:**
|
|
128
|
+
- **Spreadsheets (`.xlsx`, `.csv`):** Markdown tables with configurable row/column caps and an automated statistical data profile summary (null percentages, unique counts, numeric stats).
|
|
129
|
+
- **Jupyter Notebooks (`.ipynb`):** Strips heavy base64 image plots while preserving code cells and execution output streams.
|
|
130
|
+
- **Word Docs (`.docx`) & PDFs (`.pdf`):** Extracts headings, bullet lists, and page-by-page text.
|
|
131
|
+
|
|
132
|
+
---
|
|
133
|
+
|
|
134
|
+
## Installation
|
|
135
|
+
|
|
136
|
+
Install `folderfuse` from PyPI:
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
pip install folderfuse
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Or using `uv`:
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
uv add folderfuse
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
Or run directly without permanent installation via `pipx`:
|
|
149
|
+
|
|
150
|
+
```bash
|
|
151
|
+
pipx run folderfuse ui
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
---
|
|
155
|
+
|
|
156
|
+
## Quickstart
|
|
157
|
+
|
|
158
|
+
### 1. Headless CLI (`folderfuse pack`)
|
|
159
|
+
|
|
160
|
+
Package the current repository into an Anthropic Claude XML prompt and copy it directly to your system clipboard:
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
folderfuse pack . --model claude --clipboard
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
Stream directly to standard output for Unix pipelines:
|
|
167
|
+
|
|
168
|
+
```bash
|
|
169
|
+
# Pipe directly into macOS clipboard
|
|
170
|
+
folderfuse pack ./my-project -q | pbcopy
|
|
171
|
+
|
|
172
|
+
# Save formatted markdown context bundle to disk
|
|
173
|
+
folderfuse pack ./my-project --format markdown --output bundle.md
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
Apply AST skeletonization to secondary libraries:
|
|
177
|
+
|
|
178
|
+
```bash
|
|
179
|
+
folderfuse pack . --skeleton "src/utils/**,src/legacy/**" --strict
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
### 2. Interactive Local Web Dashboard (`folderfuse ui`)
|
|
183
|
+
|
|
184
|
+
Launch the local web server and open the dashboard in your default browser:
|
|
185
|
+
|
|
186
|
+
```bash
|
|
187
|
+
folderfuse ui /path/to/project
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
Options for UI server:
|
|
191
|
+
|
|
192
|
+
```bash
|
|
193
|
+
folderfuse ui . --port 8080 --no-browser
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
---
|
|
197
|
+
|
|
198
|
+
## CLI Flag Reference (`folderfuse pack`)
|
|
199
|
+
|
|
200
|
+
| Flag | Short | Default | Description |
|
|
201
|
+
| :----------------------- | :---- | :------------------ | :--------------------------------------------------------------------- |
|
|
202
|
+
| `PATH` | | `.` | Target directory path to inspect and package. |
|
|
203
|
+
| `--model` | `-m` | `claude-3-5-sonnet` | Tokenizer profile (`claude`, `gemini`, `openai`, `deepseek`). |
|
|
204
|
+
| `--format` | `-f` | `xml` | Output layout (`xml`, `markdown`, `json`, `raw`). |
|
|
205
|
+
| `--output` | `-o` | `None` | Destination file path to write bundle to. |
|
|
206
|
+
| `--clipboard` | `-c` | `False` | Copy generated context bundle to system clipboard. |
|
|
207
|
+
| `--skeleton` | | `None` | Glob patterns for AST skeletonization (repeatable or comma-separated). |
|
|
208
|
+
| `--exclude` | | `None` | Additional glob patterns to exclude from ingestion. |
|
|
209
|
+
| `--working` | | `None` | Globs for volatile files to place at prompt tail. |
|
|
210
|
+
| `--no-gitignore` | | `False` | Disable automatic `.gitignore` rule evaluation. |
|
|
211
|
+
| `--follow-symlinks` | | `False` | Permit traversing symlinks pointing outside root jail. |
|
|
212
|
+
| `--scrub-secrets` | | `True` | Toggle automated secret and credential masking. |
|
|
213
|
+
| `--omit-sensitive-files` | | `False` | Omit sensitive files (`.env`, `id_rsa`) instead of scrubbing. |
|
|
214
|
+
| `--max-size-kb` | | `500` | Skip text files exceeding threshold in KB. |
|
|
215
|
+
| `--instructions` | | `None` | Custom system framing instructions for prompt header. |
|
|
216
|
+
| `--strict` | | `False` | Exit with code `3` if total tokens exceed model context limit. |
|
|
217
|
+
| `--json-meta` | | `False` | Emit structured JSON diagnostic metadata to stderr. |
|
|
218
|
+
| `--quiet` | `-q` | `False` | Suppress all diagnostic banners, tables, and spinners. |
|
|
219
|
+
|
|
220
|
+
### POSIX Exit Codes
|
|
221
|
+
- `0`: Success (Context bundle generated cleanly).
|
|
222
|
+
- `1`: General runtime or filesystem write error.
|
|
223
|
+
- `2`: Target directory not found or invalid path.
|
|
224
|
+
- `3`: Context window budget overflow (when `--strict` is set).
|
|
225
|
+
|
|
226
|
+
---
|
|
227
|
+
|
|
228
|
+
## Configuration File (`folderfuse.yaml`)
|
|
229
|
+
|
|
230
|
+
FolderFuse supports repository-level configuration files (`folderfuse.yaml` or `.folderfuse.json`) located at the target directory root:
|
|
231
|
+
|
|
232
|
+
```yaml
|
|
233
|
+
version: "1"
|
|
234
|
+
settings:
|
|
235
|
+
default_model: "claude-3-5-sonnet"
|
|
236
|
+
default_format: "xml"
|
|
237
|
+
max_file_size_kb: 500
|
|
238
|
+
scrub_secrets: true
|
|
239
|
+
strict: false
|
|
240
|
+
include_instructions: true
|
|
241
|
+
instruction_text: "You are an expert systems engineer evaluating this repository."
|
|
242
|
+
|
|
243
|
+
exclusions:
|
|
244
|
+
- "dist/**"
|
|
245
|
+
- "build/**"
|
|
246
|
+
- "**/*.spec.ts"
|
|
247
|
+
|
|
248
|
+
skeletonize:
|
|
249
|
+
- "legacy/**/*.py"
|
|
250
|
+
- "src/vendor/**/*.ts"
|
|
251
|
+
|
|
252
|
+
working_files:
|
|
253
|
+
- "src/active_feature.py"
|
|
254
|
+
|
|
255
|
+
custom_redactions:
|
|
256
|
+
- name: "INTERNAL_CODENAME"
|
|
257
|
+
regex: "(?i)project-zeus"
|
|
258
|
+
replace: "[PROJECT_ZEUS]"
|
|
259
|
+
```
|
|
260
|
+
|
|
261
|
+
*Precedence:* Built-in Defaults < User Global Config (`~/.config/folderfuse/config.yaml`) < Target Repo Config (`folderfuse.yaml`) < CLI Flags.
|
|
262
|
+
|
|
263
|
+
---
|
|
264
|
+
|
|
265
|
+
## Security & Privacy
|
|
266
|
+
|
|
267
|
+
- **Air-Gapped Operation:** FolderFuse makes zero network requests, contains zero analytics SDKs, and bundles all frontend and grammar assets locally inside the Python wheel.
|
|
268
|
+
- **Localhost-Only Binding:** The embedded FastAPI server binds strictly to loopback interfaces (`127.0.0.1`).
|
|
269
|
+
- **Root Jail Containment:** Symbolic links resolving outside the designated root directory are blocked by default to prevent path traversal attacks.
|
|
270
|
+
|
|
271
|
+
---
|
|
272
|
+
|
|
273
|
+
## License
|
|
274
|
+
|
|
275
|
+
Distributed under the terms of the [MIT License](LICENSE).
|
|
276
|
+
Copyright © 2026 Latent Oxygen Studios.
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
# FolderFuse
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/folderfuse/)
|
|
4
|
+
[](https://www.python.org/downloads/)
|
|
5
|
+
[](https://github.com/Samm-G/folder-fuse/actions)
|
|
6
|
+
[](https://opensource.org/licenses/MIT)
|
|
7
|
+
[](#security--privacy)
|
|
8
|
+
|
|
9
|
+
High-performance codebase context packager, AST skeletonizer, and multi-model token optimizer engineered for Large Language Models (LLMs).
|
|
10
|
+
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
## Overview
|
|
14
|
+
|
|
15
|
+
**FolderFuse** packages local codebases, documentation repositories, and mixed-format folders into model-ready, token-optimized context bundles. It operates under a strict **dual-entry architecture**: a headless Unix-pipe CLI for terminal power users and automated CI/CD pipelines, alongside an embedded local Web UI for interactive preparation.
|
|
16
|
+
|
|
17
|
+
FolderFuse is strictly an **exporter**—it never dispatches requests directly to external LLM APIs, ensuring workflows remain 100% private, zero-cost, and completely air-gapped.
|
|
18
|
+
|
|
19
|
+
```
|
|
20
|
+
┌──────────────────────────────────────────────────────────────────┐
|
|
21
|
+
│ FolderFuse Core │
|
|
22
|
+
│ │
|
|
23
|
+
│ ┌───────────────────────┐ ┌──────────────────────┐ │
|
|
24
|
+
│ │ Headless CLI │ │ FastAPI Engine │ │
|
|
25
|
+
│ │ (Typer / Rich) │ │ (Local 127.0.0.1) │ │
|
|
26
|
+
│ └──────────┬────────────┘ └──────────┬───────────┘ │
|
|
27
|
+
│ │ │ │
|
|
28
|
+
│ │ ┌───────────┴───────────┐ │
|
|
29
|
+
│ │ │ Embedded React UI │ │
|
|
30
|
+
│ │ │ (Vite + Tailwind) │ │
|
|
31
|
+
│ │ └───────────┬───────────┘ │
|
|
32
|
+
│ └───────────────────┬─────────────────┘ │
|
|
33
|
+
│ ▼ │
|
|
34
|
+
│ ┌─────────────────────────────────────────────────────────────┐ │
|
|
35
|
+
│ │ Processing Pipeline │ │
|
|
36
|
+
│ │ │ │
|
|
37
|
+
│ │ 1. Filesystem Crawler (.gitignore, root jail containment) │ │
|
|
38
|
+
│ │ 2. Document Extractors (.pdf, .docx, .xlsx, .ipynb) │ │
|
|
39
|
+
│ │ 3. AST Skeletonizer (Tree-Sitter multi-language parser) │ │
|
|
40
|
+
│ │ 4. Security Sanitizer (Zero-leak session deduplication) │ │
|
|
41
|
+
│ │ 5. Multi-Model Tokenizers (Claude, Gemini 3, DeepSeek V4) │ │
|
|
42
|
+
│ │ 6. KV-Cache Optimizer (5-step layout ordering engine) │ │
|
|
43
|
+
│ └──────────────────────────────┬──────────────────────────────┘ │
|
|
44
|
+
│ ▼ │
|
|
45
|
+
│ Export: Stdout / File / Clipboard │
|
|
46
|
+
└──────────────────────────────────────────────────────────────────┘
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
---
|
|
50
|
+
|
|
51
|
+
## Core Capabilities
|
|
52
|
+
|
|
53
|
+
- **Dual-Entry Architecture:**
|
|
54
|
+
- **Headless POSIX CLI:** Optimized for Unix piping (`folderfuse pack . | pbcopy`). Payload directed strictly to `stdout`; progress spinners, logs, and token meters routed to `stderr`.
|
|
55
|
+
- **Embedded Web Dashboard:** Single-command boot (`folderfuse ui .`) launching an air-gapped React/Vite dashboard served directly from Python wheel assets on `127.0.0.1`.
|
|
56
|
+
- **Multi-Language AST Skeletonization (Tree-Sitter):**
|
|
57
|
+
Prunes function/method implementation bodies while preserving class definitions, method signatures, return types, docstrings, and top-level constants/type aliases. Supports Python, TypeScript, JavaScript, Go, Rust, Java, C, C++, and C#.
|
|
58
|
+
- **Prompt-Cache Optimization (Anthropic & Gemini):**
|
|
59
|
+
Structures prompt layout deterministically into a 5-step hierarchy to maximize KV-cache hit rates:
|
|
60
|
+
1. System instructions prefix
|
|
61
|
+
2. ASCII repository directory tree
|
|
62
|
+
3. Reference documentation (`*.md`, `docs/**`)
|
|
63
|
+
4. Stable source code
|
|
64
|
+
5. Ephemeral / working target files placed at the prompt tail (Git-aware + glob overrides)
|
|
65
|
+
- **Multi-Model Tokenizer Engine:**
|
|
66
|
+
100% offline exact token calculations for **Anthropic Claude** (Claude 3.5 Sonnet, Haiku), **Google Gemini** (Gemini 3.8 Flash, 3.7 Flash, 3.5 Flash), **DeepSeek** (DeepSeek-V4.1-Flash, DeepSeek-V4-Pro-0813), and **OpenAI** (GPT-4o, o200k/cl100k BPE).
|
|
67
|
+
- **Zero-Leak Secret Sanitization:**
|
|
68
|
+
Detects credentials (OpenAI, Anthropic, AWS, GitHub PATs, JWTs, private keys) and replaces them with deterministic synthetic tokens (`[REDACTED:OPENAI_KEY:1]`), preserving cross-file references without leaking raw secrets.
|
|
69
|
+
- **Rich Document Parsing & Statistical Data Profiling:**
|
|
70
|
+
- **Spreadsheets (`.xlsx`, `.csv`):** Markdown tables with configurable row/column caps and an automated statistical data profile summary (null percentages, unique counts, numeric stats).
|
|
71
|
+
- **Jupyter Notebooks (`.ipynb`):** Strips heavy base64 image plots while preserving code cells and execution output streams.
|
|
72
|
+
- **Word Docs (`.docx`) & PDFs (`.pdf`):** Extracts headings, bullet lists, and page-by-page text.
|
|
73
|
+
|
|
74
|
+
---
|
|
75
|
+
|
|
76
|
+
## Installation
|
|
77
|
+
|
|
78
|
+
Install `folderfuse` from PyPI:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
pip install folderfuse
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Or using `uv`:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
uv add folderfuse
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Or run directly without permanent installation via `pipx`:
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
pipx run folderfuse ui
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
---
|
|
97
|
+
|
|
98
|
+
## Quickstart
|
|
99
|
+
|
|
100
|
+
### 1. Headless CLI (`folderfuse pack`)
|
|
101
|
+
|
|
102
|
+
Package the current repository into an Anthropic Claude XML prompt and copy it directly to your system clipboard:
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
folderfuse pack . --model claude --clipboard
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Stream directly to standard output for Unix pipelines:
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
# Pipe directly into macOS clipboard
|
|
112
|
+
folderfuse pack ./my-project -q | pbcopy
|
|
113
|
+
|
|
114
|
+
# Save formatted markdown context bundle to disk
|
|
115
|
+
folderfuse pack ./my-project --format markdown --output bundle.md
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Apply AST skeletonization to secondary libraries:
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
folderfuse pack . --skeleton "src/utils/**,src/legacy/**" --strict
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
### 2. Interactive Local Web Dashboard (`folderfuse ui`)
|
|
125
|
+
|
|
126
|
+
Launch the local web server and open the dashboard in your default browser:
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
folderfuse ui /path/to/project
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Options for UI server:
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
folderfuse ui . --port 8080 --no-browser
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
---
|
|
139
|
+
|
|
140
|
+
## CLI Flag Reference (`folderfuse pack`)
|
|
141
|
+
|
|
142
|
+
| Flag | Short | Default | Description |
|
|
143
|
+
| :----------------------- | :---- | :------------------ | :--------------------------------------------------------------------- |
|
|
144
|
+
| `PATH` | | `.` | Target directory path to inspect and package. |
|
|
145
|
+
| `--model` | `-m` | `claude-3-5-sonnet` | Tokenizer profile (`claude`, `gemini`, `openai`, `deepseek`). |
|
|
146
|
+
| `--format` | `-f` | `xml` | Output layout (`xml`, `markdown`, `json`, `raw`). |
|
|
147
|
+
| `--output` | `-o` | `None` | Destination file path to write bundle to. |
|
|
148
|
+
| `--clipboard` | `-c` | `False` | Copy generated context bundle to system clipboard. |
|
|
149
|
+
| `--skeleton` | | `None` | Glob patterns for AST skeletonization (repeatable or comma-separated). |
|
|
150
|
+
| `--exclude` | | `None` | Additional glob patterns to exclude from ingestion. |
|
|
151
|
+
| `--working` | | `None` | Globs for volatile files to place at prompt tail. |
|
|
152
|
+
| `--no-gitignore` | | `False` | Disable automatic `.gitignore` rule evaluation. |
|
|
153
|
+
| `--follow-symlinks` | | `False` | Permit traversing symlinks pointing outside root jail. |
|
|
154
|
+
| `--scrub-secrets` | | `True` | Toggle automated secret and credential masking. |
|
|
155
|
+
| `--omit-sensitive-files` | | `False` | Omit sensitive files (`.env`, `id_rsa`) instead of scrubbing. |
|
|
156
|
+
| `--max-size-kb` | | `500` | Skip text files exceeding threshold in KB. |
|
|
157
|
+
| `--instructions` | | `None` | Custom system framing instructions for prompt header. |
|
|
158
|
+
| `--strict` | | `False` | Exit with code `3` if total tokens exceed model context limit. |
|
|
159
|
+
| `--json-meta` | | `False` | Emit structured JSON diagnostic metadata to stderr. |
|
|
160
|
+
| `--quiet` | `-q` | `False` | Suppress all diagnostic banners, tables, and spinners. |
|
|
161
|
+
|
|
162
|
+
### POSIX Exit Codes
|
|
163
|
+
- `0`: Success (Context bundle generated cleanly).
|
|
164
|
+
- `1`: General runtime or filesystem write error.
|
|
165
|
+
- `2`: Target directory not found or invalid path.
|
|
166
|
+
- `3`: Context window budget overflow (when `--strict` is set).
|
|
167
|
+
|
|
168
|
+
---
|
|
169
|
+
|
|
170
|
+
## Configuration File (`folderfuse.yaml`)
|
|
171
|
+
|
|
172
|
+
FolderFuse supports repository-level configuration files (`folderfuse.yaml` or `.folderfuse.json`) located at the target directory root:
|
|
173
|
+
|
|
174
|
+
```yaml
|
|
175
|
+
version: "1"
|
|
176
|
+
settings:
|
|
177
|
+
default_model: "claude-3-5-sonnet"
|
|
178
|
+
default_format: "xml"
|
|
179
|
+
max_file_size_kb: 500
|
|
180
|
+
scrub_secrets: true
|
|
181
|
+
strict: false
|
|
182
|
+
include_instructions: true
|
|
183
|
+
instruction_text: "You are an expert systems engineer evaluating this repository."
|
|
184
|
+
|
|
185
|
+
exclusions:
|
|
186
|
+
- "dist/**"
|
|
187
|
+
- "build/**"
|
|
188
|
+
- "**/*.spec.ts"
|
|
189
|
+
|
|
190
|
+
skeletonize:
|
|
191
|
+
- "legacy/**/*.py"
|
|
192
|
+
- "src/vendor/**/*.ts"
|
|
193
|
+
|
|
194
|
+
working_files:
|
|
195
|
+
- "src/active_feature.py"
|
|
196
|
+
|
|
197
|
+
custom_redactions:
|
|
198
|
+
- name: "INTERNAL_CODENAME"
|
|
199
|
+
regex: "(?i)project-zeus"
|
|
200
|
+
replace: "[PROJECT_ZEUS]"
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
*Precedence:* Built-in Defaults < User Global Config (`~/.config/folderfuse/config.yaml`) < Target Repo Config (`folderfuse.yaml`) < CLI Flags.
|
|
204
|
+
|
|
205
|
+
---
|
|
206
|
+
|
|
207
|
+
## Security & Privacy
|
|
208
|
+
|
|
209
|
+
- **Air-Gapped Operation:** FolderFuse makes zero network requests, contains zero analytics SDKs, and bundles all frontend and grammar assets locally inside the Python wheel.
|
|
210
|
+
- **Localhost-Only Binding:** The embedded FastAPI server binds strictly to loopback interfaces (`127.0.0.1`).
|
|
211
|
+
- **Root Jail Containment:** Symbolic links resolving outside the designated root directory are blocked by default to prevent path traversal attacks.
|
|
212
|
+
|
|
213
|
+
---
|
|
214
|
+
|
|
215
|
+
## License
|
|
216
|
+
|
|
217
|
+
Distributed under the terms of the [MIT License](LICENSE).
|
|
218
|
+
Copyright © 2026 Latent Oxygen Studios.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Local HTTP API and static asset server."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from folderfuse.api.launcher import launch_browser
|
|
6
|
+
from folderfuse.api.port_hunter import find_available_port, is_port_available
|
|
7
|
+
from folderfuse.api.server import create_app, run_server
|
|
8
|
+
|
|
9
|
+
__all__: list[str] = [
|
|
10
|
+
"create_app",
|
|
11
|
+
"find_available_port",
|
|
12
|
+
"is_port_available",
|
|
13
|
+
"launch_browser",
|
|
14
|
+
"run_server",
|
|
15
|
+
]
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Automated browser launching with headless fault tolerance."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import threading
|
|
6
|
+
import webbrowser
|
|
7
|
+
from contextlib import suppress
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def launch_browser(url: str, delay_seconds: float = 0.5) -> None:
|
|
11
|
+
"""Launch user default browser after a safety delay allowing server boot."""
|
|
12
|
+
|
|
13
|
+
def _open() -> None:
|
|
14
|
+
with suppress(Exception):
|
|
15
|
+
webbrowser.open(url)
|
|
16
|
+
|
|
17
|
+
timer = threading.Timer(delay_seconds, _open)
|
|
18
|
+
timer.daemon = True
|
|
19
|
+
timer.start()
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Socket-based port availability probing and dynamic port discovery."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import socket
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def is_port_available(host: str, port: int) -> bool:
|
|
9
|
+
"""Check if a specific host:port endpoint can be bound and listened on."""
|
|
10
|
+
try:
|
|
11
|
+
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
|
12
|
+
sock.bind((host, port))
|
|
13
|
+
sock.listen(1)
|
|
14
|
+
return True
|
|
15
|
+
except OSError:
|
|
16
|
+
return False
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def find_available_port(
|
|
20
|
+
host: str = "127.0.0.1",
|
|
21
|
+
start_port: int = 8080,
|
|
22
|
+
max_attempts: int = 20,
|
|
23
|
+
) -> int:
|
|
24
|
+
"""Probe sequentially from start_port across max_attempts for an open socket."""
|
|
25
|
+
for offset in range(max_attempts):
|
|
26
|
+
candidate = start_port + offset
|
|
27
|
+
if is_port_available(host, candidate):
|
|
28
|
+
return candidate
|
|
29
|
+
|
|
30
|
+
end_port = start_port + max_attempts - 1
|
|
31
|
+
raise RuntimeError(f"No available ports found on {host} in range {start_port}..{end_port}.")
|