tw-framework 0.2.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tw_framework/__init__.py +13 -0
- tw_framework/advanced_diagnostics.py +25 -0
- tw_framework/asset_optimizer.py +30 -0
- tw_framework/ast_nodes.py +283 -0
- tw_framework/build.py +7 -0
- tw_framework/build_performance.py +26 -0
- tw_framework/build_report.py +84 -0
- tw_framework/cli.py +1258 -0
- tw_framework/code_splitting.py +76 -0
- tw_framework/common.py +22 -0
- tw_framework/compiler.py +4817 -0
- tw_framework/compiler_stats.py +51 -0
- tw_framework/dead_code.py +44 -0
- tw_framework/dependency_graph.py +91 -0
- tw_framework/deploy.py +7 -0
- tw_framework/dev.py +9 -0
- tw_framework/diagnostics.py +98 -0
- tw_framework/dynamic_imports.py +51 -0
- tw_framework/error_formatter.py +53 -0
- tw_framework/framework.py +3216 -0
- tw_framework/hydration.py +129 -0
- tw_framework/incremental_cache.py +53 -0
- tw_framework/interpreter.py +42 -0
- tw_framework/ir.py +171 -0
- tw_framework/lexer.py +52 -0
- tw_framework/lowering.py +68 -0
- tw_framework/parser.py +167 -0
- tw_framework/partial_rebuild.py +24 -0
- tw_framework/performance_analyzer.py +37 -0
- tw_framework/plugin_runtime.py +196 -0
- tw_framework/production_optimizer.py +82 -0
- tw_framework/reactivity.py +349 -0
- tw_framework/render_css.py +22 -0
- tw_framework/render_html.py +259 -0
- tw_framework/route_optimizer.py +40 -0
- tw_framework/router.py +7 -0
- tw_framework/runtime_model.py +23 -0
- tw_framework/runtime_values.py +31 -0
- tw_framework/semantic.py +162 -0
- tw_framework/server.py +532 -0
- tw_framework/signature.py +51 -0
- tw_framework/static_dynamic_auto.py +32 -0
- tw_framework/streaming.py +135 -0
- tw_framework/test_build_cli_pytest.py +1078 -0
- tw_framework/tree_shaking.py +48 -0
- tw_framework/twm_parser.py +274 -0
- tw_framework-0.2.3.dist-info/METADATA +8 -0
- tw_framework-0.2.3.dist-info/RECORD +51 -0
- tw_framework-0.2.3.dist-info/WHEEL +5 -0
- tw_framework-0.2.3.dist-info/entry_points.txt +2 -0
- tw_framework-0.2.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,3216 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import base64
|
|
3
|
+
import contextlib
|
|
4
|
+
import fnmatch
|
|
5
|
+
import gzip
|
|
6
|
+
import hashlib
|
|
7
|
+
import hmac
|
|
8
|
+
import html
|
|
9
|
+
import http.server
|
|
10
|
+
import json
|
|
11
|
+
import logging
|
|
12
|
+
import mimetypes
|
|
13
|
+
import os
|
|
14
|
+
import posixpath
|
|
15
|
+
import re
|
|
16
|
+
import secrets
|
|
17
|
+
import shutil
|
|
18
|
+
import socketserver
|
|
19
|
+
import subprocess
|
|
20
|
+
import sys
|
|
21
|
+
import threading
|
|
22
|
+
import time
|
|
23
|
+
import ctypes
|
|
24
|
+
import select
|
|
25
|
+
import struct
|
|
26
|
+
import urllib.parse
|
|
27
|
+
from email.utils import formatdate
|
|
28
|
+
from dataclasses import dataclass
|
|
29
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
30
|
+
|
|
31
|
+
from . import compiler
|
|
32
|
+
from .common import content_hash, log
|
|
33
|
+
from .plugin_runtime import ExtensionManager
|
|
34
|
+
from .twm_parser import compile_twm_module_to_cjs, parse_twm_functions
|
|
35
|
+
|
|
36
|
+
# Deployment adapter imports
|
|
37
|
+
from .adapters.vercel import build_vercel_config, generate_vercel_output
|
|
38
|
+
from .adapters.netlify import generate_netlify_output
|
|
39
|
+
from .adapters.cloudflare import generate_cloudflare_output
|
|
40
|
+
from .adapters.vercel_functions import generate_vercel_api_functions
|
|
41
|
+
from .dependency_graph import DependencyGraph
|
|
42
|
+
from .incremental_cache import IncrementalCache
|
|
43
|
+
from .build_report import BuildReport
|
|
44
|
+
from .compiler_stats import CompilerStats
|
|
45
|
+
from .performance_analyzer import PerformanceAnalyzer
|
|
46
|
+
from .advanced_diagnostics import run_advanced_diagnostics
|
|
47
|
+
from .production_optimizer import optimize_for_production
|
|
48
|
+
from .tree_shaking import shake_project
|
|
49
|
+
from .dead_code import detect_dead_code
|
|
50
|
+
from .route_optimizer import optimize_routes
|
|
51
|
+
from .code_splitting import generate_chunks
|
|
52
|
+
from .static_dynamic_auto import determine_render_mode
|
|
53
|
+
from .hydration import wrap_interactive_nodes
|
|
54
|
+
from .streaming import render_program_streaming
|
|
55
|
+
|
|
56
|
+
logger = logging.getLogger(__name__)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
for ext, content_type in {
|
|
60
|
+
".avif": "image/avif",
|
|
61
|
+
".mjs": "application/javascript",
|
|
62
|
+
".wasm": "application/wasm",
|
|
63
|
+
".webmanifest": "application/manifest+json",
|
|
64
|
+
".woff": "font/woff",
|
|
65
|
+
".woff2": "font/woff2",
|
|
66
|
+
".yaml": "application/yaml",
|
|
67
|
+
".yml": "application/yaml",
|
|
68
|
+
".toml": "application/toml",
|
|
69
|
+
}.items():
|
|
70
|
+
mimetypes.add_type(content_type, ext)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
|
|
74
|
+
DEFAULT_PROJECT_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, compiler.PROJECT_ROOT))
|
|
75
|
+
DEFAULT_INTERNAL_OUTPUT = os.path.join(DEFAULT_PROJECT_ROOT, "dist")
|
|
76
|
+
DEFAULT_DEV_HOST = "127.0.0.1"
|
|
77
|
+
DEFAULT_DEV_PORT = 3000
|
|
78
|
+
DEFAULT_PREVIEW_PORT = 4173
|
|
79
|
+
HIDDEN_FRAMEWORK_DIR = ".tw"
|
|
80
|
+
WATCH_EXTENSIONS = {
|
|
81
|
+
".tw", ".tss", ".ts", ".twm", ".json", ".md", ".py",
|
|
82
|
+
".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".ico",
|
|
83
|
+
".env", ".txt",
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@dataclass
|
|
88
|
+
class BuildSummary:
|
|
89
|
+
built: int
|
|
90
|
+
skipped: int
|
|
91
|
+
removed: int
|
|
92
|
+
errors: int
|
|
93
|
+
output_dir: str
|
|
94
|
+
warnings: int = 0
|
|
95
|
+
route_collisions: int = 0
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@dataclass
|
|
99
|
+
class RouteMatch:
|
|
100
|
+
page_info: dict
|
|
101
|
+
params: Dict[str, str]
|
|
102
|
+
item: Optional[dict]
|
|
103
|
+
route_path: str
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def configure_compiler_paths(project_root: str):
|
|
107
|
+
project_root = os.path.abspath(project_root)
|
|
108
|
+
|
|
109
|
+
compiler.PROJECT_ROOT = project_root
|
|
110
|
+
compiler.HOME_DIR = os.path.join(project_root, "[home]")
|
|
111
|
+
compiler.COMPONENTS_DIR = os.path.join(compiler.HOME_DIR, "components")
|
|
112
|
+
compiler.PAGES_DIR = os.path.join(compiler.HOME_DIR, "pages")
|
|
113
|
+
compiler.ASSETS_DIR = os.path.join(compiler.HOME_DIR, "assets")
|
|
114
|
+
compiler.LAYOUTS_DIR = os.path.join(compiler.HOME_DIR, "layouts")
|
|
115
|
+
compiler.API_DIR = os.path.join(compiler.HOME_DIR, "api")
|
|
116
|
+
compiler.INDEX_FILE = os.path.join(compiler.HOME_DIR, "index.tw")
|
|
117
|
+
compiler.STYLE_FILE = os.path.join(compiler.HOME_DIR, "style.tss")
|
|
118
|
+
compiler.CONFIG_FILE = os.path.join(project_root, "tw.config")
|
|
119
|
+
|
|
120
|
+
compiler.INTERNAL_DIR = os.path.join(project_root, HIDDEN_FRAMEWORK_DIR)
|
|
121
|
+
compiler.CACHE_DIR = os.path.join(compiler.INTERNAL_DIR, "cache")
|
|
122
|
+
compiler.MANIFEST_DIR = os.path.join(compiler.INTERNAL_DIR, "manifest")
|
|
123
|
+
compiler.COMPILER_DIR = os.path.join(compiler.INTERNAL_DIR, "compiler")
|
|
124
|
+
|
|
125
|
+
compiler.PUBLIC_DIR = os.path.join(project_root, "dist")
|
|
126
|
+
compiler.BUILD_DIR = compiler.PUBLIC_DIR
|
|
127
|
+
compiler.PUBLIC_ASSETS_DIR = os.path.join(compiler.PUBLIC_DIR, "assets")
|
|
128
|
+
compiler.CHUNKS_DIR = os.path.join(compiler.COMPILER_DIR, "chunks")
|
|
129
|
+
compiler.CHUNKS_PUBLIC_DIR = os.path.join(compiler.PUBLIC_DIR, "_tw", "static", "chunks")
|
|
130
|
+
compiler.CHUNKS_URL_PREFIX = "/_tw/static/chunks/"
|
|
131
|
+
compiler.BUILD_MANIFEST_FILE = os.path.join(compiler.MANIFEST_DIR, "build-manifest.json")
|
|
132
|
+
compiler.HASH_DB_FILE = os.path.join(compiler.CACHE_DIR, "hash-db.json")
|
|
133
|
+
compiler.DEPENDENCY_GRAPH_FILE = os.path.join(compiler.CACHE_DIR, "dependency-graph.json")
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
@contextlib.contextmanager
|
|
137
|
+
def compiler_output_context(output_root: str):
|
|
138
|
+
old_values = {
|
|
139
|
+
"INTERNAL_DIR": getattr(compiler, "INTERNAL_DIR", None),
|
|
140
|
+
"CACHE_DIR": getattr(compiler, "CACHE_DIR", None),
|
|
141
|
+
"MANIFEST_DIR": getattr(compiler, "MANIFEST_DIR", None),
|
|
142
|
+
"COMPILER_DIR": getattr(compiler, "COMPILER_DIR", None),
|
|
143
|
+
"PUBLIC_DIR": compiler.PUBLIC_DIR,
|
|
144
|
+
"BUILD_DIR": compiler.BUILD_DIR,
|
|
145
|
+
"PUBLIC_ASSETS_DIR": compiler.PUBLIC_ASSETS_DIR,
|
|
146
|
+
"CHUNKS_DIR": compiler.CHUNKS_DIR,
|
|
147
|
+
"CHUNKS_PUBLIC_DIR": getattr(compiler, "CHUNKS_PUBLIC_DIR", None),
|
|
148
|
+
"CHUNKS_URL_PREFIX": compiler.CHUNKS_URL_PREFIX,
|
|
149
|
+
"BUILD_MANIFEST_FILE": compiler.BUILD_MANIFEST_FILE,
|
|
150
|
+
"HASH_DB_FILE": getattr(compiler, "HASH_DB_FILE", None),
|
|
151
|
+
"DEPENDENCY_GRAPH_FILE": getattr(compiler, "DEPENDENCY_GRAPH_FILE", None),
|
|
152
|
+
"MINIFY_OUTPUT": getattr(compiler, "MINIFY_OUTPUT", False),
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
output_root = os.path.abspath(output_root)
|
|
156
|
+
compiler.PUBLIC_DIR = output_root
|
|
157
|
+
compiler.BUILD_DIR = output_root
|
|
158
|
+
compiler.PUBLIC_ASSETS_DIR = os.path.join(output_root, "assets")
|
|
159
|
+
compiler.CHUNKS_PUBLIC_DIR = os.path.join(output_root, "_tw", "static", "chunks")
|
|
160
|
+
compiler.CHUNKS_URL_PREFIX = "/_tw/static/chunks/"
|
|
161
|
+
compiler._CHUNK_CACHE.clear()
|
|
162
|
+
|
|
163
|
+
try:
|
|
164
|
+
yield
|
|
165
|
+
finally:
|
|
166
|
+
compiler.INTERNAL_DIR = old_values["INTERNAL_DIR"]
|
|
167
|
+
compiler.CACHE_DIR = old_values["CACHE_DIR"]
|
|
168
|
+
compiler.MANIFEST_DIR = old_values["MANIFEST_DIR"]
|
|
169
|
+
compiler.COMPILER_DIR = old_values["COMPILER_DIR"]
|
|
170
|
+
compiler.PUBLIC_DIR = old_values["PUBLIC_DIR"]
|
|
171
|
+
compiler.BUILD_DIR = old_values["BUILD_DIR"]
|
|
172
|
+
compiler.PUBLIC_ASSETS_DIR = old_values["PUBLIC_ASSETS_DIR"]
|
|
173
|
+
compiler.CHUNKS_DIR = old_values["CHUNKS_DIR"]
|
|
174
|
+
compiler.CHUNKS_PUBLIC_DIR = old_values["CHUNKS_PUBLIC_DIR"]
|
|
175
|
+
compiler.CHUNKS_URL_PREFIX = old_values["CHUNKS_URL_PREFIX"]
|
|
176
|
+
compiler.BUILD_MANIFEST_FILE = old_values["BUILD_MANIFEST_FILE"]
|
|
177
|
+
compiler.HASH_DB_FILE = old_values["HASH_DB_FILE"]
|
|
178
|
+
compiler.DEPENDENCY_GRAPH_FILE = old_values["DEPENDENCY_GRAPH_FILE"]
|
|
179
|
+
compiler.MINIFY_OUTPUT = old_values["MINIFY_OUTPUT"]
|
|
180
|
+
compiler._CHUNK_CACHE.clear()
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def invalidate_compiler_caches():
|
|
184
|
+
compiler._CHUNK_CACHE.clear()
|
|
185
|
+
compiler._COMPONENT_AST_CACHE.clear()
|
|
186
|
+
compiler._COMPONENT_EXISTS_CACHE.clear()
|
|
187
|
+
if hasattr(compiler, "_COMPONENT_PATH_CACHE"):
|
|
188
|
+
compiler._COMPONENT_PATH_CACHE.clear()
|
|
189
|
+
compiler._LAYOUT_CACHE.clear()
|
|
190
|
+
compiler._COMPONENT_DEP_GRAPH_CACHE.clear()
|
|
191
|
+
if hasattr(compiler, "_COMPONENT_STYLESHEET_PATHS"):
|
|
192
|
+
compiler._COMPONENT_STYLESHEET_PATHS.clear()
|
|
193
|
+
compiler.INLINE_SCRIPTS.clear()
|
|
194
|
+
# Reset script placeholder growth (prevents long-lived dev sessions from leaking memory)
|
|
195
|
+
if hasattr(compiler, "_SCRIPT_COUNTER"):
|
|
196
|
+
compiler._SCRIPT_COUNTER = 0
|
|
197
|
+
# Optional caches introduced by newer versions
|
|
198
|
+
if hasattr(compiler, "_LAYOUT_META_CACHE"):
|
|
199
|
+
compiler._LAYOUT_META_CACHE.clear()
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def use_modular_pipeline(config: Optional[Dict] = None) -> bool:
|
|
203
|
+
env_value = os.environ.get("TW_USE_MODULAR_PIPELINE")
|
|
204
|
+
if env_value is not None and str(env_value).strip() != "":
|
|
205
|
+
return str(env_value).strip().lower() in {"1", "true", "yes", "on"}
|
|
206
|
+
config = config or compiler.load_config()
|
|
207
|
+
return compiler.to_bool(config.get("modular_pipeline", config.get("modularPipeline", False)))
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def normalize_url_path(path: str) -> str:
|
|
211
|
+
clean = urllib.parse.urlparse(path).path
|
|
212
|
+
clean = posixpath.normpath(clean)
|
|
213
|
+
if clean == ".":
|
|
214
|
+
clean = "/"
|
|
215
|
+
if not clean.startswith("/"):
|
|
216
|
+
clean = "/" + clean
|
|
217
|
+
return clean
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def strip_trailing_slash(path: str) -> str:
|
|
221
|
+
if path != "/" and path.endswith("/"):
|
|
222
|
+
return path[:-1]
|
|
223
|
+
return path
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def route_from_static_page(page_info: dict) -> str:
|
|
227
|
+
segments = []
|
|
228
|
+
if page_info["rel_dir"]:
|
|
229
|
+
segments.extend(page_info["rel_dir"].split(os.sep))
|
|
230
|
+
if page_info["name"] != "index":
|
|
231
|
+
segments.append(page_info["name"])
|
|
232
|
+
route = "/" + "/".join(filter(None, segments))
|
|
233
|
+
return route if route != "" else "/"
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def route_from_dynamic_page(page_info: dict, item: dict) -> str:
|
|
237
|
+
segments = []
|
|
238
|
+
if page_info["rel_dir"]:
|
|
239
|
+
segments.extend(page_info["rel_dir"].split(os.sep))
|
|
240
|
+
segments.extend(compiler.resolve_dynamic_segments(page_info, item))
|
|
241
|
+
return "/" + "/".join(filter(None, segments))
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def build_page_with_modular_pipeline(page_info: dict, css_url: str) -> List[str]:
|
|
245
|
+
tw_path = page_info["path"]
|
|
246
|
+
config = compiler.load_config()
|
|
247
|
+
pretty_urls = compiler.to_bool(config.get("pretty_urls", config.get("prettyUrls", False)))
|
|
248
|
+
|
|
249
|
+
def render_and_write(route_path: str, render_context: Dict, out_path: str) -> str:
|
|
250
|
+
artifacts = compiler.compile_file_pipeline(
|
|
251
|
+
tw_path,
|
|
252
|
+
context=render_context,
|
|
253
|
+
css_href=css_url,
|
|
254
|
+
route_path=route_path,
|
|
255
|
+
)
|
|
256
|
+
html_text = artifacts.html or ""
|
|
257
|
+
if compiler.MINIFY_OUTPUT:
|
|
258
|
+
html_text = compiler.minify_html_content(html_text)
|
|
259
|
+
os.makedirs(os.path.dirname(out_path), exist_ok=True)
|
|
260
|
+
with open(out_path, "w", encoding="utf-8") as handle:
|
|
261
|
+
handle.write(html_text)
|
|
262
|
+
return out_path
|
|
263
|
+
|
|
264
|
+
if page_info["type"] == "static":
|
|
265
|
+
page_ast = compiler.load_page_ast_from_file(tw_path)
|
|
266
|
+
route_path = compiler.route_path_from_page_info(page_info)
|
|
267
|
+
base_context = compiler.build_page_context(page_info, page_ast, tw_path, route_path=route_path)
|
|
268
|
+
out_dir = os.path.join(compiler.BUILD_DIR, page_info["rel_dir"]) if page_info["rel_dir"] else compiler.BUILD_DIR
|
|
269
|
+
if pretty_urls and page_info["name"] != "index":
|
|
270
|
+
out_dir = os.path.join(out_dir, page_info["name"])
|
|
271
|
+
out_path = os.path.join(out_dir, "index.html")
|
|
272
|
+
else:
|
|
273
|
+
out_path = os.path.join(out_dir, f"{page_info['name']}.html")
|
|
274
|
+
return [render_and_write(route_path, base_context, out_path)]
|
|
275
|
+
|
|
276
|
+
built_paths: List[str] = []
|
|
277
|
+
items = compiler.load_dynamic_items(tw_path)
|
|
278
|
+
for item in items:
|
|
279
|
+
if not isinstance(item, dict):
|
|
280
|
+
continue
|
|
281
|
+
segments = compiler.resolve_dynamic_segments(page_info, item)
|
|
282
|
+
route_path = compiler.route_path_from_page_info(page_info, item=item)
|
|
283
|
+
context = compiler.build_page_context(page_info, compiler.load_page_ast_from_file(tw_path), tw_path, item=item, route_path=route_path)
|
|
284
|
+
out_parts = [compiler.BUILD_DIR]
|
|
285
|
+
if page_info["rel_dir"]:
|
|
286
|
+
out_parts.append(page_info["rel_dir"])
|
|
287
|
+
out_parts.extend(segments)
|
|
288
|
+
out_dir = os.path.join(*out_parts)
|
|
289
|
+
built_paths.append(render_and_write(route_path or "/", context, os.path.join(out_dir, "index.html")))
|
|
290
|
+
return built_paths
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def build_page_job_modular(page_info: dict, css_url: str) -> dict:
|
|
294
|
+
outputs = build_page_with_modular_pipeline(page_info, css_url)
|
|
295
|
+
return {"page_info": page_info, "outputs": outputs}
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def special_page_name_for_status(status_code: int) -> str:
|
|
299
|
+
if status_code == 404:
|
|
300
|
+
return "404"
|
|
301
|
+
if status_code == 500:
|
|
302
|
+
return "500"
|
|
303
|
+
return str(status_code)
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def inject_dev_client(html_text: str) -> str:
|
|
307
|
+
# Robust live reload:
|
|
308
|
+
# - Auto-reconnect on background tab / network hiccups
|
|
309
|
+
# - Backoff retry to avoid CPU spikes
|
|
310
|
+
client = """
|
|
311
|
+
<script>
|
|
312
|
+
(() => {
|
|
313
|
+
let source = null;
|
|
314
|
+
let retry = 500;
|
|
315
|
+
const maxRetry = 5000;
|
|
316
|
+
|
|
317
|
+
function connect() {
|
|
318
|
+
try { if (source) source.close(); } catch (e) {}
|
|
319
|
+
source = new EventSource('/__tw/events');
|
|
320
|
+
source.onopen = () => { retry = 500; };
|
|
321
|
+
source.onmessage = (event) => {
|
|
322
|
+
if (event.data === 'reload') window.location.reload();
|
|
323
|
+
};
|
|
324
|
+
source.onerror = () => {
|
|
325
|
+
try { if (source) source.close(); } catch (e) {}
|
|
326
|
+
setTimeout(connect, retry);
|
|
327
|
+
retry = Math.min(maxRetry, retry * 2);
|
|
328
|
+
};
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
connect();
|
|
332
|
+
})();
|
|
333
|
+
</script>
|
|
334
|
+
"""
|
|
335
|
+
if "</body>" in html_text:
|
|
336
|
+
return html_text.replace("</body>", client + "\n</body>", 1)
|
|
337
|
+
return html_text + client
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def render_error_html(title: str, message: str, status_code: int = 500) -> bytes:
|
|
341
|
+
doc = f"""<!DOCTYPE html>
|
|
342
|
+
<html>
|
|
343
|
+
<head>
|
|
344
|
+
<meta charset="UTF-8">
|
|
345
|
+
<title>{html.escape(title)}</title>
|
|
346
|
+
<style>
|
|
347
|
+
body {{ margin: 0; font-family: Inter, system-ui, sans-serif; background: #0b1020; color: #f9fafb; }}
|
|
348
|
+
.overlay {{ min-height: 100vh; padding: 32px; background:
|
|
349
|
+
radial-gradient(circle at top left, rgba(220, 38, 38, 0.25), transparent 35%),
|
|
350
|
+
linear-gradient(180deg, #130d14 0%, #0b1020 100%);
|
|
351
|
+
}}
|
|
352
|
+
.card {{ max-width: 1100px; margin: 0 auto; border: 1px solid rgba(248,113,113,0.35); background: rgba(17,24,39,0.88); border-radius: 18px; box-shadow: 0 30px 80px rgba(0,0,0,0.45); overflow: hidden; }}
|
|
353
|
+
.header {{ padding: 18px 22px; border-bottom: 1px solid rgba(248,113,113,0.18); background: rgba(127,29,29,0.22); }}
|
|
354
|
+
.status {{ color: #fca5a5; font-size: 13px; letter-spacing: 0.08em; text-transform: uppercase; margin-bottom: 8px; }}
|
|
355
|
+
h1 {{ margin: 0; font-size: 28px; }}
|
|
356
|
+
pre {{ margin: 0; white-space: pre-wrap; background: #020617; padding: 22px; overflow: auto; line-height: 1.55; }}
|
|
357
|
+
</style>
|
|
358
|
+
</head>
|
|
359
|
+
<body>
|
|
360
|
+
<div class="overlay">
|
|
361
|
+
<div class="card">
|
|
362
|
+
<div class="header">
|
|
363
|
+
<div class="status">TW Compile Error · HTTP {status_code}</div>
|
|
364
|
+
<h1>{html.escape(title)}</h1>
|
|
365
|
+
</div>
|
|
366
|
+
<pre>{html.escape(message)}</pre>
|
|
367
|
+
</div>
|
|
368
|
+
</div>
|
|
369
|
+
</body>
|
|
370
|
+
</html>"""
|
|
371
|
+
return doc.encode("utf-8")
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def format_compiler_error(page_path: str, err: Exception) -> str:
|
|
375
|
+
if isinstance(err, compiler.CompilerError) and os.path.exists(page_path):
|
|
376
|
+
raw = compiler.read_text_file(page_path)
|
|
377
|
+
emitter = compiler.DiagnosticEmitter(page_path, raw)
|
|
378
|
+
return emitter.format(err)
|
|
379
|
+
return str(err)
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def safe_read_binary(path: str) -> Optional[bytes]:
|
|
383
|
+
if not os.path.exists(path) or not os.path.isfile(path):
|
|
384
|
+
return None
|
|
385
|
+
with open(path, "rb") as f:
|
|
386
|
+
return f.read()
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def is_path_within(root_path: str, candidate_path: str) -> bool:
|
|
390
|
+
try:
|
|
391
|
+
root_abs = os.path.abspath(root_path)
|
|
392
|
+
candidate_abs = os.path.abspath(candidate_path)
|
|
393
|
+
return os.path.commonpath([root_abs, candidate_abs]) == root_abs
|
|
394
|
+
except (ValueError, OSError):
|
|
395
|
+
return False
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def _normalize_config_headers(raw_value) -> List[Tuple[str, str]]:
|
|
399
|
+
if not raw_value:
|
|
400
|
+
return []
|
|
401
|
+
if isinstance(raw_value, dict):
|
|
402
|
+
return [(str(key), str(value)) for key, value in raw_value.items()]
|
|
403
|
+
if isinstance(raw_value, list):
|
|
404
|
+
normalized = []
|
|
405
|
+
for item in raw_value:
|
|
406
|
+
if isinstance(item, dict):
|
|
407
|
+
normalized.extend((str(key), str(value)) for key, value in item.items())
|
|
408
|
+
elif isinstance(item, (list, tuple)) and len(item) == 2:
|
|
409
|
+
normalized.append((str(item[0]), str(item[1])))
|
|
410
|
+
return normalized
|
|
411
|
+
return []
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def get_security_headers(config: Optional[Dict]) -> List[Tuple[str, str]]:
|
|
415
|
+
config = config or {}
|
|
416
|
+
return (
|
|
417
|
+
_normalize_config_headers(compiler.get_config_value(config, "security", "headers"))
|
|
418
|
+
or _normalize_config_headers(config.get("security.headers"))
|
|
419
|
+
or _normalize_config_headers(config.get("security_headers"))
|
|
420
|
+
or _normalize_config_headers(config.get("headers"))
|
|
421
|
+
)
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def get_cookie_secure_mode(config: Optional[Dict]) -> str:
|
|
425
|
+
config = config or {}
|
|
426
|
+
raw_value = (
|
|
427
|
+
compiler.get_config_value(config, "cookies", "secure")
|
|
428
|
+
or config.get("cookies.secure")
|
|
429
|
+
or config.get("cookies_secure")
|
|
430
|
+
or "auto"
|
|
431
|
+
)
|
|
432
|
+
value = str(raw_value).strip().lower()
|
|
433
|
+
if value in {"true", "1", "yes", "on"}:
|
|
434
|
+
return "true"
|
|
435
|
+
if value in {"false", "0", "no", "off"}:
|
|
436
|
+
return "false"
|
|
437
|
+
return "auto"
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def request_uses_https(request_headers: Optional[Dict[str, str]] = None, server_port: Optional[int] = None) -> bool:
|
|
441
|
+
request_headers = request_headers or {}
|
|
442
|
+
forwarded_proto = str(request_headers.get("X-Forwarded-Proto", "")).split(",", 1)[0].strip().lower()
|
|
443
|
+
if forwarded_proto:
|
|
444
|
+
return forwarded_proto == "https"
|
|
445
|
+
forwarded = str(request_headers.get("Forwarded", "")).lower()
|
|
446
|
+
if "proto=https" in forwarded:
|
|
447
|
+
return True
|
|
448
|
+
return server_port == 443
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def render_cookie_header(
|
|
452
|
+
name: str,
|
|
453
|
+
value: object,
|
|
454
|
+
*,
|
|
455
|
+
config: Optional[Dict] = None,
|
|
456
|
+
request_headers: Optional[Dict[str, str]] = None,
|
|
457
|
+
server_port: Optional[int] = None,
|
|
458
|
+
) -> str:
|
|
459
|
+
rendered_value = urllib.parse.quote(str(value))
|
|
460
|
+
parts = [f"{name}={rendered_value}", "Path=/", "HttpOnly", "SameSite=Lax"]
|
|
461
|
+
secure_mode = get_cookie_secure_mode(config)
|
|
462
|
+
if secure_mode == "true" or (secure_mode == "auto" and request_uses_https(request_headers, server_port=server_port)):
|
|
463
|
+
parts.append("Secure")
|
|
464
|
+
return "; ".join(parts)
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def _csrf_secret() -> str:
|
|
468
|
+
return (
|
|
469
|
+
os.environ.get("TW_CSRF_SECRET")
|
|
470
|
+
or os.environ.get("SECRET_KEY")
|
|
471
|
+
or os.environ.get("API_TOKEN")
|
|
472
|
+
or "tw-dev-csrf-secret"
|
|
473
|
+
)
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def _csrf_session_hint(request: Optional[Dict]) -> str:
|
|
477
|
+
cookies = dict((request or {}).get("cookies") or {})
|
|
478
|
+
for key in ("session_id", "session", "sid"):
|
|
479
|
+
if cookies.get(key):
|
|
480
|
+
return str(cookies[key])
|
|
481
|
+
return ""
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def generate_csrf_token(request: Optional[Dict] = None) -> str:
|
|
485
|
+
issued_at = str(int(time.time()))
|
|
486
|
+
nonce = secrets.token_urlsafe(16)
|
|
487
|
+
session_hint = _csrf_session_hint(request)
|
|
488
|
+
payload = f"{issued_at}:{nonce}:{session_hint}"
|
|
489
|
+
signature = hmac.new(_csrf_secret().encode("utf-8"), payload.encode("utf-8"), hashlib.sha256).hexdigest()
|
|
490
|
+
token = f"{payload}:{signature}".encode("utf-8")
|
|
491
|
+
return base64.urlsafe_b64encode(token).decode("ascii").rstrip("=")
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def verify_csrf_token(token: str, request: Optional[Dict] = None, *, max_age: int = 7200) -> bool:
|
|
495
|
+
if not token:
|
|
496
|
+
return False
|
|
497
|
+
try:
|
|
498
|
+
padding = "=" * (-len(token) % 4)
|
|
499
|
+
decoded = base64.urlsafe_b64decode((token + padding).encode("ascii")).decode("utf-8")
|
|
500
|
+
issued_at, nonce, session_hint, signature = decoded.split(":", 3)
|
|
501
|
+
payload = f"{issued_at}:{nonce}:{session_hint}"
|
|
502
|
+
except Exception:
|
|
503
|
+
return False
|
|
504
|
+
expected_signature = hmac.new(_csrf_secret().encode("utf-8"), payload.encode("utf-8"), hashlib.sha256).hexdigest()
|
|
505
|
+
if not hmac.compare_digest(signature, expected_signature):
|
|
506
|
+
return False
|
|
507
|
+
try:
|
|
508
|
+
if int(time.time()) - int(issued_at) > int(max_age):
|
|
509
|
+
return False
|
|
510
|
+
except (TypeError, ValueError):
|
|
511
|
+
return False
|
|
512
|
+
return session_hint == _csrf_session_hint(request)
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
class TokenBucketRateLimiter:
|
|
516
|
+
def __init__(self, capacity: int, window_seconds: float):
|
|
517
|
+
self.capacity = max(1, int(capacity))
|
|
518
|
+
self.window_seconds = max(float(window_seconds), 1.0)
|
|
519
|
+
self.refill_rate = self.capacity / self.window_seconds
|
|
520
|
+
self._state: Dict[str, Tuple[float, float]] = {}
|
|
521
|
+
self._lock = threading.Lock()
|
|
522
|
+
|
|
523
|
+
def allow(self, bucket_key: str) -> bool:
|
|
524
|
+
now = time.monotonic()
|
|
525
|
+
with self._lock:
|
|
526
|
+
tokens, updated_at = self._state.get(bucket_key, (float(self.capacity), now))
|
|
527
|
+
elapsed = max(0.0, now - updated_at)
|
|
528
|
+
tokens = min(float(self.capacity), tokens + elapsed * self.refill_rate)
|
|
529
|
+
if tokens < 1.0:
|
|
530
|
+
self._state[bucket_key] = (tokens, now)
|
|
531
|
+
return False
|
|
532
|
+
self._state[bucket_key] = (tokens - 1.0, now)
|
|
533
|
+
return True
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def parse_env_file(path: str) -> Dict[str, str]:
|
|
537
|
+
values = {}
|
|
538
|
+
if not os.path.exists(path):
|
|
539
|
+
return values
|
|
540
|
+
with open(path, "r", encoding="utf-8") as f:
|
|
541
|
+
for raw_line in f:
|
|
542
|
+
line = raw_line.strip()
|
|
543
|
+
if not line or line.startswith("#") or "=" not in line:
|
|
544
|
+
continue
|
|
545
|
+
key, value = line.split("=", 1)
|
|
546
|
+
key = key.strip()
|
|
547
|
+
value = value.strip()
|
|
548
|
+
# Support `KEY=value # comment` (comment starts only when preceded by whitespace).
|
|
549
|
+
# If value is quoted, keep everything inside quotes.
|
|
550
|
+
if value.startswith(("'", '"')) and len(value) >= 2:
|
|
551
|
+
quote = value[0]
|
|
552
|
+
# Find the matching quote; anything after it is ignored (including comments).
|
|
553
|
+
end_idx = value.find(quote, 1)
|
|
554
|
+
if end_idx != -1:
|
|
555
|
+
value = value[1:end_idx]
|
|
556
|
+
else:
|
|
557
|
+
value = value.strip(quote)
|
|
558
|
+
else:
|
|
559
|
+
value = re.split(r"\s+#", value, 1)[0].strip()
|
|
560
|
+
values[key] = value
|
|
561
|
+
return values
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def load_project_env(project_root: str, mode: str) -> Dict[str, str]:
|
|
565
|
+
env = {}
|
|
566
|
+
for name in [".env", f".env.{mode}", ".env.local"]:
|
|
567
|
+
env.update(parse_env_file(os.path.join(project_root, name)))
|
|
568
|
+
for key, value in env.items():
|
|
569
|
+
os.environ[key] = value
|
|
570
|
+
return env
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
def parse_cookie_header(raw_value: str) -> Dict[str, str]:
|
|
574
|
+
cookies = {}
|
|
575
|
+
if not raw_value:
|
|
576
|
+
return cookies
|
|
577
|
+
for part in raw_value.split(";"):
|
|
578
|
+
if "=" not in part:
|
|
579
|
+
continue
|
|
580
|
+
key, value = part.split("=", 1)
|
|
581
|
+
cookies[key.strip()] = urllib.parse.unquote(value.strip())
|
|
582
|
+
return cookies
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
def match_path_pattern(path: str, pattern: str) -> bool:
|
|
586
|
+
if not pattern or pattern in {"*", "/**"}:
|
|
587
|
+
return True
|
|
588
|
+
if pattern.endswith("/**"):
|
|
589
|
+
return path.startswith(pattern[:-3] or "/")
|
|
590
|
+
return fnmatch.fnmatch(path, pattern)
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
def middleware_file_candidates(project_root: str) -> List[str]:
|
|
594
|
+
return [
|
|
595
|
+
os.path.join(project_root, "middleware.tw"),
|
|
596
|
+
os.path.join(compiler.HOME_DIR, "middleware.tw"),
|
|
597
|
+
os.path.join(compiler.HOME_DIR, "middleware", "index.tw"),
|
|
598
|
+
]
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
def parse_value_token(tokens, i):
|
|
602
|
+
expr, j = compiler.collect_until_eol(tokens, i, stop_on_block_open=True)
|
|
603
|
+
if not expr:
|
|
604
|
+
raise RuntimeError("Expected value token")
|
|
605
|
+
return compiler.parse_literal_value(expr), j
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
def parse_middleware_rules(project_root: str) -> List[dict]:
|
|
609
|
+
source_path = next((path for path in middleware_file_candidates(project_root) if os.path.exists(path)), None)
|
|
610
|
+
if not source_path:
|
|
611
|
+
return []
|
|
612
|
+
|
|
613
|
+
tokens = compiler.tokenize_tw(compiler.read_text_file(source_path))
|
|
614
|
+
rules = []
|
|
615
|
+
|
|
616
|
+
def skip_separators(index):
|
|
617
|
+
while index < len(tokens) and compiler.is_statement_separator(tokens[index]):
|
|
618
|
+
index += 1
|
|
619
|
+
return index
|
|
620
|
+
|
|
621
|
+
def expect_block(index, label):
|
|
622
|
+
index = skip_separators(index)
|
|
623
|
+
if index >= len(tokens) or tokens[index].type != "BRACE" or tokens[index].value != "{":
|
|
624
|
+
raise RuntimeError(f"Expected `{{` after `{label}` in middleware.tw")
|
|
625
|
+
return index + 1
|
|
626
|
+
|
|
627
|
+
def ensure_list(value):
|
|
628
|
+
if value is None:
|
|
629
|
+
return []
|
|
630
|
+
if isinstance(value, (list, tuple, set)):
|
|
631
|
+
return [str(item) for item in value]
|
|
632
|
+
return [str(value)]
|
|
633
|
+
|
|
634
|
+
def parse_object_block(index):
|
|
635
|
+
index = expect_block(index, "json")
|
|
636
|
+
data = {}
|
|
637
|
+
while index < len(tokens):
|
|
638
|
+
tok = tokens[index]
|
|
639
|
+
if compiler.is_statement_separator(tok):
|
|
640
|
+
index += 1
|
|
641
|
+
continue
|
|
642
|
+
if tok.type == "BRACE" and tok.value == "}":
|
|
643
|
+
index += 1
|
|
644
|
+
break
|
|
645
|
+
if tok.type != "WORD":
|
|
646
|
+
raise RuntimeError("Invalid object key in middleware.tw")
|
|
647
|
+
key = tok.value
|
|
648
|
+
index += 1
|
|
649
|
+
index = skip_separators(index)
|
|
650
|
+
if index < len(tokens) and tokens[index].type == "BRACE" and tokens[index].value == "{":
|
|
651
|
+
value, index = parse_object_block(index - 0)
|
|
652
|
+
else:
|
|
653
|
+
value, index = parse_value_token(tokens, index)
|
|
654
|
+
data[key] = value
|
|
655
|
+
return data, index
|
|
656
|
+
|
|
657
|
+
def parse_response_block(index):
|
|
658
|
+
index = expect_block(index, "response")
|
|
659
|
+
response = {"headers": [], "cookies": []}
|
|
660
|
+
while index < len(tokens):
|
|
661
|
+
tok = tokens[index]
|
|
662
|
+
if compiler.is_statement_separator(tok):
|
|
663
|
+
index += 1
|
|
664
|
+
continue
|
|
665
|
+
if tok.type == "BRACE" and tok.value == "}":
|
|
666
|
+
index += 1
|
|
667
|
+
break
|
|
668
|
+
if tok.type != "WORD":
|
|
669
|
+
raise RuntimeError("Invalid response directive in middleware.tw")
|
|
670
|
+
key = tok.value
|
|
671
|
+
index += 1
|
|
672
|
+
if key == "status":
|
|
673
|
+
response["status"], index = parse_value_token(tokens, index)
|
|
674
|
+
elif key in {"text", "html", "content_type"}:
|
|
675
|
+
response[key], index = parse_value_token(tokens, index)
|
|
676
|
+
elif key == "json":
|
|
677
|
+
response["json"], index = parse_object_block(index)
|
|
678
|
+
elif key == "header":
|
|
679
|
+
header_name, index = parse_value_token(tokens, index)
|
|
680
|
+
header_value, index = parse_value_token(tokens, index)
|
|
681
|
+
response["headers"].append((str(header_name), str(header_value)))
|
|
682
|
+
elif key == "cookie":
|
|
683
|
+
cookie_name, index = parse_value_token(tokens, index)
|
|
684
|
+
cookie_value, index = parse_value_token(tokens, index)
|
|
685
|
+
response["cookies"].append((str(cookie_name), str(cookie_value)))
|
|
686
|
+
else:
|
|
687
|
+
raise RuntimeError(f"Unsupported response key: {key}")
|
|
688
|
+
return response, index
|
|
689
|
+
|
|
690
|
+
def parse_user_agent_block(index):
|
|
691
|
+
index = expect_block(index, "user_agent")
|
|
692
|
+
spec = {"allow": [], "block": [], "empty_is_blocked": False}
|
|
693
|
+
while index < len(tokens):
|
|
694
|
+
tok = tokens[index]
|
|
695
|
+
if compiler.is_statement_separator(tok):
|
|
696
|
+
index += 1
|
|
697
|
+
continue
|
|
698
|
+
if tok.type == "BRACE" and tok.value == "}":
|
|
699
|
+
index += 1
|
|
700
|
+
break
|
|
701
|
+
if tok.type != "WORD":
|
|
702
|
+
raise RuntimeError("Invalid user_agent directive in middleware.tw")
|
|
703
|
+
key = tok.value
|
|
704
|
+
index += 1
|
|
705
|
+
if key in {"allow", "block"}:
|
|
706
|
+
value, index = parse_value_token(tokens, index)
|
|
707
|
+
spec[key] = ensure_list(value)
|
|
708
|
+
elif key == "empty_is_blocked":
|
|
709
|
+
value, index = parse_value_token(tokens, index)
|
|
710
|
+
spec["empty_is_blocked"] = bool(value)
|
|
711
|
+
else:
|
|
712
|
+
raise RuntimeError(f"Unsupported user_agent key: {key}")
|
|
713
|
+
return spec, index
|
|
714
|
+
|
|
715
|
+
def parse_origin_block(index):
|
|
716
|
+
index = expect_block(index, "origin")
|
|
717
|
+
spec = {"allow": [], "allow_referer": True, "require": False}
|
|
718
|
+
while index < len(tokens):
|
|
719
|
+
tok = tokens[index]
|
|
720
|
+
if compiler.is_statement_separator(tok):
|
|
721
|
+
index += 1
|
|
722
|
+
continue
|
|
723
|
+
if tok.type == "BRACE" and tok.value == "}":
|
|
724
|
+
index += 1
|
|
725
|
+
break
|
|
726
|
+
if tok.type != "WORD":
|
|
727
|
+
raise RuntimeError("Invalid origin directive in middleware.tw")
|
|
728
|
+
key = tok.value
|
|
729
|
+
index += 1
|
|
730
|
+
if key == "allow":
|
|
731
|
+
value, index = parse_value_token(tokens, index)
|
|
732
|
+
spec["allow"] = ensure_list(value)
|
|
733
|
+
elif key in {"allow_referer", "require"}:
|
|
734
|
+
value, index = parse_value_token(tokens, index)
|
|
735
|
+
spec[key] = bool(value)
|
|
736
|
+
else:
|
|
737
|
+
raise RuntimeError(f"Unsupported origin key: {key}")
|
|
738
|
+
return spec, index
|
|
739
|
+
|
|
740
|
+
def parse_path_block(index):
|
|
741
|
+
index = expect_block(index, "path")
|
|
742
|
+
spec = {
|
|
743
|
+
"prefixes": [],
|
|
744
|
+
"contains": [],
|
|
745
|
+
"extensions": [],
|
|
746
|
+
"regex": [],
|
|
747
|
+
"single_segment_max": 0,
|
|
748
|
+
"deny_traversal": True,
|
|
749
|
+
"deny_null_bytes": True,
|
|
750
|
+
}
|
|
751
|
+
while index < len(tokens):
|
|
752
|
+
tok = tokens[index]
|
|
753
|
+
if compiler.is_statement_separator(tok):
|
|
754
|
+
index += 1
|
|
755
|
+
continue
|
|
756
|
+
if tok.type == "BRACE" and tok.value == "}":
|
|
757
|
+
index += 1
|
|
758
|
+
break
|
|
759
|
+
if tok.type != "WORD":
|
|
760
|
+
raise RuntimeError("Invalid path directive in middleware.tw")
|
|
761
|
+
key = tok.value
|
|
762
|
+
index += 1
|
|
763
|
+
if key in {"prefixes", "contains", "extensions", "regex"}:
|
|
764
|
+
value, index = parse_value_token(tokens, index)
|
|
765
|
+
spec[key] = ensure_list(value)
|
|
766
|
+
elif key == "single_segment_max":
|
|
767
|
+
value, index = parse_value_token(tokens, index)
|
|
768
|
+
spec[key] = int(value or 0)
|
|
769
|
+
elif key in {"deny_traversal", "deny_null_bytes"}:
|
|
770
|
+
value, index = parse_value_token(tokens, index)
|
|
771
|
+
spec[key] = bool(value)
|
|
772
|
+
else:
|
|
773
|
+
raise RuntimeError(f"Unsupported path key: {key}")
|
|
774
|
+
return spec, index
|
|
775
|
+
|
|
776
|
+
def parse_auth_block(index):
|
|
777
|
+
index = expect_block(index, "auth")
|
|
778
|
+
spec = {"cookie": "", "jwt_secret": "", "jwt_secret_env": "", "required": True}
|
|
779
|
+
while index < len(tokens):
|
|
780
|
+
tok = tokens[index]
|
|
781
|
+
if compiler.is_statement_separator(tok):
|
|
782
|
+
index += 1
|
|
783
|
+
continue
|
|
784
|
+
if tok.type == "BRACE" and tok.value == "}":
|
|
785
|
+
index += 1
|
|
786
|
+
break
|
|
787
|
+
if tok.type != "WORD":
|
|
788
|
+
raise RuntimeError("Invalid auth directive in middleware.tw")
|
|
789
|
+
key = tok.value
|
|
790
|
+
index += 1
|
|
791
|
+
if key in {"cookie", "jwt_secret", "jwt_secret_env"}:
|
|
792
|
+
spec[key], index = parse_value_token(tokens, index)
|
|
793
|
+
elif key == "required":
|
|
794
|
+
value, index = parse_value_token(tokens, index)
|
|
795
|
+
spec["required"] = bool(value)
|
|
796
|
+
else:
|
|
797
|
+
raise RuntimeError(f"Unsupported auth key: {key}")
|
|
798
|
+
return spec, index
|
|
799
|
+
|
|
800
|
+
def parse_rate_limit_block(index):
|
|
801
|
+
index = expect_block(index, "rate_limit")
|
|
802
|
+
rate_limit = {"identity": "ip", "bucket_segments": 2}
|
|
803
|
+
while index < len(tokens):
|
|
804
|
+
tok = tokens[index]
|
|
805
|
+
if compiler.is_statement_separator(tok):
|
|
806
|
+
index += 1
|
|
807
|
+
continue
|
|
808
|
+
if tok.type == "BRACE" and tok.value == "}":
|
|
809
|
+
index += 1
|
|
810
|
+
break
|
|
811
|
+
if tok.type != "WORD":
|
|
812
|
+
raise RuntimeError("Invalid rate_limit directive")
|
|
813
|
+
inner_key = tok.value
|
|
814
|
+
index += 1
|
|
815
|
+
inner_value, index = parse_value_token(tokens, index)
|
|
816
|
+
rate_limit[inner_key] = inner_value
|
|
817
|
+
requests = int(compiler.parse_config_scalar(rate_limit.get("requests", 0)) or 0)
|
|
818
|
+
window = float(compiler.parse_config_scalar(rate_limit.get("window", 0)) or 0)
|
|
819
|
+
if requests <= 0 or window <= 0:
|
|
820
|
+
raise RuntimeError("`rate_limit` requires positive `requests` and `window` values")
|
|
821
|
+
return {
|
|
822
|
+
"requests": requests,
|
|
823
|
+
"window": window,
|
|
824
|
+
"identity": str(rate_limit.get("identity", "ip") or "ip"),
|
|
825
|
+
"bucket_segments": int(rate_limit.get("bucket_segments", 2) or 2),
|
|
826
|
+
}, index
|
|
827
|
+
|
|
828
|
+
i = 0
|
|
829
|
+
while i < len(tokens):
|
|
830
|
+
token = tokens[i]
|
|
831
|
+
if compiler.is_statement_separator(token):
|
|
832
|
+
i += 1
|
|
833
|
+
continue
|
|
834
|
+
if token.type == "WORD" and token.value in {"use", "rule"}:
|
|
835
|
+
i += 1
|
|
836
|
+
i = skip_separators(i)
|
|
837
|
+
rule = {"name": "", "match": "/**", "methods": [], "headers": [], "cookies": []}
|
|
838
|
+
if i < len(tokens) and tokens[i].type in {"STRING", "WORD"}:
|
|
839
|
+
next_tok = tokens[i + 1] if i + 1 < len(tokens) else None
|
|
840
|
+
if next_tok and next_tok.type == "BRACE" and next_tok.value == "{":
|
|
841
|
+
rule["name"] = str(tokens[i].value)
|
|
842
|
+
i += 1
|
|
843
|
+
i = expect_block(i, token.value)
|
|
844
|
+
while i < len(tokens):
|
|
845
|
+
tok = tokens[i]
|
|
846
|
+
if compiler.is_statement_separator(tok):
|
|
847
|
+
i += 1
|
|
848
|
+
continue
|
|
849
|
+
if tok.type == "BRACE" and tok.value == "}":
|
|
850
|
+
i += 1
|
|
851
|
+
break
|
|
852
|
+
if tok.type != "WORD":
|
|
853
|
+
raise RuntimeError("Invalid middleware directive")
|
|
854
|
+
key = tok.value
|
|
855
|
+
i += 1
|
|
856
|
+
if key == "match":
|
|
857
|
+
rule["match"], i = parse_value_token(tokens, i)
|
|
858
|
+
elif key == "methods":
|
|
859
|
+
methods, i = parse_value_token(tokens, i)
|
|
860
|
+
rule["methods"] = [str(item).upper() for item in ensure_list(methods)]
|
|
861
|
+
elif key == "redirect":
|
|
862
|
+
rule["redirect"], i = parse_value_token(tokens, i)
|
|
863
|
+
elif key == "rewrite":
|
|
864
|
+
rule["rewrite"], i = parse_value_token(tokens, i)
|
|
865
|
+
elif key == "response":
|
|
866
|
+
rule["response"], i = parse_response_block(i)
|
|
867
|
+
elif key == "user_agent":
|
|
868
|
+
rule["user_agent"], i = parse_user_agent_block(i)
|
|
869
|
+
elif key == "origin":
|
|
870
|
+
rule["origin"], i = parse_origin_block(i)
|
|
871
|
+
elif key == "path":
|
|
872
|
+
rule["path_rule"], i = parse_path_block(i)
|
|
873
|
+
elif key == "auth_rule":
|
|
874
|
+
rule["auth_rule"], i = parse_auth_block(i)
|
|
875
|
+
elif key == "deny":
|
|
876
|
+
# Syntax:
|
|
877
|
+
# deny 403 "Forbidden"
|
|
878
|
+
# Returns an immediate response without redirect/rewrite.
|
|
879
|
+
raw_status, i = parse_value_token(tokens, i)
|
|
880
|
+
status = int(compiler.parse_config_scalar(raw_status) or raw_status)
|
|
881
|
+
# Optional message/body
|
|
882
|
+
message = "Forbidden"
|
|
883
|
+
if i < len(tokens) and not compiler.is_statement_separator(tokens[i]) and not (
|
|
884
|
+
tokens[i].type == "BRACE" and tokens[i].value == "}"
|
|
885
|
+
):
|
|
886
|
+
message, i = parse_value_token(tokens, i)
|
|
887
|
+
rule["deny"] = {"status": status, "body": str(message)}
|
|
888
|
+
elif key == "auth":
|
|
889
|
+
i = skip_separators(i)
|
|
890
|
+
if i < len(tokens) and tokens[i].type == "BRACE" and tokens[i].value == "{":
|
|
891
|
+
rule["auth_rule"], i = parse_auth_block(i)
|
|
892
|
+
else:
|
|
893
|
+
cookie_name, i = parse_value_token(tokens, i)
|
|
894
|
+
redirect_to, i = parse_value_token(tokens, i)
|
|
895
|
+
rule["auth"] = {"cookie": cookie_name, "redirect": redirect_to}
|
|
896
|
+
elif key == "header":
|
|
897
|
+
header_name, i = parse_value_token(tokens, i)
|
|
898
|
+
header_value, i = parse_value_token(tokens, i)
|
|
899
|
+
rule["headers"].append((str(header_name), str(header_value)))
|
|
900
|
+
elif key == "cookie":
|
|
901
|
+
cookie_name, i = parse_value_token(tokens, i)
|
|
902
|
+
cookie_value, i = parse_value_token(tokens, i)
|
|
903
|
+
rule["cookies"].append((str(cookie_name), str(cookie_value)))
|
|
904
|
+
elif key == "rate_limit":
|
|
905
|
+
rule["rate_limit"], i = parse_rate_limit_block(i)
|
|
906
|
+
elif key in {"status", "text", "html", "content_type"}:
|
|
907
|
+
rule.setdefault("response", {"headers": [], "cookies": []})
|
|
908
|
+
value, i = parse_value_token(tokens, i)
|
|
909
|
+
rule["response"][key] = value
|
|
910
|
+
elif key == "json":
|
|
911
|
+
rule.setdefault("response", {"headers": [], "cookies": []})
|
|
912
|
+
rule["response"]["json"], i = parse_object_block(i)
|
|
913
|
+
else:
|
|
914
|
+
raise RuntimeError(f"Unsupported middleware key: {key}")
|
|
915
|
+
rules.append(rule)
|
|
916
|
+
continue
|
|
917
|
+
i += 1
|
|
918
|
+
return rules
|
|
919
|
+
|
|
920
|
+
|
|
921
|
+
def _middleware_header(request_headers: Dict[str, str], name: str, default: str = "") -> str:
|
|
922
|
+
lname = str(name).lower()
|
|
923
|
+
for key, value in (request_headers or {}).items():
|
|
924
|
+
if str(key).lower() == lname:
|
|
925
|
+
return str(value)
|
|
926
|
+
return default
|
|
927
|
+
|
|
928
|
+
|
|
929
|
+
def _middleware_list(value) -> List[str]:
|
|
930
|
+
if value is None:
|
|
931
|
+
return []
|
|
932
|
+
if isinstance(value, (list, tuple, set)):
|
|
933
|
+
return [str(item) for item in value]
|
|
934
|
+
return [str(value)]
|
|
935
|
+
|
|
936
|
+
|
|
937
|
+
def _middleware_referer_origin(referer: str) -> str:
|
|
938
|
+
parsed = urllib.parse.urlparse(str(referer or ""))
|
|
939
|
+
if not parsed.scheme or not parsed.netloc:
|
|
940
|
+
return ""
|
|
941
|
+
return f"{parsed.scheme}://{parsed.netloc}"
|
|
942
|
+
|
|
943
|
+
|
|
944
|
+
def _jwt_b64url_decode(value: str) -> bytes:
|
|
945
|
+
padding = "=" * ((4 - (len(value) % 4)) % 4)
|
|
946
|
+
return base64.urlsafe_b64decode((value + padding).encode("ascii"))
|
|
947
|
+
|
|
948
|
+
|
|
949
|
+
def _verify_hs256_jwt(token: str, secret: str) -> bool:
|
|
950
|
+
try:
|
|
951
|
+
parts = str(token or "").split(".")
|
|
952
|
+
if len(parts) != 3:
|
|
953
|
+
return False
|
|
954
|
+
header, payload, signature = parts
|
|
955
|
+
signing_input = f"{header}.{payload}".encode("utf-8")
|
|
956
|
+
expected = hmac.new(str(secret or "").encode("utf-8"), signing_input, hashlib.sha256).digest()
|
|
957
|
+
actual = _jwt_b64url_decode(signature)
|
|
958
|
+
if not hmac.compare_digest(expected, actual):
|
|
959
|
+
return False
|
|
960
|
+
payload_obj = json.loads(_jwt_b64url_decode(payload).decode("utf-8", errors="replace") or "{}")
|
|
961
|
+
exp = payload_obj.get("exp")
|
|
962
|
+
if exp and time.time() > float(exp):
|
|
963
|
+
return False
|
|
964
|
+
return True
|
|
965
|
+
except Exception:
|
|
966
|
+
return False
|
|
967
|
+
|
|
968
|
+
|
|
969
|
+
def _build_middleware_response(
|
|
970
|
+
rule: dict,
|
|
971
|
+
result: dict,
|
|
972
|
+
*,
|
|
973
|
+
default_status: int,
|
|
974
|
+
default_text: str,
|
|
975
|
+
default_content_type: str = "text/plain; charset=utf-8",
|
|
976
|
+
extra_headers: Optional[List[Tuple[str, str]]] = None,
|
|
977
|
+
) -> dict:
|
|
978
|
+
spec = dict(rule.get("response") or {})
|
|
979
|
+
status = int(spec.get("status", default_status) or default_status)
|
|
980
|
+
headers = list(result.get("headers", [])) + list(spec.get("headers", [])) + list(extra_headers or [])
|
|
981
|
+
cookies = list(result.get("cookies", [])) + list(spec.get("cookies", []))
|
|
982
|
+
if "json" in spec:
|
|
983
|
+
body = json.dumps(spec.get("json"), ensure_ascii=False).encode("utf-8")
|
|
984
|
+
content_type = str(spec.get("content_type") or "application/json; charset=utf-8")
|
|
985
|
+
elif "html" in spec:
|
|
986
|
+
body = str(spec.get("html", default_text) or default_text).encode("utf-8")
|
|
987
|
+
content_type = str(spec.get("content_type") or "text/html; charset=utf-8")
|
|
988
|
+
else:
|
|
989
|
+
body = str(spec.get("text", default_text) or default_text).encode("utf-8")
|
|
990
|
+
content_type = str(spec.get("content_type") or default_content_type)
|
|
991
|
+
return {
|
|
992
|
+
"status": status,
|
|
993
|
+
"content_type": content_type,
|
|
994
|
+
"body": body,
|
|
995
|
+
"headers": headers,
|
|
996
|
+
"cookies": cookies,
|
|
997
|
+
}
|
|
998
|
+
|
|
999
|
+
|
|
1000
|
+
def discover_api_routes() -> List[dict]:
|
|
1001
|
+
return discover_twm_api_handlers()
|
|
1002
|
+
|
|
1003
|
+
|
|
1004
|
+
def discover_twm_api_handlers() -> List[dict]:
|
|
1005
|
+
"""
|
|
1006
|
+
Discover folder-based route handlers as server-side TW script API handlers.
|
|
1007
|
+
|
|
1008
|
+
Contract:
|
|
1009
|
+
- `get(request)`, `post(request)`, ... or `handler(request)`
|
|
1010
|
+
- No top-level statements allowed in `.twm` (enforced by parser)
|
|
1011
|
+
"""
|
|
1012
|
+
routes = []
|
|
1013
|
+
if not os.path.isdir(compiler.API_DIR):
|
|
1014
|
+
return routes
|
|
1015
|
+
for root, _, files in os.walk(compiler.API_DIR):
|
|
1016
|
+
rel_dir = os.path.relpath(root, compiler.API_DIR)
|
|
1017
|
+
rel_dir = compiler.normalize_route_directory(rel_dir)
|
|
1018
|
+
for filename in sorted(files):
|
|
1019
|
+
if filename != "route.twm":
|
|
1020
|
+
continue
|
|
1021
|
+
segments = ["api"]
|
|
1022
|
+
if rel_dir and rel_dir != ".":
|
|
1023
|
+
segments.extend(rel_dir.split(os.sep))
|
|
1024
|
+
route_path = "/" + "/".join(filter(None, segments))
|
|
1025
|
+
routes.append({"path": os.path.join(root, filename), "route": route_path, "lang": "twm"})
|
|
1026
|
+
return routes
|
|
1027
|
+
|
|
1028
|
+
|
|
1029
|
+
def _compile_twm_api_handler_to_cache(handler_path: str) -> str:
|
|
1030
|
+
cache_dir = os.path.join(compiler.CACHE_DIR, "twm_api")
|
|
1031
|
+
os.makedirs(cache_dir, exist_ok=True)
|
|
1032
|
+
with open(handler_path, "r", encoding="utf-8") as f:
|
|
1033
|
+
src = f.read()
|
|
1034
|
+
digest = content_hash(f"{handler_path}::{src}", length=16)
|
|
1035
|
+
out_path = os.path.join(cache_dir, f"{digest}.cjs")
|
|
1036
|
+
if not os.path.isfile(out_path):
|
|
1037
|
+
js = compile_twm_module_to_cjs(src, module_id=handler_path)
|
|
1038
|
+
with open(out_path, "w", encoding="utf-8") as f:
|
|
1039
|
+
f.write(js)
|
|
1040
|
+
return out_path
|
|
1041
|
+
|
|
1042
|
+
|
|
1043
|
+
def execute_twm_api_handler(handler_path: str, method: str, url_path: str,
|
|
1044
|
+
headers: Dict[str, str], body: object) -> dict:
|
|
1045
|
+
"""
|
|
1046
|
+
Execute a `.twm` API handler (server-side TW scripting) via Node.js.
|
|
1047
|
+
|
|
1048
|
+
The `.twm` module should export one of:
|
|
1049
|
+
- `get/post/put/patch/delete/options(request)`
|
|
1050
|
+
- or a generic `handler(request)`
|
|
1051
|
+
|
|
1052
|
+
Response shapes (JS):
|
|
1053
|
+
- string => text/plain
|
|
1054
|
+
- object => JSON (unless it includes {json|text|html|body,status,headers,cookies})
|
|
1055
|
+
- array => [body, status] or [body, status, headers]
|
|
1056
|
+
"""
|
|
1057
|
+
compiled = _compile_twm_api_handler_to_cache(handler_path)
|
|
1058
|
+
runner = os.path.join(SCRIPT_DIR, "twm_api_runner.js")
|
|
1059
|
+
if not os.path.isfile(runner):
|
|
1060
|
+
raise RuntimeError("Missing twm_api_runner.js (framework installation is incomplete).")
|
|
1061
|
+
|
|
1062
|
+
query = urllib.parse.parse_qs(urllib.parse.urlparse(url_path).query, keep_blank_values=True)
|
|
1063
|
+
query = {k: v[0] if len(v) == 1 else v for k, v in query.items()}
|
|
1064
|
+
cookies = parse_cookie_header(headers.get("Cookie", ""))
|
|
1065
|
+
project_root = os.path.abspath(getattr(compiler, "PROJECT_ROOT", os.getcwd()) or os.getcwd())
|
|
1066
|
+
request = {
|
|
1067
|
+
"method": method.upper(),
|
|
1068
|
+
"path": normalize_url_path(url_path),
|
|
1069
|
+
"query": query,
|
|
1070
|
+
"body": body,
|
|
1071
|
+
"headers": headers,
|
|
1072
|
+
"cookies": cookies,
|
|
1073
|
+
"env": dict(os.environ),
|
|
1074
|
+
"project_root": project_root,
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
timeout_s = float(os.environ.get("TW_TWM_TIMEOUT", "10") or "10")
|
|
1078
|
+
proc = subprocess.run(
|
|
1079
|
+
["node", runner, compiled],
|
|
1080
|
+
input=json.dumps(request, ensure_ascii=False),
|
|
1081
|
+
text=True,
|
|
1082
|
+
capture_output=True,
|
|
1083
|
+
timeout=max(1.0, timeout_s),
|
|
1084
|
+
cwd=project_root,
|
|
1085
|
+
)
|
|
1086
|
+
if proc.returncode != 0:
|
|
1087
|
+
raise RuntimeError(f"TWM handler failed (exit={proc.returncode}): {proc.stderr.strip() or proc.stdout.strip()}")
|
|
1088
|
+
|
|
1089
|
+
try:
|
|
1090
|
+
resp = json.loads(proc.stdout or "{}")
|
|
1091
|
+
except json.JSONDecodeError as err:
|
|
1092
|
+
raise RuntimeError(f"TWM runner returned invalid JSON: {err}: {proc.stdout[:2000]!r}") from err
|
|
1093
|
+
|
|
1094
|
+
status = int(resp.get("status", 200) or 200)
|
|
1095
|
+
content_type = str(resp.get("content_type") or "application/json; charset=utf-8")
|
|
1096
|
+
headers_out = resp.get("headers") or []
|
|
1097
|
+
cookies_out = resp.get("cookies") or []
|
|
1098
|
+
body_val = resp.get("body", "")
|
|
1099
|
+
|
|
1100
|
+
if isinstance(body_val, (dict, list)):
|
|
1101
|
+
body_bytes = json.dumps(body_val, ensure_ascii=False).encode("utf-8")
|
|
1102
|
+
content_type = "application/json; charset=utf-8"
|
|
1103
|
+
else:
|
|
1104
|
+
body_bytes = str(body_val).encode("utf-8")
|
|
1105
|
+
|
|
1106
|
+
return {
|
|
1107
|
+
"status": status,
|
|
1108
|
+
"content_type": content_type,
|
|
1109
|
+
"body": body_bytes,
|
|
1110
|
+
"headers": list(headers_out),
|
|
1111
|
+
"cookies": list(cookies_out),
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
|
|
1115
|
+
def parse_api_route_file(tw_path: str) -> Dict[str, dict]:
|
|
1116
|
+
tokens = compiler.tokenize_tw(compiler.read_text_file(tw_path))
|
|
1117
|
+
methods = {}
|
|
1118
|
+
i = 0
|
|
1119
|
+
while i < len(tokens):
|
|
1120
|
+
token = tokens[i]
|
|
1121
|
+
if compiler.is_statement_separator(token):
|
|
1122
|
+
i += 1
|
|
1123
|
+
continue
|
|
1124
|
+
if token.type != "WORD":
|
|
1125
|
+
i += 1
|
|
1126
|
+
continue
|
|
1127
|
+
method = token.value.upper()
|
|
1128
|
+
if method not in {"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"}:
|
|
1129
|
+
i += 1
|
|
1130
|
+
continue
|
|
1131
|
+
i += 1
|
|
1132
|
+
while i < len(tokens) and compiler.is_statement_separator(tokens[i]):
|
|
1133
|
+
i += 1
|
|
1134
|
+
if i >= len(tokens) or tokens[i].type != "BRACE" or tokens[i].value != "{":
|
|
1135
|
+
raise RuntimeError(f"Expected `{{` after `{method}` in {tw_path}")
|
|
1136
|
+
i += 1
|
|
1137
|
+
spec = {"status": 200, "headers": [], "cookies": []}
|
|
1138
|
+
while i < len(tokens):
|
|
1139
|
+
tok = tokens[i]
|
|
1140
|
+
if compiler.is_statement_separator(tok):
|
|
1141
|
+
i += 1
|
|
1142
|
+
continue
|
|
1143
|
+
if tok.type == "BRACE" and tok.value == "}":
|
|
1144
|
+
i += 1
|
|
1145
|
+
break
|
|
1146
|
+
if tok.type != "WORD":
|
|
1147
|
+
raise RuntimeError(f"Invalid API directive in {tw_path}")
|
|
1148
|
+
key = tok.value
|
|
1149
|
+
i += 1
|
|
1150
|
+
if key == "status":
|
|
1151
|
+
raw_status, i = parse_value_token(tokens, i)
|
|
1152
|
+
spec["status"] = int(raw_status)
|
|
1153
|
+
elif key in {"json", "text", "html", "redirect"}:
|
|
1154
|
+
spec[key], i = parse_value_token(tokens, i)
|
|
1155
|
+
elif key == "header":
|
|
1156
|
+
header_name, i = parse_value_token(tokens, i)
|
|
1157
|
+
header_value, i = parse_value_token(tokens, i)
|
|
1158
|
+
spec["headers"].append((header_name, header_value))
|
|
1159
|
+
elif key == "cookie":
|
|
1160
|
+
cookie_name, i = parse_value_token(tokens, i)
|
|
1161
|
+
cookie_value, i = parse_value_token(tokens, i)
|
|
1162
|
+
spec["cookies"].append((cookie_name, cookie_value))
|
|
1163
|
+
else:
|
|
1164
|
+
raise RuntimeError(f"Unsupported API key `{key}` in {tw_path}")
|
|
1165
|
+
methods[method] = spec
|
|
1166
|
+
return methods
|
|
1167
|
+
|
|
1168
|
+
|
|
1169
|
+
def render_api_value(value, context):
|
|
1170
|
+
def render_nested(item):
|
|
1171
|
+
if isinstance(item, dict):
|
|
1172
|
+
return {key: render_nested(val) for key, val in item.items()}
|
|
1173
|
+
if isinstance(item, list):
|
|
1174
|
+
return [render_nested(val) for val in item]
|
|
1175
|
+
if isinstance(item, str):
|
|
1176
|
+
return compiler.parse_literal_value(compiler.interpolate(item, context))
|
|
1177
|
+
return item
|
|
1178
|
+
|
|
1179
|
+
if isinstance(value, str):
|
|
1180
|
+
stripped = value.strip()
|
|
1181
|
+
if (stripped.startswith("{") and stripped.endswith("}")) or (stripped.startswith("[") and stripped.endswith("]")):
|
|
1182
|
+
try:
|
|
1183
|
+
return render_nested(json.loads(stripped))
|
|
1184
|
+
except json.JSONDecodeError:
|
|
1185
|
+
pass
|
|
1186
|
+
rendered = compiler.interpolate(str(value), context)
|
|
1187
|
+
return compiler.parse_literal_value(rendered)
|
|
1188
|
+
|
|
1189
|
+
|
|
1190
|
+
def decode_request_body(handler) -> object:
|
|
1191
|
+
length = int(handler.headers.get("Content-Length", "0") or "0")
|
|
1192
|
+
if length <= 0:
|
|
1193
|
+
return {}
|
|
1194
|
+
raw = handler.rfile.read(length)
|
|
1195
|
+
content_type = handler.headers.get("Content-Type", "")
|
|
1196
|
+
if "application/json" in content_type:
|
|
1197
|
+
try:
|
|
1198
|
+
return json.loads(raw.decode("utf-8"))
|
|
1199
|
+
except (json.JSONDecodeError, UnicodeDecodeError):
|
|
1200
|
+
return {"raw": raw.decode("utf-8", errors="replace")}
|
|
1201
|
+
if "application/x-www-form-urlencoded" in content_type:
|
|
1202
|
+
parsed = urllib.parse.parse_qs(raw.decode("utf-8"), keep_blank_values=True)
|
|
1203
|
+
return {key: value[0] if len(value) == 1 else value for key, value in parsed.items()}
|
|
1204
|
+
return {"raw": raw.decode("utf-8", errors="replace"), "bytes": len(raw)}
|
|
1205
|
+
|
|
1206
|
+
|
|
1207
|
+
class TWProject:
|
|
1208
|
+
def __init__(self, project_root: str):
|
|
1209
|
+
self.project_root = os.path.abspath(project_root)
|
|
1210
|
+
self._lock = threading.RLock()
|
|
1211
|
+
configure_compiler_paths(self.project_root)
|
|
1212
|
+
self.env = load_project_env(self.project_root, "development")
|
|
1213
|
+
self.config = compiler.load_config()
|
|
1214
|
+
self.modular_pipeline = use_modular_pipeline(self.config)
|
|
1215
|
+
self.extensions = ExtensionManager(self.project_root, self.config, self.env).refresh()
|
|
1216
|
+
self._rate_limiters: Dict[str, TokenBucketRateLimiter] = {}
|
|
1217
|
+
|
|
1218
|
+
@property
|
|
1219
|
+
def source_root(self) -> str:
|
|
1220
|
+
return compiler.HOME_DIR
|
|
1221
|
+
|
|
1222
|
+
def list_source_files(self) -> List[str]:
|
|
1223
|
+
files = []
|
|
1224
|
+
if os.path.exists(self.project_root):
|
|
1225
|
+
config_path = compiler.CONFIG_FILE
|
|
1226
|
+
if os.path.exists(config_path):
|
|
1227
|
+
files.append(config_path)
|
|
1228
|
+
|
|
1229
|
+
for root in [compiler.HOME_DIR]:
|
|
1230
|
+
if not os.path.exists(root):
|
|
1231
|
+
continue
|
|
1232
|
+
for dirpath, _, filenames in os.walk(root):
|
|
1233
|
+
for filename in filenames:
|
|
1234
|
+
ext = os.path.splitext(filename)[1].lower()
|
|
1235
|
+
if ext in WATCH_EXTENSIONS:
|
|
1236
|
+
files.append(os.path.join(dirpath, filename))
|
|
1237
|
+
return sorted(set(os.path.abspath(p) for p in files))
|
|
1238
|
+
|
|
1239
|
+
def source_signature(self) -> str:
|
|
1240
|
+
digest = hashlib.sha1()
|
|
1241
|
+
for path in self.list_source_files():
|
|
1242
|
+
stat = os.stat(path)
|
|
1243
|
+
digest.update(path.encode("utf-8"))
|
|
1244
|
+
digest.update(str(stat.st_mtime_ns).encode("utf-8"))
|
|
1245
|
+
digest.update(str(stat.st_size).encode("utf-8"))
|
|
1246
|
+
return digest.hexdigest()
|
|
1247
|
+
|
|
1248
|
+
def invalidate(self):
|
|
1249
|
+
invalidate_compiler_caches()
|
|
1250
|
+
self.env = load_project_env(self.project_root, "development")
|
|
1251
|
+
self.config = compiler.load_config()
|
|
1252
|
+
self.modular_pipeline = use_modular_pipeline(self.config)
|
|
1253
|
+
self.extensions.refresh(self.config, self.env)
|
|
1254
|
+
|
|
1255
|
+
def discover_pages(self) -> List[dict]:
|
|
1256
|
+
self.invalidate()
|
|
1257
|
+
return compiler.discover_pages()
|
|
1258
|
+
|
|
1259
|
+
def find_special_page(self, status_code: int) -> Optional[dict]:
|
|
1260
|
+
expected_name = special_page_name_for_status(status_code)
|
|
1261
|
+
for page in self.discover_pages():
|
|
1262
|
+
if page["type"] == "static" and page.get("name") == expected_name:
|
|
1263
|
+
return page
|
|
1264
|
+
return None
|
|
1265
|
+
|
|
1266
|
+
def resolve_asset(self, url_path: str) -> Optional[Tuple[bytes, str]]:
|
|
1267
|
+
path = normalize_url_path(url_path)
|
|
1268
|
+
|
|
1269
|
+
if path == "/_tw/search-index.json":
|
|
1270
|
+
payload = json.dumps(self.build_dev_search_index(), ensure_ascii=False, indent=2).encode("utf-8")
|
|
1271
|
+
return payload, "application/json; charset=utf-8"
|
|
1272
|
+
|
|
1273
|
+
if path.startswith("/assets/"):
|
|
1274
|
+
candidate = os.path.abspath(os.path.join(compiler.ASSETS_DIR, path[len("/assets/"):]))
|
|
1275
|
+
if is_path_within(compiler.ASSETS_DIR, candidate):
|
|
1276
|
+
payload = safe_read_binary(candidate)
|
|
1277
|
+
if payload is not None:
|
|
1278
|
+
content_type = mimetypes.guess_type(candidate)[0] or "application/octet-stream"
|
|
1279
|
+
return payload, content_type
|
|
1280
|
+
|
|
1281
|
+
if path.startswith("/_tw/static/chunks/"):
|
|
1282
|
+
candidate = os.path.abspath(os.path.join(compiler.CHUNKS_DIR, path.split("/_tw/static/chunks/", 1)[1]))
|
|
1283
|
+
if is_path_within(compiler.CHUNKS_DIR, candidate):
|
|
1284
|
+
payload = safe_read_binary(candidate)
|
|
1285
|
+
if payload is not None:
|
|
1286
|
+
content_type = mimetypes.guess_type(candidate)[0] or "application/javascript"
|
|
1287
|
+
return payload, content_type
|
|
1288
|
+
|
|
1289
|
+
return None
|
|
1290
|
+
|
|
1291
|
+
def build_dev_search_index(self) -> List[dict]:
|
|
1292
|
+
# Dev-only: compile current static pages and extract searchable text.
|
|
1293
|
+
items = []
|
|
1294
|
+
for page in self.discover_pages():
|
|
1295
|
+
if page["type"] != "static":
|
|
1296
|
+
continue
|
|
1297
|
+
route = route_from_static_page(page)
|
|
1298
|
+
match = RouteMatch(page_info=page, params={}, item=None, route_path=route or "/")
|
|
1299
|
+
html_text = self.compile_match_to_html(match, dev_mode=False)
|
|
1300
|
+
text = strip_html_to_text(html_text)
|
|
1301
|
+
if not text:
|
|
1302
|
+
continue
|
|
1303
|
+
page_ast = compiler.load_page_ast_from_file(page["path"])
|
|
1304
|
+
title = page_ast.title or route
|
|
1305
|
+
items.append({
|
|
1306
|
+
"route": route,
|
|
1307
|
+
"title": title,
|
|
1308
|
+
"excerpt": text[:240],
|
|
1309
|
+
"content": text[:4000],
|
|
1310
|
+
})
|
|
1311
|
+
return items
|
|
1312
|
+
|
|
1313
|
+
def resolve_route(self, raw_path: str) -> Optional[RouteMatch]:
|
|
1314
|
+
path = strip_trailing_slash(normalize_url_path(raw_path))
|
|
1315
|
+
pages = self.discover_pages()
|
|
1316
|
+
|
|
1317
|
+
for page in pages:
|
|
1318
|
+
if page["type"] == "static":
|
|
1319
|
+
route = strip_trailing_slash(route_from_static_page(page))
|
|
1320
|
+
html_route = route + ".html" if route != "/" else "/index.html"
|
|
1321
|
+
pretty_html_route = (route + "/index.html") if route != "/" else "/index.html"
|
|
1322
|
+
if path in {route, html_route, strip_trailing_slash(pretty_html_route)}:
|
|
1323
|
+
return RouteMatch(page_info=page, params={}, item=None, route_path=route or "/")
|
|
1324
|
+
continue
|
|
1325
|
+
|
|
1326
|
+
items = compiler.load_dynamic_items(page["path"])
|
|
1327
|
+
for item in items:
|
|
1328
|
+
if not isinstance(item, dict):
|
|
1329
|
+
continue
|
|
1330
|
+
route = strip_trailing_slash(route_from_dynamic_page(page, item))
|
|
1331
|
+
html_route = route + "/index.html" if route != "/" else "/index.html"
|
|
1332
|
+
if path in {route, strip_trailing_slash(html_route)}:
|
|
1333
|
+
param = page["param"]
|
|
1334
|
+
segments = compiler.resolve_dynamic_segments(page, item)
|
|
1335
|
+
params = {param: "/".join(segments)}
|
|
1336
|
+
if page.get("route_kind") != "single":
|
|
1337
|
+
params[param + "Segments"] = segments
|
|
1338
|
+
return RouteMatch(page_info=page, params=params, item=item, route_path=route or "/")
|
|
1339
|
+
|
|
1340
|
+
return None
|
|
1341
|
+
|
|
1342
|
+
def _get_rate_limiter(self, rule: dict) -> Optional[TokenBucketRateLimiter]:
|
|
1343
|
+
rate_limit = rule.get("rate_limit")
|
|
1344
|
+
if not rate_limit:
|
|
1345
|
+
return None
|
|
1346
|
+
limiter_key = f"{rule.get('match', '/**')}::{rate_limit['requests']}::{rate_limit['window']}"
|
|
1347
|
+
limiter = self._rate_limiters.get(limiter_key)
|
|
1348
|
+
if limiter is None:
|
|
1349
|
+
limiter = TokenBucketRateLimiter(rate_limit["requests"], rate_limit["window"])
|
|
1350
|
+
self._rate_limiters[limiter_key] = limiter
|
|
1351
|
+
return limiter
|
|
1352
|
+
|
|
1353
|
+
def _resolve_rate_limit_identity(self, request_headers: Dict[str, str], request_meta: Optional[Dict[str, str]] = None) -> str:
|
|
1354
|
+
request_meta = request_meta or {}
|
|
1355
|
+
forwarded_for = str(request_headers.get("X-Forwarded-For", "")).split(",", 1)[0].strip()
|
|
1356
|
+
return (
|
|
1357
|
+
request_meta.get("client_ip")
|
|
1358
|
+
or forwarded_for
|
|
1359
|
+
or str(request_headers.get("X-Real-IP", "")).strip()
|
|
1360
|
+
or "anonymous"
|
|
1361
|
+
)
|
|
1362
|
+
|
|
1363
|
+
def apply_middleware(
|
|
1364
|
+
self,
|
|
1365
|
+
raw_path: str,
|
|
1366
|
+
request_headers: Optional[Dict[str, str]] = None,
|
|
1367
|
+
request_meta: Optional[Dict[str, str]] = None,
|
|
1368
|
+
method: str = "GET",
|
|
1369
|
+
) -> dict:
|
|
1370
|
+
path = normalize_url_path(raw_path)
|
|
1371
|
+
request_headers = request_headers or {}
|
|
1372
|
+
method = str(method or "GET").upper()
|
|
1373
|
+
cookies = parse_cookie_header(_middleware_header(request_headers, "Cookie", ""))
|
|
1374
|
+
result = {
|
|
1375
|
+
"path": path,
|
|
1376
|
+
"headers": list(get_security_headers(self.config)),
|
|
1377
|
+
"cookies": [],
|
|
1378
|
+
"redirect": None,
|
|
1379
|
+
"response": None,
|
|
1380
|
+
}
|
|
1381
|
+
|
|
1382
|
+
for rule in parse_middleware_rules(self.project_root):
|
|
1383
|
+
if not match_path_pattern(path, str(rule.get("match", "/**"))):
|
|
1384
|
+
continue
|
|
1385
|
+
allowed_methods = [str(item).upper() for item in rule.get("methods", [])]
|
|
1386
|
+
if allowed_methods and method not in allowed_methods:
|
|
1387
|
+
continue
|
|
1388
|
+
|
|
1389
|
+
# Modern rule engine for root `middleware.tw`
|
|
1390
|
+
path_rule = rule.get("path_rule") or {}
|
|
1391
|
+
if path_rule:
|
|
1392
|
+
url_lower = path.lower()
|
|
1393
|
+
block_match = False
|
|
1394
|
+
if path_rule.get("deny_traversal", True) and ".." in path:
|
|
1395
|
+
block_match = True
|
|
1396
|
+
if path_rule.get("deny_null_bytes", True) and "%00" in url_lower:
|
|
1397
|
+
block_match = True
|
|
1398
|
+
for prefix in _middleware_list(path_rule.get("prefixes")):
|
|
1399
|
+
if url_lower.startswith(str(prefix).lower()):
|
|
1400
|
+
block_match = True
|
|
1401
|
+
break
|
|
1402
|
+
if not block_match:
|
|
1403
|
+
for part in _middleware_list(path_rule.get("contains")):
|
|
1404
|
+
if str(part).lower() in url_lower:
|
|
1405
|
+
block_match = True
|
|
1406
|
+
break
|
|
1407
|
+
if not block_match:
|
|
1408
|
+
for ext in _middleware_list(path_rule.get("extensions")):
|
|
1409
|
+
if url_lower.endswith(str(ext).lower()):
|
|
1410
|
+
block_match = True
|
|
1411
|
+
break
|
|
1412
|
+
if not block_match:
|
|
1413
|
+
for pattern in _middleware_list(path_rule.get("regex")):
|
|
1414
|
+
if re.search(str(pattern), path):
|
|
1415
|
+
block_match = True
|
|
1416
|
+
break
|
|
1417
|
+
single_segment_max = int(path_rule.get("single_segment_max", 0) or 0)
|
|
1418
|
+
if not block_match and single_segment_max > 0:
|
|
1419
|
+
m = re.match(r"^/([^/?#]+)$", path)
|
|
1420
|
+
if m and len(m.group(1)) > single_segment_max:
|
|
1421
|
+
block_match = True
|
|
1422
|
+
if block_match:
|
|
1423
|
+
result["response"] = _build_middleware_response(rule, result, default_status=404, default_text="Not Found")
|
|
1424
|
+
return result
|
|
1425
|
+
|
|
1426
|
+
user_agent_rule = rule.get("user_agent") or {}
|
|
1427
|
+
if user_agent_rule:
|
|
1428
|
+
ua = _middleware_header(request_headers, "User-Agent", "")
|
|
1429
|
+
ua_lower = ua.lower()
|
|
1430
|
+
allow_patterns = [item.lower() for item in _middleware_list(user_agent_rule.get("allow"))]
|
|
1431
|
+
block_patterns = [item.lower() for item in _middleware_list(user_agent_rule.get("block"))]
|
|
1432
|
+
exempt = bool(ua and any(pattern in ua_lower for pattern in allow_patterns))
|
|
1433
|
+
blocked = (not ua and bool(user_agent_rule.get("empty_is_blocked"))) or (
|
|
1434
|
+
(not exempt) and any(pattern in ua_lower for pattern in block_patterns)
|
|
1435
|
+
)
|
|
1436
|
+
if blocked:
|
|
1437
|
+
result["response"] = _build_middleware_response(rule, result, default_status=403, default_text="Forbidden")
|
|
1438
|
+
return result
|
|
1439
|
+
|
|
1440
|
+
origin_rule = rule.get("origin") or {}
|
|
1441
|
+
if origin_rule:
|
|
1442
|
+
allowed = {str(item) for item in _middleware_list(origin_rule.get("allow"))}
|
|
1443
|
+
origin = _middleware_header(request_headers, "Origin", "")
|
|
1444
|
+
referer_origin = _middleware_referer_origin(_middleware_header(request_headers, "Referer", ""))
|
|
1445
|
+
require_origin = bool(origin_rule.get("require"))
|
|
1446
|
+
allow_referer = bool(origin_rule.get("allow_referer", True))
|
|
1447
|
+
is_allowed = False
|
|
1448
|
+
if origin and origin in allowed:
|
|
1449
|
+
is_allowed = True
|
|
1450
|
+
elif allow_referer and referer_origin and referer_origin in allowed:
|
|
1451
|
+
is_allowed = True
|
|
1452
|
+
elif not origin and not referer_origin and not require_origin:
|
|
1453
|
+
is_allowed = True
|
|
1454
|
+
if allowed and not is_allowed:
|
|
1455
|
+
result["response"] = _build_middleware_response(rule, result, default_status=403, default_text="Access Denied")
|
|
1456
|
+
return result
|
|
1457
|
+
|
|
1458
|
+
auth_rule = rule.get("auth_rule") or {}
|
|
1459
|
+
if auth_rule:
|
|
1460
|
+
cookie_name = str(auth_rule.get("cookie") or "").strip()
|
|
1461
|
+
if cookie_name:
|
|
1462
|
+
token = cookies.get(cookie_name, "")
|
|
1463
|
+
if not token and bool(auth_rule.get("required", True)):
|
|
1464
|
+
result["response"] = _build_middleware_response(rule, result, default_status=401, default_text="Unauthorized")
|
|
1465
|
+
return result
|
|
1466
|
+
if token:
|
|
1467
|
+
secret = str(auth_rule.get("jwt_secret") or "")
|
|
1468
|
+
secret_env = str(auth_rule.get("jwt_secret_env") or "")
|
|
1469
|
+
if secret_env:
|
|
1470
|
+
secret = os.environ.get(secret_env, secret)
|
|
1471
|
+
if secret and not _verify_hs256_jwt(token, secret):
|
|
1472
|
+
result["response"] = _build_middleware_response(rule, result, default_status=401, default_text="Invalid token")
|
|
1473
|
+
return result
|
|
1474
|
+
|
|
1475
|
+
auth = rule.get("auth")
|
|
1476
|
+
if auth and not cookies.get(auth["cookie"]):
|
|
1477
|
+
result["redirect"] = auth["redirect"]
|
|
1478
|
+
return result
|
|
1479
|
+
limiter = self._get_rate_limiter(rule)
|
|
1480
|
+
if limiter is not None:
|
|
1481
|
+
identity = self._resolve_rate_limit_identity(request_headers, request_meta=request_meta)
|
|
1482
|
+
if str(rule["rate_limit"].get("identity", "ip")).lower() == "path":
|
|
1483
|
+
bucket_segments = max(1, int(rule["rate_limit"].get("bucket_segments", 2) or 2))
|
|
1484
|
+
bucket = "/".join(path.split("/")[: bucket_segments + 1]) or "/"
|
|
1485
|
+
identity = f"{identity}::{bucket}"
|
|
1486
|
+
if not limiter.allow(f"{rule.get('match', '/**')}::{identity}"):
|
|
1487
|
+
result["response"] = _build_middleware_response(
|
|
1488
|
+
rule,
|
|
1489
|
+
result,
|
|
1490
|
+
default_status=429,
|
|
1491
|
+
default_text="Too Many Requests",
|
|
1492
|
+
extra_headers=[("Retry-After", str(int(rule["rate_limit"]["window"])))],
|
|
1493
|
+
)
|
|
1494
|
+
return result
|
|
1495
|
+
deny = rule.get("deny")
|
|
1496
|
+
if deny:
|
|
1497
|
+
body = str(deny.get("body", "Forbidden") or "Forbidden").encode("utf-8")
|
|
1498
|
+
result["response"] = {
|
|
1499
|
+
"status": int(deny.get("status", 403) or 403),
|
|
1500
|
+
"content_type": "text/plain; charset=utf-8",
|
|
1501
|
+
"body": body,
|
|
1502
|
+
"headers": list(result["headers"]),
|
|
1503
|
+
"cookies": list(result["cookies"]),
|
|
1504
|
+
}
|
|
1505
|
+
return result
|
|
1506
|
+
if rule.get("response") and not any(
|
|
1507
|
+
rule.get(key)
|
|
1508
|
+
for key in ["user_agent", "origin", "path_rule", "auth_rule", "rate_limit", "deny", "redirect", "rewrite", "auth"]
|
|
1509
|
+
):
|
|
1510
|
+
result["response"] = _build_middleware_response(rule, result, default_status=403, default_text="Forbidden")
|
|
1511
|
+
return result
|
|
1512
|
+
if rule.get("rewrite"):
|
|
1513
|
+
result["path"] = normalize_url_path(rule["rewrite"])
|
|
1514
|
+
if rule.get("redirect"):
|
|
1515
|
+
result["redirect"] = rule["redirect"]
|
|
1516
|
+
return result
|
|
1517
|
+
result["headers"].extend(rule.get("headers", []))
|
|
1518
|
+
result["cookies"].extend(rule.get("cookies", []))
|
|
1519
|
+
|
|
1520
|
+
return result
|
|
1521
|
+
|
|
1522
|
+
def resolve_api_route(self, raw_path: str) -> Optional[dict]:
|
|
1523
|
+
path = strip_trailing_slash(normalize_url_path(raw_path))
|
|
1524
|
+
for route in discover_twm_api_handlers():
|
|
1525
|
+
if strip_trailing_slash(route["route"]) == path:
|
|
1526
|
+
return route
|
|
1527
|
+
return None
|
|
1528
|
+
|
|
1529
|
+
def execute_api_route(self, api_route: dict, method: str, url_path: str, headers: Dict[str, str], body: object) -> dict:
|
|
1530
|
+
if api_route.get("lang") == "twm":
|
|
1531
|
+
return execute_twm_api_handler(api_route["path"], method, url_path, headers, body)
|
|
1532
|
+
methods = parse_api_route_file(api_route["path"])
|
|
1533
|
+
spec = methods.get(method.upper())
|
|
1534
|
+
if not spec:
|
|
1535
|
+
allowed = ", ".join(sorted(methods))
|
|
1536
|
+
payload = json.dumps({"error": "Method not allowed", "allowed": sorted(methods)})
|
|
1537
|
+
return {
|
|
1538
|
+
"status": 405,
|
|
1539
|
+
"content_type": "application/json; charset=utf-8",
|
|
1540
|
+
"body": payload.encode("utf-8"),
|
|
1541
|
+
"headers": [("Allow", allowed)],
|
|
1542
|
+
"cookies": [],
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1545
|
+
query = urllib.parse.parse_qs(urllib.parse.urlparse(url_path).query, keep_blank_values=True)
|
|
1546
|
+
query = {key: value[0] if len(value) == 1 else value for key, value in query.items()}
|
|
1547
|
+
cookies = parse_cookie_header(headers.get("Cookie", ""))
|
|
1548
|
+
context = {
|
|
1549
|
+
"env": dict(os.environ),
|
|
1550
|
+
"query": query,
|
|
1551
|
+
"body": body,
|
|
1552
|
+
"headers": headers,
|
|
1553
|
+
"cookies": cookies,
|
|
1554
|
+
"request": {
|
|
1555
|
+
"method": method.upper(),
|
|
1556
|
+
"path": normalize_url_path(url_path),
|
|
1557
|
+
"query": query,
|
|
1558
|
+
"body": body,
|
|
1559
|
+
"headers": headers,
|
|
1560
|
+
"cookies": cookies,
|
|
1561
|
+
},
|
|
1562
|
+
}
|
|
1563
|
+
context["generate_csrf_token"] = lambda: generate_csrf_token(context["request"])
|
|
1564
|
+
context["verify_csrf_token"] = lambda token, max_age=7200: verify_csrf_token(token, context["request"], max_age=max_age)
|
|
1565
|
+
|
|
1566
|
+
response_headers = list(spec.get("headers", []))
|
|
1567
|
+
response_cookies = list(spec.get("cookies", []))
|
|
1568
|
+
if "redirect" in spec:
|
|
1569
|
+
location = str(render_api_value(spec["redirect"], context))
|
|
1570
|
+
return {
|
|
1571
|
+
"status": int(spec.get("status", 302) or 302),
|
|
1572
|
+
"content_type": "text/plain; charset=utf-8",
|
|
1573
|
+
"body": f"Redirecting to {location}".encode("utf-8"),
|
|
1574
|
+
"headers": response_headers + [("Location", location)],
|
|
1575
|
+
"cookies": response_cookies,
|
|
1576
|
+
}
|
|
1577
|
+
if "json" in spec:
|
|
1578
|
+
payload = render_api_value(spec["json"], context)
|
|
1579
|
+
body_bytes = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
|
1580
|
+
return {
|
|
1581
|
+
"status": int(spec.get("status", 200)),
|
|
1582
|
+
"content_type": "application/json; charset=utf-8",
|
|
1583
|
+
"body": body_bytes,
|
|
1584
|
+
"headers": response_headers,
|
|
1585
|
+
"cookies": response_cookies,
|
|
1586
|
+
}
|
|
1587
|
+
if "html" in spec:
|
|
1588
|
+
payload = str(render_api_value(spec["html"], context))
|
|
1589
|
+
return {
|
|
1590
|
+
"status": int(spec.get("status", 200)),
|
|
1591
|
+
"content_type": "text/html; charset=utf-8",
|
|
1592
|
+
"body": payload.encode("utf-8"),
|
|
1593
|
+
"headers": response_headers,
|
|
1594
|
+
"cookies": response_cookies,
|
|
1595
|
+
}
|
|
1596
|
+
|
|
1597
|
+
payload = str(render_api_value(spec.get("text", ""), context))
|
|
1598
|
+
return {
|
|
1599
|
+
"status": int(spec.get("status", 200)),
|
|
1600
|
+
"content_type": "text/plain; charset=utf-8",
|
|
1601
|
+
"body": payload.encode("utf-8"),
|
|
1602
|
+
"headers": response_headers,
|
|
1603
|
+
"cookies": response_cookies,
|
|
1604
|
+
}
|
|
1605
|
+
|
|
1606
|
+
def compile_match_response(self, match: RouteMatch, dev_mode: bool = False, depth: int = 0) -> dict:
|
|
1607
|
+
if depth > 5:
|
|
1608
|
+
raise RuntimeError("Rewrite loop detected")
|
|
1609
|
+
|
|
1610
|
+
page_info = match.page_info
|
|
1611
|
+
tw_path = page_info["path"]
|
|
1612
|
+
page_ast = compiler.load_page_ast_from_file(tw_path)
|
|
1613
|
+
css_url, _ = compiler.read_global_stylesheet()
|
|
1614
|
+
|
|
1615
|
+
context = compiler.build_page_context(
|
|
1616
|
+
page_info,
|
|
1617
|
+
page_ast,
|
|
1618
|
+
tw_path,
|
|
1619
|
+
item=match.item,
|
|
1620
|
+
route_path=match.route_path,
|
|
1621
|
+
request_params=match.params,
|
|
1622
|
+
)
|
|
1623
|
+
hook_state = self.extensions.emit(
|
|
1624
|
+
"beforeRoute",
|
|
1625
|
+
match=match,
|
|
1626
|
+
page_info=page_info,
|
|
1627
|
+
page_ast=page_ast,
|
|
1628
|
+
context=context,
|
|
1629
|
+
dev_mode=dev_mode,
|
|
1630
|
+
)
|
|
1631
|
+
context = hook_state.get("context", context)
|
|
1632
|
+
|
|
1633
|
+
if page_ast.rewrite_to:
|
|
1634
|
+
rewritten_path = compiler.interpolate(page_ast.rewrite_to, context)
|
|
1635
|
+
rewritten_match = self.resolve_route(rewritten_path)
|
|
1636
|
+
if rewritten_match:
|
|
1637
|
+
rewritten_match = RouteMatch(
|
|
1638
|
+
page_info=rewritten_match.page_info,
|
|
1639
|
+
params=rewritten_match.params,
|
|
1640
|
+
item=rewritten_match.item,
|
|
1641
|
+
route_path=match.route_path,
|
|
1642
|
+
)
|
|
1643
|
+
return self.compile_match_response(rewritten_match, dev_mode=dev_mode, depth=depth + 1)
|
|
1644
|
+
|
|
1645
|
+
if self.modular_pipeline:
|
|
1646
|
+
artifacts = compiler.compile_file_pipeline(
|
|
1647
|
+
tw_path,
|
|
1648
|
+
context=context,
|
|
1649
|
+
css_href=css_url,
|
|
1650
|
+
route_path=match.route_path,
|
|
1651
|
+
)
|
|
1652
|
+
rendered = artifacts.html or ""
|
|
1653
|
+
program = artifacts.program
|
|
1654
|
+
redirect_to = None
|
|
1655
|
+
if program is not None:
|
|
1656
|
+
redirect_to = program.meta.redirect_to or getattr(getattr(program, "legacy_page", None), "redirect_to", None)
|
|
1657
|
+
rewrite_to = program.meta.rewrite_to or getattr(getattr(program, "legacy_page", None), "rewrite_to", None)
|
|
1658
|
+
if rewrite_to:
|
|
1659
|
+
rewritten_path = compiler.interpolate(rewrite_to, context)
|
|
1660
|
+
rewritten_match = self.resolve_route(rewritten_path)
|
|
1661
|
+
if rewritten_match:
|
|
1662
|
+
rewritten_match = RouteMatch(
|
|
1663
|
+
page_info=rewritten_match.page_info,
|
|
1664
|
+
params=rewritten_match.params,
|
|
1665
|
+
item=rewritten_match.item,
|
|
1666
|
+
route_path=match.route_path,
|
|
1667
|
+
)
|
|
1668
|
+
return self.compile_match_response(rewritten_match, dev_mode=dev_mode, depth=depth + 1)
|
|
1669
|
+
status = 302 if redirect_to else 200
|
|
1670
|
+
else:
|
|
1671
|
+
rendered = compiler.render_html(page_ast, context, css_url)
|
|
1672
|
+
redirect_to = page_ast.redirect_to
|
|
1673
|
+
status = 302 if redirect_to else 200
|
|
1674
|
+
headers = []
|
|
1675
|
+
if redirect_to:
|
|
1676
|
+
headers.append(("Location", compiler.interpolate(redirect_to, context)))
|
|
1677
|
+
|
|
1678
|
+
if dev_mode:
|
|
1679
|
+
rendered = inject_dev_client(rendered)
|
|
1680
|
+
response = {"html": rendered, "status": status, "headers": headers}
|
|
1681
|
+
hook_state = self.extensions.emit(
|
|
1682
|
+
"afterRoute",
|
|
1683
|
+
match=match,
|
|
1684
|
+
page_info=page_info,
|
|
1685
|
+
page_ast=page_ast,
|
|
1686
|
+
context=context,
|
|
1687
|
+
response=response,
|
|
1688
|
+
dev_mode=dev_mode,
|
|
1689
|
+
)
|
|
1690
|
+
return hook_state.get("response", response)
|
|
1691
|
+
|
|
1692
|
+
def compile_match_to_html(self, match: RouteMatch, dev_mode: bool = False) -> str:
|
|
1693
|
+
return self.compile_match_response(match, dev_mode=dev_mode)["html"]
|
|
1694
|
+
|
|
1695
|
+
def compile_special_page(self, status_code: int, dev_mode: bool = False) -> Optional[str]:
|
|
1696
|
+
page_info = self.find_special_page(status_code)
|
|
1697
|
+
if not page_info:
|
|
1698
|
+
return None
|
|
1699
|
+
return self.compile_match_to_html(RouteMatch(page_info=page_info, params={}, item=None, route_path=f"/{status_code}"), dev_mode=dev_mode)
|
|
1700
|
+
|
|
1701
|
+
|
|
1702
|
+
class TWDevState:
|
|
1703
|
+
def __init__(self, project: TWProject):
|
|
1704
|
+
self.project = project
|
|
1705
|
+
self.version = 0
|
|
1706
|
+
self.stop_event = threading.Event()
|
|
1707
|
+
self.lock = threading.Lock()
|
|
1708
|
+
|
|
1709
|
+
def bump(self):
|
|
1710
|
+
with self.lock:
|
|
1711
|
+
self.version += 1
|
|
1712
|
+
|
|
1713
|
+
def current_version(self) -> int:
|
|
1714
|
+
with self.lock:
|
|
1715
|
+
return self.version
|
|
1716
|
+
|
|
1717
|
+
|
|
1718
|
+
class TWFileWatcher(threading.Thread):
|
|
1719
|
+
def __init__(self, state: TWDevState, interval: float = 1.0):
|
|
1720
|
+
super().__init__(daemon=True)
|
|
1721
|
+
self.state = state
|
|
1722
|
+
# Allow tuning:
|
|
1723
|
+
# - env: TW_WATCH_INTERVAL
|
|
1724
|
+
# - config: watch_interval
|
|
1725
|
+
configured = None
|
|
1726
|
+
try:
|
|
1727
|
+
configured = float(os.environ.get("TW_WATCH_INTERVAL", "") or "")
|
|
1728
|
+
except (ValueError, TypeError):
|
|
1729
|
+
configured = None
|
|
1730
|
+
if configured is None:
|
|
1731
|
+
try:
|
|
1732
|
+
configured = float(state.project.config.get("watch_interval", state.project.config.get("watchInterval", interval)))
|
|
1733
|
+
except (ValueError, TypeError):
|
|
1734
|
+
configured = None
|
|
1735
|
+
self.interval = max(0.2, min(2.0, configured if configured is not None else interval))
|
|
1736
|
+
# How often to rescan the full file list (for new/deleted files).
|
|
1737
|
+
# Default is fairly aggressive to reduce "new file not detected" latency.
|
|
1738
|
+
rescan_ticks = None
|
|
1739
|
+
try:
|
|
1740
|
+
rescan_ticks = int(os.environ.get("TW_WATCH_RESCAN_TICKS", "") or "")
|
|
1741
|
+
except (ValueError, TypeError):
|
|
1742
|
+
rescan_ticks = None
|
|
1743
|
+
if rescan_ticks is None:
|
|
1744
|
+
try:
|
|
1745
|
+
rescan_ticks = int(state.project.config.get("watch_rescan_ticks", state.project.config.get("watchRescanTicks", 2)))
|
|
1746
|
+
except (ValueError, TypeError):
|
|
1747
|
+
rescan_ticks = None
|
|
1748
|
+
self.rescan_ticks = max(1, int(rescan_ticks if rescan_ticks is not None else 2))
|
|
1749
|
+
|
|
1750
|
+
self._tick = 0
|
|
1751
|
+
self._files = []
|
|
1752
|
+
self._stats = {}
|
|
1753
|
+
self.backend = str(os.environ.get("TW_WATCH_BACKEND", "auto") or "auto").strip().lower()
|
|
1754
|
+
self._inotify_fd: Optional[int] = None
|
|
1755
|
+
self._wd_to_dir: Dict[int, str] = {}
|
|
1756
|
+
self._debounce_s = max(0.05, float(os.environ.get("TW_WATCH_DEBOUNCE", "0.15") or 0.15))
|
|
1757
|
+
self._refresh_file_list()
|
|
1758
|
+
|
|
1759
|
+
def _refresh_file_list(self):
|
|
1760
|
+
self._files = self.state.project.list_source_files()
|
|
1761
|
+
self._stats = {}
|
|
1762
|
+
for path in self._files:
|
|
1763
|
+
try:
|
|
1764
|
+
st = os.stat(path)
|
|
1765
|
+
self._stats[path] = (st.st_mtime_ns, st.st_size)
|
|
1766
|
+
except OSError:
|
|
1767
|
+
self._stats[path] = None
|
|
1768
|
+
|
|
1769
|
+
def _can_use_inotify(self) -> bool:
|
|
1770
|
+
if self.backend == "poll":
|
|
1771
|
+
return False
|
|
1772
|
+
if self.backend == "inotify":
|
|
1773
|
+
return True
|
|
1774
|
+
# auto
|
|
1775
|
+
return sys.platform.startswith("linux")
|
|
1776
|
+
|
|
1777
|
+
def _inotify_setup(self) -> bool:
|
|
1778
|
+
if not self._can_use_inotify():
|
|
1779
|
+
return False
|
|
1780
|
+
if self._inotify_fd is not None:
|
|
1781
|
+
return True
|
|
1782
|
+
try:
|
|
1783
|
+
libc = ctypes.CDLL("libc.so.6", use_errno=True)
|
|
1784
|
+
except OSError:
|
|
1785
|
+
return False
|
|
1786
|
+
|
|
1787
|
+
inotify_init1 = getattr(libc, "inotify_init1", None)
|
|
1788
|
+
inotify_add_watch = getattr(libc, "inotify_add_watch", None)
|
|
1789
|
+
if inotify_init1 is None or inotify_add_watch is None:
|
|
1790
|
+
return False
|
|
1791
|
+
|
|
1792
|
+
# Flags: IN_NONBLOCK (0x800) | IN_CLOEXEC (0x80000)
|
|
1793
|
+
fd = int(inotify_init1(0x800 | 0x80000))
|
|
1794
|
+
if fd < 0:
|
|
1795
|
+
return False
|
|
1796
|
+
self._inotify_fd = fd
|
|
1797
|
+
|
|
1798
|
+
# NOTE: We watch directories (not individual files). For new directories,
|
|
1799
|
+
# we also add a watch dynamically when created.
|
|
1800
|
+
try:
|
|
1801
|
+
self._inotify_refresh_watches(libc)
|
|
1802
|
+
except Exception:
|
|
1803
|
+
logger.exception("Failed to setup inotify watches, falling back to polling")
|
|
1804
|
+
try:
|
|
1805
|
+
os.close(fd)
|
|
1806
|
+
except OSError:
|
|
1807
|
+
pass
|
|
1808
|
+
self._inotify_fd = None
|
|
1809
|
+
self._wd_to_dir = {}
|
|
1810
|
+
return False
|
|
1811
|
+
return True
|
|
1812
|
+
|
|
1813
|
+
def _inotify_refresh_watches(self, libc):
|
|
1814
|
+
inotify_add_watch = libc.inotify_add_watch
|
|
1815
|
+
inotify_add_watch.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_uint32]
|
|
1816
|
+
inotify_add_watch.restype = ctypes.c_int
|
|
1817
|
+
|
|
1818
|
+
# Watch events: create/delete/move/modify/attrib on dirs and files
|
|
1819
|
+
mask = (
|
|
1820
|
+
0x00000100 # IN_CREATE
|
|
1821
|
+
| 0x00000200 # IN_DELETE
|
|
1822
|
+
| 0x00000002 # IN_MODIFY
|
|
1823
|
+
| 0x00000004 # IN_ATTRIB
|
|
1824
|
+
| 0x00000040 # IN_MOVED_FROM
|
|
1825
|
+
| 0x00000080 # IN_MOVED_TO
|
|
1826
|
+
| 0x00000400 # IN_DELETE_SELF
|
|
1827
|
+
| 0x00000800 # IN_MOVE_SELF
|
|
1828
|
+
)
|
|
1829
|
+
|
|
1830
|
+
# Clear and rebuild watch list (simple and robust).
|
|
1831
|
+
self._wd_to_dir = {}
|
|
1832
|
+
roots = [self.state.project.project_root, os.path.join(self.state.project.project_root, "[home]")]
|
|
1833
|
+
for root in roots:
|
|
1834
|
+
if not root or not os.path.isdir(root):
|
|
1835
|
+
continue
|
|
1836
|
+
for dirpath, dirnames, _ in os.walk(root):
|
|
1837
|
+
# skip hidden caches inside project (don't waste watches)
|
|
1838
|
+
dirnames[:] = [d for d in dirnames if d not in {".tw", "dist", "__pycache__", ".pytest_cache"}]
|
|
1839
|
+
wd = int(inotify_add_watch(self._inotify_fd, dirpath.encode("utf-8"), mask))
|
|
1840
|
+
if wd >= 0:
|
|
1841
|
+
self._wd_to_dir[wd] = dirpath
|
|
1842
|
+
|
|
1843
|
+
def _inotify_read_events(self) -> List[tuple]:
|
|
1844
|
+
"""Returns list of (mask, name, watched_dir)."""
|
|
1845
|
+
assert self._inotify_fd is not None
|
|
1846
|
+
try:
|
|
1847
|
+
data = os.read(self._inotify_fd, 65536)
|
|
1848
|
+
except BlockingIOError:
|
|
1849
|
+
return []
|
|
1850
|
+
except OSError:
|
|
1851
|
+
logger.exception("inotify read failed")
|
|
1852
|
+
return []
|
|
1853
|
+
|
|
1854
|
+
events = []
|
|
1855
|
+
off = 0
|
|
1856
|
+
# struct inotify_event { int wd; uint32_t mask; uint32_t cookie; uint32_t len; char name[]; }
|
|
1857
|
+
while off + 16 <= len(data):
|
|
1858
|
+
wd, mask, _cookie, name_len = struct.unpack_from("iIII", data, off)
|
|
1859
|
+
off += 16
|
|
1860
|
+
name = b""
|
|
1861
|
+
if name_len:
|
|
1862
|
+
name = data[off : off + name_len].split(b"\x00", 1)[0]
|
|
1863
|
+
off += name_len
|
|
1864
|
+
watched_dir = self._wd_to_dir.get(int(wd), "")
|
|
1865
|
+
events.append((int(mask), name.decode("utf-8", errors="replace"), watched_dir))
|
|
1866
|
+
return events
|
|
1867
|
+
|
|
1868
|
+
def run(self):
|
|
1869
|
+
# Prefer inotify on Linux for better latency and battery usage.
|
|
1870
|
+
if self._inotify_setup():
|
|
1871
|
+
log("👀 File watcher: inotify mode")
|
|
1872
|
+
try:
|
|
1873
|
+
libc = ctypes.CDLL("libc.so.6", use_errno=True)
|
|
1874
|
+
except OSError:
|
|
1875
|
+
libc = None
|
|
1876
|
+
last_change = 0.0
|
|
1877
|
+
while not self.state.stop_event.is_set():
|
|
1878
|
+
fd = self._inotify_fd
|
|
1879
|
+
if fd is None:
|
|
1880
|
+
break
|
|
1881
|
+
rlist, _, _ = select.select([fd], [], [], 0.5)
|
|
1882
|
+
if not rlist:
|
|
1883
|
+
continue
|
|
1884
|
+
events = self._inotify_read_events()
|
|
1885
|
+
if not events:
|
|
1886
|
+
continue
|
|
1887
|
+
changed = True
|
|
1888
|
+
|
|
1889
|
+
# If new dirs created, refresh watches (simple, avoids edge cases).
|
|
1890
|
+
if libc is not None:
|
|
1891
|
+
for mask, name, watched_dir in events:
|
|
1892
|
+
# IN_ISDIR (0x40000000) + IN_CREATE
|
|
1893
|
+
if (mask & 0x40000000) and (mask & 0x00000100):
|
|
1894
|
+
try:
|
|
1895
|
+
self._inotify_refresh_watches(libc)
|
|
1896
|
+
except Exception:
|
|
1897
|
+
logger.exception("Failed to refresh inotify watches")
|
|
1898
|
+
break
|
|
1899
|
+
|
|
1900
|
+
now = time.monotonic()
|
|
1901
|
+
if changed:
|
|
1902
|
+
# Debounce to avoid bumping multiple times during a save burst.
|
|
1903
|
+
if (now - last_change) < self._debounce_s:
|
|
1904
|
+
continue
|
|
1905
|
+
last_change = now
|
|
1906
|
+
self._refresh_file_list()
|
|
1907
|
+
self.state.project.invalidate()
|
|
1908
|
+
self.state.bump()
|
|
1909
|
+
|
|
1910
|
+
if self._inotify_fd is not None:
|
|
1911
|
+
try:
|
|
1912
|
+
os.close(self._inotify_fd)
|
|
1913
|
+
except OSError:
|
|
1914
|
+
pass
|
|
1915
|
+
self._inotify_fd = None
|
|
1916
|
+
return
|
|
1917
|
+
|
|
1918
|
+
# Polling fallback (portable)
|
|
1919
|
+
log("👀 File watcher: polling mode")
|
|
1920
|
+
while not self.state.stop_event.is_set():
|
|
1921
|
+
time.sleep(self.interval)
|
|
1922
|
+
|
|
1923
|
+
changed = False
|
|
1924
|
+
for path in list(self._files):
|
|
1925
|
+
try:
|
|
1926
|
+
st = os.stat(path)
|
|
1927
|
+
sig = (st.st_mtime_ns, st.st_size)
|
|
1928
|
+
except OSError:
|
|
1929
|
+
sig = None
|
|
1930
|
+
if self._stats.get(path) != sig:
|
|
1931
|
+
changed = True
|
|
1932
|
+
break
|
|
1933
|
+
|
|
1934
|
+
# Periodically rescan to detect new files / deletions (low overhead)
|
|
1935
|
+
if not changed and (self._tick % self.rescan_ticks == 0):
|
|
1936
|
+
new_files = self.state.project.list_source_files()
|
|
1937
|
+
if set(new_files) != set(self._files):
|
|
1938
|
+
changed = True
|
|
1939
|
+
|
|
1940
|
+
if changed:
|
|
1941
|
+
self._refresh_file_list()
|
|
1942
|
+
with self.state.project._lock:
|
|
1943
|
+
self.state.project.invalidate()
|
|
1944
|
+
self.state.bump()
|
|
1945
|
+
|
|
1946
|
+
|
|
1947
|
+
def make_dev_handler(state: TWDevState):
|
|
1948
|
+
class TWDevHandler(http.server.BaseHTTPRequestHandler):
|
|
1949
|
+
server_version = "TWDevServer/1.0"
|
|
1950
|
+
|
|
1951
|
+
def log_message(self, fmt, *args):
|
|
1952
|
+
log(f"[dev] {self.address_string()} - {fmt % args}")
|
|
1953
|
+
|
|
1954
|
+
def do_GET(self):
|
|
1955
|
+
self.handle_request("GET")
|
|
1956
|
+
|
|
1957
|
+
def do_POST(self):
|
|
1958
|
+
self.handle_request("POST")
|
|
1959
|
+
|
|
1960
|
+
def do_PUT(self):
|
|
1961
|
+
self.handle_request("PUT")
|
|
1962
|
+
|
|
1963
|
+
def do_PATCH(self):
|
|
1964
|
+
self.handle_request("PATCH")
|
|
1965
|
+
|
|
1966
|
+
def do_DELETE(self):
|
|
1967
|
+
self.handle_request("DELETE")
|
|
1968
|
+
|
|
1969
|
+
def do_OPTIONS(self):
|
|
1970
|
+
self.handle_request("OPTIONS")
|
|
1971
|
+
|
|
1972
|
+
def handle_request(self, method: str):
|
|
1973
|
+
path = normalize_url_path(self.path)
|
|
1974
|
+
if path == "/__tw/events":
|
|
1975
|
+
self.handle_events()
|
|
1976
|
+
return
|
|
1977
|
+
|
|
1978
|
+
if path == "/__tw/health":
|
|
1979
|
+
self.respond_bytes(200, b"ok", "text/plain; charset=utf-8")
|
|
1980
|
+
return
|
|
1981
|
+
|
|
1982
|
+
request_headers = {key: value for key, value in self.headers.items()}
|
|
1983
|
+
# ── Modern request middleware hook (extensions) ───────────────
|
|
1984
|
+
# Allows project code to intercept/block/redirect/rewrite before TW middleware/api/routes.
|
|
1985
|
+
hook_state = state.project.extensions.emit(
|
|
1986
|
+
"beforeRequest",
|
|
1987
|
+
method=method,
|
|
1988
|
+
raw_path=self.path,
|
|
1989
|
+
url_path=path,
|
|
1990
|
+
request_headers=request_headers,
|
|
1991
|
+
request_meta={"client_ip": self.client_address[0] if self.client_address else ""},
|
|
1992
|
+
dev_mode=True,
|
|
1993
|
+
)
|
|
1994
|
+
if hook_state.get("response"):
|
|
1995
|
+
response = hook_state["response"]
|
|
1996
|
+
self.respond_bytes(
|
|
1997
|
+
response["status"],
|
|
1998
|
+
response["body"],
|
|
1999
|
+
response["content_type"],
|
|
2000
|
+
headers=response.get("headers", []),
|
|
2001
|
+
cookies=response.get("cookies", []),
|
|
2002
|
+
)
|
|
2003
|
+
return
|
|
2004
|
+
if hook_state.get("redirect"):
|
|
2005
|
+
location = str(hook_state["redirect"])
|
|
2006
|
+
self.respond_bytes(
|
|
2007
|
+
302,
|
|
2008
|
+
b"",
|
|
2009
|
+
"text/plain; charset=utf-8",
|
|
2010
|
+
headers=[("Location", location)] + list(hook_state.get("headers", [])),
|
|
2011
|
+
cookies=list(hook_state.get("cookies", [])),
|
|
2012
|
+
)
|
|
2013
|
+
return
|
|
2014
|
+
if hook_state.get("rewrite"):
|
|
2015
|
+
path = normalize_url_path(str(hook_state["rewrite"]))
|
|
2016
|
+
request_headers = hook_state.get("request_headers", request_headers)
|
|
2017
|
+
middleware = state.project.apply_middleware(
|
|
2018
|
+
self.path,
|
|
2019
|
+
request_headers,
|
|
2020
|
+
request_meta={"client_ip": self.client_address[0] if self.client_address else ""},
|
|
2021
|
+
method=method,
|
|
2022
|
+
)
|
|
2023
|
+
if middleware.get("response"):
|
|
2024
|
+
response = middleware["response"]
|
|
2025
|
+
self.respond_bytes(
|
|
2026
|
+
response["status"],
|
|
2027
|
+
response["body"],
|
|
2028
|
+
response["content_type"],
|
|
2029
|
+
headers=response.get("headers", []),
|
|
2030
|
+
cookies=response.get("cookies", []),
|
|
2031
|
+
)
|
|
2032
|
+
return
|
|
2033
|
+
if middleware.get("redirect"):
|
|
2034
|
+
payload = f"Redirecting to {middleware['redirect']}".encode("utf-8")
|
|
2035
|
+
self.respond_bytes(
|
|
2036
|
+
302,
|
|
2037
|
+
payload,
|
|
2038
|
+
"text/plain; charset=utf-8",
|
|
2039
|
+
headers=[("Location", middleware["redirect"])] + middleware.get("headers", []),
|
|
2040
|
+
cookies=middleware.get("cookies", []),
|
|
2041
|
+
)
|
|
2042
|
+
return
|
|
2043
|
+
path = normalize_url_path(middleware.get("path", path))
|
|
2044
|
+
with state.project._lock:
|
|
2045
|
+
api_route = state.project.resolve_api_route(path)
|
|
2046
|
+
if api_route is not None:
|
|
2047
|
+
body = decode_request_body(self) if method in {"POST", "PUT", "PATCH"} else {}
|
|
2048
|
+
try:
|
|
2049
|
+
api_response = state.project.execute_api_route(api_route, method, self.path, request_headers, body)
|
|
2050
|
+
self.respond_bytes(
|
|
2051
|
+
api_response["status"],
|
|
2052
|
+
api_response["body"],
|
|
2053
|
+
api_response["content_type"],
|
|
2054
|
+
headers=middleware.get("headers", []) + api_response.get("headers", []),
|
|
2055
|
+
cookies=middleware.get("cookies", []) + api_response.get("cookies", []),
|
|
2056
|
+
)
|
|
2057
|
+
except Exception as err:
|
|
2058
|
+
logger.exception("Unhandled API route error (dev): %s %s -> %s", method, self.path, api_route)
|
|
2059
|
+
body = render_error_html("API route error", str(err), 500)
|
|
2060
|
+
self.respond_bytes(500, body, "text/html; charset=utf-8", headers=middleware.get("headers", []), cookies=middleware.get("cookies", []))
|
|
2061
|
+
return
|
|
2062
|
+
|
|
2063
|
+
asset = state.project.resolve_asset(path)
|
|
2064
|
+
if asset is not None:
|
|
2065
|
+
payload, content_type = asset
|
|
2066
|
+
self.respond_bytes(200, payload, content_type, headers=middleware.get("headers", []), cookies=middleware.get("cookies", []))
|
|
2067
|
+
return
|
|
2068
|
+
|
|
2069
|
+
match = state.project.resolve_route(path)
|
|
2070
|
+
if not match:
|
|
2071
|
+
custom_404 = state.project.compile_special_page(404, dev_mode=True)
|
|
2072
|
+
if custom_404 is not None:
|
|
2073
|
+
self.respond_bytes(404, custom_404.encode("utf-8"), "text/html; charset=utf-8", headers=middleware.get("headers", []), cookies=middleware.get("cookies", []))
|
|
2074
|
+
return
|
|
2075
|
+
body = render_error_html("Page not found", f"Route not found: {path}", 404)
|
|
2076
|
+
self.respond_bytes(404, body, "text/html; charset=utf-8", headers=middleware.get("headers", []), cookies=middleware.get("cookies", []))
|
|
2077
|
+
return
|
|
2078
|
+
|
|
2079
|
+
try:
|
|
2080
|
+
response = state.project.compile_match_response(match, dev_mode=True)
|
|
2081
|
+
self.respond_bytes(
|
|
2082
|
+
response["status"],
|
|
2083
|
+
response["html"].encode("utf-8"),
|
|
2084
|
+
"text/html; charset=utf-8",
|
|
2085
|
+
headers=middleware.get("headers", []) + response.get("headers", []),
|
|
2086
|
+
cookies=middleware.get("cookies", []),
|
|
2087
|
+
)
|
|
2088
|
+
except Exception as err:
|
|
2089
|
+
try:
|
|
2090
|
+
custom_500 = state.project.compile_special_page(500, dev_mode=True)
|
|
2091
|
+
if custom_500 is not None:
|
|
2092
|
+
self.respond_bytes(500, custom_500.encode("utf-8"), "text/html; charset=utf-8", headers=middleware.get("headers", []), cookies=middleware.get("cookies", []))
|
|
2093
|
+
return
|
|
2094
|
+
except Exception:
|
|
2095
|
+
logger.exception("Failed to compile custom 500 page (dev)")
|
|
2096
|
+
message = format_compiler_error(match.page_info["path"], err)
|
|
2097
|
+
body = render_error_html("Compile error", message, 500)
|
|
2098
|
+
self.respond_bytes(500, body, "text/html; charset=utf-8", headers=middleware.get("headers", []), cookies=middleware.get("cookies", []))
|
|
2099
|
+
|
|
2100
|
+
def handle_events(self):
|
|
2101
|
+
self.send_response(200)
|
|
2102
|
+
self.send_header("Content-Type", "text/event-stream")
|
|
2103
|
+
self.send_header("Cache-Control", "no-cache")
|
|
2104
|
+
self.send_header("Connection", "keep-alive")
|
|
2105
|
+
self.end_headers()
|
|
2106
|
+
|
|
2107
|
+
last_seen = state.current_version()
|
|
2108
|
+
try:
|
|
2109
|
+
# EventSource reconnection hint (ms)
|
|
2110
|
+
self.wfile.write(b"retry: 500\n\n")
|
|
2111
|
+
self.wfile.flush()
|
|
2112
|
+
while not state.stop_event.is_set():
|
|
2113
|
+
current = state.current_version()
|
|
2114
|
+
if current != last_seen:
|
|
2115
|
+
self.wfile.write(b"data: reload\n\n")
|
|
2116
|
+
self.wfile.flush()
|
|
2117
|
+
last_seen = current
|
|
2118
|
+
else:
|
|
2119
|
+
self.wfile.write(b": ping\n\n")
|
|
2120
|
+
self.wfile.flush()
|
|
2121
|
+
time.sleep(1)
|
|
2122
|
+
except (BrokenPipeError, ConnectionResetError):
|
|
2123
|
+
return
|
|
2124
|
+
|
|
2125
|
+
def respond_bytes(self, status: int, payload: bytes, content_type: str, headers: Optional[List[Tuple[str, str]]] = None, cookies: Optional[List[Tuple[str, str]]] = None):
|
|
2126
|
+
self.send_response(status)
|
|
2127
|
+
self.send_header("Content-Type", content_type)
|
|
2128
|
+
self.send_header("Content-Length", str(len(payload)))
|
|
2129
|
+
# Dev cache hardening: avoid stale HTML/CSS/JS when browser caches aggressively
|
|
2130
|
+
if content_type.startswith("text/html") or content_type.startswith("text/css") or "javascript" in content_type:
|
|
2131
|
+
self.send_header("Cache-Control", "no-store")
|
|
2132
|
+
else:
|
|
2133
|
+
self.send_header("Cache-Control", "no-cache")
|
|
2134
|
+
for header_name, header_value in headers or []:
|
|
2135
|
+
self.send_header(header_name, header_value)
|
|
2136
|
+
for cookie_name, cookie_value in cookies or []:
|
|
2137
|
+
self.send_header(
|
|
2138
|
+
"Set-Cookie",
|
|
2139
|
+
render_cookie_header(
|
|
2140
|
+
cookie_name,
|
|
2141
|
+
cookie_value,
|
|
2142
|
+
config=state.project.config,
|
|
2143
|
+
request_headers={key: value for key, value in self.headers.items()},
|
|
2144
|
+
server_port=self.server.server_address[1],
|
|
2145
|
+
),
|
|
2146
|
+
)
|
|
2147
|
+
self.end_headers()
|
|
2148
|
+
try:
|
|
2149
|
+
self.wfile.write(payload)
|
|
2150
|
+
except (BrokenPipeError, ConnectionResetError):
|
|
2151
|
+
return
|
|
2152
|
+
|
|
2153
|
+
return TWDevHandler
|
|
2154
|
+
|
|
2155
|
+
|
|
2156
|
+
class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
|
|
2157
|
+
allow_reuse_address = True
|
|
2158
|
+
daemon_threads = True
|
|
2159
|
+
|
|
2160
|
+
|
|
2161
|
+
def run_dev_server(project_root: str, host: str, port: int):
|
|
2162
|
+
project = TWProject(project_root)
|
|
2163
|
+
state = TWDevState(project)
|
|
2164
|
+
watcher = TWFileWatcher(state)
|
|
2165
|
+
watcher.start()
|
|
2166
|
+
|
|
2167
|
+
handler = make_dev_handler(state)
|
|
2168
|
+
try:
|
|
2169
|
+
server = ThreadedTCPServer((host, port), handler)
|
|
2170
|
+
except OSError as err:
|
|
2171
|
+
state.stop_event.set()
|
|
2172
|
+
raise RuntimeError(
|
|
2173
|
+
f"Could not bind to port {port} ({err}). Is the port already in use? "
|
|
2174
|
+
f"Try: `tw dev --port {port + 1}` or choose a free port."
|
|
2175
|
+
) from err
|
|
2176
|
+
|
|
2177
|
+
log(f"TW dev server running at http://{host}:{port}")
|
|
2178
|
+
log("Source workflow active: edit `.tw`, `.tss`, `.ts`; the browser will auto-reload.")
|
|
2179
|
+
|
|
2180
|
+
try:
|
|
2181
|
+
server.serve_forever()
|
|
2182
|
+
except KeyboardInterrupt:
|
|
2183
|
+
log("\nStopping dev server...")
|
|
2184
|
+
finally:
|
|
2185
|
+
state.stop_event.set()
|
|
2186
|
+
server.shutdown()
|
|
2187
|
+
server.server_close()
|
|
2188
|
+
|
|
2189
|
+
|
|
2190
|
+
def build_preview_candidates(output_dir: str, path: str) -> List[str]:
|
|
2191
|
+
if path == "/":
|
|
2192
|
+
return [os.path.join(output_dir, "index.html")]
|
|
2193
|
+
|
|
2194
|
+
trimmed = path.lstrip("/")
|
|
2195
|
+
candidates = [os.path.join(output_dir, trimmed)]
|
|
2196
|
+
if not os.path.splitext(trimmed)[1]:
|
|
2197
|
+
candidates.append(os.path.join(output_dir, trimmed + ".html"))
|
|
2198
|
+
candidates.append(os.path.join(output_dir, trimmed, "index.html"))
|
|
2199
|
+
return candidates
|
|
2200
|
+
|
|
2201
|
+
|
|
2202
|
+
def make_preview_handler(output_dir: str):
|
|
2203
|
+
output_dir = os.path.abspath(output_dir)
|
|
2204
|
+
|
|
2205
|
+
class TWPreviewHandler(http.server.BaseHTTPRequestHandler):
|
|
2206
|
+
server_version = "TWPreviewServer/1.0"
|
|
2207
|
+
|
|
2208
|
+
def log_message(self, fmt, *args):
|
|
2209
|
+
log(f"[preview] {self.address_string()} - {fmt % args}")
|
|
2210
|
+
|
|
2211
|
+
def do_GET(self):
|
|
2212
|
+
path = normalize_url_path(self.path)
|
|
2213
|
+
for candidate in build_preview_candidates(output_dir, path):
|
|
2214
|
+
candidate = os.path.abspath(candidate)
|
|
2215
|
+
if not is_path_within(output_dir, candidate):
|
|
2216
|
+
continue
|
|
2217
|
+
payload = safe_read_binary(candidate)
|
|
2218
|
+
if payload is not None:
|
|
2219
|
+
content_type = mimetypes.guess_type(candidate)[0] or "application/octet-stream"
|
|
2220
|
+
self.respond_bytes(200, payload, content_type)
|
|
2221
|
+
return
|
|
2222
|
+
|
|
2223
|
+
custom_404 = safe_read_binary(os.path.join(output_dir, "404.html"))
|
|
2224
|
+
if custom_404 is not None:
|
|
2225
|
+
self.respond_bytes(404, custom_404, "text/html; charset=utf-8")
|
|
2226
|
+
return
|
|
2227
|
+
|
|
2228
|
+
body = render_error_html("Page not found", f"Route not found: {path}", 404)
|
|
2229
|
+
self.respond_bytes(404, body, "text/html; charset=utf-8")
|
|
2230
|
+
|
|
2231
|
+
def respond_bytes(self, status: int, payload: bytes, content_type: str):
|
|
2232
|
+
if (
|
|
2233
|
+
(
|
|
2234
|
+
content_type.startswith("text/html")
|
|
2235
|
+
or content_type.startswith("text/css")
|
|
2236
|
+
or "javascript" in content_type
|
|
2237
|
+
)
|
|
2238
|
+
and "charset=" not in content_type.lower()
|
|
2239
|
+
):
|
|
2240
|
+
content_type = f"{content_type}; charset=utf-8"
|
|
2241
|
+
self.send_response(status)
|
|
2242
|
+
self.send_header("Content-Type", content_type)
|
|
2243
|
+
self.send_header("Content-Length", str(len(payload)))
|
|
2244
|
+
# Preview: prefer fresh content during local testing
|
|
2245
|
+
self.send_header("Cache-Control", "no-cache")
|
|
2246
|
+
self.end_headers()
|
|
2247
|
+
try:
|
|
2248
|
+
self.wfile.write(payload)
|
|
2249
|
+
except (BrokenPipeError, ConnectionResetError):
|
|
2250
|
+
return
|
|
2251
|
+
|
|
2252
|
+
return TWPreviewHandler
|
|
2253
|
+
|
|
2254
|
+
|
|
2255
|
+
def run_preview_server(output_dir: str, host: str, port: int):
|
|
2256
|
+
output_dir = os.path.abspath(output_dir)
|
|
2257
|
+
if not os.path.isdir(output_dir):
|
|
2258
|
+
raise RuntimeError(f"Preview output missing: {output_dir}. Run `tw build` or `tw export` first.")
|
|
2259
|
+
|
|
2260
|
+
handler = make_preview_handler(output_dir)
|
|
2261
|
+
server = ThreadedTCPServer((host, port), handler)
|
|
2262
|
+
log(f"TW preview server running at http://{host}:{port}")
|
|
2263
|
+
log(f"Serving static output from: {output_dir}")
|
|
2264
|
+
try:
|
|
2265
|
+
server.serve_forever()
|
|
2266
|
+
except KeyboardInterrupt:
|
|
2267
|
+
log("\nStopping preview server...")
|
|
2268
|
+
finally:
|
|
2269
|
+
server.shutdown()
|
|
2270
|
+
server.server_close()
|
|
2271
|
+
|
|
2272
|
+
|
|
2273
|
+
def compile_typescript_sources(project_root: str, output_dir: str) -> List[str]:
|
|
2274
|
+
ts_files = []
|
|
2275
|
+
for dirpath, _, filenames in os.walk(os.path.join(project_root, "[home]")):
|
|
2276
|
+
for filename in filenames:
|
|
2277
|
+
if filename.endswith(".ts"):
|
|
2278
|
+
ts_files.append(os.path.join(dirpath, filename))
|
|
2279
|
+
|
|
2280
|
+
if not ts_files:
|
|
2281
|
+
return []
|
|
2282
|
+
|
|
2283
|
+
tsc_bin = shutil.which("tsc")
|
|
2284
|
+
if not tsc_bin:
|
|
2285
|
+
log(" ⚠️ TypeScript files found, but `tsc` was not found. Skipping TS compilation.", level="warning")
|
|
2286
|
+
return []
|
|
2287
|
+
|
|
2288
|
+
out_dir = os.path.join(output_dir, "_tw", "ts")
|
|
2289
|
+
os.makedirs(out_dir, exist_ok=True)
|
|
2290
|
+
|
|
2291
|
+
command = [
|
|
2292
|
+
tsc_bin,
|
|
2293
|
+
"--module", "esnext",
|
|
2294
|
+
"--target", "es2020",
|
|
2295
|
+
"--sourceMap",
|
|
2296
|
+
"--outDir", out_dir,
|
|
2297
|
+
"--rootDir", os.path.join(project_root, "[home]"),
|
|
2298
|
+
"--pretty", "false",
|
|
2299
|
+
"--skipLibCheck",
|
|
2300
|
+
] + ts_files
|
|
2301
|
+
|
|
2302
|
+
result = subprocess.run(command, capture_output=True, text=True)
|
|
2303
|
+
if result.returncode != 0:
|
|
2304
|
+
raise RuntimeError(f"TypeScript compile failed:\n{result.stdout}\n{result.stderr}".strip())
|
|
2305
|
+
|
|
2306
|
+
emitted = []
|
|
2307
|
+
for root, _, files in os.walk(out_dir):
|
|
2308
|
+
for filename in files:
|
|
2309
|
+
emitted.append(os.path.join(root, filename))
|
|
2310
|
+
return emitted
|
|
2311
|
+
|
|
2312
|
+
|
|
2313
|
+
def ensure_project_metadata(project_root: str):
|
|
2314
|
+
package_json_path = os.path.join(project_root, "package.json")
|
|
2315
|
+
if not os.path.exists(package_json_path):
|
|
2316
|
+
package_name = os.path.basename(os.path.abspath(project_root)).lower().replace(" ", "-")
|
|
2317
|
+
with open(package_json_path, "w", encoding="utf-8") as f:
|
|
2318
|
+
json.dump(
|
|
2319
|
+
{
|
|
2320
|
+
"name": package_name or "tw-site",
|
|
2321
|
+
"private": True,
|
|
2322
|
+
"version": "0.1.0",
|
|
2323
|
+
"engines": {
|
|
2324
|
+
"node": ">=18",
|
|
2325
|
+
},
|
|
2326
|
+
"scripts": {
|
|
2327
|
+
"dev": "tw dev",
|
|
2328
|
+
"build": "tw build",
|
|
2329
|
+
"export": "tw export",
|
|
2330
|
+
"preview": "tw preview",
|
|
2331
|
+
"clean": "tw clean",
|
|
2332
|
+
"doctor": "tw doctor",
|
|
2333
|
+
"info": "tw info",
|
|
2334
|
+
"deploy": "tw deploy",
|
|
2335
|
+
},
|
|
2336
|
+
"dependencies": {},
|
|
2337
|
+
"devDependencies": {},
|
|
2338
|
+
},
|
|
2339
|
+
f,
|
|
2340
|
+
indent=2,
|
|
2341
|
+
)
|
|
2342
|
+
f.write("\n")
|
|
2343
|
+
|
|
2344
|
+
vercel_json_path = os.path.join(project_root, "vercel.json")
|
|
2345
|
+
if not os.path.exists(vercel_json_path):
|
|
2346
|
+
with open(vercel_json_path, "w", encoding="utf-8") as f:
|
|
2347
|
+
json.dump(
|
|
2348
|
+
{
|
|
2349
|
+
"buildCommand": "tw build",
|
|
2350
|
+
"outputDirectory": "dist",
|
|
2351
|
+
},
|
|
2352
|
+
f,
|
|
2353
|
+
indent=2,
|
|
2354
|
+
)
|
|
2355
|
+
f.write("\n")
|
|
2356
|
+
|
|
2357
|
+
|
|
2358
|
+
def sync_runtime_chunks_to_output():
|
|
2359
|
+
os.makedirs(compiler.CHUNKS_DIR, exist_ok=True)
|
|
2360
|
+
os.makedirs(compiler.CHUNKS_PUBLIC_DIR, exist_ok=True)
|
|
2361
|
+
|
|
2362
|
+
for filename in os.listdir(compiler.CHUNKS_DIR):
|
|
2363
|
+
src = os.path.join(compiler.CHUNKS_DIR, filename)
|
|
2364
|
+
dst = os.path.join(compiler.CHUNKS_PUBLIC_DIR, filename)
|
|
2365
|
+
if os.path.isfile(src):
|
|
2366
|
+
shutil.copy2(src, dst)
|
|
2367
|
+
|
|
2368
|
+
|
|
2369
|
+
def write_hidden_cache_files(dependency_map: Dict[str, List[str]], metadata_map: Optional[Dict[str, dict]] = None):
|
|
2370
|
+
os.makedirs(compiler.CACHE_DIR, exist_ok=True)
|
|
2371
|
+
os.makedirs(compiler.MANIFEST_DIR, exist_ok=True)
|
|
2372
|
+
os.makedirs(compiler.COMPILER_DIR, exist_ok=True)
|
|
2373
|
+
|
|
2374
|
+
hash_db = {}
|
|
2375
|
+
for page_key, dependencies in dependency_map.items():
|
|
2376
|
+
hash_db[page_key] = {
|
|
2377
|
+
"signature": compiler.compute_dependency_signature(dependencies),
|
|
2378
|
+
"dependencies": dependencies,
|
|
2379
|
+
"metadata": dict((metadata_map or {}).get(page_key) or {}),
|
|
2380
|
+
}
|
|
2381
|
+
|
|
2382
|
+
with open(compiler.HASH_DB_FILE, "w", encoding="utf-8") as f:
|
|
2383
|
+
json.dump(hash_db, f, indent=2, sort_keys=True)
|
|
2384
|
+
|
|
2385
|
+
|
|
2386
|
+
def write_text_file(path: str, content: str):
|
|
2387
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
2388
|
+
with open(path, "w", encoding="utf-8") as f:
|
|
2389
|
+
f.write(content)
|
|
2390
|
+
|
|
2391
|
+
|
|
2392
|
+
def route_records_for_build() -> List[dict]:
|
|
2393
|
+
records = []
|
|
2394
|
+
config = compiler.load_config()
|
|
2395
|
+
pipeline_name = "modular" if use_modular_pipeline(config) else "legacy"
|
|
2396
|
+
for page in compiler.discover_pages():
|
|
2397
|
+
page_ast = compiler.load_page_ast_from_file(page["path"])
|
|
2398
|
+
if page["type"] == "static":
|
|
2399
|
+
route = route_from_static_page(page)
|
|
2400
|
+
metadata = compiler.collect_page_metadata(page, page_ast=page_ast, route_path=route, pipeline=pipeline_name)
|
|
2401
|
+
render_mode = determine_render_mode(page["path"])
|
|
2402
|
+
records.append({
|
|
2403
|
+
"route": route,
|
|
2404
|
+
"type": "page",
|
|
2405
|
+
"render": render_mode,
|
|
2406
|
+
"revalidate": page_ast.revalidate,
|
|
2407
|
+
"source": compiler.safe_relpath(page["path"], compiler.PROJECT_ROOT),
|
|
2408
|
+
"title": page_ast.title,
|
|
2409
|
+
"layouts": metadata["layouts"],
|
|
2410
|
+
"components": metadata["components"],
|
|
2411
|
+
"pipeline": metadata["pipeline"],
|
|
2412
|
+
})
|
|
2413
|
+
continue
|
|
2414
|
+
for item in compiler.load_dynamic_items(page["path"]):
|
|
2415
|
+
if not isinstance(item, dict):
|
|
2416
|
+
continue
|
|
2417
|
+
route = route_from_dynamic_page(page, item)
|
|
2418
|
+
metadata = compiler.collect_page_metadata(page, page_ast=page_ast, route_path=route, pipeline=pipeline_name, item=item)
|
|
2419
|
+
render_mode = determine_render_mode(page["path"])
|
|
2420
|
+
records.append({
|
|
2421
|
+
"route": route,
|
|
2422
|
+
"type": "page",
|
|
2423
|
+
"render": render_mode,
|
|
2424
|
+
"revalidate": page_ast.revalidate,
|
|
2425
|
+
"source": compiler.safe_relpath(page["path"], compiler.PROJECT_ROOT),
|
|
2426
|
+
"title": page_ast.title,
|
|
2427
|
+
"layouts": metadata["layouts"],
|
|
2428
|
+
"components": metadata["components"],
|
|
2429
|
+
"pipeline": metadata["pipeline"],
|
|
2430
|
+
"route_kind": metadata["route_kind"],
|
|
2431
|
+
})
|
|
2432
|
+
return sorted(records, key=lambda item: item["route"])
|
|
2433
|
+
|
|
2434
|
+
|
|
2435
|
+
def write_route_artifacts(output_dir: str):
|
|
2436
|
+
config = compiler.load_config()
|
|
2437
|
+
site_url = str(config.get("site_url", "") or config.get("siteUrl", "") or "").rstrip("/")
|
|
2438
|
+
routes = [item for item in route_records_for_build() if item["route"] not in {"/404", "/500"}]
|
|
2439
|
+
# Route collision detection (warn by default; can be escalated by callers).
|
|
2440
|
+
# Collision = same final route emitted by multiple source pages/items.
|
|
2441
|
+
collisions = {}
|
|
2442
|
+
for entry in routes:
|
|
2443
|
+
route = entry.get("route") or "/"
|
|
2444
|
+
src = entry.get("source") or ""
|
|
2445
|
+
collisions.setdefault(route, []).append(src)
|
|
2446
|
+
collisions = {r: srcs for r, srcs in collisions.items() if len(set(srcs)) > 1}
|
|
2447
|
+
if collisions:
|
|
2448
|
+
for route, srcs in sorted(collisions.items()):
|
|
2449
|
+
sources = ", ".join(sorted(set(srcs)))
|
|
2450
|
+
log(f" ⚠️ Route collision: {route} <- {sources}", level="warning")
|
|
2451
|
+
write_text_file(os.path.join(output_dir, "_tw", "route-manifest.json"), json.dumps(routes, indent=2, ensure_ascii=False) + "\n")
|
|
2452
|
+
|
|
2453
|
+
api_manifest = []
|
|
2454
|
+
for api in discover_api_routes():
|
|
2455
|
+
if api.get("lang") == "twm":
|
|
2456
|
+
with open(api["path"], "r", encoding="utf-8") as handle:
|
|
2457
|
+
funcs = parse_twm_functions(handle.read())
|
|
2458
|
+
method_names = {"get", "post", "put", "patch", "delete", "options"}
|
|
2459
|
+
methods = sorted({fn["name"].upper() for fn in funcs if fn["name"].lower() in method_names})
|
|
2460
|
+
if not methods and any(fn["name"].lower() == "handler" for fn in funcs):
|
|
2461
|
+
methods = ["ANY"]
|
|
2462
|
+
else:
|
|
2463
|
+
methods = sorted(parse_api_route_file(api["path"]).keys())
|
|
2464
|
+
api_manifest.append({
|
|
2465
|
+
"route": api["route"],
|
|
2466
|
+
"methods": methods,
|
|
2467
|
+
"source": compiler.safe_relpath(api["path"], compiler.PROJECT_ROOT),
|
|
2468
|
+
})
|
|
2469
|
+
write_text_file(os.path.join(output_dir, "_tw", "api-manifest.json"), json.dumps(api_manifest, indent=2, ensure_ascii=False) + "\n")
|
|
2470
|
+
|
|
2471
|
+
if site_url:
|
|
2472
|
+
lines = ['<?xml version="1.0" encoding="UTF-8"?>', '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">']
|
|
2473
|
+
for route in routes:
|
|
2474
|
+
lines.append(" <url>")
|
|
2475
|
+
lines.append(f" <loc>{html.escape(site_url + route['route'])}</loc>")
|
|
2476
|
+
lines.append(" </url>")
|
|
2477
|
+
lines.append("</urlset>")
|
|
2478
|
+
write_text_file(os.path.join(output_dir, "sitemap.xml"), "\n".join(lines) + "\n")
|
|
2479
|
+
|
|
2480
|
+
rss_lines = [
|
|
2481
|
+
'<?xml version="1.0" encoding="UTF-8"?>',
|
|
2482
|
+
'<rss version="2.0">',
|
|
2483
|
+
"<channel>",
|
|
2484
|
+
f" <title>{html.escape(str(config.get('name', 'TW Site')))}</title>",
|
|
2485
|
+
f" <link>{html.escape(site_url)}</link>",
|
|
2486
|
+
f" <description>{html.escape(str(config.get('description', 'TW generated feed')))}</description>",
|
|
2487
|
+
]
|
|
2488
|
+
for route in routes:
|
|
2489
|
+
rss_lines.append(" <item>")
|
|
2490
|
+
rss_lines.append(f" <title>{html.escape(str(route.get('title') or route['route']))}</title>")
|
|
2491
|
+
rss_lines.append(f" <link>{html.escape(site_url + route['route'])}</link>")
|
|
2492
|
+
rss_lines.append(f" <guid>{html.escape(site_url + route['route'])}</guid>")
|
|
2493
|
+
rss_lines.append(" <pubDate>" + formatdate(usegmt=True) + "</pubDate>")
|
|
2494
|
+
rss_lines.append(" </item>")
|
|
2495
|
+
rss_lines.extend(["</channel>", "</rss>"])
|
|
2496
|
+
write_text_file(os.path.join(output_dir, "rss.xml"), "\n".join(rss_lines) + "\n")
|
|
2497
|
+
|
|
2498
|
+
robots_content = "User-agent: *\nAllow: /\n"
|
|
2499
|
+
if site_url:
|
|
2500
|
+
robots_content += f"Sitemap: {site_url}/sitemap.xml\n"
|
|
2501
|
+
write_text_file(os.path.join(output_dir, "robots.txt"), robots_content)
|
|
2502
|
+
|
|
2503
|
+
# Optional: lightweight client-side search index
|
|
2504
|
+
search_enabled = bool(config.get("search", config.get("search_index", config.get("searchIndex", False))))
|
|
2505
|
+
if search_enabled:
|
|
2506
|
+
try:
|
|
2507
|
+
write_search_index(output_dir, routes)
|
|
2508
|
+
except Exception as err:
|
|
2509
|
+
log(f" ⚠️ Search index build skipped: {err}", level="warning")
|
|
2510
|
+
return len(collisions)
|
|
2511
|
+
|
|
2512
|
+
|
|
2513
|
+
def strip_html_to_text(html_text: str) -> str:
|
|
2514
|
+
# Remove script/style blocks
|
|
2515
|
+
html_text = re.sub(r"<script[\\s\\S]*?</script>", " ", html_text, flags=re.I)
|
|
2516
|
+
html_text = re.sub(r"<style[\\s\\S]*?</style>", " ", html_text, flags=re.I)
|
|
2517
|
+
# Remove tags
|
|
2518
|
+
html_text = re.sub(r"<[^>]+>", " ", html_text)
|
|
2519
|
+
# Decode entities + normalize whitespace
|
|
2520
|
+
html_text = html.unescape(html_text)
|
|
2521
|
+
html_text = re.sub(r"\\s+", " ", html_text).strip()
|
|
2522
|
+
return html_text
|
|
2523
|
+
|
|
2524
|
+
|
|
2525
|
+
def write_search_index(output_dir: str, routes: List[dict]):
|
|
2526
|
+
"""
|
|
2527
|
+
Generates: dist/_tw/search-index.json
|
|
2528
|
+
Format: [{route,title,excerpt,content}]
|
|
2529
|
+
"""
|
|
2530
|
+
items = []
|
|
2531
|
+
for route in routes:
|
|
2532
|
+
route_path = str(route.get("route") or "/")
|
|
2533
|
+
title = str(route.get("title") or route_path)
|
|
2534
|
+
source_html = None
|
|
2535
|
+
for candidate in build_preview_candidates(output_dir, route_path):
|
|
2536
|
+
if os.path.exists(candidate) and os.path.isfile(candidate):
|
|
2537
|
+
source_html = compiler.read_text_file(candidate)
|
|
2538
|
+
break
|
|
2539
|
+
if not source_html:
|
|
2540
|
+
continue
|
|
2541
|
+
content = strip_html_to_text(source_html)
|
|
2542
|
+
if not content:
|
|
2543
|
+
continue
|
|
2544
|
+
excerpt = content[:240]
|
|
2545
|
+
items.append({
|
|
2546
|
+
"route": route_path,
|
|
2547
|
+
"title": title,
|
|
2548
|
+
"excerpt": excerpt,
|
|
2549
|
+
"content": content[:4000],
|
|
2550
|
+
})
|
|
2551
|
+
|
|
2552
|
+
write_text_file(
|
|
2553
|
+
os.path.join(output_dir, "_tw", "search-index.json"),
|
|
2554
|
+
json.dumps(items, indent=2, ensure_ascii=False) + "\n",
|
|
2555
|
+
)
|
|
2556
|
+
|
|
2557
|
+
|
|
2558
|
+
def precompress_output(output_dir: str):
|
|
2559
|
+
try:
|
|
2560
|
+
import brotli # type: ignore
|
|
2561
|
+
except ImportError:
|
|
2562
|
+
brotli = None
|
|
2563
|
+
|
|
2564
|
+
compress_exts = {".html", ".css", ".js", ".json", ".xml", ".txt", ".svg"}
|
|
2565
|
+
brotli_warned = False
|
|
2566
|
+
for root, _, files in os.walk(output_dir):
|
|
2567
|
+
for filename in files:
|
|
2568
|
+
ext = os.path.splitext(filename)[1].lower()
|
|
2569
|
+
if ext not in compress_exts:
|
|
2570
|
+
continue
|
|
2571
|
+
path = os.path.join(root, filename)
|
|
2572
|
+
with open(path, "rb") as f:
|
|
2573
|
+
payload = f.read()
|
|
2574
|
+
with gzip.open(path + ".gz", "wb", compresslevel=9) as gz:
|
|
2575
|
+
gz.write(payload)
|
|
2576
|
+
if brotli is not None:
|
|
2577
|
+
with open(path + ".br", "wb") as br:
|
|
2578
|
+
br.write(brotli.compress(payload))
|
|
2579
|
+
elif not brotli_warned:
|
|
2580
|
+
log("⚠️ Brotli dependency not installed; generated gzip precompression only.", level="warning")
|
|
2581
|
+
brotli_warned = True
|
|
2582
|
+
|
|
2583
|
+
|
|
2584
|
+
def ensure_deploy_support_files(project_root: str):
|
|
2585
|
+
netlify_toml = os.path.join(project_root, "netlify.toml")
|
|
2586
|
+
if not os.path.exists(netlify_toml):
|
|
2587
|
+
write_text_file(netlify_toml, '[build]\ncommand = "tw build --prod"\npublish = "dist"\n')
|
|
2588
|
+
|
|
2589
|
+
dockerfile = os.path.join(project_root, "Dockerfile")
|
|
2590
|
+
if not os.path.exists(dockerfile):
|
|
2591
|
+
write_text_file(
|
|
2592
|
+
dockerfile,
|
|
2593
|
+
"FROM nginx:alpine\nCOPY dist /usr/share/nginx/html\nEXPOSE 80\nCMD [\"nginx\", \"-g\", \"daemon off;\"]\n",
|
|
2594
|
+
)
|
|
2595
|
+
|
|
2596
|
+
github_workflow = os.path.join(project_root, ".github", "workflows", "tw-pages.yml")
|
|
2597
|
+
if not os.path.exists(github_workflow):
|
|
2598
|
+
write_text_file(
|
|
2599
|
+
github_workflow,
|
|
2600
|
+
"""name: Deploy TW Pages
|
|
2601
|
+
on:
|
|
2602
|
+
push:
|
|
2603
|
+
branches: [main]
|
|
2604
|
+
jobs:
|
|
2605
|
+
build:
|
|
2606
|
+
runs-on: ubuntu-latest
|
|
2607
|
+
steps:
|
|
2608
|
+
- uses: actions/checkout@v4
|
|
2609
|
+
- uses: actions/setup-python@v5
|
|
2610
|
+
with:
|
|
2611
|
+
python-version: "3.11"
|
|
2612
|
+
- run: pip install .
|
|
2613
|
+
- run: tw build --prod
|
|
2614
|
+
- uses: actions/upload-pages-artifact@v3
|
|
2615
|
+
with:
|
|
2616
|
+
path: dist
|
|
2617
|
+
deploy:
|
|
2618
|
+
needs: build
|
|
2619
|
+
permissions:
|
|
2620
|
+
pages: write
|
|
2621
|
+
id-token: write
|
|
2622
|
+
runs-on: ubuntu-latest
|
|
2623
|
+
environment:
|
|
2624
|
+
name: github-pages
|
|
2625
|
+
steps:
|
|
2626
|
+
- uses: actions/deploy-pages@v4
|
|
2627
|
+
""",
|
|
2628
|
+
)
|
|
2629
|
+
|
|
2630
|
+
|
|
2631
|
+
def generate_deploy_metadata(output_dir: str, config: Dict[str, Any]) -> None:
|
|
2632
|
+
"""Write tw.deploy.json to the output directory for platform detection."""
|
|
2633
|
+
metadata = {
|
|
2634
|
+
"framework": "tw",
|
|
2635
|
+
"version": compiler.get_framework_version(),
|
|
2636
|
+
"build": "tw build",
|
|
2637
|
+
"output": "dist",
|
|
2638
|
+
"runtime": "ssr",
|
|
2639
|
+
"detect": {
|
|
2640
|
+
"config": "tw.config",
|
|
2641
|
+
"pages": "pages/",
|
|
2642
|
+
"components": "components/",
|
|
2643
|
+
},
|
|
2644
|
+
}
|
|
2645
|
+
deploy_path = os.path.join(output_dir, "tw.deploy.json")
|
|
2646
|
+
os.makedirs(os.path.dirname(deploy_path), exist_ok=True)
|
|
2647
|
+
with open(deploy_path, "w", encoding="utf-8") as f:
|
|
2648
|
+
json.dump(metadata, f, indent=2)
|
|
2649
|
+
log(f" ✅ Deployment metadata written: {deploy_path}")
|
|
2650
|
+
|
|
2651
|
+
|
|
2652
|
+
def build_hidden_site(project_root: str, output_dir: str, force: bool = False, workers: Optional[int] = None, minify: bool = True, strict: bool = False, adapters: Optional[List[str]] = None, debug: bool = False) -> BuildSummary:
|
|
2653
|
+
project_root = os.path.abspath(project_root)
|
|
2654
|
+
output_dir = os.path.abspath(output_dir)
|
|
2655
|
+
workers = workers or compiler.DEFAULT_WORKERS
|
|
2656
|
+
|
|
2657
|
+
configure_compiler_paths(project_root)
|
|
2658
|
+
invalidate_compiler_caches()
|
|
2659
|
+
env = load_project_env(project_root, "production")
|
|
2660
|
+
ensure_project_metadata(project_root)
|
|
2661
|
+
ensure_deploy_support_files(project_root)
|
|
2662
|
+
config = compiler.load_config()
|
|
2663
|
+
modular_pipeline = use_modular_pipeline(config)
|
|
2664
|
+
extensions = ExtensionManager(project_root, config, env).refresh()
|
|
2665
|
+
previous_minify = getattr(compiler, "MINIFY_OUTPUT", False)
|
|
2666
|
+
|
|
2667
|
+
# Initialize incremental cache and dependency graph
|
|
2668
|
+
cache = IncrementalCache(project_root)
|
|
2669
|
+
graph = DependencyGraph(project_root)
|
|
2670
|
+
|
|
2671
|
+
try:
|
|
2672
|
+
with compiler_output_context(output_dir):
|
|
2673
|
+
compiler.MINIFY_OUTPUT = minify
|
|
2674
|
+
os.makedirs(output_dir, exist_ok=True)
|
|
2675
|
+
os.makedirs(compiler.PUBLIC_ASSETS_DIR, exist_ok=True)
|
|
2676
|
+
os.makedirs(compiler.CACHE_DIR, exist_ok=True)
|
|
2677
|
+
os.makedirs(compiler.MANIFEST_DIR, exist_ok=True)
|
|
2678
|
+
os.makedirs(compiler.COMPILER_DIR, exist_ok=True)
|
|
2679
|
+
|
|
2680
|
+
css_url, _ = compiler.read_global_stylesheet()
|
|
2681
|
+
manifest = compiler.load_build_manifest()
|
|
2682
|
+
|
|
2683
|
+
compiler.copy_assets()
|
|
2684
|
+
compiler.verify_api_isolated()
|
|
2685
|
+
|
|
2686
|
+
pages = compiler.discover_pages()
|
|
2687
|
+
current_page_keys = {compiler.page_cache_key(page) for page in pages}
|
|
2688
|
+
removed = compiler.remove_deleted_page_outputs(manifest, current_page_keys)
|
|
2689
|
+
|
|
2690
|
+
# Load dependency graph
|
|
2691
|
+
graph.build_from_project()
|
|
2692
|
+
|
|
2693
|
+
dependency_map = {}
|
|
2694
|
+
page_metadata_map = {}
|
|
2695
|
+
pages_to_build = []
|
|
2696
|
+
skipped = 0
|
|
2697
|
+
errors = 0
|
|
2698
|
+
built = 0
|
|
2699
|
+
warnings = 0
|
|
2700
|
+
|
|
2701
|
+
options = compiler.BuildOptions(force=force, workers=workers)
|
|
2702
|
+
shared_dependencies = extensions.dependency_paths()
|
|
2703
|
+
if extensions.errors:
|
|
2704
|
+
for message in extensions.errors:
|
|
2705
|
+
log(f" ❌ Extension error: {message}", level="error")
|
|
2706
|
+
errors += len(extensions.errors)
|
|
2707
|
+
extensions.emit(
|
|
2708
|
+
"beforeBuild",
|
|
2709
|
+
output_dir=output_dir,
|
|
2710
|
+
force=force,
|
|
2711
|
+
workers=workers,
|
|
2712
|
+
minify=minify,
|
|
2713
|
+
)
|
|
2714
|
+
|
|
2715
|
+
for page_info in pages:
|
|
2716
|
+
try:
|
|
2717
|
+
page_ast = compiler.load_page_ast_from_file(page_info["path"])
|
|
2718
|
+
analysis_context = compiler.build_page_context(page_info, page_ast, page_info["path"], route_path=compiler.route_path_from_page_info(page_info))
|
|
2719
|
+
if page_info["type"] == "dynamic":
|
|
2720
|
+
items = compiler.load_dynamic_items(page_info["path"])
|
|
2721
|
+
sample_item = next((item for item in items if isinstance(item, dict)), None)
|
|
2722
|
+
if sample_item:
|
|
2723
|
+
analysis_context = compiler.build_page_context(
|
|
2724
|
+
page_info,
|
|
2725
|
+
page_ast,
|
|
2726
|
+
page_info["path"],
|
|
2727
|
+
item=sample_item,
|
|
2728
|
+
route_path=compiler.route_path_from_page_info(page_info, item=sample_item),
|
|
2729
|
+
)
|
|
2730
|
+
if modular_pipeline:
|
|
2731
|
+
analysis_route = analysis_context.get("_tw_route", "/")
|
|
2732
|
+
artifacts = compiler.compile_file_pipeline(
|
|
2733
|
+
page_info["path"],
|
|
2734
|
+
context=analysis_context,
|
|
2735
|
+
css_href=css_url,
|
|
2736
|
+
route_path=analysis_route,
|
|
2737
|
+
)
|
|
2738
|
+
diagnostics = [compiler.Diagnostic(**item) for item in artifacts.diagnostics]
|
|
2739
|
+
else:
|
|
2740
|
+
diagnostics = compiler.analyze_page_semantics(page_ast, analysis_context, page_info["path"], page_info=page_info)
|
|
2741
|
+
for diagnostic in diagnostics:
|
|
2742
|
+
if diagnostic.severity == "warning":
|
|
2743
|
+
warnings += 1
|
|
2744
|
+
compiler.print_diagnostic(diagnostic)
|
|
2745
|
+
dependencies = compiler.collect_page_dependencies(page_info["path"]) + shared_dependencies
|
|
2746
|
+
dependencies = sorted(set(dependencies))
|
|
2747
|
+
dependency_map[compiler.page_cache_key(page_info)] = dependencies
|
|
2748
|
+
|
|
2749
|
+
# Update dependency graph
|
|
2750
|
+
for dep in dependencies:
|
|
2751
|
+
graph.add_dependency(compiler.page_cache_key(page_info), dep)
|
|
2752
|
+
|
|
2753
|
+
page_metadata = compiler.collect_page_metadata(
|
|
2754
|
+
page_info,
|
|
2755
|
+
page_ast=page_ast,
|
|
2756
|
+
route_path=analysis_context.get("_tw_route", "/"),
|
|
2757
|
+
pipeline="modular" if modular_pipeline else "legacy",
|
|
2758
|
+
)
|
|
2759
|
+
page_metadata["dependency_count"] = len(dependencies)
|
|
2760
|
+
page_metadata_map[compiler.page_cache_key(page_info)] = page_metadata
|
|
2761
|
+
|
|
2762
|
+
# Check incremental cache
|
|
2763
|
+
cached = cache.get(compiler.page_cache_key(page_info))
|
|
2764
|
+
if cached is not None:
|
|
2765
|
+
cached_sig = cached.get("signature")
|
|
2766
|
+
current_sig = compiler.compute_dependency_signature(dependencies)
|
|
2767
|
+
if cached_sig == current_sig:
|
|
2768
|
+
# Cache hit: skip compilation
|
|
2769
|
+
skipped += 1
|
|
2770
|
+
log(f" ⏭️ {compiler.safe_relpath(page_info['path'], compiler.PROJECT_ROOT)} (cache hit)")
|
|
2771
|
+
continue
|
|
2772
|
+
|
|
2773
|
+
needs_build, reason = compiler.should_rebuild_page(page_info, dependencies, manifest, options)
|
|
2774
|
+
if needs_build:
|
|
2775
|
+
pages_to_build.append(page_info)
|
|
2776
|
+
else:
|
|
2777
|
+
skipped += 1
|
|
2778
|
+
log(f" ⏭️ {compiler.safe_relpath(page_info['path'], compiler.PROJECT_ROOT)} ({reason})")
|
|
2779
|
+
except Exception as err:
|
|
2780
|
+
errors += 1
|
|
2781
|
+
compiler.print_compiler_error(page_info, err, debug=debug)
|
|
2782
|
+
|
|
2783
|
+
if pages_to_build:
|
|
2784
|
+
with compiler.concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
|
|
2785
|
+
future_map = {
|
|
2786
|
+
executor.submit(build_page_job_modular if modular_pipeline else compiler.build_page_job, page_info, css_url): page_info
|
|
2787
|
+
for page_info in pages_to_build
|
|
2788
|
+
}
|
|
2789
|
+
for future in compiler.concurrent.futures.as_completed(future_map):
|
|
2790
|
+
page_info = future_map[future]
|
|
2791
|
+
try:
|
|
2792
|
+
result = future.result()
|
|
2793
|
+
outputs = result["outputs"]
|
|
2794
|
+
compiler.update_page_manifest_entry(
|
|
2795
|
+
manifest,
|
|
2796
|
+
page_info,
|
|
2797
|
+
dependency_map[compiler.page_cache_key(page_info)],
|
|
2798
|
+
outputs,
|
|
2799
|
+
metadata=page_metadata_map.get(compiler.page_cache_key(page_info)),
|
|
2800
|
+
)
|
|
2801
|
+
# Save to incremental cache
|
|
2802
|
+
cache.set(compiler.page_cache_key(page_info), {
|
|
2803
|
+
"dependencies": dependency_map.get(compiler.page_cache_key(page_info), []),
|
|
2804
|
+
"outputs": outputs,
|
|
2805
|
+
"signature": compiler.compute_dependency_signature(
|
|
2806
|
+
dependency_map.get(compiler.page_cache_key(page_info), [])
|
|
2807
|
+
),
|
|
2808
|
+
})
|
|
2809
|
+
for out_path in outputs:
|
|
2810
|
+
rel_out = os.path.relpath(out_path, output_dir)
|
|
2811
|
+
log(f" ✅ {rel_out}")
|
|
2812
|
+
built += 1
|
|
2813
|
+
except Exception as err:
|
|
2814
|
+
errors += 1
|
|
2815
|
+
compiler.print_compiler_error(page_info, err, debug=debug)
|
|
2816
|
+
|
|
2817
|
+
try:
|
|
2818
|
+
ts_outputs = compile_typescript_sources(project_root, output_dir)
|
|
2819
|
+
if ts_outputs:
|
|
2820
|
+
log(f" ✅ TypeScript emitted: {len(ts_outputs)} file(s)")
|
|
2821
|
+
except Exception as err:
|
|
2822
|
+
errors += 1
|
|
2823
|
+
log(f" ❌ {err}", level="error")
|
|
2824
|
+
|
|
2825
|
+
sync_runtime_chunks_to_output()
|
|
2826
|
+
write_hidden_cache_files(dependency_map, page_metadata_map)
|
|
2827
|
+
compiler.save_build_manifest(manifest)
|
|
2828
|
+
compiler.save_dependency_graph({"forward": dependency_map, "metadata": page_metadata_map})
|
|
2829
|
+
route_collisions = 0
|
|
2830
|
+
try:
|
|
2831
|
+
route_collisions = write_route_artifacts(output_dir)
|
|
2832
|
+
except Exception as err:
|
|
2833
|
+
# Route artifact generation should not take down builds silently.
|
|
2834
|
+
errors += 1
|
|
2835
|
+
log(f" ❌ Failed to write route artifacts: {err}", level="error")
|
|
2836
|
+
if route_collisions:
|
|
2837
|
+
if strict:
|
|
2838
|
+
errors += route_collisions
|
|
2839
|
+
log(f" ❌ Route collisions treated as errors (--strict): {route_collisions}", level="error")
|
|
2840
|
+
else:
|
|
2841
|
+
warnings += route_collisions
|
|
2842
|
+
log(f" ⚠️ Route collisions: {route_collisions}", level="warning")
|
|
2843
|
+
|
|
2844
|
+
# Log static/dynamic route counts
|
|
2845
|
+
static_count = sum(1 for p in pages if p.get("type") == "static")
|
|
2846
|
+
dynamic_count = sum(1 for p in pages if p.get("type") == "dynamic")
|
|
2847
|
+
log(f" 📄 Static routes: {static_count}")
|
|
2848
|
+
log(f" 📄 Dynamic routes: {dynamic_count}")
|
|
2849
|
+
|
|
2850
|
+
precompress_output(output_dir)
|
|
2851
|
+
|
|
2852
|
+
# ── Connect deployment adapters ──────────────────────────────
|
|
2853
|
+
# Generate provider-specific configuration files after build.
|
|
2854
|
+
selected_adapters = adapters or []
|
|
2855
|
+
for adapter_name in selected_adapters:
|
|
2856
|
+
try:
|
|
2857
|
+
if adapter_name == "vercel":
|
|
2858
|
+
generate_vercel_output(output_dir, config, project_root)
|
|
2859
|
+
generate_vercel_api_functions(output_dir, project_root, discover_api_routes())
|
|
2860
|
+
adapter_file_path = os.path.join(output_dir, "vercel.json")
|
|
2861
|
+
elif adapter_name == "netlify":
|
|
2862
|
+
generate_netlify_output(output_dir, config, project_root)
|
|
2863
|
+
adapter_file_path = os.path.join(output_dir, "netlify.toml")
|
|
2864
|
+
elif adapter_name == "cloudflare":
|
|
2865
|
+
generate_cloudflare_output(output_dir, config, project_root)
|
|
2866
|
+
adapter_file_path = os.path.join(output_dir, "wrangler.toml")
|
|
2867
|
+
else:
|
|
2868
|
+
adapter_file_path = None
|
|
2869
|
+
if adapter_file_path:
|
|
2870
|
+
if os.path.isfile(adapter_file_path):
|
|
2871
|
+
log(f" ✅ {adapter_name} adapter config generated: {adapter_file_path}")
|
|
2872
|
+
else:
|
|
2873
|
+
log(f" ⚠️ {adapter_name} adapter config file not found: {adapter_file_path}", level="warning")
|
|
2874
|
+
except Exception as err:
|
|
2875
|
+
log(f" ⚠️ {adapter_name} adapter config generation failed: {err}", level="warning")
|
|
2876
|
+
|
|
2877
|
+
extensions.emit(
|
|
2878
|
+
"afterBuild",
|
|
2879
|
+
output_dir=output_dir,
|
|
2880
|
+
summary={
|
|
2881
|
+
"built": built,
|
|
2882
|
+
"skipped": skipped,
|
|
2883
|
+
"removed": removed,
|
|
2884
|
+
"errors": errors,
|
|
2885
|
+
"warnings": warnings,
|
|
2886
|
+
},
|
|
2887
|
+
)
|
|
2888
|
+
if warnings:
|
|
2889
|
+
log(f" ⚠️ Semantic warnings: {warnings}", level="warning")
|
|
2890
|
+
|
|
2891
|
+
# Save dependency graph
|
|
2892
|
+
try:
|
|
2893
|
+
graph.save()
|
|
2894
|
+
except Exception as e:
|
|
2895
|
+
log(f" ⚠️ Failed to save dependency graph: {e}", level="warning")
|
|
2896
|
+
|
|
2897
|
+
# Generate deployment metadata for zero‑config platforms
|
|
2898
|
+
try:
|
|
2899
|
+
generate_deploy_metadata(output_dir, config)
|
|
2900
|
+
except Exception as e:
|
|
2901
|
+
log(f" ⚠️ Failed to write deployment metadata: {e}", level="warning")
|
|
2902
|
+
|
|
2903
|
+
# Generate build report
|
|
2904
|
+
try:
|
|
2905
|
+
report = BuildReport(project_root)
|
|
2906
|
+
report.pages = [{"path": p["path"], "size": 0, "duration": 0} for p in pages]
|
|
2907
|
+
report.save()
|
|
2908
|
+
except Exception as e:
|
|
2909
|
+
log(f" ⚠️ Failed to generate build report: {e}", level="warning")
|
|
2910
|
+
|
|
2911
|
+
# Compiler statistics
|
|
2912
|
+
try:
|
|
2913
|
+
stats = CompilerStats()
|
|
2914
|
+
stats.pages_compiled = built
|
|
2915
|
+
stats.files_reused_from_cache = skipped
|
|
2916
|
+
stats.files_rebuilt = built
|
|
2917
|
+
stats.cache_hits = skipped
|
|
2918
|
+
stats.cache_misses = built
|
|
2919
|
+
log(f" 📊 Compiler stats: {stats.to_dict()}")
|
|
2920
|
+
except Exception as e:
|
|
2921
|
+
log(f" ⚠️ Failed to compute compiler stats: {e}", level="warning")
|
|
2922
|
+
|
|
2923
|
+
# Performance analysis
|
|
2924
|
+
try:
|
|
2925
|
+
analyzer = PerformanceAnalyzer()
|
|
2926
|
+
analyzer.measure("build", lambda: None)
|
|
2927
|
+
report_data = analyzer.get_report()
|
|
2928
|
+
log(f" 📊 Performance: {report_data}")
|
|
2929
|
+
except Exception as e:
|
|
2930
|
+
log(f" ⚠️ Performance analysis failed: {e}", level="warning")
|
|
2931
|
+
|
|
2932
|
+
# Advanced diagnostics
|
|
2933
|
+
try:
|
|
2934
|
+
diag_bag = run_advanced_diagnostics(project_root)
|
|
2935
|
+
if diag_bag.has_errors:
|
|
2936
|
+
log(f" ❌ Advanced diagnostics found errors", level="error")
|
|
2937
|
+
except Exception as e:
|
|
2938
|
+
log(f" ⚠️ Advanced diagnostics failed: {e}", level="warning")
|
|
2939
|
+
|
|
2940
|
+
# Production optimization (only for production builds)
|
|
2941
|
+
if minify:
|
|
2942
|
+
try:
|
|
2943
|
+
optimize_for_production(output_dir)
|
|
2944
|
+
log(" ✅ Production optimization applied")
|
|
2945
|
+
except Exception as e:
|
|
2946
|
+
log(f" ⚠️ Production optimization failed: {e}", level="warning")
|
|
2947
|
+
|
|
2948
|
+
# Tree shaking
|
|
2949
|
+
try:
|
|
2950
|
+
unused = shake_project(project_root)
|
|
2951
|
+
if unused.get("components"):
|
|
2952
|
+
log(f" ⚠️ Unused components: {', '.join(unused['components'])}", level="warning")
|
|
2953
|
+
except Exception as e:
|
|
2954
|
+
log(f" ⚠️ Tree shaking failed: {e}", level="warning")
|
|
2955
|
+
|
|
2956
|
+
# Dead code detection
|
|
2957
|
+
try:
|
|
2958
|
+
dead = detect_dead_code(project_root)
|
|
2959
|
+
for category, items in dead.items():
|
|
2960
|
+
if items:
|
|
2961
|
+
log(f" ⚠️ Dead {category}: {', '.join(items)}", level="warning")
|
|
2962
|
+
except Exception as e:
|
|
2963
|
+
log(f" ⚠️ Dead code detection failed: {e}", level="warning")
|
|
2964
|
+
|
|
2965
|
+
# Route optimization
|
|
2966
|
+
try:
|
|
2967
|
+
route_issues = optimize_routes(project_root)
|
|
2968
|
+
if route_issues.get("duplicate_routes"):
|
|
2969
|
+
for issue in route_issues["duplicate_routes"]:
|
|
2970
|
+
log(f" ⚠️ Duplicate route: {issue['route']} from {', '.join(issue['paths'])}", level="warning")
|
|
2971
|
+
except Exception as e:
|
|
2972
|
+
log(f" ⚠️ Route optimization failed: {e}", level="warning")
|
|
2973
|
+
|
|
2974
|
+
# Code splitting
|
|
2975
|
+
try:
|
|
2976
|
+
chunk_map = generate_chunks(pages, output_dir)
|
|
2977
|
+
if chunk_map:
|
|
2978
|
+
log(f" ✅ Code splitting generated {len(chunk_map)} chunk(s)")
|
|
2979
|
+
except Exception as e:
|
|
2980
|
+
log(f" ⚠️ Code splitting failed: {e}", level="warning")
|
|
2981
|
+
|
|
2982
|
+
return BuildSummary(
|
|
2983
|
+
built=built,
|
|
2984
|
+
skipped=skipped,
|
|
2985
|
+
removed=removed,
|
|
2986
|
+
errors=errors,
|
|
2987
|
+
warnings=warnings,
|
|
2988
|
+
route_collisions=route_collisions,
|
|
2989
|
+
output_dir=output_dir,
|
|
2990
|
+
)
|
|
2991
|
+
finally:
|
|
2992
|
+
compiler.MINIFY_OUTPUT = previous_minify
|
|
2993
|
+
|
|
2994
|
+
|
|
2995
|
+
def clean_project_outputs(project_root: str):
|
|
2996
|
+
project_root = os.path.abspath(project_root)
|
|
2997
|
+
configure_compiler_paths(project_root)
|
|
2998
|
+
|
|
2999
|
+
for path in [compiler.PUBLIC_DIR, compiler.INTERNAL_DIR]:
|
|
3000
|
+
if os.path.exists(path):
|
|
3001
|
+
shutil.rmtree(path)
|
|
3002
|
+
|
|
3003
|
+
os.makedirs(compiler.CACHE_DIR, exist_ok=True)
|
|
3004
|
+
os.makedirs(compiler.MANIFEST_DIR, exist_ok=True)
|
|
3005
|
+
os.makedirs(compiler.COMPILER_DIR, exist_ok=True)
|
|
3006
|
+
os.makedirs(compiler.PUBLIC_DIR, exist_ok=True)
|
|
3007
|
+
|
|
3008
|
+
|
|
3009
|
+
def inspect_project(project_root: str) -> dict:
|
|
3010
|
+
project = TWProject(project_root)
|
|
3011
|
+
pages = project.discover_pages()
|
|
3012
|
+
components_dir = os.path.join(project_root, "[home]", "components")
|
|
3013
|
+
component_count = 0
|
|
3014
|
+
if os.path.isdir(components_dir):
|
|
3015
|
+
for name in os.listdir(components_dir):
|
|
3016
|
+
if name.endswith(".tw"):
|
|
3017
|
+
component_count += 1
|
|
3018
|
+
|
|
3019
|
+
dynamic_routes = sum(1 for page in pages if page["type"] == "dynamic")
|
|
3020
|
+
static_routes = sum(1 for page in pages if page["type"] == "static")
|
|
3021
|
+
return {
|
|
3022
|
+
"project_root": project_root,
|
|
3023
|
+
"source_root": compiler.HOME_DIR,
|
|
3024
|
+
"output_dir": compiler.PUBLIC_DIR,
|
|
3025
|
+
"hidden_dir": compiler.INTERNAL_DIR,
|
|
3026
|
+
"page_count": len(pages),
|
|
3027
|
+
"static_routes": static_routes,
|
|
3028
|
+
"dynamic_routes": dynamic_routes,
|
|
3029
|
+
"component_count": component_count,
|
|
3030
|
+
"has_404": project.find_special_page(404) is not None,
|
|
3031
|
+
"has_500": project.find_special_page(500) is not None,
|
|
3032
|
+
"modular_pipeline": project.modular_pipeline,
|
|
3033
|
+
}
|
|
3034
|
+
|
|
3035
|
+
|
|
3036
|
+
def doctor_project(project_root: str) -> List[dict]:
|
|
3037
|
+
project_root = os.path.abspath(project_root)
|
|
3038
|
+
configure_compiler_paths(project_root)
|
|
3039
|
+
checks = []
|
|
3040
|
+
|
|
3041
|
+
def add_check(name: str, ok: bool, detail: str):
|
|
3042
|
+
checks.append({"name": name, "ok": ok, "detail": detail})
|
|
3043
|
+
|
|
3044
|
+
add_check("tw.config", os.path.exists(compiler.CONFIG_FILE), compiler.CONFIG_FILE)
|
|
3045
|
+
add_check("[home]", os.path.isdir(compiler.HOME_DIR), compiler.HOME_DIR)
|
|
3046
|
+
add_check("package.json", os.path.exists(os.path.join(project_root, "package.json")), os.path.join(project_root, "package.json"))
|
|
3047
|
+
add_check("vercel.json", os.path.exists(os.path.join(project_root, "vercel.json")), os.path.join(project_root, "vercel.json"))
|
|
3048
|
+
add_check("netlify.toml", os.path.exists(os.path.join(project_root, "netlify.toml")), os.path.join(project_root, "netlify.toml"))
|
|
3049
|
+
add_check("Dockerfile", os.path.exists(os.path.join(project_root, "Dockerfile")), os.path.join(project_root, "Dockerfile"))
|
|
3050
|
+
add_check("TypeScript compiler", shutil.which("tsc") is not None, shutil.which("tsc") or "`tsc` not installed")
|
|
3051
|
+
add_check("Vercel CLI", shutil.which("vercel") is not None, shutil.which("vercel") or "`vercel` not installed")
|
|
3052
|
+
add_check("Cloudflare Wrangler", shutil.which("wrangler") is not None, shutil.which("wrangler") or "`wrangler` not installed")
|
|
3053
|
+
|
|
3054
|
+
project_info = inspect_project(project_root)
|
|
3055
|
+
add_check("Route discovery", project_info["page_count"] > 0, f"{project_info['page_count']} route(s) discovered")
|
|
3056
|
+
add_check(
|
|
3057
|
+
"Custom 404 page",
|
|
3058
|
+
project_info["has_404"],
|
|
3059
|
+
"`[home]/pages/404.tw` detected" if project_info["has_404"] else "Add `[home]/pages/404.tw` for branded not-found pages",
|
|
3060
|
+
)
|
|
3061
|
+
add_check(
|
|
3062
|
+
"Custom 500 page",
|
|
3063
|
+
project_info["has_500"],
|
|
3064
|
+
"`[home]/pages/500.tw` detected" if project_info["has_500"] else "Add `[home]/pages/500.tw` for branded server error pages",
|
|
3065
|
+
)
|
|
3066
|
+
add_check("API routes", len(discover_api_routes()) >= 0, f"{len(discover_api_routes())} API route(s) discovered")
|
|
3067
|
+
return checks
|
|
3068
|
+
|
|
3069
|
+
|
|
3070
|
+
def deploy_with_vercel(output_dir: str, production: bool):
|
|
3071
|
+
vercel_bin = shutil.which("vercel")
|
|
3072
|
+
if not vercel_bin:
|
|
3073
|
+
raise RuntimeError("`vercel` CLI is not installed. Please install the Vercel CLI first.")
|
|
3074
|
+
|
|
3075
|
+
command = [vercel_bin, "deploy", output_dir, "--yes"]
|
|
3076
|
+
if production:
|
|
3077
|
+
command.append("--prod")
|
|
3078
|
+
|
|
3079
|
+
token = os.environ.get("VERCEL_TOKEN")
|
|
3080
|
+
if token:
|
|
3081
|
+
command.extend(["--token", token])
|
|
3082
|
+
|
|
3083
|
+
subprocess.run(command, check=True)
|
|
3084
|
+
|
|
3085
|
+
|
|
3086
|
+
def deploy_with_cloudflare(output_dir: str, project_name: str):
|
|
3087
|
+
wrangler_bin = shutil.which("wrangler")
|
|
3088
|
+
if not wrangler_bin:
|
|
3089
|
+
raise RuntimeError("`wrangler` CLI is not installed. Please install Cloudflare Wrangler first.")
|
|
3090
|
+
|
|
3091
|
+
command = [
|
|
3092
|
+
wrangler_bin,
|
|
3093
|
+
"pages",
|
|
3094
|
+
"deploy",
|
|
3095
|
+
output_dir,
|
|
3096
|
+
"--project-name",
|
|
3097
|
+
project_name,
|
|
3098
|
+
]
|
|
3099
|
+
subprocess.run(command, check=True)
|
|
3100
|
+
|
|
3101
|
+
|
|
3102
|
+
def deploy_with_netlify(output_dir: str):
|
|
3103
|
+
netlify_bin = shutil.which("netlify")
|
|
3104
|
+
if not netlify_bin:
|
|
3105
|
+
log("`netlify` CLI not found. Static output and `netlify.toml` are ready.", level="warning")
|
|
3106
|
+
return
|
|
3107
|
+
subprocess.run([netlify_bin, "deploy", "--dir", output_dir, "--prod"], check=True)
|
|
3108
|
+
|
|
3109
|
+
|
|
3110
|
+
def deploy_with_docker(project_root: str, production: bool):
|
|
3111
|
+
docker_bin = shutil.which("docker")
|
|
3112
|
+
if not docker_bin:
|
|
3113
|
+
log("`docker` CLI not found. `Dockerfile` is ready.", level="warning")
|
|
3114
|
+
return
|
|
3115
|
+
tag = os.path.basename(os.path.abspath(project_root)).lower().replace("_", "-") + (":prod" if production else ":latest")
|
|
3116
|
+
subprocess.run([docker_bin, "build", "-t", tag, project_root], check=True)
|
|
3117
|
+
|
|
3118
|
+
|
|
3119
|
+
def run_deploy(project_root: str, output_dir: str, provider: str, production: bool, dry_run: bool = False):
|
|
3120
|
+
ensure_deploy_support_files(project_root)
|
|
3121
|
+
checks = doctor_project(project_root)
|
|
3122
|
+
blocking_failures = [check for check in checks if not check["ok"] and check["name"] in {"tw.config", "[home]", "Route discovery"}]
|
|
3123
|
+
if blocking_failures:
|
|
3124
|
+
raise RuntimeError("Project checks failed. Fix the config or route structure first.")
|
|
3125
|
+
summary = build_hidden_site(project_root, output_dir, force=True, minify=production)
|
|
3126
|
+
if summary.errors:
|
|
3127
|
+
raise RuntimeError("Build failed with errors. Deployment was aborted.")
|
|
3128
|
+
if dry_run:
|
|
3129
|
+
log("✔ Deploy dry-run: checks + build OK (no upload performed).")
|
|
3130
|
+
return
|
|
3131
|
+
|
|
3132
|
+
if provider == "local":
|
|
3133
|
+
log(f"Local deploy package ready: {summary.output_dir}")
|
|
3134
|
+
return
|
|
3135
|
+
|
|
3136
|
+
if provider == "vercel":
|
|
3137
|
+
deploy_with_vercel(summary.output_dir, production)
|
|
3138
|
+
return
|
|
3139
|
+
|
|
3140
|
+
if provider == "cloudflare":
|
|
3141
|
+
project_name = os.path.basename(os.path.abspath(project_root)).lower().replace("_", "-")
|
|
3142
|
+
deploy_with_cloudflare(summary.output_dir, project_name)
|
|
3143
|
+
return
|
|
3144
|
+
|
|
3145
|
+
if provider == "netlify":
|
|
3146
|
+
deploy_with_netlify(summary.output_dir)
|
|
3147
|
+
return
|
|
3148
|
+
|
|
3149
|
+
if provider == "github-pages":
|
|
3150
|
+
log("GitHub Pages workflow file is ready. Deployment will run after you push the repo.")
|
|
3151
|
+
return
|
|
3152
|
+
|
|
3153
|
+
if provider == "docker":
|
|
3154
|
+
deploy_with_docker(project_root, production)
|
|
3155
|
+
return
|
|
3156
|
+
|
|
3157
|
+
raise RuntimeError(f"Unsupported provider: {provider}")
|
|
3158
|
+
|
|
3159
|
+
|
|
3160
|
+
def parse_args():
|
|
3161
|
+
parser = argparse.ArgumentParser(description="TW framework CLI")
|
|
3162
|
+
parser.add_argument("--project-root", default=DEFAULT_PROJECT_ROOT, help="TW project root")
|
|
3163
|
+
|
|
3164
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
3165
|
+
|
|
3166
|
+
dev_parser = subparsers.add_parser("dev", help="Run the in-memory dev server")
|
|
3167
|
+
dev_parser.add_argument("--host", default=DEFAULT_DEV_HOST)
|
|
3168
|
+
dev_parser.add_argument("--port", type=int, default=DEFAULT_DEV_PORT)
|
|
3169
|
+
|
|
3170
|
+
build_parser = subparsers.add_parser("build", help="Generate a hidden production build")
|
|
3171
|
+
build_parser.add_argument("--out-dir", default=DEFAULT_INTERNAL_OUTPUT)
|
|
3172
|
+
build_parser.add_argument("--force", action="store_true")
|
|
3173
|
+
build_parser.add_argument("--workers", type=int, default=compiler.DEFAULT_WORKERS)
|
|
3174
|
+
|
|
3175
|
+
deploy_parser = subparsers.add_parser("deploy", help="Build and deploy to a hosting provider")
|
|
3176
|
+
deploy_parser.add_argument("--out-dir", default=DEFAULT_INTERNAL_OUTPUT)
|
|
3177
|
+
deploy_parser.add_argument("--provider", default="local", choices=["local", "vercel", "cloudflare", "netlify", "github-pages", "docker"])
|
|
3178
|
+
deploy_parser.add_argument("--prod", action="store_true")
|
|
3179
|
+
|
|
3180
|
+
return parser.parse_args()
|
|
3181
|
+
|
|
3182
|
+
|
|
3183
|
+
def main():
|
|
3184
|
+
args = parse_args()
|
|
3185
|
+
|
|
3186
|
+
if args.command == "dev":
|
|
3187
|
+
run_dev_server(args.project_root, args.host, args.port)
|
|
3188
|
+
return
|
|
3189
|
+
|
|
3190
|
+
if args.command == "build":
|
|
3191
|
+
summary = build_hidden_site(
|
|
3192
|
+
project_root=args.project_root,
|
|
3193
|
+
output_dir=args.out_dir,
|
|
3194
|
+
force=args.force,
|
|
3195
|
+
workers=args.workers,
|
|
3196
|
+
)
|
|
3197
|
+
log(
|
|
3198
|
+
f"\nTW build complete — {summary.built} generated, "
|
|
3199
|
+
f"{summary.skipped} skipped, {summary.removed} removed, {summary.errors} errors"
|
|
3200
|
+
)
|
|
3201
|
+
log(f"Internal output: {summary.output_dir}")
|
|
3202
|
+
return
|
|
3203
|
+
|
|
3204
|
+
if args.command == "deploy":
|
|
3205
|
+
run_deploy(
|
|
3206
|
+
project_root=args.project_root,
|
|
3207
|
+
output_dir=args.out_dir,
|
|
3208
|
+
provider=args.provider,
|
|
3209
|
+
production=args.prod,
|
|
3210
|
+
)
|
|
3211
|
+
log("Deploy complete.")
|
|
3212
|
+
return
|
|
3213
|
+
|
|
3214
|
+
|
|
3215
|
+
if __name__ == "__main__":
|
|
3216
|
+
main()
|