takos-runtime-service 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +29 -0
- package/src/__tests__/middleware/rate-limit.test.ts +33 -0
- package/src/__tests__/middleware/workspace-scope-extended.test.ts +163 -0
- package/src/__tests__/routes/actions-start-limits.test.ts +139 -0
- package/src/__tests__/routes/actions-step-warnings.test.ts +194 -0
- package/src/__tests__/routes/cli-proxy.test.ts +72 -0
- package/src/__tests__/routes/git-http.test.ts +218 -0
- package/src/__tests__/routes/git-lfs-policy.test.ts +112 -0
- package/src/__tests__/routes/sessions/store.test.ts +72 -0
- package/src/__tests__/routes/workspace-scope.test.ts +45 -0
- package/src/__tests__/runtime/action-registry.test.ts +208 -0
- package/src/__tests__/runtime/action-result-helpers.test.ts +129 -0
- package/src/__tests__/runtime/actions/executor.test.ts +131 -0
- package/src/__tests__/runtime/composite-expression.test.ts +294 -0
- package/src/__tests__/runtime/file-parsers.test.ts +129 -0
- package/src/__tests__/runtime/logging.test.ts +65 -0
- package/src/__tests__/runtime/paths.test.ts +236 -0
- package/src/__tests__/runtime/secrets.test.ts +247 -0
- package/src/__tests__/runtime/validation.test.ts +516 -0
- package/src/__tests__/setup.ts +126 -0
- package/src/__tests__/shared/errors.test.ts +117 -0
- package/src/__tests__/storage/r2.test.ts +106 -0
- package/src/__tests__/utils/audit-log.test.ts +163 -0
- package/src/__tests__/utils/error-message.test.ts +38 -0
- package/src/__tests__/utils/sandbox-env.test.ts +74 -0
- package/src/app.ts +245 -0
- package/src/index.ts +1 -0
- package/src/middleware/rate-limit.ts +91 -0
- package/src/middleware/space-scope.ts +95 -0
- package/src/routes/actions/action-types.ts +20 -0
- package/src/routes/actions/execution.ts +229 -0
- package/src/routes/actions/index.ts +17 -0
- package/src/routes/actions/job-lifecycle.ts +242 -0
- package/src/routes/actions/job-queries.ts +52 -0
- package/src/routes/cli/proxy.ts +105 -0
- package/src/routes/git/http.ts +565 -0
- package/src/routes/git/init.ts +88 -0
- package/src/routes/repos/branches.ts +160 -0
- package/src/routes/repos/content.ts +209 -0
- package/src/routes/repos/read.ts +130 -0
- package/src/routes/repos/repo-validation.ts +136 -0
- package/src/routes/repos/write.ts +274 -0
- package/src/routes/runtime/exec.ts +147 -0
- package/src/routes/runtime/tools.ts +113 -0
- package/src/routes/sessions/execution.ts +263 -0
- package/src/routes/sessions/files.ts +326 -0
- package/src/routes/sessions/session-routes.ts +241 -0
- package/src/routes/sessions/session-utils.ts +88 -0
- package/src/routes/sessions/snapshot.ts +208 -0
- package/src/routes/sessions/storage.ts +329 -0
- package/src/runtime/actions/action-registry.ts +450 -0
- package/src/runtime/actions/action-result-converter.ts +31 -0
- package/src/runtime/actions/builtin/artifacts.ts +292 -0
- package/src/runtime/actions/builtin/cache-operations.ts +358 -0
- package/src/runtime/actions/builtin/checkout.ts +58 -0
- package/src/runtime/actions/builtin/index.ts +5 -0
- package/src/runtime/actions/builtin/setup-node.ts +86 -0
- package/src/runtime/actions/builtin/tar-parser.ts +175 -0
- package/src/runtime/actions/composite-executor.ts +192 -0
- package/src/runtime/actions/composite-expression.ts +190 -0
- package/src/runtime/actions/executor.ts +578 -0
- package/src/runtime/actions/file-parsers.ts +51 -0
- package/src/runtime/actions/job-manager.ts +213 -0
- package/src/runtime/actions/process-spawner.ts +275 -0
- package/src/runtime/actions/secrets.ts +162 -0
- package/src/runtime/command.ts +120 -0
- package/src/runtime/exec-runner.ts +309 -0
- package/src/runtime/git-http-backend.ts +145 -0
- package/src/runtime/git.ts +98 -0
- package/src/runtime/heartbeat.ts +57 -0
- package/src/runtime/logging.ts +26 -0
- package/src/runtime/paths.ts +264 -0
- package/src/runtime/secure-fs.ts +82 -0
- package/src/runtime/tools/network.ts +161 -0
- package/src/runtime/tools/worker.ts +335 -0
- package/src/runtime/validation.ts +292 -0
- package/src/shared/config.ts +149 -0
- package/src/shared/errors.ts +65 -0
- package/src/shared/temp-id.ts +10 -0
- package/src/storage/r2.ts +287 -0
- package/src/types/hono.d.ts +23 -0
- package/src/utils/audit-log.ts +92 -0
- package/src/utils/process-kill.ts +18 -0
- package/src/utils/sandbox-env.ts +136 -0
- package/src/utils/temp-dir.ts +74 -0
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
const CORE_SAFE_ENV: Set<string> = new Set([
|
|
2
|
+
'PATH', 'HOME', 'USER', 'USERNAME', 'SHELL', 'TERM',
|
|
3
|
+
'LANG', 'LC_ALL', 'LC_CTYPE', 'LC_MESSAGES', 'LANGUAGE', 'TZ',
|
|
4
|
+
'COLORTERM', 'FORCE_COLOR', 'NO_COLOR',
|
|
5
|
+
'TEMP', 'TMP', 'TMPDIR',
|
|
6
|
+
'NODE_ENV', 'NODE_VERSION', 'NPM_CONFIG_REGISTRY',
|
|
7
|
+
'CI',
|
|
8
|
+
'EDITOR', 'VISUAL', 'PAGER',
|
|
9
|
+
]);
|
|
10
|
+
|
|
11
|
+
const GIT_ENV: Set<string> = new Set([
|
|
12
|
+
'GIT_AUTHOR_NAME', 'GIT_AUTHOR_EMAIL',
|
|
13
|
+
'GIT_COMMITTER_NAME', 'GIT_COMMITTER_EMAIL',
|
|
14
|
+
'GIT_TERMINAL_PROMPT',
|
|
15
|
+
]);
|
|
16
|
+
|
|
17
|
+
const CI_ENV: Set<string> = new Set([
|
|
18
|
+
'RUNNER_TEMP', 'RUNNER_TOOL_CACHE',
|
|
19
|
+
'PNPM_HOME', 'YARN_CACHE_FOLDER', 'npm_config_cache',
|
|
20
|
+
]);
|
|
21
|
+
|
|
22
|
+
const TAKOS_ACTIONS_ENV_ALLOWLIST: Set<string> = new Set([
|
|
23
|
+
'TAKOS_API_URL',
|
|
24
|
+
'TAKOS_TOKEN',
|
|
25
|
+
'TAKOS_WORKSPACE_ID',
|
|
26
|
+
'TAKOS_SPACE_ID',
|
|
27
|
+
'TAKOS_REPO_ID',
|
|
28
|
+
'TAKOS_SESSION_ID',
|
|
29
|
+
]);
|
|
30
|
+
|
|
31
|
+
const SENSITIVE_PATTERNS: RegExp[] = [
|
|
32
|
+
// Exact-match service tokens
|
|
33
|
+
/^SERVICE_TOKEN$/i,
|
|
34
|
+
// Prefix-based: cloud, DB, crypto, internal
|
|
35
|
+
/^(R2|S3|JWT|TAKOS|AWS|AZURE|GCP|GOOGLE|CLOUDFLARE|DATABASE|DB|POSTGRES|MYSQL|MONGO|REDIS|SSL|TLS|SSH|PGP|GPG)_/i,
|
|
36
|
+
// Keyword-based: secrets, auth, crypto
|
|
37
|
+
/SECRET|PASSWORD|PASSWD|CREDENTIAL|PRIVATE|AUTH|ENCRYPTION|SIGNING|CERTIFICATE/i,
|
|
38
|
+
// Positional matches
|
|
39
|
+
/^API_KEY/i, /TOKEN$/i,
|
|
40
|
+
];
|
|
41
|
+
|
|
42
|
+
export const BLOCKED_ENV: Set<string> = new Set([
|
|
43
|
+
'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN', 'AWS_SECURITY_TOKEN',
|
|
44
|
+
'GOOGLE_APPLICATION_CREDENTIALS', 'AZURE_STORAGE_KEY',
|
|
45
|
+
'S3_ENDPOINT', 'S3_REGION', 'S3_ACCESS_KEY_ID', 'S3_SECRET_ACCESS_KEY', 'S3_BUCKET',
|
|
46
|
+
'R2_ACCOUNT_ID', 'R2_ACCESS_KEY_ID', 'R2_SECRET_ACCESS_KEY',
|
|
47
|
+
'SERVICE_TOKEN', 'JWT_PUBLIC_KEY',
|
|
48
|
+
'DATABASE_URL', 'REDIS_URL', 'SECRET_KEY', 'PRIVATE_KEY', 'API_KEY', 'AUTH_TOKEN',
|
|
49
|
+
]);
|
|
50
|
+
|
|
51
|
+
const VALID_ENV_VAR_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
52
|
+
const MAX_EXEC_ENV_VAR_VALUE_LENGTH = 10 * 1024 * 1024;
|
|
53
|
+
|
|
54
|
+
export type RuntimeExecEnvValidationResult =
|
|
55
|
+
| { ok: true; env: Record<string, string> }
|
|
56
|
+
| { ok: false; error: string };
|
|
57
|
+
|
|
58
|
+
export function isSensitiveEnvVar(name: string): boolean {
|
|
59
|
+
if (BLOCKED_ENV.has(name)) return true;
|
|
60
|
+
if (CORE_SAFE_ENV.has(name)) return false;
|
|
61
|
+
return SENSITIVE_PATTERNS.some((pattern) => pattern.test(name));
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export function validateRuntimeExecEnv(
|
|
65
|
+
env: Record<string, string> | undefined
|
|
66
|
+
): RuntimeExecEnvValidationResult {
|
|
67
|
+
if (!env) {
|
|
68
|
+
return { ok: true, env: {} };
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const filteredEnv: Record<string, string> = {};
|
|
72
|
+
|
|
73
|
+
for (const [key, value] of Object.entries(env)) {
|
|
74
|
+
if (!VALID_ENV_VAR_NAME_PATTERN.test(key)) {
|
|
75
|
+
return { ok: false, error: `Invalid environment variable name: ${key}` };
|
|
76
|
+
}
|
|
77
|
+
if (isSensitiveEnvVar(key)) {
|
|
78
|
+
return { ok: false, error: `Sensitive environment variable is not allowed: ${key}` };
|
|
79
|
+
}
|
|
80
|
+
if (value.length > MAX_EXEC_ENV_VAR_VALUE_LENGTH) {
|
|
81
|
+
return { ok: false, error: `Environment variable value too long: ${key}` };
|
|
82
|
+
}
|
|
83
|
+
// eslint-disable-next-line no-control-regex
|
|
84
|
+
if (/[\x00\r\n]/.test(value)) {
|
|
85
|
+
return { ok: false, error: `Environment variable contains invalid characters: ${key}` };
|
|
86
|
+
}
|
|
87
|
+
filteredEnv[key] = value;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
return { ok: true, env: filteredEnv };
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export function filterSafeEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
|
|
94
|
+
const filtered: NodeJS.ProcessEnv = {};
|
|
95
|
+
for (const key of Object.keys(env)) {
|
|
96
|
+
if (CORE_SAFE_ENV.has(key) && !isSensitiveEnvVar(key)) {
|
|
97
|
+
filtered[key] = env[key];
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return filtered;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export function createSandboxEnv(
|
|
104
|
+
baseEnv: Record<string, string>,
|
|
105
|
+
maxValueLength: number = 1024 * 1024
|
|
106
|
+
): Record<string, string> {
|
|
107
|
+
const sandboxEnv: Record<string, string> = { CI: 'true' };
|
|
108
|
+
|
|
109
|
+
const allAllowed = new Set([...CORE_SAFE_ENV, ...GIT_ENV, ...CI_ENV]);
|
|
110
|
+
|
|
111
|
+
for (const key of allAllowed) {
|
|
112
|
+
if (process.env[key] && !BLOCKED_ENV.has(key)) {
|
|
113
|
+
sandboxEnv[key] = process.env[key]!;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
for (const [key, value] of Object.entries(baseEnv)) {
|
|
118
|
+
if (value.length > maxValueLength) continue;
|
|
119
|
+
|
|
120
|
+
// Keep process.env inheritance filtered, but trust explicitly provided
|
|
121
|
+
// workflow/job/step env values even when key names look sensitive.
|
|
122
|
+
const allowTakos = TAKOS_ACTIONS_ENV_ALLOWLIST.has(key);
|
|
123
|
+
if (
|
|
124
|
+
key.startsWith('GITHUB_') ||
|
|
125
|
+
key.startsWith('INPUT_') ||
|
|
126
|
+
key.startsWith('RUNNER_') ||
|
|
127
|
+
allAllowed.has(key) ||
|
|
128
|
+
allowTakos
|
|
129
|
+
) {
|
|
130
|
+
sandboxEnv[key] = value;
|
|
131
|
+
}
|
|
132
|
+
// Keys not matching any allowlist pattern are silently dropped
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
return sandboxEnv;
|
|
136
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import * as crypto from 'crypto';
|
|
2
|
+
import * as fs from 'fs/promises';
|
|
3
|
+
import * as os from 'os';
|
|
4
|
+
import * as path from 'path';
|
|
5
|
+
import { createLogger } from 'takos-common/logger';
|
|
6
|
+
|
|
7
|
+
const logger = createLogger({ service: 'takos-runtime' });
|
|
8
|
+
|
|
9
|
+
interface TempDirManagerOptions {
|
|
10
|
+
/** Cleanup timeout in milliseconds (default: 5 minutes) */
|
|
11
|
+
cleanupTimeoutMs?: number;
|
|
12
|
+
/** Log prefix for cleanup messages */
|
|
13
|
+
logPrefix?: string;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
interface TempDirEntry {
|
|
17
|
+
path: string;
|
|
18
|
+
createdAt: number;
|
|
19
|
+
timer: ReturnType<typeof setTimeout>;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
class TempDirManager {
|
|
23
|
+
private readonly activeTempDirs = new Map<string, TempDirEntry>();
|
|
24
|
+
private readonly cleanupTimeoutMs: number;
|
|
25
|
+
private readonly logPrefix: string;
|
|
26
|
+
|
|
27
|
+
constructor(options: TempDirManagerOptions = {}) {
|
|
28
|
+
this.cleanupTimeoutMs = options.cleanupTimeoutMs ?? 5 * 60 * 1000;
|
|
29
|
+
this.logPrefix = options.logPrefix ?? 'temp';
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
async createTempDirWithCleanup(prefix: string): Promise<string> {
|
|
33
|
+
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), prefix));
|
|
34
|
+
const id = crypto.randomUUID();
|
|
35
|
+
|
|
36
|
+
const timer = setTimeout(async () => {
|
|
37
|
+
try {
|
|
38
|
+
await fs.rm(tempDir, { recursive: true, force: true });
|
|
39
|
+
this.activeTempDirs.delete(id);
|
|
40
|
+
logger.info('Cleanup: removed stale temp dir', { prefix: this.logPrefix, tempDir });
|
|
41
|
+
} catch {
|
|
42
|
+
// Ignore cleanup errors
|
|
43
|
+
}
|
|
44
|
+
}, this.cleanupTimeoutMs);
|
|
45
|
+
|
|
46
|
+
this.activeTempDirs.set(id, { path: tempDir, createdAt: Date.now(), timer });
|
|
47
|
+
|
|
48
|
+
return tempDir;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async cleanupTempDir(tempDir: string): Promise<void> {
|
|
52
|
+
for (const [id, entry] of this.activeTempDirs.entries()) {
|
|
53
|
+
if (entry.path === tempDir) {
|
|
54
|
+
clearTimeout(entry.timer);
|
|
55
|
+
this.activeTempDirs.delete(id);
|
|
56
|
+
break;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
await fs.rm(tempDir, { recursive: true, force: true }).catch((err) => {
|
|
60
|
+
logger.debug('Failed to clean up temp dir', { tempDir, error: err as Error });
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// Default instances for common use cases
|
|
66
|
+
export const execTempDirManager = new TempDirManager({
|
|
67
|
+
cleanupTimeoutMs: 5 * 60 * 1000, // 5 minutes
|
|
68
|
+
logPrefix: 'exec',
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
export const mergeTempDirManager = new TempDirManager({
|
|
72
|
+
cleanupTimeoutMs: 2 * 60 * 60 * 1000, // 2 hours for large merge operations
|
|
73
|
+
logPrefix: 'merge',
|
|
74
|
+
});
|