qwenproxy-cli 1.0.0 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +14 -14
- package/README.md +906 -906
- package/bin/qwenproxy.js +5 -1
- package/package.json +77 -78
- package/src/api/error-classifier.ts +159 -159
- package/src/api/error-helpers.ts +118 -118
- package/src/api/models.ts +261 -261
- package/src/api/server.ts +860 -859
- package/src/cache/memory-cache.ts +385 -385
- package/src/clean-cache.ts +204 -204
- package/src/core/account-concurrency.ts +671 -671
- package/src/core/account-manager.ts +301 -297
- package/src/core/account-priority.ts +163 -163
- package/src/core/accounts.ts +186 -186
- package/src/core/config.ts +383 -383
- package/src/core/crypto-utils.ts +79 -79
- package/src/core/database.ts +276 -276
- package/src/core/errors.ts +118 -118
- package/src/core/logger.ts +269 -269
- package/src/core/memory-usage.ts +84 -84
- package/src/core/metrics.ts +291 -291
- package/src/core/model-alias.ts +77 -77
- package/src/core/model-registry.ts +544 -544
- package/src/core/mutex.ts +119 -119
- package/src/core/paths.ts +199 -199
- package/src/core/prompt-limits.ts +214 -214
- package/src/core/reasoning-effort.ts +102 -102
- package/src/core/stream-registry.ts +96 -96
- package/src/core/waf-isolation.ts +117 -117
- package/src/core/watchdog.ts +195 -195
- package/src/delete-chats.ts +23 -23
- package/src/index.ts +65 -64
- package/src/login.ts +147 -147
- package/src/reset-cooldowns.ts +11 -11
- package/src/routes/anthropic/index.ts +355 -355
- package/src/routes/anthropic/translate.ts +522 -522
- package/src/routes/anthropic/types.ts +154 -154
- package/src/routes/anthropic/validation.ts +144 -144
- package/src/routes/chat/account.ts +1817 -1817
- package/src/routes/chat/context.ts +241 -241
- package/src/routes/chat/errors.ts +85 -85
- package/src/routes/chat/helpers.ts +268 -268
- package/src/routes/chat/index.ts +618 -618
- package/src/routes/chat/media.ts +285 -285
- package/src/routes/chat/retry-policy.ts +754 -754
- package/src/routes/chat/stop.ts +98 -98
- package/src/routes/chat/streaming.ts +2710 -2710
- package/src/routes/chat/validation.ts +526 -526
- package/src/routes/chat.ts +2 -2
- package/src/routes/completions.ts +290 -290
- package/src/routes/images.ts +139 -139
- package/src/routes/responses/adapter.ts +503 -503
- package/src/routes/responses/index.ts +405 -405
- package/src/routes/responses/state.ts +230 -230
- package/src/routes/responses/streaming.ts +528 -528
- package/src/routes/responses/types.ts +285 -285
- package/src/routes/responses/validation.ts +202 -202
- package/src/routes/upload.ts +731 -731
- package/src/routes/videos.ts +214 -214
- package/src/services/auth-playwright.ts +173 -173
- package/src/services/captcha-coordinator.ts +161 -161
- package/src/services/captcha-solver.ts +553 -553
- package/src/services/chat-cleanup.ts +80 -80
- package/src/services/context-meter.ts +317 -317
- package/src/services/fingerprint.ts +242 -242
- package/src/services/human-behavior.ts +173 -173
- package/src/services/media-generation.ts +1748 -1748
- package/src/services/playwright.ts +2878 -2800
- package/src/services/qwen-chat-pool.ts +345 -345
- package/src/services/qwen-errors.ts +133 -133
- package/src/services/qwen-headers.ts +79 -79
- package/src/services/qwen-thread-state.ts +393 -393
- package/src/services/qwen-url.ts +19 -19
- package/src/services/qwen.ts +3126 -3126
- package/src/services/session-keeper.ts +88 -88
- package/src/services/token-estimation-metrics.ts +118 -118
- package/src/sync/claude-code.ts +75 -75
- package/src/sync/codex.ts +123 -123
- package/src/sync/index.ts +362 -362
- package/src/sync/omp.ts +105 -105
- package/src/sync/opencode.ts +214 -214
- package/src/sync/types.ts +53 -53
- package/src/sync/utils.ts +27 -27
- package/src/sync-clients.ts +189 -189
- package/src/tools/instructions.ts +137 -137
- package/src/tools/manifest.ts +81 -81
- package/src/tools/parser.ts +2989 -2989
- package/src/tools/toolcall-tags.ts +142 -142
- package/src/tui/app.ts +259 -264
- package/src/tui/index.ts +61 -61
- package/src/tui/markdown.ts +258 -258
- package/src/tui/proxy-client.ts +331 -326
- package/src/tui/screen.ts +294 -278
- package/src/tui/server-manager.ts +270 -270
- package/src/tui/theme.ts +432 -432
- package/src/tui/types.ts +33 -33
- package/src/tui/views/accounts-view.ts +656 -656
- package/src/tui/views/chat-view.ts +1018 -823
- package/src/tui/views/logs-view.ts +479 -413
- package/src/tui/views/status-view.ts +204 -204
- package/src/tui/views/storage-view.ts +304 -291
- package/src/tui/views/sync-view.ts +409 -409
- package/src/types/ali-oss.d.ts +32 -32
- package/src/update-cli.ts +121 -0
- package/src/utils/context-truncation.ts +84 -84
- package/src/utils/json.ts +380 -380
- package/src/utils/session-id.ts +37 -37
- package/src/utils/tool-call-guard.ts +84 -84
- package/src/utils/types.ts +109 -109
package/src/types/ali-oss.d.ts
CHANGED
|
@@ -1,32 +1,32 @@
|
|
|
1
|
-
declare module "ali-oss" {
|
|
2
|
-
interface OSSOptions {
|
|
3
|
-
region?: string;
|
|
4
|
-
accessKeyId: string;
|
|
5
|
-
accessKeySecret: string;
|
|
6
|
-
stsToken?: string;
|
|
7
|
-
bucket?: string;
|
|
8
|
-
endpoint?: string;
|
|
9
|
-
secure?: boolean;
|
|
10
|
-
refreshSTSToken?: () => Promise<{
|
|
11
|
-
accessKeyId: string;
|
|
12
|
-
accessKeySecret: string;
|
|
13
|
-
stsToken: string;
|
|
14
|
-
}>;
|
|
15
|
-
refreshSTSTokenInterval?: number;
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
interface PutOptions {
|
|
19
|
-
headers?: Record<string, string>;
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
class OSS {
|
|
23
|
-
constructor(options: OSSOptions);
|
|
24
|
-
put(
|
|
25
|
-
name: string,
|
|
26
|
-
file: Buffer | string,
|
|
27
|
-
options?: PutOptions,
|
|
28
|
-
): Promise<any>;
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
export default OSS;
|
|
32
|
-
}
|
|
1
|
+
declare module "ali-oss" {
|
|
2
|
+
interface OSSOptions {
|
|
3
|
+
region?: string;
|
|
4
|
+
accessKeyId: string;
|
|
5
|
+
accessKeySecret: string;
|
|
6
|
+
stsToken?: string;
|
|
7
|
+
bucket?: string;
|
|
8
|
+
endpoint?: string;
|
|
9
|
+
secure?: boolean;
|
|
10
|
+
refreshSTSToken?: () => Promise<{
|
|
11
|
+
accessKeyId: string;
|
|
12
|
+
accessKeySecret: string;
|
|
13
|
+
stsToken: string;
|
|
14
|
+
}>;
|
|
15
|
+
refreshSTSTokenInterval?: number;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
interface PutOptions {
|
|
19
|
+
headers?: Record<string, string>;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
class OSS {
|
|
23
|
+
constructor(options: OSSOptions);
|
|
24
|
+
put(
|
|
25
|
+
name: string,
|
|
26
|
+
file: Buffer | string,
|
|
27
|
+
options?: PutOptions,
|
|
28
|
+
): Promise<any>;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export default OSS;
|
|
32
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process";
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
|
|
6
|
+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
7
|
+
const packageRoot = path.resolve(__dirname, "..");
|
|
8
|
+
const packageJsonPath = path.join(packageRoot, "package.json");
|
|
9
|
+
|
|
10
|
+
export type PackageManager = "bun" | "pnpm" | "yarn" | "npm";
|
|
11
|
+
|
|
12
|
+
export function detectPackageManager(): PackageManager {
|
|
13
|
+
const userAgent = process.env.npm_config_user_agent || "";
|
|
14
|
+
if (userAgent.startsWith("bun")) return "bun";
|
|
15
|
+
if (userAgent.startsWith("pnpm")) return "pnpm";
|
|
16
|
+
if (userAgent.startsWith("yarn")) return "yarn";
|
|
17
|
+
|
|
18
|
+
// Check which executable is actually running the script or installed in PATH
|
|
19
|
+
const execPath = process.execPath.toLowerCase();
|
|
20
|
+
if (execPath.includes("bun")) return "bun";
|
|
21
|
+
|
|
22
|
+
// Check if installation path contains pnpm / bun / yarn markers
|
|
23
|
+
const currentPath = packageRoot.toLowerCase();
|
|
24
|
+
if (currentPath.includes(".pnpm") || currentPath.includes("pnpm")) return "pnpm";
|
|
25
|
+
if (currentPath.includes(".bun") || currentPath.includes("bun")) return "bun";
|
|
26
|
+
if (currentPath.includes("yarn")) return "yarn";
|
|
27
|
+
|
|
28
|
+
return "npm";
|
|
29
|
+
}
|
|
30
|
+
export function getUpdateArgs(pm: PackageManager, packageName: string): { cmd: string; args: string[] } {
|
|
31
|
+
switch (pm) {
|
|
32
|
+
case "bun":
|
|
33
|
+
return { cmd: "bun", args: ["add", "-g", `${packageName}@latest`] };
|
|
34
|
+
case "pnpm":
|
|
35
|
+
return { cmd: "pnpm", args: ["update", "-g", packageName] };
|
|
36
|
+
case "yarn":
|
|
37
|
+
return { cmd: "yarn", args: ["global", "upgrade", packageName] };
|
|
38
|
+
case "npm":
|
|
39
|
+
default:
|
|
40
|
+
return { cmd: "npm", args: ["install", "-g", `${packageName}@latest`] };
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function isNewerVersion(current: string, latest: string): boolean {
|
|
45
|
+
const c = current.replace(/^v/i, "").split(".").map((n) => parseInt(n, 10));
|
|
46
|
+
const l = latest.replace(/^v/i, "").split(".").map((n) => parseInt(n, 10));
|
|
47
|
+
for (let i = 0; i < 3; i++) {
|
|
48
|
+
const cv = c[i] || 0;
|
|
49
|
+
const lv = l[i] || 0;
|
|
50
|
+
if (lv > cv) return true;
|
|
51
|
+
if (lv < cv) return false;
|
|
52
|
+
}
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export async function runUpdateCommand(): Promise<void> {
|
|
57
|
+
let pkg: any = { name: "qwenproxy-cli", version: "1.0.0" };
|
|
58
|
+
try {
|
|
59
|
+
pkg = JSON.parse(fs.readFileSync(packageJsonPath, "utf-8"));
|
|
60
|
+
} catch {}
|
|
61
|
+
|
|
62
|
+
const currentVersion = pkg.version || "1.0.0";
|
|
63
|
+
const packageName = pkg.name || "qwenproxy-cli";
|
|
64
|
+
const pm = detectPackageManager();
|
|
65
|
+
|
|
66
|
+
console.log(`\n📦 [QwenProxy] Versão local instalada: v${currentVersion}`);
|
|
67
|
+
console.log(`⚙️ [QwenProxy] Gerenciador de pacotes detectado: ${pm}`);
|
|
68
|
+
console.log(`🔍 [QwenProxy] Verificando se há novas versões de ${packageName} no npm registry...`);
|
|
69
|
+
|
|
70
|
+
let latestVersion = "";
|
|
71
|
+
try {
|
|
72
|
+
const fullCmd = `npm view ${packageName} version`;
|
|
73
|
+
const res = spawnSync(fullCmd, {
|
|
74
|
+
encoding: "utf-8",
|
|
75
|
+
shell: true,
|
|
76
|
+
timeout: 10000,
|
|
77
|
+
});
|
|
78
|
+
latestVersion = res.stdout ? res.stdout.trim() : "";
|
|
79
|
+
} catch {}
|
|
80
|
+
|
|
81
|
+
if (!latestVersion) {
|
|
82
|
+
console.warn("⚠️ [QwenProxy] Não foi possível consultar o registro online.");
|
|
83
|
+
const manual = getUpdateArgs(pm, packageName);
|
|
84
|
+
console.log(`👉 Você pode forçar a atualização manualmente com:\n ${manual.cmd} ${manual.args.join(" ")}\n`);
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
console.log(`🌐 [QwenProxy] Versão mais recente disponível: v${latestVersion}`);
|
|
89
|
+
|
|
90
|
+
if (!isNewerVersion(currentVersion, latestVersion)) {
|
|
91
|
+
console.log(`\n✨ Você já está utilizando a versão mais recente (v${currentVersion})!\n`);
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
console.log(`\n🚀 Nova versão disponível: v${currentVersion} ➔ v${latestVersion}`);
|
|
96
|
+
const { cmd, args } = getUpdateArgs(pm, packageName);
|
|
97
|
+
console.log(`⏳ Atualizando globalmente via ${pm} (${cmd} ${args.join(" ")})...`);
|
|
98
|
+
const fullUpdateCmd = `${cmd} ${args.join(" ")}`;
|
|
99
|
+
const updateProc = spawnSync(fullUpdateCmd, {
|
|
100
|
+
stdio: "inherit",
|
|
101
|
+
shell: true,
|
|
102
|
+
});
|
|
103
|
+
if (updateProc.status === 0) {
|
|
104
|
+
console.log(`\n✅ [QwenProxy] Atualizado com sucesso para a versão v${latestVersion}!`);
|
|
105
|
+
console.log("👉 Digite 'qpx' para iniciar a nova versão.\n");
|
|
106
|
+
} else {
|
|
107
|
+
console.error(`\n❌ [QwenProxy] Falha ao atualizar automaticamente com ${cmd}.`);
|
|
108
|
+
console.log(`👉 Tente executar manualmente:\n ${cmd} ${args.join(" ")}\n`);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// Execute if run directly via CLI runner, not when imported in unit tests
|
|
113
|
+
const isDirectRun =
|
|
114
|
+
Boolean(process.argv[1]) &&
|
|
115
|
+
(fileURLToPath(import.meta.url) === path.resolve(process.argv[1]) ||
|
|
116
|
+
process.argv[1].endsWith("update-cli.ts") ||
|
|
117
|
+
process.argv[1].endsWith("update-cli.js"));
|
|
118
|
+
|
|
119
|
+
if (isDirectRun) {
|
|
120
|
+
void runUpdateCommand();
|
|
121
|
+
}
|
|
@@ -1,84 +1,84 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Heuristic token estimation.
|
|
3
|
-
*
|
|
4
|
-
* Accepts one or more string parts so callers can avoid concatenating large
|
|
5
|
-
* strings just to estimate them (the parts are accumulated before a single
|
|
6
|
-
* final Math.ceil, keeping the result identical to the concatenated form).
|
|
7
|
-
*/
|
|
8
|
-
export function estimateTokenCount(...parts: string[]): number {
|
|
9
|
-
let tokens = 0;
|
|
10
|
-
for (const part of parts) {
|
|
11
|
-
if (part) tokens += estimatePart(part);
|
|
12
|
-
}
|
|
13
|
-
return Math.ceil(tokens);
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
function estimatePart(text: string): number {
|
|
17
|
-
let tokens = 0;
|
|
18
|
-
let i = 0;
|
|
19
|
-
const n = text.length;
|
|
20
|
-
|
|
21
|
-
while (i < n) {
|
|
22
|
-
const c = text.charCodeAt(i);
|
|
23
|
-
|
|
24
|
-
// Fast path: ASCII (the overwhelmingly common case in OpenAI payloads).
|
|
25
|
-
// charCodeAt avoids the per-char string allocation and codePointAt decode.
|
|
26
|
-
if (c < 0x80) {
|
|
27
|
-
if (c >= 0x20 && c <= 0x7e) {
|
|
28
|
-
// Printable ASCII: structural characters weigh more than prose.
|
|
29
|
-
tokens +=
|
|
30
|
-
c === 0x7b || // {
|
|
31
|
-
c === 0x7d || // }
|
|
32
|
-
c === 0x5b || // [
|
|
33
|
-
c === 0x5d || // ]
|
|
34
|
-
c === 0x22 || // "
|
|
35
|
-
c === 0x3a || // :
|
|
36
|
-
c === 0x2c || // ,
|
|
37
|
-
c === 0x3b || // ;
|
|
38
|
-
c === 0x28 || // (
|
|
39
|
-
c === 0x29 || // )
|
|
40
|
-
c === 0x2f || // /
|
|
41
|
-
c === 0x5c // \
|
|
42
|
-
? 0.4
|
|
43
|
-
: 0.25;
|
|
44
|
-
} else if (c === 0x0a || c === 0x0d || c === 0x09) {
|
|
45
|
-
tokens += 0.2; // \n \r \t
|
|
46
|
-
} else {
|
|
47
|
-
tokens += 1.0; // remaining control characters
|
|
48
|
-
}
|
|
49
|
-
i += 1;
|
|
50
|
-
continue;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
// Non-ASCII path: code-point based heuristics (CJK/kana/hangul/other).
|
|
54
|
-
const codePoint = text.codePointAt(i) || 0;
|
|
55
|
-
|
|
56
|
-
// CJK Unified Ideographs (U+4E00-U+9FFF)
|
|
57
|
-
if (codePoint >= 0x4e00 && codePoint <= 0x9fff) {
|
|
58
|
-
tokens += 1.5;
|
|
59
|
-
i += 1;
|
|
60
|
-
}
|
|
61
|
-
// CJK Extension A/B (U+3400-U+2A6DF)
|
|
62
|
-
else if (codePoint >= 0x3400 && codePoint <= 0x2a6df) {
|
|
63
|
-
tokens += 1.5;
|
|
64
|
-
i += codePoint > 0xffff ? 2 : 1;
|
|
65
|
-
}
|
|
66
|
-
// Hiragana/Katakana (U+3040-U+30FF)
|
|
67
|
-
else if (codePoint >= 0x3040 && codePoint <= 0x30ff) {
|
|
68
|
-
tokens += 1.2;
|
|
69
|
-
i += 1;
|
|
70
|
-
}
|
|
71
|
-
// Hangul (U+AC00-U+D7AF)
|
|
72
|
-
else if (codePoint >= 0xac00 && codePoint <= 0xd7af) {
|
|
73
|
-
tokens += 1.3;
|
|
74
|
-
i += 1;
|
|
75
|
-
}
|
|
76
|
-
// Other Unicode (emoji, symbols, etc.)
|
|
77
|
-
else {
|
|
78
|
-
tokens += 1.0;
|
|
79
|
-
i += codePoint > 0xffff ? 2 : 1;
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
return tokens;
|
|
84
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Heuristic token estimation.
|
|
3
|
+
*
|
|
4
|
+
* Accepts one or more string parts so callers can avoid concatenating large
|
|
5
|
+
* strings just to estimate them (the parts are accumulated before a single
|
|
6
|
+
* final Math.ceil, keeping the result identical to the concatenated form).
|
|
7
|
+
*/
|
|
8
|
+
export function estimateTokenCount(...parts: string[]): number {
|
|
9
|
+
let tokens = 0;
|
|
10
|
+
for (const part of parts) {
|
|
11
|
+
if (part) tokens += estimatePart(part);
|
|
12
|
+
}
|
|
13
|
+
return Math.ceil(tokens);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function estimatePart(text: string): number {
|
|
17
|
+
let tokens = 0;
|
|
18
|
+
let i = 0;
|
|
19
|
+
const n = text.length;
|
|
20
|
+
|
|
21
|
+
while (i < n) {
|
|
22
|
+
const c = text.charCodeAt(i);
|
|
23
|
+
|
|
24
|
+
// Fast path: ASCII (the overwhelmingly common case in OpenAI payloads).
|
|
25
|
+
// charCodeAt avoids the per-char string allocation and codePointAt decode.
|
|
26
|
+
if (c < 0x80) {
|
|
27
|
+
if (c >= 0x20 && c <= 0x7e) {
|
|
28
|
+
// Printable ASCII: structural characters weigh more than prose.
|
|
29
|
+
tokens +=
|
|
30
|
+
c === 0x7b || // {
|
|
31
|
+
c === 0x7d || // }
|
|
32
|
+
c === 0x5b || // [
|
|
33
|
+
c === 0x5d || // ]
|
|
34
|
+
c === 0x22 || // "
|
|
35
|
+
c === 0x3a || // :
|
|
36
|
+
c === 0x2c || // ,
|
|
37
|
+
c === 0x3b || // ;
|
|
38
|
+
c === 0x28 || // (
|
|
39
|
+
c === 0x29 || // )
|
|
40
|
+
c === 0x2f || // /
|
|
41
|
+
c === 0x5c // \
|
|
42
|
+
? 0.4
|
|
43
|
+
: 0.25;
|
|
44
|
+
} else if (c === 0x0a || c === 0x0d || c === 0x09) {
|
|
45
|
+
tokens += 0.2; // \n \r \t
|
|
46
|
+
} else {
|
|
47
|
+
tokens += 1.0; // remaining control characters
|
|
48
|
+
}
|
|
49
|
+
i += 1;
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Non-ASCII path: code-point based heuristics (CJK/kana/hangul/other).
|
|
54
|
+
const codePoint = text.codePointAt(i) || 0;
|
|
55
|
+
|
|
56
|
+
// CJK Unified Ideographs (U+4E00-U+9FFF)
|
|
57
|
+
if (codePoint >= 0x4e00 && codePoint <= 0x9fff) {
|
|
58
|
+
tokens += 1.5;
|
|
59
|
+
i += 1;
|
|
60
|
+
}
|
|
61
|
+
// CJK Extension A/B (U+3400-U+2A6DF)
|
|
62
|
+
else if (codePoint >= 0x3400 && codePoint <= 0x2a6df) {
|
|
63
|
+
tokens += 1.5;
|
|
64
|
+
i += codePoint > 0xffff ? 2 : 1;
|
|
65
|
+
}
|
|
66
|
+
// Hiragana/Katakana (U+3040-U+30FF)
|
|
67
|
+
else if (codePoint >= 0x3040 && codePoint <= 0x30ff) {
|
|
68
|
+
tokens += 1.2;
|
|
69
|
+
i += 1;
|
|
70
|
+
}
|
|
71
|
+
// Hangul (U+AC00-U+D7AF)
|
|
72
|
+
else if (codePoint >= 0xac00 && codePoint <= 0xd7af) {
|
|
73
|
+
tokens += 1.3;
|
|
74
|
+
i += 1;
|
|
75
|
+
}
|
|
76
|
+
// Other Unicode (emoji, symbols, etc.)
|
|
77
|
+
else {
|
|
78
|
+
tokens += 1.0;
|
|
79
|
+
i += codePoint > 0xffff ? 2 : 1;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
return tokens;
|
|
84
|
+
}
|