aicorpusx 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aicorpusx-0.1.0/LICENSE +22 -0
- aicorpusx-0.1.0/PKG-INFO +144 -0
- aicorpusx-0.1.0/README.md +128 -0
- aicorpusx-0.1.0/pyproject.toml +28 -0
- aicorpusx-0.1.0/setup.cfg +4 -0
- aicorpusx-0.1.0/src/aicorpusx/__init__.py +7 -0
- aicorpusx-0.1.0/src/aicorpusx/_core.py +665 -0
- aicorpusx-0.1.0/src/aicorpusx/_glossary.py +144 -0
- aicorpusx-0.1.0/src/aicorpusx/_progress.py +68 -0
- aicorpusx-0.1.0/src/aicorpusx/_providers.py +204 -0
- aicorpusx-0.1.0/src/aicorpusx/_tables.py +161 -0
- aicorpusx-0.1.0/src/aicorpusx.egg-info/PKG-INFO +144 -0
- aicorpusx-0.1.0/src/aicorpusx.egg-info/SOURCES.txt +14 -0
- aicorpusx-0.1.0/src/aicorpusx.egg-info/dependency_links.txt +1 -0
- aicorpusx-0.1.0/src/aicorpusx.egg-info/requires.txt +2 -0
- aicorpusx-0.1.0/src/aicorpusx.egg-info/top_level.txt +1 -0
aicorpusx-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 aicorpusx contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
22
|
+
|
aicorpusx-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: aicorpusx
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Resumable, multi-key translation for CSV and XLSX corpora.
|
|
5
|
+
Author-email: FENG YIFAN <yifan.f.academic@icloud.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Classifier: Development Status :: 3 - Alpha
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
10
|
+
Requires-Python: >=3.9
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Requires-Dist: openpyxl>=3.1
|
|
14
|
+
Requires-Dist: rich>=13.0
|
|
15
|
+
Dynamic: license-file
|
|
16
|
+
|
|
17
|
+
# aicorpusx
|
|
18
|
+
|
|
19
|
+
面向 CSV / XLSX 语料表的可恢复、多 API Key 并发翻译。任意走 OpenAI 兼容 `/chat/completions` 的服务,只需提供 `apis`、`model`、`base_url`:
|
|
20
|
+
|
|
21
|
+
```python
|
|
22
|
+
import aicorpusx
|
|
23
|
+
|
|
24
|
+
aicorpusx.trans(
|
|
25
|
+
"terms.xlsx",
|
|
26
|
+
source_column="中文",
|
|
27
|
+
targets={"ar": "阿拉伯语", "en": "英语"},
|
|
28
|
+
apis=["key-1", "key-2"],
|
|
29
|
+
model="deepseek-chat",
|
|
30
|
+
base_url="https://api.deepseek.com",
|
|
31
|
+
)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
默认输出到 `terms_translated.xlsx`。CSV 输入自动输出 CSV;XLSX 输入自动输出 XLSX。
|
|
35
|
+
|
|
36
|
+
`base_url` 写到 API 根路径即可(例如 `https://api.deepseek.com` 或 `https://api.openai.com/v1`),库会自动补上 `/chat/completions`。若已写成完整 endpoint,则按原样请求。
|
|
37
|
+
|
|
38
|
+
## 安装
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
python -m pip install .
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
## 常用示例
|
|
45
|
+
|
|
46
|
+
指定工作表、源语言与内容模式:
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
aicorpusx.trans(
|
|
50
|
+
"corpus.xlsx",
|
|
51
|
+
sheet_name="Data",
|
|
52
|
+
source_column="原文",
|
|
53
|
+
source_language="zh",
|
|
54
|
+
targets={"ar": "译文"},
|
|
55
|
+
mode="sentence", # auto / term / sentence / text
|
|
56
|
+
apis=["key-1", "key-2"],
|
|
57
|
+
model="deepseek-chat",
|
|
58
|
+
base_url="https://api.deepseek.com",
|
|
59
|
+
)
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
直接传术语表:
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
aicorpusx.trans(
|
|
66
|
+
"corpus.csv",
|
|
67
|
+
source_column="原文",
|
|
68
|
+
targets={"ar": "阿拉伯语"},
|
|
69
|
+
glossary={
|
|
70
|
+
"阴阳": "اليِن واليانغ",
|
|
71
|
+
"五行": "العناصر الخمسة",
|
|
72
|
+
},
|
|
73
|
+
glossary_mode="strict", # strict / prefer / off
|
|
74
|
+
apis=["key-1"],
|
|
75
|
+
model="deepseek-chat",
|
|
76
|
+
base_url="https://api.deepseek.com",
|
|
77
|
+
)
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
多语言术语文件可使用 `source,ar,en,de` 这样的列:
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
aicorpusx.trans(
|
|
84
|
+
"corpus.xlsx",
|
|
85
|
+
source_column="原文",
|
|
86
|
+
targets={"ar": "阿拉伯语", "en": "英语", "de": "德语"},
|
|
87
|
+
glossary="glossary.xlsx",
|
|
88
|
+
apis=["key-1", "key-2"],
|
|
89
|
+
model="deepseek-chat",
|
|
90
|
+
base_url="https://api.deepseek.com",
|
|
91
|
+
)
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
列名不规则时可以显式映射:
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
aicorpusx.trans(
|
|
98
|
+
"corpus.xlsx",
|
|
99
|
+
source_column="原文",
|
|
100
|
+
targets={"ar": "阿拉伯语"},
|
|
101
|
+
glossary="glossary.xlsx",
|
|
102
|
+
glossary_source_column="中文术语",
|
|
103
|
+
glossary_target_columns={"ar": "阿语标准译名"},
|
|
104
|
+
apis=["key-1"],
|
|
105
|
+
model="deepseek-chat",
|
|
106
|
+
base_url="https://api.deepseek.com",
|
|
107
|
+
)
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
术语在句中按最长优先做局部匹配。`strict` 会在返回后校验匹配到的目标术语,未满足时按重试规则重新请求;`prefer` 只向模型提供约束;`off` 忽略术语表。
|
|
111
|
+
|
|
112
|
+
## 调度与恢复
|
|
113
|
+
|
|
114
|
+
- `strategy="dynamic"`(默认):所有可用 API 从公共队列抢任务,快的 key 自动多处理。
|
|
115
|
+
- `strategy="balanced"`:开始时平均切分,每个 API 显示固定进度;key 失效后,未完成任务仍会交给其他 API。
|
|
116
|
+
- 正常请求间隔默认为 `sleep=0.2` 秒。
|
|
117
|
+
- 429、5xx、超时和连接错误会指数退避并加入 jitter;401/403 会停用该 key;普通 400 不会无限重试。
|
|
118
|
+
- `checkpoint=True`(默认)会保存成功结果和失败任务。重新运行且 `overwrite=False` 时,已有输出和 checkpoint 中的结果都会跳过。
|
|
119
|
+
- Rich 进度只显示 `API 1`、`API 2` 等编号,日志和 checkpoint 都不会保存 API key。
|
|
120
|
+
|
|
121
|
+
主要容错参数及默认值:
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
aicorpusx.trans(
|
|
125
|
+
"corpus.csv",
|
|
126
|
+
source_column="source",
|
|
127
|
+
targets={"en": "English"},
|
|
128
|
+
apis=["key-1", "key-2"],
|
|
129
|
+
model="deepseek-chat",
|
|
130
|
+
base_url="https://api.deepseek.com",
|
|
131
|
+
max_retries=5,
|
|
132
|
+
backoff_base=1,
|
|
133
|
+
max_backoff=60,
|
|
134
|
+
api_failure_threshold=5,
|
|
135
|
+
api_cooldown=30,
|
|
136
|
+
max_api_cooldown=300,
|
|
137
|
+
)
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
## 非兼容接口
|
|
141
|
+
|
|
142
|
+
默认按 OpenAI 兼容协议发请求。`temperature` 等额外字段可用 `provider_options` 传入。
|
|
143
|
+
|
|
144
|
+
若服务不是 `/chat/completions`,可传入实现 `translate(...)` 的对象作为 `provider`。
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# aicorpusx
|
|
2
|
+
|
|
3
|
+
面向 CSV / XLSX 语料表的可恢复、多 API Key 并发翻译。任意走 OpenAI 兼容 `/chat/completions` 的服务,只需提供 `apis`、`model`、`base_url`:
|
|
4
|
+
|
|
5
|
+
```python
|
|
6
|
+
import aicorpusx
|
|
7
|
+
|
|
8
|
+
aicorpusx.trans(
|
|
9
|
+
"terms.xlsx",
|
|
10
|
+
source_column="中文",
|
|
11
|
+
targets={"ar": "阿拉伯语", "en": "英语"},
|
|
12
|
+
apis=["key-1", "key-2"],
|
|
13
|
+
model="deepseek-chat",
|
|
14
|
+
base_url="https://api.deepseek.com",
|
|
15
|
+
)
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
默认输出到 `terms_translated.xlsx`。CSV 输入自动输出 CSV;XLSX 输入自动输出 XLSX。
|
|
19
|
+
|
|
20
|
+
`base_url` 写到 API 根路径即可(例如 `https://api.deepseek.com` 或 `https://api.openai.com/v1`),库会自动补上 `/chat/completions`。若已写成完整 endpoint,则按原样请求。
|
|
21
|
+
|
|
22
|
+
## 安装
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
python -m pip install .
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## 常用示例
|
|
29
|
+
|
|
30
|
+
指定工作表、源语言与内容模式:
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
aicorpusx.trans(
|
|
34
|
+
"corpus.xlsx",
|
|
35
|
+
sheet_name="Data",
|
|
36
|
+
source_column="原文",
|
|
37
|
+
source_language="zh",
|
|
38
|
+
targets={"ar": "译文"},
|
|
39
|
+
mode="sentence", # auto / term / sentence / text
|
|
40
|
+
apis=["key-1", "key-2"],
|
|
41
|
+
model="deepseek-chat",
|
|
42
|
+
base_url="https://api.deepseek.com",
|
|
43
|
+
)
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
直接传术语表:
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
aicorpusx.trans(
|
|
50
|
+
"corpus.csv",
|
|
51
|
+
source_column="原文",
|
|
52
|
+
targets={"ar": "阿拉伯语"},
|
|
53
|
+
glossary={
|
|
54
|
+
"阴阳": "اليِن واليانغ",
|
|
55
|
+
"五行": "العناصر الخمسة",
|
|
56
|
+
},
|
|
57
|
+
glossary_mode="strict", # strict / prefer / off
|
|
58
|
+
apis=["key-1"],
|
|
59
|
+
model="deepseek-chat",
|
|
60
|
+
base_url="https://api.deepseek.com",
|
|
61
|
+
)
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
多语言术语文件可使用 `source,ar,en,de` 这样的列:
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
aicorpusx.trans(
|
|
68
|
+
"corpus.xlsx",
|
|
69
|
+
source_column="原文",
|
|
70
|
+
targets={"ar": "阿拉伯语", "en": "英语", "de": "德语"},
|
|
71
|
+
glossary="glossary.xlsx",
|
|
72
|
+
apis=["key-1", "key-2"],
|
|
73
|
+
model="deepseek-chat",
|
|
74
|
+
base_url="https://api.deepseek.com",
|
|
75
|
+
)
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
列名不规则时可以显式映射:
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
aicorpusx.trans(
|
|
82
|
+
"corpus.xlsx",
|
|
83
|
+
source_column="原文",
|
|
84
|
+
targets={"ar": "阿拉伯语"},
|
|
85
|
+
glossary="glossary.xlsx",
|
|
86
|
+
glossary_source_column="中文术语",
|
|
87
|
+
glossary_target_columns={"ar": "阿语标准译名"},
|
|
88
|
+
apis=["key-1"],
|
|
89
|
+
model="deepseek-chat",
|
|
90
|
+
base_url="https://api.deepseek.com",
|
|
91
|
+
)
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
术语在句中按最长优先做局部匹配。`strict` 会在返回后校验匹配到的目标术语,未满足时按重试规则重新请求;`prefer` 只向模型提供约束;`off` 忽略术语表。
|
|
95
|
+
|
|
96
|
+
## 调度与恢复
|
|
97
|
+
|
|
98
|
+
- `strategy="dynamic"`(默认):所有可用 API 从公共队列抢任务,快的 key 自动多处理。
|
|
99
|
+
- `strategy="balanced"`:开始时平均切分,每个 API 显示固定进度;key 失效后,未完成任务仍会交给其他 API。
|
|
100
|
+
- 正常请求间隔默认为 `sleep=0.2` 秒。
|
|
101
|
+
- 429、5xx、超时和连接错误会指数退避并加入 jitter;401/403 会停用该 key;普通 400 不会无限重试。
|
|
102
|
+
- `checkpoint=True`(默认)会保存成功结果和失败任务。重新运行且 `overwrite=False` 时,已有输出和 checkpoint 中的结果都会跳过。
|
|
103
|
+
- Rich 进度只显示 `API 1`、`API 2` 等编号,日志和 checkpoint 都不会保存 API key。
|
|
104
|
+
|
|
105
|
+
主要容错参数及默认值:
|
|
106
|
+
|
|
107
|
+
```python
|
|
108
|
+
aicorpusx.trans(
|
|
109
|
+
"corpus.csv",
|
|
110
|
+
source_column="source",
|
|
111
|
+
targets={"en": "English"},
|
|
112
|
+
apis=["key-1", "key-2"],
|
|
113
|
+
model="deepseek-chat",
|
|
114
|
+
base_url="https://api.deepseek.com",
|
|
115
|
+
max_retries=5,
|
|
116
|
+
backoff_base=1,
|
|
117
|
+
max_backoff=60,
|
|
118
|
+
api_failure_threshold=5,
|
|
119
|
+
api_cooldown=30,
|
|
120
|
+
max_api_cooldown=300,
|
|
121
|
+
)
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
## 非兼容接口
|
|
125
|
+
|
|
126
|
+
默认按 OpenAI 兼容协议发请求。`temperature` 等额外字段可用 `provider_options` 传入。
|
|
127
|
+
|
|
128
|
+
若服务不是 `/chat/completions`,可传入实现 `translate(...)` 的对象作为 `provider`。
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
authors = [{name = "FENG YIFAN", email = "yifan.f.academic@icloud.com"}]
|
|
7
|
+
name = "aicorpusx"
|
|
8
|
+
version = "0.1.0"
|
|
9
|
+
description = "Resumable, multi-key translation for CSV and XLSX corpora."
|
|
10
|
+
readme = "README.md"
|
|
11
|
+
requires-python = ">=3.9"
|
|
12
|
+
license = "MIT"
|
|
13
|
+
license-files = ["LICENSE"]
|
|
14
|
+
dependencies = [
|
|
15
|
+
"openpyxl>=3.1",
|
|
16
|
+
"rich>=13.0",
|
|
17
|
+
]
|
|
18
|
+
classifiers = [
|
|
19
|
+
"Development Status :: 3 - Alpha",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
[tool.setuptools]
|
|
25
|
+
package-dir = {"" = "src"}
|
|
26
|
+
|
|
27
|
+
[tool.setuptools.packages.find]
|
|
28
|
+
where = ["src"]
|