Autarkia-Io 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- autarkia_io-0.1.0/.gitignore +11 -0
- autarkia_io-0.1.0/LICENSE +21 -0
- autarkia_io-0.1.0/PKG-INFO +239 -0
- autarkia_io-0.1.0/README.md +210 -0
- autarkia_io-0.1.0/pyproject.toml +56 -0
- autarkia_io-0.1.0/src/autarkia_io/__init__.py +54 -0
- autarkia_io-0.1.0/src/autarkia_io/atomic.py +369 -0
- autarkia_io-0.1.0/src/autarkia_io/jsontools.py +231 -0
- autarkia_io-0.1.0/src/autarkia_io/py.typed +0 -0
- autarkia_io-0.1.0/tests/test_atomic.py +257 -0
- autarkia_io-0.1.0/tests/test_jsontools.py +111 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Shihao Qi
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: Autarkia-Io
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Atomic file I/O for long-running agents: thread/process-safe temp files, exponential retry for Windows locks, optional fsync, sync + async APIs, and resilient JSON extraction.
|
|
5
|
+
Project-URL: Homepage, https://qishihao.icu/projects/autarkia
|
|
6
|
+
Project-URL: Repository, https://github.com/hakituo/Autarkia-Io
|
|
7
|
+
Project-URL: Roadmap, https://github.com/hakituo/Autarkia-tool/blob/main/ROADMAP.md
|
|
8
|
+
Project-URL: Autarkia Runtime, https://github.com/hakituo/Autarkia
|
|
9
|
+
Project-URL: Author, https://qishihao.icu
|
|
10
|
+
Author-email: Shihao Qi <qishihao512@gmail.com>
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: atomic-write,autarkia,durability,file-io,json,retry,windows
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.10
|
|
26
|
+
Provides-Extra: async
|
|
27
|
+
Requires-Dist: aiofiles>=23.0; extra == 'async'
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
<p align="center">
|
|
31
|
+
<picture>
|
|
32
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/hakituo/Autarkia-Io/main/autarkia-dark.svg">
|
|
33
|
+
<img src="https://raw.githubusercontent.com/hakituo/Autarkia-Io/main/autarkia.svg" alt="Autarkia" width="300">
|
|
34
|
+
</picture>
|
|
35
|
+
</p>
|
|
36
|
+
|
|
37
|
+
<p align="center">
|
|
38
|
+
<b>Autarkia-Io</b><br>
|
|
39
|
+
<sub>长跑程序的原子文件 IO · Atomic file I/O for long-running agents</sub>
|
|
40
|
+
</p>
|
|
41
|
+
|
|
42
|
+
<p align="center">
|
|
43
|
+
<a href="https://pypi.org/project/Autarkia-Io/"><img alt="PyPI" src="https://img.shields.io/pypi/v/autarkia-io.svg"></a>
|
|
44
|
+
<a href="https://pypi.org/project/Autarkia-Io/"><img alt="Python" src="https://img.shields.io/pypi/pyversions/autarkia-io.svg"></a>
|
|
45
|
+
<a href="https://github.com/hakituo/Autarkia-Io/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/hakituo/Autarkia-Io/actions/workflows/ci.yml/badge.svg"></a>
|
|
46
|
+
<a href="LICENSE"><img alt="License: MIT" src="https://img.shields.io/badge/license-MIT-blue.svg"></a>
|
|
47
|
+
</p>
|
|
48
|
+
|
|
49
|
+
<p align="center">
|
|
50
|
+
<a href="#中文">简体中文</a> · <a href="#english">English</a>
|
|
51
|
+
</p>
|
|
52
|
+
|
|
53
|
+
---
|
|
54
|
+
|
|
55
|
+
## 中文
|
|
56
|
+
|
|
57
|
+
**崩溃安全的写入、Windows 文件占用重试、可选 fsync、sync/async 双套 API,以及从 LLM 输出里抠 JSON。**
|
|
58
|
+
|
|
59
|
+
来自在单台 Windows 机器上持续持久化状态数月的 [Autarkia](https://qishihao.icu/projects/autarkia) agent 运行时。整个系列见[组件库路线图](https://github.com/hakituo/Autarkia-tool/blob/main/ROADMAP.md)。
|
|
60
|
+
|
|
61
|
+
### 为什么
|
|
62
|
+
|
|
63
|
+
`open(path, "w")` 不是一种持久化策略:
|
|
64
|
+
|
|
65
|
+
- **截断窗口**:以写模式打开会先清空文件,新字节还没落盘,崩溃或断电就留下一个空文件。
|
|
66
|
+
- **临时名竞争**:固定的 `.tmp` 名字在线程和 asyncio 任务之间撞车,两个写入方替换同一个临时文件,其中一个的数据就没了。
|
|
67
|
+
- **Windows 文件锁**:杀毒软件和搜索索引器会短暂占用文件,`os.replace` 抛 `WinError 5` 或 `32`——重试一次就能过。
|
|
68
|
+
- **缓冲区会骗你**:没有 `fsync`,操作系统可能确认了一次其实没到磁盘的写入。
|
|
69
|
+
- **模型会给 JSON 包壳**:代码块、解释文字、模板里带出来的双层花括号,甚至直接拒答。
|
|
70
|
+
|
|
71
|
+
### 安装
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
pip install autarkia-io
|
|
75
|
+
# 可选的异步文件后端
|
|
76
|
+
pip install "autarkia-io[async]"
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
没装 `aiofiles` 时,异步函数在工作线程里跑同步实现。
|
|
80
|
+
|
|
81
|
+
### 快速开始
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
from autarkia_io import safe_json_dump, safe_json_load, safe_write_text
|
|
85
|
+
|
|
86
|
+
safe_json_dump({"mood": "calm"}, "data/state.json")
|
|
87
|
+
state = safe_json_load("data/state.json", default={})
|
|
88
|
+
|
|
89
|
+
safe_write_text("line 1\nline 2", "data/notes.txt", use_fsync=True)
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
异步:
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
from autarkia_io import async_safe_json_dump, async_safe_json_load
|
|
96
|
+
|
|
97
|
+
await async_safe_json_dump({"a": 1}, "data/state.json")
|
|
98
|
+
state = await async_safe_json_load("data/state.json", default={})
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
从模型回复里取 JSON:
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
from autarkia_io import extract_json_object, looks_like_refusal, unwrap_redundant_brackets
|
|
105
|
+
|
|
106
|
+
extract_json_object('好的:```json\n{"mood": "happy"}\n```')
|
|
107
|
+
# {"mood": "happy"}
|
|
108
|
+
|
|
109
|
+
extract_json_object('{{"people": [{{"name": "a"}}]}}')
|
|
110
|
+
# 整串都被包了一层时,先 unwrap_redundant_brackets
|
|
111
|
+
|
|
112
|
+
looks_like_refusal("抱歉,我无法完成这个请求") # True
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
### 接口一览
|
|
116
|
+
|
|
117
|
+
| 函数 | 保证 |
|
|
118
|
+
| --- | --- |
|
|
119
|
+
| `safe_write_text` / `safe_json_dump` | 临时文件 + `os.replace`,每次调用唯一 |
|
|
120
|
+
| `retry_replace` | `WinError 5/32` / `EACCES` 的指数退避重试 |
|
|
121
|
+
| `cleanup_stale_temp_files` | 清理 5 分钟以上的孤儿临时文件 |
|
|
122
|
+
| `safe_read_text` / `safe_json_load` | 缺失或损坏时返回默认值 |
|
|
123
|
+
| `async_*` | 同样的保证,`aiofiles` 或线程回退 |
|
|
124
|
+
| `extract_json_object` | 第一个配平的 object,否则第一个数组 |
|
|
125
|
+
| `unwrap_redundant_brackets` | 剥掉从提示模板抄来的 `{{...}}` |
|
|
126
|
+
| `looks_like_refusal` | 区分「拒答」与「JSON 写坏了」 |
|
|
127
|
+
|
|
128
|
+
### 设计说明
|
|
129
|
+
|
|
130
|
+
- **重试有上限**:同步约 25 秒、异步约 2 秒,之后错误照常抛出。
|
|
131
|
+
- **最终还是 `PermissionError` 时回退**:同步写入退回直接写,避免数据丢失;这是文档化的行为,不是静默降级。
|
|
132
|
+
- **没有全局状态**:路径全是参数,不 import 任何配置。
|
|
133
|
+
- **测试确定性**:`retry_replace` 接受 `sleep=`,重试路径测试瞬间跑完。
|
|
134
|
+
|
|
135
|
+
### 系列与链接
|
|
136
|
+
|
|
137
|
+
- 项目页:<https://qishihao.icu/projects/autarkia>
|
|
138
|
+
- 运行时仓库:<https://github.com/hakituo/Autarkia>
|
|
139
|
+
- 本包:<https://github.com/hakituo/Autarkia-Io>
|
|
140
|
+
- 组件库路线图:<https://github.com/hakituo/Autarkia-tool/blob/main/ROADMAP.md>
|
|
141
|
+
- 作者:Shihao Qi — <https://qishihao.icu>
|
|
142
|
+
|
|
143
|
+
### 许可
|
|
144
|
+
|
|
145
|
+
MIT
|
|
146
|
+
|
|
147
|
+
---
|
|
148
|
+
|
|
149
|
+
## English
|
|
150
|
+
|
|
151
|
+
**Crash-safe writes, Windows lock retries, optional fsync, sync + async APIs, and resilient JSON extraction from LLM output.**
|
|
152
|
+
|
|
153
|
+
Extracted from the [Autarkia](https://qishihao.icu/projects/autarkia) agent runtime, which has persisted state on a single Windows machine for months. See the [component roadmap](https://github.com/hakituo/Autarkia-tool/blob/main/ROADMAP.md).
|
|
154
|
+
|
|
155
|
+
### Why
|
|
156
|
+
|
|
157
|
+
Naive `open(path, "w")` is not a persistence strategy:
|
|
158
|
+
|
|
159
|
+
- **Truncation window.** Opening for write empties the file before new bytes land. A crash or power loss leaves an empty file.
|
|
160
|
+
- **Temp-name races.** A fixed `.tmp` name collides between threads and asyncio tasks. Two writers replace the same temp file and one loses its data.
|
|
161
|
+
- **Windows locks.** Antivirus and the search indexer briefly hold files. `os.replace` fails with `WinError 5` or `32`, which a retry resolves.
|
|
162
|
+
- **Buffers lie.** Without `fsync`, the OS may acknowledge a write that never reaches disk.
|
|
163
|
+
- **Models wrap JSON.** LLMs return JSON inside fences, prose, doubled template braces, or a refusal.
|
|
164
|
+
|
|
165
|
+
### Install
|
|
166
|
+
|
|
167
|
+
```bash
|
|
168
|
+
pip install autarkia-io
|
|
169
|
+
# optional async file backend
|
|
170
|
+
pip install "autarkia-io[async]"
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
Without `aiofiles`, async functions run the sync implementation in a worker thread.
|
|
174
|
+
|
|
175
|
+
### Quick start
|
|
176
|
+
|
|
177
|
+
```python
|
|
178
|
+
from autarkia_io import safe_json_dump, safe_json_load, safe_write_text
|
|
179
|
+
|
|
180
|
+
safe_json_dump({"mood": "calm"}, "data/state.json")
|
|
181
|
+
state = safe_json_load("data/state.json", default={})
|
|
182
|
+
|
|
183
|
+
safe_write_text("line 1\nline 2", "data/notes.txt", use_fsync=True)
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
#### Async
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
from autarkia_io import async_safe_json_dump, async_safe_json_load
|
|
190
|
+
|
|
191
|
+
await async_safe_json_dump({"a": 1}, "data/state.json")
|
|
192
|
+
state = await async_safe_json_load("data/state.json", default={})
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
#### JSON from an LLM reply
|
|
196
|
+
|
|
197
|
+
```python
|
|
198
|
+
from autarkia_io import extract_json_object, looks_like_refusal, unwrap_redundant_brackets
|
|
199
|
+
|
|
200
|
+
extract_json_object('好的:```json\n{"mood": "happy"}\n```')
|
|
201
|
+
# {"mood": "happy"}
|
|
202
|
+
|
|
203
|
+
extract_json_object('{{"people": [{{"name": "a"}}]}}')
|
|
204
|
+
# ... use unwrap_redundant_brackets first if the whole string is doubled
|
|
205
|
+
|
|
206
|
+
looks_like_refusal("抱歉,我无法完成这个请求") # True
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
### Concepts
|
|
210
|
+
|
|
211
|
+
| Function | What it guarantees |
|
|
212
|
+
| --- | --- |
|
|
213
|
+
| `safe_write_text` / `safe_json_dump` | temp file + `os.replace`, unique per call |
|
|
214
|
+
| `retry_replace` | exponential backoff for `WinError 5/32` / `EACCES` |
|
|
215
|
+
| `cleanup_stale_temp_files` | removes orphaned temp files older than 5 minutes |
|
|
216
|
+
| `safe_read_text` / `safe_json_load` | default value for missing/corrupt data |
|
|
217
|
+
| `async_*` | same guarantees, `aiofiles` or thread fallback |
|
|
218
|
+
| `extract_json_object` | first balanced object, else first array |
|
|
219
|
+
| `unwrap_redundant_brackets` | strips `{{...}}` copied from prompt templates |
|
|
220
|
+
| `looks_like_refusal` | distinguishes a refusal from malformed JSON |
|
|
221
|
+
|
|
222
|
+
### Design notes
|
|
223
|
+
|
|
224
|
+
- **Retry is bounded.** Sync waits about 25 seconds total; async about 2 seconds. After that, errors surface.
|
|
225
|
+
- **Fallback on final `PermissionError`.** The sync write falls back to a direct write so data is not lost when replacement is impossible; this is documented, not silent.
|
|
226
|
+
- **No global state.** All paths are arguments; no configuration import.
|
|
227
|
+
- **Deterministic tests.** `retry_replace` accepts `sleep=` so retry tests run instantly.
|
|
228
|
+
|
|
229
|
+
### The bigger picture
|
|
230
|
+
|
|
231
|
+
- Project page: <https://qishihao.icu/projects/autarkia>
|
|
232
|
+
- Runtime repository: <https://github.com/hakituo/Autarkia>
|
|
233
|
+
- This package: <https://github.com/hakituo/Autarkia-Io>
|
|
234
|
+
- Roadmap: <https://github.com/hakituo/Autarkia-tool/blob/main/ROADMAP.md>
|
|
235
|
+
- Author: Shihao Qi — <https://qishihao.icu>
|
|
236
|
+
|
|
237
|
+
### License
|
|
238
|
+
|
|
239
|
+
MIT
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<picture>
|
|
3
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/hakituo/Autarkia-Io/main/autarkia-dark.svg">
|
|
4
|
+
<img src="https://raw.githubusercontent.com/hakituo/Autarkia-Io/main/autarkia.svg" alt="Autarkia" width="300">
|
|
5
|
+
</picture>
|
|
6
|
+
</p>
|
|
7
|
+
|
|
8
|
+
<p align="center">
|
|
9
|
+
<b>Autarkia-Io</b><br>
|
|
10
|
+
<sub>长跑程序的原子文件 IO · Atomic file I/O for long-running agents</sub>
|
|
11
|
+
</p>
|
|
12
|
+
|
|
13
|
+
<p align="center">
|
|
14
|
+
<a href="https://pypi.org/project/Autarkia-Io/"><img alt="PyPI" src="https://img.shields.io/pypi/v/autarkia-io.svg"></a>
|
|
15
|
+
<a href="https://pypi.org/project/Autarkia-Io/"><img alt="Python" src="https://img.shields.io/pypi/pyversions/autarkia-io.svg"></a>
|
|
16
|
+
<a href="https://github.com/hakituo/Autarkia-Io/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/hakituo/Autarkia-Io/actions/workflows/ci.yml/badge.svg"></a>
|
|
17
|
+
<a href="LICENSE"><img alt="License: MIT" src="https://img.shields.io/badge/license-MIT-blue.svg"></a>
|
|
18
|
+
</p>
|
|
19
|
+
|
|
20
|
+
<p align="center">
|
|
21
|
+
<a href="#中文">简体中文</a> · <a href="#english">English</a>
|
|
22
|
+
</p>
|
|
23
|
+
|
|
24
|
+
---
|
|
25
|
+
|
|
26
|
+
## 中文
|
|
27
|
+
|
|
28
|
+
**崩溃安全的写入、Windows 文件占用重试、可选 fsync、sync/async 双套 API,以及从 LLM 输出里抠 JSON。**
|
|
29
|
+
|
|
30
|
+
来自在单台 Windows 机器上持续持久化状态数月的 [Autarkia](https://qishihao.icu/projects/autarkia) agent 运行时。整个系列见[组件库路线图](https://github.com/hakituo/Autarkia-tool/blob/main/ROADMAP.md)。
|
|
31
|
+
|
|
32
|
+
### 为什么
|
|
33
|
+
|
|
34
|
+
`open(path, "w")` 不是一种持久化策略:
|
|
35
|
+
|
|
36
|
+
- **截断窗口**:以写模式打开会先清空文件,新字节还没落盘,崩溃或断电就留下一个空文件。
|
|
37
|
+
- **临时名竞争**:固定的 `.tmp` 名字在线程和 asyncio 任务之间撞车,两个写入方替换同一个临时文件,其中一个的数据就没了。
|
|
38
|
+
- **Windows 文件锁**:杀毒软件和搜索索引器会短暂占用文件,`os.replace` 抛 `WinError 5` 或 `32`——重试一次就能过。
|
|
39
|
+
- **缓冲区会骗你**:没有 `fsync`,操作系统可能确认了一次其实没到磁盘的写入。
|
|
40
|
+
- **模型会给 JSON 包壳**:代码块、解释文字、模板里带出来的双层花括号,甚至直接拒答。
|
|
41
|
+
|
|
42
|
+
### 安装
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pip install autarkia-io
|
|
46
|
+
# 可选的异步文件后端
|
|
47
|
+
pip install "autarkia-io[async]"
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
没装 `aiofiles` 时,异步函数在工作线程里跑同步实现。
|
|
51
|
+
|
|
52
|
+
### 快速开始
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
from autarkia_io import safe_json_dump, safe_json_load, safe_write_text
|
|
56
|
+
|
|
57
|
+
safe_json_dump({"mood": "calm"}, "data/state.json")
|
|
58
|
+
state = safe_json_load("data/state.json", default={})
|
|
59
|
+
|
|
60
|
+
safe_write_text("line 1\nline 2", "data/notes.txt", use_fsync=True)
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
异步:
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
from autarkia_io import async_safe_json_dump, async_safe_json_load
|
|
67
|
+
|
|
68
|
+
await async_safe_json_dump({"a": 1}, "data/state.json")
|
|
69
|
+
state = await async_safe_json_load("data/state.json", default={})
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
从模型回复里取 JSON:
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
from autarkia_io import extract_json_object, looks_like_refusal, unwrap_redundant_brackets
|
|
76
|
+
|
|
77
|
+
extract_json_object('好的:```json\n{"mood": "happy"}\n```')
|
|
78
|
+
# {"mood": "happy"}
|
|
79
|
+
|
|
80
|
+
extract_json_object('{{"people": [{{"name": "a"}}]}}')
|
|
81
|
+
# 整串都被包了一层时,先 unwrap_redundant_brackets
|
|
82
|
+
|
|
83
|
+
looks_like_refusal("抱歉,我无法完成这个请求") # True
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
### 接口一览
|
|
87
|
+
|
|
88
|
+
| 函数 | 保证 |
|
|
89
|
+
| --- | --- |
|
|
90
|
+
| `safe_write_text` / `safe_json_dump` | 临时文件 + `os.replace`,每次调用唯一 |
|
|
91
|
+
| `retry_replace` | `WinError 5/32` / `EACCES` 的指数退避重试 |
|
|
92
|
+
| `cleanup_stale_temp_files` | 清理 5 分钟以上的孤儿临时文件 |
|
|
93
|
+
| `safe_read_text` / `safe_json_load` | 缺失或损坏时返回默认值 |
|
|
94
|
+
| `async_*` | 同样的保证,`aiofiles` 或线程回退 |
|
|
95
|
+
| `extract_json_object` | 第一个配平的 object,否则第一个数组 |
|
|
96
|
+
| `unwrap_redundant_brackets` | 剥掉从提示模板抄来的 `{{...}}` |
|
|
97
|
+
| `looks_like_refusal` | 区分「拒答」与「JSON 写坏了」 |
|
|
98
|
+
|
|
99
|
+
### 设计说明
|
|
100
|
+
|
|
101
|
+
- **重试有上限**:同步约 25 秒、异步约 2 秒,之后错误照常抛出。
|
|
102
|
+
- **最终还是 `PermissionError` 时回退**:同步写入退回直接写,避免数据丢失;这是文档化的行为,不是静默降级。
|
|
103
|
+
- **没有全局状态**:路径全是参数,不 import 任何配置。
|
|
104
|
+
- **测试确定性**:`retry_replace` 接受 `sleep=`,重试路径测试瞬间跑完。
|
|
105
|
+
|
|
106
|
+
### 系列与链接
|
|
107
|
+
|
|
108
|
+
- 项目页:<https://qishihao.icu/projects/autarkia>
|
|
109
|
+
- 运行时仓库:<https://github.com/hakituo/Autarkia>
|
|
110
|
+
- 本包:<https://github.com/hakituo/Autarkia-Io>
|
|
111
|
+
- 组件库路线图:<https://github.com/hakituo/Autarkia-tool/blob/main/ROADMAP.md>
|
|
112
|
+
- 作者:Shihao Qi — <https://qishihao.icu>
|
|
113
|
+
|
|
114
|
+
### 许可
|
|
115
|
+
|
|
116
|
+
MIT
|
|
117
|
+
|
|
118
|
+
---
|
|
119
|
+
|
|
120
|
+
## English
|
|
121
|
+
|
|
122
|
+
**Crash-safe writes, Windows lock retries, optional fsync, sync + async APIs, and resilient JSON extraction from LLM output.**
|
|
123
|
+
|
|
124
|
+
Extracted from the [Autarkia](https://qishihao.icu/projects/autarkia) agent runtime, which has persisted state on a single Windows machine for months. See the [component roadmap](https://github.com/hakituo/Autarkia-tool/blob/main/ROADMAP.md).
|
|
125
|
+
|
|
126
|
+
### Why
|
|
127
|
+
|
|
128
|
+
Naive `open(path, "w")` is not a persistence strategy:
|
|
129
|
+
|
|
130
|
+
- **Truncation window.** Opening for write empties the file before new bytes land. A crash or power loss leaves an empty file.
|
|
131
|
+
- **Temp-name races.** A fixed `.tmp` name collides between threads and asyncio tasks. Two writers replace the same temp file and one loses its data.
|
|
132
|
+
- **Windows locks.** Antivirus and the search indexer briefly hold files. `os.replace` fails with `WinError 5` or `32`, which a retry resolves.
|
|
133
|
+
- **Buffers lie.** Without `fsync`, the OS may acknowledge a write that never reaches disk.
|
|
134
|
+
- **Models wrap JSON.** LLMs return JSON inside fences, prose, doubled template braces, or a refusal.
|
|
135
|
+
|
|
136
|
+
### Install
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
pip install autarkia-io
|
|
140
|
+
# optional async file backend
|
|
141
|
+
pip install "autarkia-io[async]"
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
Without `aiofiles`, async functions run the sync implementation in a worker thread.
|
|
145
|
+
|
|
146
|
+
### Quick start
|
|
147
|
+
|
|
148
|
+
```python
|
|
149
|
+
from autarkia_io import safe_json_dump, safe_json_load, safe_write_text
|
|
150
|
+
|
|
151
|
+
safe_json_dump({"mood": "calm"}, "data/state.json")
|
|
152
|
+
state = safe_json_load("data/state.json", default={})
|
|
153
|
+
|
|
154
|
+
safe_write_text("line 1\nline 2", "data/notes.txt", use_fsync=True)
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
#### Async
|
|
158
|
+
|
|
159
|
+
```python
|
|
160
|
+
from autarkia_io import async_safe_json_dump, async_safe_json_load
|
|
161
|
+
|
|
162
|
+
await async_safe_json_dump({"a": 1}, "data/state.json")
|
|
163
|
+
state = await async_safe_json_load("data/state.json", default={})
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
#### JSON from an LLM reply
|
|
167
|
+
|
|
168
|
+
```python
|
|
169
|
+
from autarkia_io import extract_json_object, looks_like_refusal, unwrap_redundant_brackets
|
|
170
|
+
|
|
171
|
+
extract_json_object('好的:```json\n{"mood": "happy"}\n```')
|
|
172
|
+
# {"mood": "happy"}
|
|
173
|
+
|
|
174
|
+
extract_json_object('{{"people": [{{"name": "a"}}]}}')
|
|
175
|
+
# ... use unwrap_redundant_brackets first if the whole string is doubled
|
|
176
|
+
|
|
177
|
+
looks_like_refusal("抱歉,我无法完成这个请求") # True
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
### Concepts
|
|
181
|
+
|
|
182
|
+
| Function | What it guarantees |
|
|
183
|
+
| --- | --- |
|
|
184
|
+
| `safe_write_text` / `safe_json_dump` | temp file + `os.replace`, unique per call |
|
|
185
|
+
| `retry_replace` | exponential backoff for `WinError 5/32` / `EACCES` |
|
|
186
|
+
| `cleanup_stale_temp_files` | removes orphaned temp files older than 5 minutes |
|
|
187
|
+
| `safe_read_text` / `safe_json_load` | default value for missing/corrupt data |
|
|
188
|
+
| `async_*` | same guarantees, `aiofiles` or thread fallback |
|
|
189
|
+
| `extract_json_object` | first balanced object, else first array |
|
|
190
|
+
| `unwrap_redundant_brackets` | strips `{{...}}` copied from prompt templates |
|
|
191
|
+
| `looks_like_refusal` | distinguishes a refusal from malformed JSON |
|
|
192
|
+
|
|
193
|
+
### Design notes
|
|
194
|
+
|
|
195
|
+
- **Retry is bounded.** Sync waits about 25 seconds total; async about 2 seconds. After that, errors surface.
|
|
196
|
+
- **Fallback on final `PermissionError`.** The sync write falls back to a direct write so data is not lost when replacement is impossible; this is documented, not silent.
|
|
197
|
+
- **No global state.** All paths are arguments; no configuration import.
|
|
198
|
+
- **Deterministic tests.** `retry_replace` accepts `sleep=` so retry tests run instantly.
|
|
199
|
+
|
|
200
|
+
### The bigger picture
|
|
201
|
+
|
|
202
|
+
- Project page: <https://qishihao.icu/projects/autarkia>
|
|
203
|
+
- Runtime repository: <https://github.com/hakituo/Autarkia>
|
|
204
|
+
- This package: <https://github.com/hakituo/Autarkia-Io>
|
|
205
|
+
- Roadmap: <https://github.com/hakituo/Autarkia-tool/blob/main/ROADMAP.md>
|
|
206
|
+
- Author: Shihao Qi — <https://qishihao.icu>
|
|
207
|
+
|
|
208
|
+
### License
|
|
209
|
+
|
|
210
|
+
MIT
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "Autarkia-Io"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Atomic file I/O for long-running agents: thread/process-safe temp files, exponential retry for Windows locks, optional fsync, sync + async APIs, and resilient JSON extraction."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "Shihao Qi", email = "qishihao512@gmail.com" },
|
|
14
|
+
]
|
|
15
|
+
keywords = [
|
|
16
|
+
"atomic-write",
|
|
17
|
+
"file-io",
|
|
18
|
+
"json",
|
|
19
|
+
"windows",
|
|
20
|
+
"retry",
|
|
21
|
+
"durability",
|
|
22
|
+
"autarkia",
|
|
23
|
+
]
|
|
24
|
+
classifiers = [
|
|
25
|
+
"Development Status :: 4 - Beta",
|
|
26
|
+
"Intended Audience :: Developers",
|
|
27
|
+
"License :: OSI Approved :: MIT License",
|
|
28
|
+
"Operating System :: OS Independent",
|
|
29
|
+
"Programming Language :: Python :: 3",
|
|
30
|
+
"Programming Language :: Python :: 3.10",
|
|
31
|
+
"Programming Language :: Python :: 3.11",
|
|
32
|
+
"Programming Language :: Python :: 3.12",
|
|
33
|
+
"Programming Language :: Python :: 3.13",
|
|
34
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
35
|
+
"Typing :: Typed",
|
|
36
|
+
]
|
|
37
|
+
dependencies = []
|
|
38
|
+
|
|
39
|
+
[project.optional-dependencies]
|
|
40
|
+
async = ["aiofiles>=23.0"]
|
|
41
|
+
|
|
42
|
+
[project.urls]
|
|
43
|
+
Homepage = "https://qishihao.icu/projects/autarkia"
|
|
44
|
+
Repository = "https://github.com/hakituo/Autarkia-Io"
|
|
45
|
+
Roadmap = "https://github.com/hakituo/Autarkia-tool/blob/main/ROADMAP.md"
|
|
46
|
+
"Autarkia Runtime" = "https://github.com/hakituo/Autarkia"
|
|
47
|
+
Author = "https://qishihao.icu"
|
|
48
|
+
|
|
49
|
+
[tool.pytest.ini_options]
|
|
50
|
+
testpaths = ["tests"]
|
|
51
|
+
|
|
52
|
+
[tool.hatch.build.targets.wheel]
|
|
53
|
+
packages = ["src/autarkia_io"]
|
|
54
|
+
|
|
55
|
+
[tool.hatch.build.targets.sdist]
|
|
56
|
+
include = ["src", "tests", "README.md", "LICENSE"]
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Autarkia-Io: atomic file I/O and resilient JSON extraction."""
|
|
2
|
+
|
|
3
|
+
from .atomic import (
|
|
4
|
+
AIOFILES_AVAILABLE,
|
|
5
|
+
ASYNC_RETRY_DELAYS,
|
|
6
|
+
STALE_TEMP_FILE_TTL_SECONDS,
|
|
7
|
+
SYNC_RETRY_DELAYS,
|
|
8
|
+
async_safe_json_dump,
|
|
9
|
+
async_safe_json_load,
|
|
10
|
+
async_safe_read_text,
|
|
11
|
+
async_safe_write_text,
|
|
12
|
+
cleanup_stale_temp_files,
|
|
13
|
+
ensure_parent_dir,
|
|
14
|
+
generate_temp_path,
|
|
15
|
+
retry_replace,
|
|
16
|
+
safe_json_dump,
|
|
17
|
+
safe_json_load,
|
|
18
|
+
safe_read_text,
|
|
19
|
+
safe_write_text,
|
|
20
|
+
should_retry_fs_error,
|
|
21
|
+
)
|
|
22
|
+
from .jsontools import (
|
|
23
|
+
extract_json_block,
|
|
24
|
+
extract_json_object,
|
|
25
|
+
looks_like_refusal,
|
|
26
|
+
unwrap_redundant_brackets,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
__version__ = "0.1.0"
|
|
30
|
+
|
|
31
|
+
__all__ = [
|
|
32
|
+
"__version__",
|
|
33
|
+
"AIOFILES_AVAILABLE",
|
|
34
|
+
"SYNC_RETRY_DELAYS",
|
|
35
|
+
"ASYNC_RETRY_DELAYS",
|
|
36
|
+
"STALE_TEMP_FILE_TTL_SECONDS",
|
|
37
|
+
"should_retry_fs_error",
|
|
38
|
+
"ensure_parent_dir",
|
|
39
|
+
"generate_temp_path",
|
|
40
|
+
"cleanup_stale_temp_files",
|
|
41
|
+
"retry_replace",
|
|
42
|
+
"safe_write_text",
|
|
43
|
+
"safe_read_text",
|
|
44
|
+
"safe_json_dump",
|
|
45
|
+
"safe_json_load",
|
|
46
|
+
"async_safe_write_text",
|
|
47
|
+
"async_safe_read_text",
|
|
48
|
+
"async_safe_json_dump",
|
|
49
|
+
"async_safe_json_load",
|
|
50
|
+
"unwrap_redundant_brackets",
|
|
51
|
+
"looks_like_refusal",
|
|
52
|
+
"extract_json_block",
|
|
53
|
+
"extract_json_object",
|
|
54
|
+
]
|