jobpicky 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jobpicky-0.2.1/LICENSE +21 -0
- jobpicky-0.2.1/PKG-INFO +39 -0
- jobpicky-0.2.1/README.md +13 -0
- jobpicky-0.2.1/pyproject.toml +54 -0
- jobpicky-0.2.1/setup.cfg +4 -0
- jobpicky-0.2.1/src/jobpicky/__init__.py +3 -0
- jobpicky-0.2.1/src/jobpicky/__main__.py +4 -0
- jobpicky-0.2.1/src/jobpicky/alerts.py +32 -0
- jobpicky-0.2.1/src/jobpicky/audit.py +120 -0
- jobpicky-0.2.1/src/jobpicky/backup.py +109 -0
- jobpicky-0.2.1/src/jobpicky/cli.py +833 -0
- jobpicky-0.2.1/src/jobpicky/config.py +320 -0
- jobpicky-0.2.1/src/jobpicky/diagnostics.py +28 -0
- jobpicky-0.2.1/src/jobpicky/error_safety.py +107 -0
- jobpicky-0.2.1/src/jobpicky/exporters.py +144 -0
- jobpicky-0.2.1/src/jobpicky/feishu.py +326 -0
- jobpicky-0.2.1/src/jobpicky/feishu_records.py +60 -0
- jobpicky-0.2.1/src/jobpicky/launcher.py +49 -0
- jobpicky-0.2.1/src/jobpicky/logging_utils.py +59 -0
- jobpicky-0.2.1/src/jobpicky/matcher.py +266 -0
- jobpicky-0.2.1/src/jobpicky/models.py +118 -0
- jobpicky-0.2.1/src/jobpicky/normalizer.py +147 -0
- jobpicky-0.2.1/src/jobpicky/official_search.py +329 -0
- jobpicky-0.2.1/src/jobpicky/onboarding.py +113 -0
- jobpicky-0.2.1/src/jobpicky/paths.py +87 -0
- jobpicky-0.2.1/src/jobpicky/pipeline.py +266 -0
- jobpicky-0.2.1/src/jobpicky/resources/__init__.py +1 -0
- jobpicky-0.2.1/src/jobpicky/resources/jobs_seed.sqlite +0 -0
- jobpicky-0.2.1/src/jobpicky/run_guard.py +132 -0
- jobpicky-0.2.1/src/jobpicky/runtime.py +88 -0
- jobpicky-0.2.1/src/jobpicky/seed.py +58 -0
- jobpicky-0.2.1/src/jobpicky/services/__init__.py +1 -0
- jobpicky-0.2.1/src/jobpicky/services/initialization.py +86 -0
- jobpicky-0.2.1/src/jobpicky/services/migration.py +78 -0
- jobpicky-0.2.1/src/jobpicky/services/scanning.py +619 -0
- jobpicky-0.2.1/src/jobpicky/services/synchronization.py +118 -0
- jobpicky-0.2.1/src/jobpicky/services/web_state.py +137 -0
- jobpicky-0.2.1/src/jobpicky/storage.py +771 -0
- jobpicky-0.2.1/src/jobpicky/web/__init__.py +1 -0
- jobpicky-0.2.1/src/jobpicky/web/app.py +391 -0
- jobpicky-0.2.1/src/jobpicky/web/templates/index.html +449 -0
- jobpicky-0.2.1/src/jobpicky/wondercv.py +685 -0
- jobpicky-0.2.1/src/jobpicky/workspace_provisioner.py +219 -0
- jobpicky-0.2.1/src/jobpicky/workspace_schema.py +112 -0
- jobpicky-0.2.1/src/jobpicky.egg-info/PKG-INFO +39 -0
- jobpicky-0.2.1/src/jobpicky.egg-info/SOURCES.txt +89 -0
- jobpicky-0.2.1/src/jobpicky.egg-info/dependency_links.txt +1 -0
- jobpicky-0.2.1/src/jobpicky.egg-info/entry_points.txt +2 -0
- jobpicky-0.2.1/src/jobpicky.egg-info/requires.txt +10 -0
- jobpicky-0.2.1/src/jobpicky.egg-info/top_level.txt +1 -0
- jobpicky-0.2.1/tests/test_alerts.py +22 -0
- jobpicky-0.2.1/tests/test_backup.py +79 -0
- jobpicky-0.2.1/tests/test_city_inference.py +30 -0
- jobpicky-0.2.1/tests/test_cli.py +539 -0
- jobpicky-0.2.1/tests/test_config.py +72 -0
- jobpicky-0.2.1/tests/test_crawler_pagination.py +146 -0
- jobpicky-0.2.1/tests/test_daily_workflow.py +679 -0
- jobpicky-0.2.1/tests/test_detail_backfill.py +62 -0
- jobpicky-0.2.1/tests/test_diagnostics.py +11 -0
- jobpicky-0.2.1/tests/test_error_safety_paths.py +418 -0
- jobpicky-0.2.1/tests/test_export_order.py +31 -0
- jobpicky-0.2.1/tests/test_exporters.py +65 -0
- jobpicky-0.2.1/tests/test_feishu.py +361 -0
- jobpicky-0.2.1/tests/test_feishu_audit.py +120 -0
- jobpicky-0.2.1/tests/test_feishu_records.py +50 -0
- jobpicky-0.2.1/tests/test_feishu_workspace_api.py +165 -0
- jobpicky-0.2.1/tests/test_incremental_init.py +37 -0
- jobpicky-0.2.1/tests/test_initialization_cli.py +158 -0
- jobpicky-0.2.1/tests/test_launcher.py +54 -0
- jobpicky-0.2.1/tests/test_matcher.py +389 -0
- jobpicky-0.2.1/tests/test_migration.py +40 -0
- jobpicky-0.2.1/tests/test_mvp_onboarding.py +227 -0
- jobpicky-0.2.1/tests/test_normalizer.py +60 -0
- jobpicky-0.2.1/tests/test_official_pipeline.py +61 -0
- jobpicky-0.2.1/tests/test_official_search.py +132 -0
- jobpicky-0.2.1/tests/test_onboarding.py +166 -0
- jobpicky-0.2.1/tests/test_paths.py +32 -0
- jobpicky-0.2.1/tests/test_pipeline.py +211 -0
- jobpicky-0.2.1/tests/test_public_config.py +95 -0
- jobpicky-0.2.1/tests/test_recommendation_refresh.py +38 -0
- jobpicky-0.2.1/tests/test_run_guard.py +57 -0
- jobpicky-0.2.1/tests/test_runtime.py +36 -0
- jobpicky-0.2.1/tests/test_schema_optimization.py +113 -0
- jobpicky-0.2.1/tests/test_seed.py +42 -0
- jobpicky-0.2.1/tests/test_storage.py +186 -0
- jobpicky-0.2.1/tests/test_sync_candidates.py +34 -0
- jobpicky-0.2.1/tests/test_user_tables.py +146 -0
- jobpicky-0.2.1/tests/test_web.py +72 -0
- jobpicky-0.2.1/tests/test_wondercv.py +226 -0
- jobpicky-0.2.1/tests/test_workspace_provisioner.py +176 -0
- jobpicky-0.2.1/tests/test_workspace_schema.py +62 -0
jobpicky-0.2.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 JobPicky contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
jobpicky-0.2.1/PKG-INFO
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: jobpicky
|
|
3
|
+
Version: 0.2.1
|
|
4
|
+
Summary: JobPicky - Your personalized job radar
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/mushang0/JobPicky
|
|
7
|
+
Project-URL: Repository, https://github.com/mushang0/JobPicky
|
|
8
|
+
Keywords: job-search,personalized-job-search,feishu,career,招聘,求职
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
12
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
13
|
+
Requires-Python: >=3.11
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Requires-Dist: beautifulsoup4>=4.12
|
|
17
|
+
Requires-Dist: fastapi<0.116,>=0.115
|
|
18
|
+
Requires-Dist: openpyxl>=3.1
|
|
19
|
+
Requires-Dist: PyYAML>=6.0
|
|
20
|
+
Requires-Dist: requests>=2.31
|
|
21
|
+
Requires-Dist: uvicorn>=0.32
|
|
22
|
+
Provides-Extra: test
|
|
23
|
+
Requires-Dist: httpx<0.29,>=0.27; extra == "test"
|
|
24
|
+
Requires-Dist: pytest<9,>=8; extra == "test"
|
|
25
|
+
Dynamic: license-file
|
|
26
|
+
|
|
27
|
+
# JobPicky
|
|
28
|
+
|
|
29
|
+
Your personalized job radar
|
|
30
|
+
|
|
31
|
+
懂你偏好的个性化岗位雷达:选择你的偏好,快速发现匹配岗位;飞书是可选的,连接后即可同步推荐。
|
|
32
|
+
|
|
33
|
+
## 启动
|
|
34
|
+
|
|
35
|
+
~~~powershell
|
|
36
|
+
uvx --python 3.12 jobpicky
|
|
37
|
+
~~~
|
|
38
|
+
|
|
39
|
+
启动后浏览器会自动打开配置页面;连接飞书后会自动创建或修复工作台并同步推荐。源码安装、开发环境、测试、打包和发布流程见 [DEVELOPER.md](DEVELOPER.md)。
|
jobpicky-0.2.1/README.md
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "jobpicky"
|
|
3
|
+
version = "0.2.1"
|
|
4
|
+
description = "JobPicky - Your personalized job radar"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
keywords = ["job-search", "personalized-job-search", "feishu", "career", "招聘", "求职"]
|
|
8
|
+
classifiers = [
|
|
9
|
+
"Programming Language :: Python :: 3",
|
|
10
|
+
"Programming Language :: Python :: 3.11",
|
|
11
|
+
"Programming Language :: Python :: 3.12",
|
|
12
|
+
"Operating System :: Microsoft :: Windows",
|
|
13
|
+
]
|
|
14
|
+
requires-python = ">=3.11"
|
|
15
|
+
dependencies = [
|
|
16
|
+
"beautifulsoup4>=4.12",
|
|
17
|
+
"fastapi>=0.115,<0.116",
|
|
18
|
+
"openpyxl>=3.1",
|
|
19
|
+
"PyYAML>=6.0",
|
|
20
|
+
"requests>=2.31",
|
|
21
|
+
"uvicorn>=0.32",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
[project.optional-dependencies]
|
|
25
|
+
test = [
|
|
26
|
+
"httpx>=0.27,<0.29",
|
|
27
|
+
"pytest>=8,<9",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
[project.scripts]
|
|
31
|
+
jobpicky = "jobpicky.launcher:main"
|
|
32
|
+
|
|
33
|
+
[project.urls]
|
|
34
|
+
Homepage = "https://github.com/mushang0/JobPicky"
|
|
35
|
+
Repository = "https://github.com/mushang0/JobPicky"
|
|
36
|
+
|
|
37
|
+
[build-system]
|
|
38
|
+
requires = ["setuptools>=77"]
|
|
39
|
+
build-backend = "setuptools.build_meta"
|
|
40
|
+
|
|
41
|
+
[tool.setuptools.packages.find]
|
|
42
|
+
where = ["src"]
|
|
43
|
+
|
|
44
|
+
[tool.setuptools.package-data]
|
|
45
|
+
jobpicky = [
|
|
46
|
+
"resources/*.sqlite",
|
|
47
|
+
"resources/defaults/*",
|
|
48
|
+
"web/templates/**/*",
|
|
49
|
+
"web/static/**/*",
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
[tool.pytest.ini_options]
|
|
53
|
+
pythonpath = ["src"]
|
|
54
|
+
testpaths = ["tests"]
|
jobpicky-0.2.1/setup.cfg
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from .error_safety import redact_text
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def build_daily_message(total_new: int, relevant_rows: list[dict[str, Any]], error: str | None = None) -> str:
|
|
9
|
+
lines = [f"今日新增秋招信息:{total_new} 条", f"推荐岗位:{len(relevant_rows)} 条"]
|
|
10
|
+
if error:
|
|
11
|
+
error = redact_text(error)
|
|
12
|
+
lines.append(f"抓取异常:{error}")
|
|
13
|
+
if not relevant_rows:
|
|
14
|
+
lines.append("暂无需要推送的推荐岗位。")
|
|
15
|
+
return "\n".join(lines)
|
|
16
|
+
|
|
17
|
+
for index, row in enumerate(relevant_rows[:20], start=1):
|
|
18
|
+
company = row.get("company") or "未知公司"
|
|
19
|
+
title = row.get("title") or row.get("clean_title") or "未命名公告"
|
|
20
|
+
city = row.get("city") or "城市待确认"
|
|
21
|
+
link = row.get("original_url") or row.get("detail_url") or row.get("apply_url") or ""
|
|
22
|
+
lines.extend(
|
|
23
|
+
[
|
|
24
|
+
"",
|
|
25
|
+
f"{index}. {company} - {title} - {city}",
|
|
26
|
+
]
|
|
27
|
+
)
|
|
28
|
+
if link:
|
|
29
|
+
lines.append(str(link))
|
|
30
|
+
if len(relevant_rows) > 20:
|
|
31
|
+
lines.append(f"\n其余 {len(relevant_rows) - 20} 条请在飞书视图查看。")
|
|
32
|
+
return "\n".join(lines)
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from typing import Any, Iterable
|
|
5
|
+
|
|
6
|
+
from .storage import JobRepository
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
KNOWN_STATUSES = frozenset({"待处理", "收藏", "不合适", "已投递", "笔试中", "面试中", "Offer", "已结束"})
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True, slots=True)
|
|
13
|
+
class FeishuAuditReport:
|
|
14
|
+
local_job_count: int
|
|
15
|
+
remote_record_count: int
|
|
16
|
+
only_local_job_ids: list[int] = field(default_factory=list)
|
|
17
|
+
only_remote_record_ids: list[str] = field(default_factory=list)
|
|
18
|
+
duplicate_job_ids: list[int] = field(default_factory=list)
|
|
19
|
+
blank_record_ids: list[str] = field(default_factory=list)
|
|
20
|
+
unmatched_record_ids: list[str] = field(default_factory=list)
|
|
21
|
+
unknown_statuses: dict[str, str] = field(default_factory=dict)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass(frozen=True, slots=True)
|
|
25
|
+
class StateRecoveryResult:
|
|
26
|
+
updated_count: int
|
|
27
|
+
unknown_statuses: dict[str, str]
|
|
28
|
+
skipped_record_ids: list[str]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def audit_feishu_records(repo: JobRepository, records: Iterable[dict[str, Any]]) -> FeishuAuditReport:
|
|
32
|
+
record_list = list(records)
|
|
33
|
+
record_to_job = repo.sync_job_ids_by_record_id()
|
|
34
|
+
local_ids = set(record_to_job.values())
|
|
35
|
+
matched_ids: set[int] = set()
|
|
36
|
+
seen_ids: set[int] = set()
|
|
37
|
+
duplicates: set[int] = set()
|
|
38
|
+
only_remote: list[str] = []
|
|
39
|
+
blank: list[str] = []
|
|
40
|
+
unmatched: list[str] = []
|
|
41
|
+
unknown: dict[str, str] = {}
|
|
42
|
+
|
|
43
|
+
for index, record in enumerate(record_list):
|
|
44
|
+
record_id = str(record.get("record_id") or f"record-{index + 1}")
|
|
45
|
+
if not record.get("record_id"):
|
|
46
|
+
blank.append(record_id)
|
|
47
|
+
continue
|
|
48
|
+
job_id = record_to_job.get(record_id)
|
|
49
|
+
if job_id in seen_ids:
|
|
50
|
+
duplicates.add(job_id)
|
|
51
|
+
seen_ids.add(job_id)
|
|
52
|
+
if job_id is None:
|
|
53
|
+
only_remote.append(record_id)
|
|
54
|
+
else:
|
|
55
|
+
matched_ids.add(job_id)
|
|
56
|
+
status = _text(_fields(record).get("求职状态")).strip()
|
|
57
|
+
if status and normalize_status(status) is None:
|
|
58
|
+
unknown[record_id] = status
|
|
59
|
+
|
|
60
|
+
return FeishuAuditReport(
|
|
61
|
+
local_job_count=len(local_ids),
|
|
62
|
+
remote_record_count=len(record_list),
|
|
63
|
+
only_local_job_ids=sorted(local_ids - matched_ids),
|
|
64
|
+
only_remote_record_ids=only_remote,
|
|
65
|
+
duplicate_job_ids=sorted(duplicates),
|
|
66
|
+
blank_record_ids=blank,
|
|
67
|
+
unmatched_record_ids=unmatched,
|
|
68
|
+
unknown_statuses=unknown,
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def recover_user_states(repo: JobRepository, records: Iterable[dict[str, Any]]) -> StateRecoveryResult:
|
|
73
|
+
record_list = list(records)
|
|
74
|
+
report = audit_feishu_records(repo, record_list)
|
|
75
|
+
record_to_job = repo.sync_job_ids_by_record_id()
|
|
76
|
+
skipped = set(report.only_remote_record_ids + report.blank_record_ids + report.unmatched_record_ids)
|
|
77
|
+
duplicate_ids = set(report.duplicate_job_ids)
|
|
78
|
+
updated = 0
|
|
79
|
+
for index, record in enumerate(record_list):
|
|
80
|
+
record_id = str(record.get("record_id") or f"record-{index + 1}")
|
|
81
|
+
fields = _fields(record)
|
|
82
|
+
job_id = record_to_job.get(record_id)
|
|
83
|
+
if job_id in duplicate_ids:
|
|
84
|
+
skipped.add(record_id)
|
|
85
|
+
if record_id in skipped or record_id in report.unknown_statuses:
|
|
86
|
+
continue
|
|
87
|
+
status = normalize_status(_text(fields.get("求职状态")).strip())
|
|
88
|
+
if job_id is None or status is None:
|
|
89
|
+
continue
|
|
90
|
+
repo.update_user_state(
|
|
91
|
+
job_id,
|
|
92
|
+
status,
|
|
93
|
+
_text(fields.get("备注")),
|
|
94
|
+
apply_url_manual=None,
|
|
95
|
+
)
|
|
96
|
+
if record.get("record_id"):
|
|
97
|
+
repo.mark_sync(job_id, "synced", record_id=str(record["record_id"]))
|
|
98
|
+
updated += 1
|
|
99
|
+
return StateRecoveryResult(updated_count=updated, unknown_statuses=report.unknown_statuses, skipped_record_ids=sorted(skipped))
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def normalize_status(value: str) -> str | None:
|
|
103
|
+
return value if value in KNOWN_STATUSES else None
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _fields(record: dict[str, Any]) -> dict[str, Any]:
|
|
107
|
+
fields = record.get("fields", {})
|
|
108
|
+
return fields if isinstance(fields, dict) else {}
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _text(value: Any) -> str:
|
|
112
|
+
if isinstance(value, list):
|
|
113
|
+
return "".join(_text(item) for item in value)
|
|
114
|
+
if isinstance(value, dict):
|
|
115
|
+
return str(value.get("text") or value.get("link") or "")
|
|
116
|
+
return "" if value is None else str(value)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _link_or_text(value: Any) -> str:
|
|
120
|
+
return _text(value)
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
import sqlite3
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from datetime import datetime, timezone
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any, Iterable
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True, slots=True)
|
|
15
|
+
class SQLiteBackup:
|
|
16
|
+
backup_path: Path
|
|
17
|
+
metadata_path: Path
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class BackupService:
|
|
21
|
+
"""Create self-describing, restorable snapshots without touching the source."""
|
|
22
|
+
|
|
23
|
+
def __init__(self, backup_directory: str | Path):
|
|
24
|
+
self.backup_directory = Path(backup_directory)
|
|
25
|
+
|
|
26
|
+
def backup_sqlite(self, database_path: str | Path, *, source: str) -> SQLiteBackup:
|
|
27
|
+
database_path = Path(database_path)
|
|
28
|
+
if not database_path.is_file():
|
|
29
|
+
raise FileNotFoundError(database_path)
|
|
30
|
+
|
|
31
|
+
self.backup_directory.mkdir(parents=True, exist_ok=True)
|
|
32
|
+
created_at = _utc_now()
|
|
33
|
+
filename = f"{_safe_name(source)}-{database_path.stem}-{created_at.replace(':', '').replace('-', '')}.sqlite"
|
|
34
|
+
backup_path = self.backup_directory / filename
|
|
35
|
+
temporary_path = backup_path.with_suffix(".sqlite.tmp")
|
|
36
|
+
try:
|
|
37
|
+
source_connection = sqlite3.connect(database_path)
|
|
38
|
+
target_connection = sqlite3.connect(temporary_path)
|
|
39
|
+
try:
|
|
40
|
+
source_connection.backup(target_connection)
|
|
41
|
+
finally:
|
|
42
|
+
target_connection.close()
|
|
43
|
+
source_connection.close()
|
|
44
|
+
os.replace(temporary_path, backup_path)
|
|
45
|
+
finally:
|
|
46
|
+
if temporary_path.exists():
|
|
47
|
+
temporary_path.unlink()
|
|
48
|
+
|
|
49
|
+
metadata = {
|
|
50
|
+
"format_version": 1,
|
|
51
|
+
"created_at": created_at,
|
|
52
|
+
"source": source,
|
|
53
|
+
"backup_file": backup_path.name,
|
|
54
|
+
"sha256": _sha256(backup_path),
|
|
55
|
+
}
|
|
56
|
+
metadata_path = backup_path.with_suffix(".json")
|
|
57
|
+
_write_json(metadata_path, metadata)
|
|
58
|
+
return SQLiteBackup(backup_path=backup_path, metadata_path=metadata_path)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def write_feishu_backup(
|
|
62
|
+
pages: Iterable[Iterable[dict[str, Any]]], backup_directory: str | Path, *, source: str = "feishu"
|
|
63
|
+
) -> Path:
|
|
64
|
+
"""Persist already-fetched Feishu pages while removing credential-shaped fields."""
|
|
65
|
+
directory = Path(backup_directory)
|
|
66
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
67
|
+
created_at = _utc_now()
|
|
68
|
+
output = directory / f"{_safe_name(source)}-records-{created_at.replace(':', '').replace('-', '')}.json"
|
|
69
|
+
records = [_redact_credentials(record) for page in pages for record in page]
|
|
70
|
+
_write_json(output, {"format_version": 1, "created_at": created_at, "source": source, "records": records})
|
|
71
|
+
return output
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _utc_now() -> str:
|
|
75
|
+
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _safe_name(value: str) -> str:
|
|
79
|
+
return re.sub(r"[^A-Za-z0-9_.-]+", "-", value).strip("-") or "backup"
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _sha256(path: Path) -> str:
|
|
83
|
+
digest = hashlib.sha256()
|
|
84
|
+
with path.open("rb") as stream:
|
|
85
|
+
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
86
|
+
digest.update(chunk)
|
|
87
|
+
return digest.hexdigest()
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _write_json(path: Path, payload: dict[str, Any]) -> None:
|
|
91
|
+
temporary_path = path.with_suffix(path.suffix + ".tmp")
|
|
92
|
+
try:
|
|
93
|
+
temporary_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
94
|
+
os.replace(temporary_path, path)
|
|
95
|
+
finally:
|
|
96
|
+
if temporary_path.exists():
|
|
97
|
+
temporary_path.unlink()
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _redact_credentials(value: Any) -> Any:
|
|
101
|
+
if isinstance(value, dict):
|
|
102
|
+
return {
|
|
103
|
+
key: _redact_credentials(item)
|
|
104
|
+
for key, item in value.items()
|
|
105
|
+
if not any(marker in key.lower() for marker in ("secret", "token", "authorization", "webhook", "password"))
|
|
106
|
+
}
|
|
107
|
+
if isinstance(value, list):
|
|
108
|
+
return [_redact_credentials(item) for item in value]
|
|
109
|
+
return value
|