bearcli 1.0.1__tar.gz → 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bearcli-1.0.1 → bearcli-1.2.0}/PKG-INFO +15 -2
- {bearcli-1.0.1 → bearcli-1.2.0}/README.md +13 -1
- {bearcli-1.0.1 → bearcli-1.2.0}/pyproject.toml +2 -1
- {bearcli-1.0.1 → bearcli-1.2.0}/src/bearcli/cli.py +87 -32
- {bearcli-1.0.1 → bearcli-1.2.0}/src/bearcli/db.py +55 -19
- {bearcli-1.0.1 → bearcli-1.2.0}/src/bearcli/export.py +37 -36
- bearcli-1.2.0/src/bearcli/gitsync.py +106 -0
- bearcli-1.2.0/src/bearcli/markdown.py +25 -0
- bearcli-1.2.0/src/bearcli/secrets.py +203 -0
- {bearcli-1.0.1 → bearcli-1.2.0}/LICENSE +0 -0
- {bearcli-1.0.1 → bearcli-1.2.0}/src/bearcli/__init__.py +0 -0
- {bearcli-1.0.1 → bearcli-1.2.0}/src/bearcli/actions.py +0 -0
- {bearcli-1.0.1 → bearcli-1.2.0}/src/bearcli/search.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: bearcli
|
|
3
|
-
Version: 1.0
|
|
3
|
+
Version: 1.2.0
|
|
4
4
|
Summary: The missing CLI for Bear notes — read, search, export, and manage your notes from the terminal
|
|
5
5
|
Keywords: bear,notes,cli,markdown,macos
|
|
6
6
|
Author: Michel Tricot
|
|
@@ -12,6 +12,7 @@ Classifier: Environment :: Console
|
|
|
12
12
|
Classifier: Operating System :: MacOS
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.13
|
|
14
14
|
Classifier: Topic :: Utilities
|
|
15
|
+
Requires-Dist: detect-secrets>=1.5.0
|
|
15
16
|
Requires-Dist: rapidfuzz>=3.14.5
|
|
16
17
|
Requires-Dist: typer>=0.27.0
|
|
17
18
|
Requires-Python: >=3.13
|
|
@@ -73,9 +74,10 @@ bearcli note list --only pinned # or: encrypted, trashed, archived
|
|
|
73
74
|
bearcli note list --all --trashed --archived
|
|
74
75
|
bearcli note list --ids # only identifiers, one per line
|
|
75
76
|
|
|
76
|
-
bearcli get C44D09DC
|
|
77
|
+
bearcli get C44D09DC # a unique id prefix (4+ chars) works everywhere
|
|
77
78
|
bearcli get C44D09DC-... --meta # with YAML-style frontmatter
|
|
78
79
|
bearcli get C44D09DC-... -r # rewrite attachment refs to absolute paths
|
|
80
|
+
bearcli get C44D09DC-... --redact-secrets # secrets replaced by placeholders
|
|
79
81
|
bearcli open C44D09DC-... # open in the Bear app
|
|
80
82
|
```
|
|
81
83
|
|
|
@@ -117,9 +119,20 @@ Every note becomes a self-contained directory — `<slug>/README.md` plus its
|
|
|
117
119
|
attachments — with a generated index, so GitHub renders the whole export as a
|
|
118
120
|
browsable tree.
|
|
119
121
|
|
|
122
|
+
Before anything is written, the notes are scanned for potential secrets
|
|
123
|
+
(token formats, key blocks, credential assignments); findings block the export
|
|
124
|
+
with a list of the affected notes (`--allow-secrets` overrides).
|
|
125
|
+
|
|
120
126
|
```sh
|
|
121
127
|
bearcli export ~/bear-backup
|
|
122
128
|
bearcli export ~/bear-backup --sync # only rewrite notes that changed
|
|
129
|
+
bearcli export ~/bear-backup --redact-secrets # secrets become [redacted: <rule>]
|
|
130
|
+
|
|
131
|
+
# Mirror to a git repository (clone it first; use a *private* repo — these are
|
|
132
|
+
# your notes). Bear is the source of truth: remote or manual edits are kept in
|
|
133
|
+
# git history but overwritten in HEAD. Never force-pushes, never gets stuck.
|
|
134
|
+
git clone git@github.com:you/bear-notes.git ~/bear-notes
|
|
135
|
+
bearcli export ~/bear-notes --sync --push
|
|
123
136
|
```
|
|
124
137
|
|
|
125
138
|
## Scripting
|
|
@@ -51,9 +51,10 @@ bearcli note list --only pinned # or: encrypted, trashed, archived
|
|
|
51
51
|
bearcli note list --all --trashed --archived
|
|
52
52
|
bearcli note list --ids # only identifiers, one per line
|
|
53
53
|
|
|
54
|
-
bearcli get C44D09DC
|
|
54
|
+
bearcli get C44D09DC # a unique id prefix (4+ chars) works everywhere
|
|
55
55
|
bearcli get C44D09DC-... --meta # with YAML-style frontmatter
|
|
56
56
|
bearcli get C44D09DC-... -r # rewrite attachment refs to absolute paths
|
|
57
|
+
bearcli get C44D09DC-... --redact-secrets # secrets replaced by placeholders
|
|
57
58
|
bearcli open C44D09DC-... # open in the Bear app
|
|
58
59
|
```
|
|
59
60
|
|
|
@@ -95,9 +96,20 @@ Every note becomes a self-contained directory — `<slug>/README.md` plus its
|
|
|
95
96
|
attachments — with a generated index, so GitHub renders the whole export as a
|
|
96
97
|
browsable tree.
|
|
97
98
|
|
|
99
|
+
Before anything is written, the notes are scanned for potential secrets
|
|
100
|
+
(token formats, key blocks, credential assignments); findings block the export
|
|
101
|
+
with a list of the affected notes (`--allow-secrets` overrides).
|
|
102
|
+
|
|
98
103
|
```sh
|
|
99
104
|
bearcli export ~/bear-backup
|
|
100
105
|
bearcli export ~/bear-backup --sync # only rewrite notes that changed
|
|
106
|
+
bearcli export ~/bear-backup --redact-secrets # secrets become [redacted: <rule>]
|
|
107
|
+
|
|
108
|
+
# Mirror to a git repository (clone it first; use a *private* repo — these are
|
|
109
|
+
# your notes). Bear is the source of truth: remote or manual edits are kept in
|
|
110
|
+
# git history but overwritten in HEAD. Never force-pushes, never gets stuck.
|
|
111
|
+
git clone git@github.com:you/bear-notes.git ~/bear-notes
|
|
112
|
+
bearcli export ~/bear-notes --sync --push
|
|
101
113
|
```
|
|
102
114
|
|
|
103
115
|
## Scripting
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "bearcli"
|
|
3
|
-
version = "1.0
|
|
3
|
+
version = "1.2.0"
|
|
4
4
|
description = "The missing CLI for Bear notes — read, search, export, and manage your notes from the terminal"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
keywords = ["bear", "notes", "cli", "markdown", "macos"]
|
|
@@ -18,6 +18,7 @@ license = "MIT"
|
|
|
18
18
|
license-files = ["LICENSE"]
|
|
19
19
|
requires-python = ">=3.13"
|
|
20
20
|
dependencies = [
|
|
21
|
+
"detect-secrets>=1.5.0",
|
|
21
22
|
"rapidfuzz>=3.14.5",
|
|
22
23
|
"typer>=0.27.0",
|
|
23
24
|
]
|
|
@@ -11,7 +11,6 @@ from datetime import UTC, datetime, timedelta
|
|
|
11
11
|
from enum import StrEnum
|
|
12
12
|
from pathlib import Path
|
|
13
13
|
from typing import Annotated
|
|
14
|
-
from urllib.parse import quote
|
|
15
14
|
|
|
16
15
|
import typer
|
|
17
16
|
from rich import box
|
|
@@ -20,9 +19,12 @@ from rich.markup import escape as rich_escape
|
|
|
20
19
|
from rich.table import Table
|
|
21
20
|
|
|
22
21
|
from bearcli import actions
|
|
23
|
-
from bearcli.db import DEFAULT_DB_PATH, BearDB, Note
|
|
22
|
+
from bearcli.db import DEFAULT_DB_PATH, AmbiguousNoteId, BearDB, Note, note_metadata
|
|
24
23
|
from bearcli.export import export_notes
|
|
24
|
+
from bearcli.gitsync import GitError, export_and_push
|
|
25
|
+
from bearcli.markdown import rewrite_attachment_refs
|
|
25
26
|
from bearcli.search import naive_search, search_notes
|
|
27
|
+
from bearcli.secrets import SecretFinding, redact_text, redaction_map, scan_notes
|
|
26
28
|
|
|
27
29
|
app = typer.Typer(help="Read notes from the Bear note app.", no_args_is_help=True, add_completion=False)
|
|
28
30
|
note_app = typer.Typer(help="Create, read, and modify notes.", no_args_is_help=True)
|
|
@@ -46,17 +48,7 @@ class OnlyFilter(StrEnum):
|
|
|
46
48
|
|
|
47
49
|
|
|
48
50
|
def _note_to_dict(note: Note, with_text: bool = False) -> dict:
|
|
49
|
-
data =
|
|
50
|
-
"id": note.id,
|
|
51
|
-
"title": note.title,
|
|
52
|
-
"tags": note.tags,
|
|
53
|
-
"created": note.created.isoformat() if note.created else None,
|
|
54
|
-
"modified": note.modified.isoformat() if note.modified else None,
|
|
55
|
-
"pinned": note.pinned,
|
|
56
|
-
"encrypted": note.encrypted,
|
|
57
|
-
"archived": note.archived,
|
|
58
|
-
"trashed": note.trashed,
|
|
59
|
-
}
|
|
51
|
+
data = note_metadata(note)
|
|
60
52
|
if with_text:
|
|
61
53
|
data["text"] = note.text
|
|
62
54
|
data["attachments"] = [
|
|
@@ -82,17 +74,8 @@ def _note_status(note: Note) -> str:
|
|
|
82
74
|
|
|
83
75
|
|
|
84
76
|
def _resolve_attachments(note: Note) -> str:
|
|
85
|
-
"""Rewrite
|
|
86
|
-
|
|
87
|
-
Bear percent-encodes filenames in the note text (e.g. spaces as %20), so match
|
|
88
|
-
both the raw and encoded forms; emit encoded paths to keep the links valid.
|
|
89
|
-
"""
|
|
90
|
-
text = note.text or ""
|
|
91
|
-
for att in note.attachments:
|
|
92
|
-
target = quote(str(att.path))
|
|
93
|
-
for ref in {att.filename, quote(att.filename)}:
|
|
94
|
-
text = text.replace(f"]({ref})", f"]({target})")
|
|
95
|
-
return text
|
|
77
|
+
"""Rewrite attachment links to the files' absolute paths on disk."""
|
|
78
|
+
return rewrite_attachment_refs(note, lambda att: str(att.path))
|
|
96
79
|
|
|
97
80
|
|
|
98
81
|
DbPathOption = Annotated[
|
|
@@ -221,6 +204,10 @@ def get(
|
|
|
221
204
|
help="Rewrite attachment references in the content to absolute file paths.",
|
|
222
205
|
),
|
|
223
206
|
] = False,
|
|
207
|
+
redact_secrets: Annotated[
|
|
208
|
+
bool,
|
|
209
|
+
typer.Option("--redact-secrets", help="Replace detected secrets with a [redacted: <rule>] placeholder."),
|
|
210
|
+
] = False,
|
|
224
211
|
fmt: Annotated[
|
|
225
212
|
OutputFormat,
|
|
226
213
|
typer.Option(
|
|
@@ -234,18 +221,18 @@ def get(
|
|
|
234
221
|
"""Print the content of a note."""
|
|
235
222
|
db = _open_db(db_path)
|
|
236
223
|
try:
|
|
237
|
-
note = db
|
|
224
|
+
note = _require_note(db, note_id)
|
|
238
225
|
finally:
|
|
239
226
|
db.close()
|
|
240
227
|
|
|
241
|
-
if note is None:
|
|
242
|
-
console.print(f"[red]Error:[/red] no note with id {note_id!r}")
|
|
243
|
-
raise typer.Exit(1)
|
|
244
228
|
if note.encrypted or note.text is None:
|
|
245
229
|
console.print(f"[red]Error:[/red] note {note.id} is encrypted; its content is unavailable")
|
|
246
230
|
raise typer.Exit(1)
|
|
247
231
|
|
|
248
232
|
text = _resolve_attachments(note) if resolve_attachments else note.text
|
|
233
|
+
if redact_secrets and text is not None:
|
|
234
|
+
note_secrets = redaction_map(scan_notes([note])).get(note.id, {})
|
|
235
|
+
text = redact_text(text, note_secrets)
|
|
249
236
|
|
|
250
237
|
if fmt is OutputFormat.json:
|
|
251
238
|
data = _note_to_dict(note, with_text=True)
|
|
@@ -288,6 +275,24 @@ def get(
|
|
|
288
275
|
print(text)
|
|
289
276
|
|
|
290
277
|
|
|
278
|
+
def _report_secrets(findings: list[SecretFinding]) -> None:
|
|
279
|
+
table = Table(box=box.ROUNDED, header_style="bold")
|
|
280
|
+
table.add_column("Note", overflow="ellipsis", max_width=30)
|
|
281
|
+
table.add_column("ID", style="dim", no_wrap=True)
|
|
282
|
+
table.add_column("Rule", style="yellow")
|
|
283
|
+
table.add_column("Line", justify="right")
|
|
284
|
+
table.add_column("Match", style="red")
|
|
285
|
+
for f in findings:
|
|
286
|
+
table.add_row(f.note_title, f.note_id, f.rule, str(f.line), f.excerpt)
|
|
287
|
+
console.print(table)
|
|
288
|
+
notes = len({f.note_id for f in findings})
|
|
289
|
+
console.print(
|
|
290
|
+
f"[red]Export blocked:[/red] {len(findings)} potential secret(s) in {notes} note(s). "
|
|
291
|
+
"Move them somewhere safe (or into an encrypted note), re-run with --redact-secrets "
|
|
292
|
+
"to export with placeholders, or with --allow-secrets to export as-is."
|
|
293
|
+
)
|
|
294
|
+
|
|
295
|
+
|
|
291
296
|
@app.command()
|
|
292
297
|
def export(
|
|
293
298
|
dest: Annotated[Path, typer.Argument(help="Destination directory for the markdown files.")],
|
|
@@ -298,17 +303,60 @@ def export(
|
|
|
298
303
|
help="Only rewrite notes that changed since the last export instead of everything.",
|
|
299
304
|
),
|
|
300
305
|
] = False,
|
|
306
|
+
push: Annotated[
|
|
307
|
+
bool,
|
|
308
|
+
typer.Option(
|
|
309
|
+
"--push",
|
|
310
|
+
help="Treat DEST as a git clone: commit the export and push. Bear is the source of "
|
|
311
|
+
"truth — remote or manual edits are kept in history but overwritten in HEAD.",
|
|
312
|
+
),
|
|
313
|
+
] = False,
|
|
314
|
+
allow_secrets: Annotated[
|
|
315
|
+
bool,
|
|
316
|
+
typer.Option("--allow-secrets", help="Export even if the secret scan finds potential credentials."),
|
|
317
|
+
] = False,
|
|
318
|
+
redact_secrets: Annotated[
|
|
319
|
+
bool,
|
|
320
|
+
typer.Option(
|
|
321
|
+
"--redact-secrets",
|
|
322
|
+
help="Export with detected secrets replaced by a [redacted: <rule>] placeholder "
|
|
323
|
+
"(notes in Bear are untouched).",
|
|
324
|
+
),
|
|
325
|
+
] = False,
|
|
301
326
|
db_path: DbPathOption = DEFAULT_DB_PATH,
|
|
302
327
|
) -> None:
|
|
303
328
|
"""Export all notes as markdown files with frontmatter and attachments."""
|
|
329
|
+
if allow_secrets and redact_secrets:
|
|
330
|
+
console.print("[red]Error:[/red] --allow-secrets and --redact-secrets are mutually exclusive")
|
|
331
|
+
raise typer.Exit(2)
|
|
304
332
|
db = _open_db(db_path)
|
|
305
333
|
try:
|
|
334
|
+
redactions: dict[str, dict[str, str]] | None = None
|
|
335
|
+
if not allow_secrets:
|
|
336
|
+
with console.status("Scanning notes for secrets…", spinner="dots"):
|
|
337
|
+
candidates = db.list_notes(limit=None, include_archived=True, with_text=True)
|
|
338
|
+
findings = scan_notes(candidates)
|
|
339
|
+
if findings and not redact_secrets:
|
|
340
|
+
_report_secrets(findings)
|
|
341
|
+
raise typer.Exit(1)
|
|
342
|
+
if findings:
|
|
343
|
+
redactions = redaction_map(findings)
|
|
306
344
|
with console.status("Exporting…", spinner="dots") as status:
|
|
307
|
-
|
|
345
|
+
update = lambda msg: status.update(rich_escape(msg)) # noqa: E731
|
|
346
|
+
if push:
|
|
347
|
+
try:
|
|
348
|
+
result, outcome = export_and_push(db, dest, sync=sync, progress=update, redactions=redactions)
|
|
349
|
+
except GitError as exc:
|
|
350
|
+
console.print(f"[red]Error:[/red] {exc}")
|
|
351
|
+
raise typer.Exit(1) from None
|
|
352
|
+
else:
|
|
353
|
+
result = export_notes(db, dest, sync=sync, progress=update, redactions=redactions)
|
|
308
354
|
finally:
|
|
309
355
|
db.close()
|
|
310
356
|
|
|
311
357
|
parts = [f"{result.written} written"]
|
|
358
|
+
if push:
|
|
359
|
+
parts.append(outcome)
|
|
312
360
|
if sync:
|
|
313
361
|
parts.append(f"{result.unchanged} unchanged")
|
|
314
362
|
if result.removed:
|
|
@@ -317,9 +365,10 @@ def export(
|
|
|
317
365
|
parts.append(f"{result.skipped_encrypted} encrypted skipped")
|
|
318
366
|
if result.index_updated:
|
|
319
367
|
parts.append("index updated")
|
|
368
|
+
if redactions:
|
|
369
|
+
secrets_count = sum(len(v) for v in redactions.values())
|
|
370
|
+
parts.append(f"{secrets_count} secret(s) redacted in {len(redactions)} note(s)")
|
|
320
371
|
console.print(f"Exported to {dest}: " + ", ".join(parts))
|
|
321
|
-
if result.index_skipped:
|
|
322
|
-
console.print("[yellow]Warning:[/yellow] README.md exists but was not generated by bearcli; left untouched")
|
|
323
372
|
|
|
324
373
|
|
|
325
374
|
def _text_or_stdin(text: str | None) -> str | None:
|
|
@@ -329,7 +378,13 @@ def _text_or_stdin(text: str | None) -> str | None:
|
|
|
329
378
|
|
|
330
379
|
|
|
331
380
|
def _require_note(db: BearDB, note_id: str) -> Note:
|
|
332
|
-
|
|
381
|
+
try:
|
|
382
|
+
note = db.get_note(note_id)
|
|
383
|
+
except AmbiguousNoteId as exc:
|
|
384
|
+
console.print(f"[red]Error:[/red] note id prefix {exc.prefix!r} matches several notes:")
|
|
385
|
+
for full_id, title in exc.matches:
|
|
386
|
+
console.print(f" {full_id} {title}")
|
|
387
|
+
raise typer.Exit(1) from None
|
|
333
388
|
if note is None:
|
|
334
389
|
console.print(f"[red]Error:[/red] no note with id {note_id!r}")
|
|
335
390
|
raise typer.Exit(1)
|
|
@@ -52,6 +52,33 @@ class Note:
|
|
|
52
52
|
attachments: list[Attachment] = field(default_factory=list)
|
|
53
53
|
|
|
54
54
|
|
|
55
|
+
def note_metadata(note: Note) -> dict:
|
|
56
|
+
"""The canonical serializable metadata for a note (everything but content)."""
|
|
57
|
+
return {
|
|
58
|
+
"id": note.id,
|
|
59
|
+
"title": note.title,
|
|
60
|
+
"tags": note.tags,
|
|
61
|
+
"created": note.created.isoformat() if note.created else None,
|
|
62
|
+
"modified": note.modified.isoformat() if note.modified else None,
|
|
63
|
+
"pinned": note.pinned,
|
|
64
|
+
"encrypted": note.encrypted,
|
|
65
|
+
"archived": note.archived,
|
|
66
|
+
"trashed": note.trashed,
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class AmbiguousNoteId(Exception):
|
|
71
|
+
"""A note id prefix matched more than one note."""
|
|
72
|
+
|
|
73
|
+
def __init__(self, prefix: str, matches: list[tuple[str, str]]):
|
|
74
|
+
super().__init__(f"note id prefix {prefix!r} is ambiguous")
|
|
75
|
+
self.prefix = prefix
|
|
76
|
+
self.matches = matches # (id, title) pairs
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
MIN_ID_PREFIX = 4
|
|
80
|
+
|
|
81
|
+
|
|
55
82
|
class BearDB:
|
|
56
83
|
def __init__(self, path: Path = DEFAULT_DB_PATH):
|
|
57
84
|
if not path.exists():
|
|
@@ -215,25 +242,13 @@ class BearDB:
|
|
|
215
242
|
query += " LIMIT ?"
|
|
216
243
|
params.append(limit)
|
|
217
244
|
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
id=row["ZUNIQUEIDENTIFIER"],
|
|
223
|
-
title=row["ZTITLE"] or "(untitled)",
|
|
224
|
-
created=core_data_to_datetime(row["ZCREATIONDATE"]),
|
|
225
|
-
modified=core_data_to_datetime(row["ZMODIFICATIONDATE"]),
|
|
226
|
-
pinned=bool(row["ZPINNED"]),
|
|
227
|
-
encrypted=bool(row["ZENCRYPTED"]),
|
|
228
|
-
archived=bool(row["ZARCHIVED"]),
|
|
229
|
-
trashed=bool(row["ZTRASHED"]),
|
|
230
|
-
tags=self._tags_for_note(row["Z_PK"]),
|
|
231
|
-
text=row["ZTEXT"] if with_text else None,
|
|
232
|
-
)
|
|
233
|
-
)
|
|
234
|
-
return notes
|
|
245
|
+
return [
|
|
246
|
+
self._note_from_row(row, text=row["ZTEXT"] if with_text else None)
|
|
247
|
+
for row in self.conn.execute(query, params)
|
|
248
|
+
]
|
|
235
249
|
|
|
236
250
|
def get_note(self, note_id: str) -> Note | None:
|
|
251
|
+
"""Fetch a note by id; a unique prefix (>= 4 chars) works like a full id."""
|
|
237
252
|
row = self.conn.execute(
|
|
238
253
|
"""
|
|
239
254
|
SELECT Z_PK, ZUNIQUEIDENTIFIER, ZTITLE, ZTEXT, ZCREATIONDATE,
|
|
@@ -243,8 +258,29 @@ class BearDB:
|
|
|
243
258
|
""",
|
|
244
259
|
(note_id,),
|
|
245
260
|
).fetchone()
|
|
261
|
+
if row is None and len(note_id) >= MIN_ID_PREFIX:
|
|
262
|
+
escaped = note_id.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
|
263
|
+
matches = self.conn.execute(
|
|
264
|
+
"""
|
|
265
|
+
SELECT Z_PK, ZUNIQUEIDENTIFIER, ZTITLE, ZTEXT, ZCREATIONDATE,
|
|
266
|
+
ZMODIFICATIONDATE, ZPINNED, ZENCRYPTED, ZARCHIVED, ZTRASHED
|
|
267
|
+
FROM ZSFNOTE
|
|
268
|
+
WHERE ZUNIQUEIDENTIFIER LIKE ? ESCAPE '\\' COLLATE NOCASE AND ZPERMANENTLYDELETED = 0
|
|
269
|
+
LIMIT 6
|
|
270
|
+
""",
|
|
271
|
+
(f"{escaped}%",),
|
|
272
|
+
).fetchall()
|
|
273
|
+
if len(matches) > 1:
|
|
274
|
+
raise AmbiguousNoteId(note_id, [(r["ZUNIQUEIDENTIFIER"], r["ZTITLE"] or "(untitled)") for r in matches])
|
|
275
|
+
if matches:
|
|
276
|
+
row = matches[0]
|
|
246
277
|
if row is None:
|
|
247
278
|
return None
|
|
279
|
+
return self._note_from_row(row, text=row["ZTEXT"], attachments=self._attachments_for_note(row["Z_PK"]))
|
|
280
|
+
|
|
281
|
+
def _note_from_row(
|
|
282
|
+
self, row: sqlite3.Row, text: str | None = None, attachments: list[Attachment] | None = None
|
|
283
|
+
) -> Note:
|
|
248
284
|
return Note(
|
|
249
285
|
id=row["ZUNIQUEIDENTIFIER"],
|
|
250
286
|
title=row["ZTITLE"] or "(untitled)",
|
|
@@ -255,6 +291,6 @@ class BearDB:
|
|
|
255
291
|
archived=bool(row["ZARCHIVED"]),
|
|
256
292
|
trashed=bool(row["ZTRASHED"]),
|
|
257
293
|
tags=self._tags_for_note(row["Z_PK"]),
|
|
258
|
-
text=
|
|
259
|
-
attachments=
|
|
294
|
+
text=text,
|
|
295
|
+
attachments=attachments or [],
|
|
260
296
|
)
|
|
@@ -9,9 +9,10 @@ from collections import Counter
|
|
|
9
9
|
from collections.abc import Callable
|
|
10
10
|
from dataclasses import dataclass
|
|
11
11
|
from pathlib import Path
|
|
12
|
-
from urllib.parse import quote
|
|
13
12
|
|
|
14
|
-
from bearcli.db import BearDB, Note
|
|
13
|
+
from bearcli.db import BearDB, Note, note_metadata
|
|
14
|
+
from bearcli.markdown import rewrite_attachment_refs
|
|
15
|
+
from bearcli.secrets import redact_text
|
|
15
16
|
|
|
16
17
|
NOTE_FILENAME = "README.md"
|
|
17
18
|
ATTACHMENTS_DIRNAME = "attachments"
|
|
@@ -24,7 +25,6 @@ class ExportResult:
|
|
|
24
25
|
removed: int = 0
|
|
25
26
|
skipped_encrypted: int = 0
|
|
26
27
|
index_updated: bool = False
|
|
27
|
-
index_skipped: bool = False
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
def slugify(title: str, max_length: int = 60) -> str:
|
|
@@ -44,7 +44,7 @@ def _dirnames(notes: list[Note]) -> dict[str, str]:
|
|
|
44
44
|
return {n.id: short[n.id] if counts[short[n.id]] == 1 else f"{slugify(n.title)}-{n.id.lower()}" for n in notes}
|
|
45
45
|
|
|
46
46
|
|
|
47
|
-
def _frontmatter(note: Note) -> str:
|
|
47
|
+
def _frontmatter(note: Note, redacted: bool = False) -> str:
|
|
48
48
|
lines = [
|
|
49
49
|
"---",
|
|
50
50
|
f"id: {note.id}",
|
|
@@ -55,6 +55,8 @@ def _frontmatter(note: Note) -> str:
|
|
|
55
55
|
lines.append(f"created: {note.created.isoformat()}")
|
|
56
56
|
if note.modified:
|
|
57
57
|
lines.append(f"modified: {note.modified.isoformat()}")
|
|
58
|
+
if redacted:
|
|
59
|
+
lines.append("redacted: true")
|
|
58
60
|
lines.append("---")
|
|
59
61
|
return "\n".join(lines)
|
|
60
62
|
|
|
@@ -90,13 +92,20 @@ def _entry_flags(entry: dict) -> str:
|
|
|
90
92
|
return flags
|
|
91
93
|
|
|
92
94
|
|
|
95
|
+
_INDEX_TITLE_LENGTH = 48
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _ellipsize(value: str, length: int) -> str:
|
|
99
|
+
return value if len(value) <= length else value[: length - 1].rstrip() + "…"
|
|
100
|
+
|
|
101
|
+
|
|
93
102
|
def _index_rows(entries: list[dict]) -> list[str]:
|
|
94
|
-
rows = ["|
|
|
103
|
+
rows = ["| ID | Modified | | Note |", "|---|---|---|---|"]
|
|
95
104
|
for e in entries:
|
|
96
|
-
title = e["title"].replace("|", "\\|")
|
|
105
|
+
title = _ellipsize(e["title"].replace("|", "\\|"), _INDEX_TITLE_LENGTH)
|
|
97
106
|
link = f"[{title}]({e['path']})" if e["path"] else title
|
|
98
|
-
|
|
99
|
-
rows.append(f"| {
|
|
107
|
+
# The short id resolves anywhere a note id is accepted (git-style prefix).
|
|
108
|
+
rows.append(f"| `{e['id'][:8].lower()}` | {(e['modified'] or '')[:10]} | {_entry_flags(e)} | {link} |")
|
|
100
109
|
return rows
|
|
101
110
|
|
|
102
111
|
|
|
@@ -112,16 +121,20 @@ def _index_markdown(entries: list[dict]) -> str:
|
|
|
112
121
|
f"{len(entries)} notes · 📎 attachments · 🔒 encrypted · 🗄 archived",
|
|
113
122
|
"",
|
|
114
123
|
]
|
|
115
|
-
|
|
124
|
+
archived = [e for e in entries if e["archived"]]
|
|
125
|
+
active = [e for e in entries if not e["archived"]]
|
|
126
|
+
pinned = [e for e in active if e["pinned"]]
|
|
116
127
|
if pinned:
|
|
117
128
|
lines += ["## 📌 Pinned", ""] + _index_rows(pinned) + [""]
|
|
118
|
-
rest = [e for e in
|
|
129
|
+
rest = [e for e in active if not e["pinned"]]
|
|
119
130
|
by_year: dict[str, list[dict]] = {}
|
|
120
131
|
for e in rest:
|
|
121
132
|
year = e["modified"][:4] if e["modified"] else "Undated"
|
|
122
133
|
by_year.setdefault(year, []).append(e)
|
|
123
134
|
for year in sorted(by_year, reverse=True):
|
|
124
135
|
lines += [f"## {year}", ""] + _index_rows(by_year[year]) + [""]
|
|
136
|
+
if archived:
|
|
137
|
+
lines += ["## 🗄 Archived", ""] + _index_rows(archived) + [""]
|
|
125
138
|
return "\n".join(lines)
|
|
126
139
|
|
|
127
140
|
|
|
@@ -136,12 +149,7 @@ def _write_if_changed(path: Path, content: str) -> bool:
|
|
|
136
149
|
|
|
137
150
|
|
|
138
151
|
def _rewrite_refs(note: Note) -> str:
|
|
139
|
-
|
|
140
|
-
for att in note.attachments:
|
|
141
|
-
target = quote(f"{ATTACHMENTS_DIRNAME}/{att.filename}")
|
|
142
|
-
for ref in {att.filename, quote(att.filename)}:
|
|
143
|
-
text = text.replace(f"]({ref})", f"]({target})")
|
|
144
|
-
return text
|
|
152
|
+
return rewrite_attachment_refs(note, lambda att: f"{ATTACHMENTS_DIRNAME}/{att.filename}")
|
|
145
153
|
|
|
146
154
|
|
|
147
155
|
def export_notes(
|
|
@@ -149,6 +157,7 @@ def export_notes(
|
|
|
149
157
|
dest: Path,
|
|
150
158
|
sync: bool = False,
|
|
151
159
|
progress: Callable[[str], None] | None = None,
|
|
160
|
+
redactions: dict[str, dict[str, str]] | None = None,
|
|
152
161
|
) -> ExportResult:
|
|
153
162
|
"""Write every non-trashed note as dest/<slug>/README.md plus attachments/.
|
|
154
163
|
|
|
@@ -171,20 +180,7 @@ def export_notes(
|
|
|
171
180
|
entries: list[dict] = []
|
|
172
181
|
|
|
173
182
|
def add_entry(note: Note, path: str | None, attachments: bool) -> None:
|
|
174
|
-
entries.append(
|
|
175
|
-
{
|
|
176
|
-
"id": note.id,
|
|
177
|
-
"title": note.title,
|
|
178
|
-
"path": path,
|
|
179
|
-
"tags": note.tags,
|
|
180
|
-
"created": note.created.isoformat() if note.created else None,
|
|
181
|
-
"modified": note.modified.isoformat() if note.modified else None,
|
|
182
|
-
"pinned": note.pinned,
|
|
183
|
-
"encrypted": note.encrypted,
|
|
184
|
-
"archived": note.archived,
|
|
185
|
-
"attachments": attachments,
|
|
186
|
-
}
|
|
187
|
-
)
|
|
183
|
+
entries.append({**note_metadata(note), "path": path, "attachments": attachments})
|
|
188
184
|
|
|
189
185
|
for position, summary in enumerate(summaries, start=1):
|
|
190
186
|
report(f"[{position}/{len(summaries)}] {summary.title}")
|
|
@@ -197,10 +193,15 @@ def export_notes(
|
|
|
197
193
|
note_dir = dest / slug
|
|
198
194
|
note_path = note_dir / NOTE_FILENAME
|
|
199
195
|
|
|
196
|
+
note_secrets = (redactions or {}).get(summary.id, {})
|
|
200
197
|
modified_iso = summary.modified.isoformat() if summary.modified else ""
|
|
201
198
|
if sync and note_path.exists():
|
|
202
199
|
existing = _parse_frontmatter(note_path)
|
|
203
|
-
|
|
200
|
+
# A change in redaction state must rewrite the file even though the
|
|
201
|
+
# note itself is unchanged — otherwise a previously exported secret
|
|
202
|
+
# would survive a later --redact-secrets run (and vice versa).
|
|
203
|
+
same_redaction = (existing.get("redacted") == "true") == bool(note_secrets)
|
|
204
|
+
if existing.get("id") == summary.id and existing.get("modified") == modified_iso and same_redaction:
|
|
204
205
|
result.unchanged += 1
|
|
205
206
|
add_entry(summary, f"{slug}/", (note_dir / ATTACHMENTS_DIRNAME).exists())
|
|
206
207
|
continue
|
|
@@ -211,8 +212,11 @@ def export_notes(
|
|
|
211
212
|
add_entry(summary, None, False)
|
|
212
213
|
continue
|
|
213
214
|
|
|
215
|
+
text = _rewrite_refs(note)
|
|
216
|
+
if note_secrets:
|
|
217
|
+
text = redact_text(text, note_secrets)
|
|
214
218
|
note_dir.mkdir(exist_ok=True)
|
|
215
|
-
note_path.write_text(f"{_frontmatter(note)}\n{
|
|
219
|
+
note_path.write_text(f"{_frontmatter(note, redacted=bool(note_secrets))}\n{text}\n")
|
|
216
220
|
|
|
217
221
|
attach_dir = note_dir / ATTACHMENTS_DIRNAME
|
|
218
222
|
if attach_dir.exists():
|
|
@@ -238,10 +242,7 @@ def export_notes(
|
|
|
238
242
|
|
|
239
243
|
report("Writing index…")
|
|
240
244
|
index_path = dest / "README.md"
|
|
241
|
-
|
|
242
|
-
result.index_skipped = True
|
|
243
|
-
else:
|
|
244
|
-
result.index_updated = _write_if_changed(index_path, _index_markdown(entries))
|
|
245
|
+
result.index_updated = _write_if_changed(index_path, _index_markdown(entries))
|
|
245
246
|
result.index_updated |= _write_if_changed(
|
|
246
247
|
dest / "index.json",
|
|
247
248
|
json.dumps(sorted(entries, key=lambda e: e["title"].lower()), indent=2, ensure_ascii=False) + "\n",
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Export into a git worktree and push, converging without manual intervention.
|
|
2
|
+
|
|
3
|
+
The repository is a one-way mirror owned by bearcli: Bear is the source of
|
|
4
|
+
truth, and HEAD always converges to Bear's state. Nothing is ever lost —
|
|
5
|
+
local or remote edits are committed/merged before being overwritten, so they
|
|
6
|
+
remain in history — and nothing is ever force-pushed. Every failure mode
|
|
7
|
+
either self-heals (merge preferring local, reset to origin as last resort,
|
|
8
|
+
bounded push retries) or raises a clear error; there is no state that
|
|
9
|
+
requires manual git surgery to resume.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import subprocess
|
|
15
|
+
from collections.abc import Callable
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
from bearcli.db import BearDB
|
|
19
|
+
from bearcli.export import ExportResult, export_notes
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class GitError(RuntimeError):
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _git(dest: Path, *args: str) -> subprocess.CompletedProcess[str]:
|
|
27
|
+
return subprocess.run(["git", "-C", str(dest), *args], capture_output=True, text=True, check=False)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _run(dest: Path, *args: str) -> str:
|
|
31
|
+
proc = _git(dest, *args)
|
|
32
|
+
if proc.returncode != 0:
|
|
33
|
+
raise GitError(f"git {' '.join(args)} failed: {proc.stderr.strip()}")
|
|
34
|
+
return proc.stdout.strip()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _ok(dest: Path, *args: str) -> bool:
|
|
38
|
+
return _git(dest, *args).returncode == 0
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _commit_if_dirty(dest: Path, message: str) -> None:
|
|
42
|
+
_run(dest, "add", "-A")
|
|
43
|
+
if not _ok(dest, "diff", "--cached", "--quiet"):
|
|
44
|
+
_run(dest, "commit", "-m", message)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def export_and_push(
|
|
48
|
+
db: BearDB,
|
|
49
|
+
dest: Path,
|
|
50
|
+
sync: bool = True,
|
|
51
|
+
progress: Callable[[str], None] | None = None,
|
|
52
|
+
redactions: dict[str, dict[str, str]] | None = None,
|
|
53
|
+
attempts: int = 3,
|
|
54
|
+
) -> tuple[ExportResult, str]:
|
|
55
|
+
def report(message: str) -> None:
|
|
56
|
+
if progress is not None:
|
|
57
|
+
progress(message)
|
|
58
|
+
|
|
59
|
+
if not _ok(dest, "rev-parse", "--is-inside-work-tree"):
|
|
60
|
+
raise GitError(f"{dest} is not a git repository — clone your notes repo (or `git init`) first")
|
|
61
|
+
branch = _run(dest, "symbolic-ref", "--short", "HEAD")
|
|
62
|
+
has_remote = _ok(dest, "remote", "get-url", "origin")
|
|
63
|
+
|
|
64
|
+
last_error = "unknown"
|
|
65
|
+
result = ExportResult()
|
|
66
|
+
for _ in range(attempts):
|
|
67
|
+
# Anything lying around (manual edits, a previous crashed run) gets
|
|
68
|
+
# committed first: merges start from a clean tree and nothing is lost.
|
|
69
|
+
_commit_if_dirty(dest, "bear export: local changes")
|
|
70
|
+
|
|
71
|
+
full = not sync
|
|
72
|
+
if has_remote:
|
|
73
|
+
report("Fetching origin…")
|
|
74
|
+
_run(dest, "fetch", "origin")
|
|
75
|
+
if (
|
|
76
|
+
_ok(dest, "rev-parse", "--verify", f"origin/{branch}")
|
|
77
|
+
and _run(dest, "rev-list", "--count", f"HEAD..origin/{branch}") != "0"
|
|
78
|
+
):
|
|
79
|
+
report("Integrating remote changes…")
|
|
80
|
+
if not (
|
|
81
|
+
_ok(dest, "merge", "--ff-only", f"origin/{branch}")
|
|
82
|
+
or _ok(dest, "merge", "-X", "ours", "--no-edit", f"origin/{branch}")
|
|
83
|
+
):
|
|
84
|
+
_git(dest, "merge", "--abort")
|
|
85
|
+
_run(dest, "reset", "--hard", f"origin/{branch}")
|
|
86
|
+
# Whatever was pulled, Bear's state must win in the worktree:
|
|
87
|
+
# a full export re-asserts every note, not just changed ones.
|
|
88
|
+
full = True
|
|
89
|
+
|
|
90
|
+
result = export_notes(db, dest, sync=not full, progress=progress, redactions=redactions)
|
|
91
|
+
|
|
92
|
+
report("Committing…")
|
|
93
|
+
parts = [f"{result.written} written"]
|
|
94
|
+
if result.removed:
|
|
95
|
+
parts.append(f"{result.removed} removed")
|
|
96
|
+
_commit_if_dirty(dest, "bear export: " + ", ".join(parts))
|
|
97
|
+
|
|
98
|
+
if not has_remote:
|
|
99
|
+
return result, "committed (no remote configured)"
|
|
100
|
+
report("Pushing…")
|
|
101
|
+
push = _git(dest, "push", "-u", "origin", branch)
|
|
102
|
+
if push.returncode == 0:
|
|
103
|
+
return result, "pushed"
|
|
104
|
+
last_error = push.stderr.strip() # racing push; refetch and retry
|
|
105
|
+
|
|
106
|
+
raise GitError(f"push failed after {attempts} attempts: {last_error}")
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Helpers for Bear's markdown conventions, shared by get and export."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
from urllib.parse import quote
|
|
7
|
+
|
|
8
|
+
from bearcli.db import Attachment, Note
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def rewrite_attachment_refs(note: Note, target_for: Callable[[Attachment], str]) -> str:
|
|
12
|
+
"""Rewrite bare attachment filename links to per-attachment targets.
|
|
13
|
+
|
|
14
|
+
Bear references attachments by bare filename and percent-encodes it in the
|
|
15
|
+
note text (spaces as %20), so both the raw and encoded forms must be
|
|
16
|
+
matched; targets are emitted percent-encoded to keep the links valid.
|
|
17
|
+
Only filenames known from the attachment records are rewritten — regular
|
|
18
|
+
URLs in the text are never touched.
|
|
19
|
+
"""
|
|
20
|
+
text = note.text or ""
|
|
21
|
+
for att in note.attachments:
|
|
22
|
+
target = quote(target_for(att))
|
|
23
|
+
for ref in {att.filename, quote(att.filename)}:
|
|
24
|
+
text = text.replace(f"]({ref})", f"]({target})")
|
|
25
|
+
return text
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
"""Secret detection over note text, powered by detect-secrets (Yelp) plus
|
|
2
|
+
note-specific detectors.
|
|
3
|
+
|
|
4
|
+
A first line of defense before notes leave the machine via export. Everything
|
|
5
|
+
runs offline — note content is never sent anywhere. detect-secrets provides
|
|
6
|
+
the format detectors (AWS, GitHub, Slack, Stripe, private keys, JWTs, keyword
|
|
7
|
+
assignments, …); two gaps matter for prose notes and are covered here:
|
|
8
|
+
|
|
9
|
+
- detect-secrets' entropy plugins only match *quoted* strings, which pasted
|
|
10
|
+
keys in notes never are — so unquoted token runs get their own entropy scan.
|
|
11
|
+
- Credentials in notes often sit on the line *below* a label ("Client
|
|
12
|
+
secret:" then the value, often inside a code fence) — a look-ahead rule
|
|
13
|
+
catches those.
|
|
14
|
+
|
|
15
|
+
Secrets written as prose are still undetectable; for deeper auditing run
|
|
16
|
+
gitleaks over an --allow-secrets export.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import math
|
|
22
|
+
import re
|
|
23
|
+
from dataclasses import dataclass
|
|
24
|
+
|
|
25
|
+
from detect_secrets.core.plugins.util import get_mapping_from_secret_type_to_class
|
|
26
|
+
from detect_secrets.filters import heuristic
|
|
27
|
+
from detect_secrets.plugins.base import BasePlugin
|
|
28
|
+
|
|
29
|
+
from bearcli.db import Note
|
|
30
|
+
|
|
31
|
+
# IPPublicDetector: an IP address in a note is not a credential.
|
|
32
|
+
# The entropy plugins only match quoted strings — replaced by _scan_entropy.
|
|
33
|
+
_EXCLUDED_PLUGINS = {"IPPublicDetector", "Base64HighEntropyString", "HexHighEntropyString"}
|
|
34
|
+
|
|
35
|
+
_BASE64_ENTROPY_LIMIT = 4.5 # detect-secrets' default
|
|
36
|
+
_HEX_ENTROPY_LIMIT = 3.5 # above default (3.0): notes are full of UUID/hash fragments
|
|
37
|
+
|
|
38
|
+
# A contiguous credential-looking run: base64/urlsafe chars plus the '.' that
|
|
39
|
+
# joins JWT segments. 20+ chars keeps ordinary words and short ids out.
|
|
40
|
+
_TOKEN_RUN = re.compile(r"[A-Za-z0-9+/\-_=.]{20,}")
|
|
41
|
+
_HEX_ONLY = re.compile(r"^[0-9a-fA-F]+$")
|
|
42
|
+
|
|
43
|
+
# "Client secret:" / "Access token" style label with nothing after it — the
|
|
44
|
+
# value is expected on the next content line (code fences skipped).
|
|
45
|
+
_LABEL_ONLY = re.compile(
|
|
46
|
+
r"(?i)^\s*\**(?:client\s+secret|secret\s+access\s+key|access\s+token|auth\s+token|refresh\s+token"
|
|
47
|
+
r"|api\s*-?\s*key|access\s+key|private\s+key|secret|password|passwd|pwd|token)\s*\**\s*[:=]?\s*$"
|
|
48
|
+
)
|
|
49
|
+
_FENCE = re.compile(r"^\s*(?:```|~~~)")
|
|
50
|
+
_TRIM_PUNCT = "()[]{}<>«»\"'`,;|"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _build_plugins() -> list[BasePlugin]:
|
|
54
|
+
return [cls() for cls in get_mapping_from_secret_type_to_class().values() if cls.__name__ not in _EXCLUDED_PLUGINS]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _entropy(value: str) -> float:
|
|
58
|
+
counts = {char: value.count(char) for char in set(value)}
|
|
59
|
+
return -sum(n / len(value) * math.log2(n / len(value)) for n in counts.values())
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _is_false_positive(value: str, line: str) -> bool:
|
|
63
|
+
return (
|
|
64
|
+
heuristic.is_potential_uuid(value)
|
|
65
|
+
or heuristic.is_sequential_string(value)
|
|
66
|
+
or heuristic.is_templated_secret(value)
|
|
67
|
+
or heuristic.is_likely_id_string(value, line)
|
|
68
|
+
or heuristic.is_not_alphanumeric_string(value)
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _expand_to_token(line: str, value: str) -> str:
|
|
73
|
+
"""Grow a detected value to the full unbroken token around it.
|
|
74
|
+
|
|
75
|
+
Format detectors can match only part of a longer credential (a JWT's
|
|
76
|
+
first two segments, say); redacting the partial match would leave the
|
|
77
|
+
rest behind.
|
|
78
|
+
"""
|
|
79
|
+
start = line.find(value)
|
|
80
|
+
if start < 0:
|
|
81
|
+
return value
|
|
82
|
+
end = start + len(value)
|
|
83
|
+
while start > 0 and not line[start - 1].isspace():
|
|
84
|
+
start -= 1
|
|
85
|
+
while end < len(line) and not line[end].isspace():
|
|
86
|
+
end += 1
|
|
87
|
+
return line[start:end].strip(_TRIM_PUNCT)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _in_url(line: str, start: int) -> bool:
|
|
91
|
+
"""Whether the token starting at `start` is part of a URL.
|
|
92
|
+
|
|
93
|
+
Long ids inside links (Google Docs, Notion, …) are high-entropy but are
|
|
94
|
+
shareable references, not credentials — flagging them would drown real
|
|
95
|
+
findings. Token-bearing URLs with known formats (Slack webhooks, …) are
|
|
96
|
+
still caught by their format detectors.
|
|
97
|
+
"""
|
|
98
|
+
token_start = start
|
|
99
|
+
while token_start > 0 and not line[token_start - 1].isspace():
|
|
100
|
+
token_start -= 1
|
|
101
|
+
return "://" in line[token_start : start + 3]
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _scan_entropy(line: str) -> list[str]:
|
|
105
|
+
hits = []
|
|
106
|
+
for match in _TOKEN_RUN.finditer(line):
|
|
107
|
+
if _in_url(line, match.start()):
|
|
108
|
+
continue
|
|
109
|
+
value = match.group().strip(_TRIM_PUNCT + ".=")
|
|
110
|
+
limit = _HEX_ENTROPY_LIMIT if _HEX_ONLY.fullmatch(value) else _BASE64_ENTROPY_LIMIT
|
|
111
|
+
if len(value) >= 20 and _entropy(value) >= limit:
|
|
112
|
+
hits.append(match.group())
|
|
113
|
+
return hits
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _labeled_value(lines: list[str], label_index: int) -> tuple[int, str] | None:
|
|
117
|
+
"""The first content line after a bare credential label, if it looks like a value."""
|
|
118
|
+
for offset in range(1, 4):
|
|
119
|
+
index = label_index + offset
|
|
120
|
+
if index >= len(lines):
|
|
121
|
+
return None
|
|
122
|
+
candidate = lines[index].strip()
|
|
123
|
+
if not candidate or _FENCE.match(candidate):
|
|
124
|
+
continue
|
|
125
|
+
candidate = candidate.strip(_TRIM_PUNCT)
|
|
126
|
+
if " " not in candidate and 8 <= len(candidate) <= 200:
|
|
127
|
+
return index, candidate
|
|
128
|
+
return None
|
|
129
|
+
return None
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
@dataclass
|
|
133
|
+
class SecretFinding:
|
|
134
|
+
note_id: str
|
|
135
|
+
note_title: str
|
|
136
|
+
rule: str
|
|
137
|
+
line: int
|
|
138
|
+
excerpt: str
|
|
139
|
+
secret: str # raw value, for redaction only — never print this
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _redact(value: str) -> str:
|
|
143
|
+
"""Show enough to locate the secret in the note, never the secret itself."""
|
|
144
|
+
return f"{value[:8]}…" if len(value) > 12 else "…"
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _scan_note(note: Note, plugins: list[BasePlugin]) -> list[SecretFinding]:
|
|
148
|
+
lines = list((note.text or "").splitlines())
|
|
149
|
+
seen: set[str] = set()
|
|
150
|
+
findings: list[SecretFinding] = []
|
|
151
|
+
|
|
152
|
+
def add(lineno: int, value: str, rule: str) -> None:
|
|
153
|
+
if value and value not in seen and not _is_false_positive(value, lines[lineno - 1]):
|
|
154
|
+
seen.add(value)
|
|
155
|
+
findings.append(
|
|
156
|
+
SecretFinding(
|
|
157
|
+
note_id=note.id,
|
|
158
|
+
note_title=note.title,
|
|
159
|
+
rule=rule,
|
|
160
|
+
line=lineno,
|
|
161
|
+
excerpt=_redact(value),
|
|
162
|
+
secret=value,
|
|
163
|
+
)
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
for lineno, line in enumerate(lines, start=1):
|
|
167
|
+
for plugin in plugins:
|
|
168
|
+
# The yaml filetype hint makes KeywordDetector accept unquoted
|
|
169
|
+
# `password: value` assignments, which is how notes write them.
|
|
170
|
+
for secret in plugin.analyze_line(filename="note.yaml", line=line, line_number=lineno):
|
|
171
|
+
add(lineno, _expand_to_token(line, secret.secret_value or ""), secret.type)
|
|
172
|
+
for value in _scan_entropy(line):
|
|
173
|
+
add(lineno, value, "High Entropy String")
|
|
174
|
+
if _LABEL_ONLY.match(line):
|
|
175
|
+
if labeled := _labeled_value(lines, lineno - 1):
|
|
176
|
+
value_index, value = labeled
|
|
177
|
+
add(value_index + 1, value, "Labeled Credential")
|
|
178
|
+
return findings
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def scan_notes(notes: list[Note]) -> list[SecretFinding]:
|
|
182
|
+
plugins = _build_plugins()
|
|
183
|
+
findings = []
|
|
184
|
+
for note in notes:
|
|
185
|
+
findings.extend(_scan_note(note, plugins))
|
|
186
|
+
return findings
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def redaction_map(findings: list[SecretFinding]) -> dict[str, dict[str, str]]:
|
|
190
|
+
"""Per note id, the secret values to replace and the rule that found them."""
|
|
191
|
+
by_note: dict[str, dict[str, str]] = {}
|
|
192
|
+
for finding in findings:
|
|
193
|
+
if finding.secret:
|
|
194
|
+
by_note.setdefault(finding.note_id, {}).setdefault(finding.secret, finding.rule)
|
|
195
|
+
return by_note
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def redact_text(text: str, secrets: dict[str, str]) -> str:
|
|
199
|
+
"""Replace each secret value with a placeholder naming the rule that found it."""
|
|
200
|
+
# Longest first, in case one detected value contains another.
|
|
201
|
+
for value in sorted(secrets, key=len, reverse=True):
|
|
202
|
+
text = text.replace(value, f"[redacted: {secrets[value]}]")
|
|
203
|
+
return text
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|