sourceloop 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +401 -0
- package/dist/commands/attach.d.ts +2 -0
- package/dist/commands/attach.js +103 -0
- package/dist/commands/attach.js.map +1 -0
- package/dist/commands/auth.d.ts +2 -0
- package/dist/commands/auth.js +33 -0
- package/dist/commands/auth.js.map +1 -0
- package/dist/commands/chrome.d.ts +2 -0
- package/dist/commands/chrome.js +30 -0
- package/dist/commands/chrome.js.map +1 -0
- package/dist/commands/compose.d.ts +2 -0
- package/dist/commands/compose.js +14 -0
- package/dist/commands/compose.js.map +1 -0
- package/dist/commands/doctor.d.ts +2 -0
- package/dist/commands/doctor.js +15 -0
- package/dist/commands/doctor.js.map +1 -0
- package/dist/commands/import-latest.d.ts +2 -0
- package/dist/commands/import-latest.js +42 -0
- package/dist/commands/import-latest.js.map +1 -0
- package/dist/commands/ingest.d.ts +2 -0
- package/dist/commands/ingest.js +14 -0
- package/dist/commands/ingest.js.map +1 -0
- package/dist/commands/init.d.ts +2 -0
- package/dist/commands/init.js +24 -0
- package/dist/commands/init.js.map +1 -0
- package/dist/commands/notebook-bind.d.ts +2 -0
- package/dist/commands/notebook-bind.js +39 -0
- package/dist/commands/notebook-bind.js.map +1 -0
- package/dist/commands/notebook-create.d.ts +2 -0
- package/dist/commands/notebook-create.js +39 -0
- package/dist/commands/notebook-create.js.map +1 -0
- package/dist/commands/notebook-import.d.ts +2 -0
- package/dist/commands/notebook-import.js +39 -0
- package/dist/commands/notebook-import.js.map +1 -0
- package/dist/commands/notebook-source.d.ts +2 -0
- package/dist/commands/notebook-source.js +71 -0
- package/dist/commands/notebook-source.js.map +1 -0
- package/dist/commands/plan.d.ts +9 -0
- package/dist/commands/plan.js +59 -0
- package/dist/commands/plan.js.map +1 -0
- package/dist/commands/run.d.ts +2 -0
- package/dist/commands/run.js +76 -0
- package/dist/commands/run.js.map +1 -0
- package/dist/commands/status.d.ts +2 -0
- package/dist/commands/status.js +15 -0
- package/dist/commands/status.js.map +1 -0
- package/dist/commands/topic.d.ts +2 -0
- package/dist/commands/topic.js +53 -0
- package/dist/commands/topic.js.map +1 -0
- package/dist/core/attach/launch-managed-chrome.d.ts +27 -0
- package/dist/core/attach/launch-managed-chrome.js +136 -0
- package/dist/core/attach/launch-managed-chrome.js.map +1 -0
- package/dist/core/attach/manage-targets.d.ts +49 -0
- package/dist/core/attach/manage-targets.js +179 -0
- package/dist/core/attach/manage-targets.js.map +1 -0
- package/dist/core/ingest/frontmatter.d.ts +4 -0
- package/dist/core/ingest/frontmatter.js +30 -0
- package/dist/core/ingest/frontmatter.js.map +1 -0
- package/dist/core/ingest/html-to-markdown.d.ts +5 -0
- package/dist/core/ingest/html-to-markdown.js +53 -0
- package/dist/core/ingest/html-to-markdown.js.map +1 -0
- package/dist/core/ingest/ingest-source.d.ts +11 -0
- package/dist/core/ingest/ingest-source.js +115 -0
- package/dist/core/ingest/ingest-source.js.map +1 -0
- package/dist/core/notebooklm/adapter.d.ts +17 -0
- package/dist/core/notebooklm/adapter.js +2 -0
- package/dist/core/notebooklm/adapter.js.map +1 -0
- package/dist/core/notebooklm/auth.d.ts +30 -0
- package/dist/core/notebooklm/auth.js +105 -0
- package/dist/core/notebooklm/auth.js.map +1 -0
- package/dist/core/notebooklm/browser-agent-adapter.d.ts +21 -0
- package/dist/core/notebooklm/browser-agent-adapter.js +37 -0
- package/dist/core/notebooklm/browser-agent-adapter.js.map +1 -0
- package/dist/core/notebooklm/browser-agent.d.ts +121 -0
- package/dist/core/notebooklm/browser-agent.js +1604 -0
- package/dist/core/notebooklm/browser-agent.js.map +1 -0
- package/dist/core/notebooklm/config.d.ts +20 -0
- package/dist/core/notebooklm/config.js +133 -0
- package/dist/core/notebooklm/config.js.map +1 -0
- package/dist/core/notebooklm/fixture-adapter.d.ts +13 -0
- package/dist/core/notebooklm/fixture-adapter.js +32 -0
- package/dist/core/notebooklm/fixture-adapter.js.map +1 -0
- package/dist/core/notebooklm/response-extraction.d.ts +23 -0
- package/dist/core/notebooklm/response-extraction.js +348 -0
- package/dist/core/notebooklm/response-extraction.js.map +1 -0
- package/dist/core/notebooks/bind-notebook.d.ts +21 -0
- package/dist/core/notebooks/bind-notebook.js +95 -0
- package/dist/core/notebooks/bind-notebook.js.map +1 -0
- package/dist/core/notebooks/manage-managed-notebooks.d.ts +70 -0
- package/dist/core/notebooks/manage-managed-notebooks.js +491 -0
- package/dist/core/notebooks/manage-managed-notebooks.js.map +1 -0
- package/dist/core/notebooks/manage-notebook-source-manifests.d.ts +25 -0
- package/dist/core/notebooks/manage-notebook-source-manifests.js +127 -0
- package/dist/core/notebooks/manage-notebook-source-manifests.js.map +1 -0
- package/dist/core/operator/workspace-operator.d.ts +82 -0
- package/dist/core/operator/workspace-operator.js +610 -0
- package/dist/core/operator/workspace-operator.js.map +1 -0
- package/dist/core/outputs/compose-run.d.ts +11 -0
- package/dist/core/outputs/compose-run.js +98 -0
- package/dist/core/outputs/compose-run.js.map +1 -0
- package/dist/core/runs/load-artifacts.d.ts +14 -0
- package/dist/core/runs/load-artifacts.js +51 -0
- package/dist/core/runs/load-artifacts.js.map +1 -0
- package/dist/core/runs/question-planner.d.ts +20 -0
- package/dist/core/runs/question-planner.js +276 -0
- package/dist/core/runs/question-planner.js.map +1 -0
- package/dist/core/runs/render-run-note.d.ts +13 -0
- package/dist/core/runs/render-run-note.js +111 -0
- package/dist/core/runs/render-run-note.js.map +1 -0
- package/dist/core/runs/run-qa.d.ts +28 -0
- package/dist/core/runs/run-qa.js +393 -0
- package/dist/core/runs/run-qa.js.map +1 -0
- package/dist/core/topics/manage-topics.d.ts +27 -0
- package/dist/core/topics/manage-topics.js +314 -0
- package/dist/core/topics/manage-topics.js.map +1 -0
- package/dist/core/vault/notes.d.ts +29 -0
- package/dist/core/vault/notes.js +147 -0
- package/dist/core/vault/notes.js.map +1 -0
- package/dist/core/vault/paths.d.ts +31 -0
- package/dist/core/vault/paths.js +44 -0
- package/dist/core/vault/paths.js.map +1 -0
- package/dist/core/workspace/bootstrap.d.ts +16 -0
- package/dist/core/workspace/bootstrap.js +443 -0
- package/dist/core/workspace/bootstrap.js.map +1 -0
- package/dist/core/workspace/constants.d.ts +3 -0
- package/dist/core/workspace/constants.js +16 -0
- package/dist/core/workspace/constants.js.map +1 -0
- package/dist/core/workspace/init-workspace.d.ts +15 -0
- package/dist/core/workspace/init-workspace.js +86 -0
- package/dist/core/workspace/init-workspace.js.map +1 -0
- package/dist/core/workspace/load-workspace.d.ts +6 -0
- package/dist/core/workspace/load-workspace.js +51 -0
- package/dist/core/workspace/load-workspace.js.map +1 -0
- package/dist/core/workspace/schema.d.ts +19 -0
- package/dist/core/workspace/schema.js +19 -0
- package/dist/core/workspace/schema.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +41 -0
- package/dist/index.js.map +1 -0
- package/dist/lib/cli-output.d.ts +2 -0
- package/dist/lib/cli-output.js +7 -0
- package/dist/lib/cli-output.js.map +1 -0
- package/dist/lib/obsidian.d.ts +4 -0
- package/dist/lib/obsidian.js +23 -0
- package/dist/lib/obsidian.js.map +1 -0
- package/dist/lib/slugify.d.ts +1 -0
- package/dist/lib/slugify.js +10 -0
- package/dist/lib/slugify.js.map +1 -0
- package/dist/lib/write-json.d.ts +1 -0
- package/dist/lib/write-json.js +5 -0
- package/dist/lib/write-json.js.map +1 -0
- package/dist/schemas/attach.d.ts +118 -0
- package/dist/schemas/attach.js +33 -0
- package/dist/schemas/attach.js.map +1 -0
- package/dist/schemas/managed-notebook.d.ts +47 -0
- package/dist/schemas/managed-notebook.js +30 -0
- package/dist/schemas/managed-notebook.js.map +1 -0
- package/dist/schemas/notebook-source.d.ts +31 -0
- package/dist/schemas/notebook-source.js +23 -0
- package/dist/schemas/notebook-source.js.map +1 -0
- package/dist/schemas/notebook.d.ts +26 -0
- package/dist/schemas/notebook.js +18 -0
- package/dist/schemas/notebook.js.map +1 -0
- package/dist/schemas/run.d.ts +169 -0
- package/dist/schemas/run.js +80 -0
- package/dist/schemas/run.js.map +1 -0
- package/dist/schemas/source.d.ts +18 -0
- package/dist/schemas/source.js +13 -0
- package/dist/schemas/source.js.map +1 -0
- package/dist/schemas/topic.d.ts +37 -0
- package/dist/schemas/topic.js +25 -0
- package/dist/schemas/topic.js.map +1 -0
- package/package.json +44 -0
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { type ChildProcess } from "node:child_process";
|
|
2
|
+
import { type RegisterChromeAttachTargetResult } from "./manage-targets.js";
|
|
3
|
+
import { allocateFreePort, resolveChromeExecutablePath, waitForRemoteDebuggingEndpoint } from "../notebooklm/browser-agent.js";
|
|
4
|
+
export type LaunchManagedChromeInput = {
|
|
5
|
+
name?: string;
|
|
6
|
+
chromeExecutablePath?: string;
|
|
7
|
+
remoteDebuggingPort?: number;
|
|
8
|
+
launchArgs?: string[];
|
|
9
|
+
description?: string;
|
|
10
|
+
force?: boolean;
|
|
11
|
+
cwd?: string;
|
|
12
|
+
};
|
|
13
|
+
export type LaunchManagedChromeResult = RegisterChromeAttachTargetResult & {
|
|
14
|
+
endpoint: string;
|
|
15
|
+
profileDirPath: string;
|
|
16
|
+
launched: boolean;
|
|
17
|
+
reusedTarget: boolean;
|
|
18
|
+
};
|
|
19
|
+
type SpawnedChromeProcess = Pick<ChildProcess, "kill" | "unref">;
|
|
20
|
+
type LaunchManagedChromeDeps = {
|
|
21
|
+
resolveChromeExecutablePath: typeof resolveChromeExecutablePath;
|
|
22
|
+
allocateFreePort: typeof allocateFreePort;
|
|
23
|
+
waitForRemoteDebuggingEndpoint: typeof waitForRemoteDebuggingEndpoint;
|
|
24
|
+
spawnChromeProcess: (executablePath: string, args: string[]) => SpawnedChromeProcess;
|
|
25
|
+
};
|
|
26
|
+
export declare function launchManagedChrome(input: LaunchManagedChromeInput, deps?: LaunchManagedChromeDeps): Promise<LaunchManagedChromeResult>;
|
|
27
|
+
export {};
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { mkdir } from "node:fs/promises";
|
|
2
|
+
import { spawn } from "node:child_process";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { getVaultPaths } from "../vault/paths.js";
|
|
5
|
+
import { loadWorkspace } from "../workspace/load-workspace.js";
|
|
6
|
+
import { slugify } from "../../lib/slugify.js";
|
|
7
|
+
import { loadChromeAttachTarget, registerChromeProfileTarget } from "./manage-targets.js";
|
|
8
|
+
import { allocateFreePort, resolveChromeExecutablePath, waitForRemoteDebuggingEndpoint } from "../notebooklm/browser-agent.js";
|
|
9
|
+
const defaultDeps = {
|
|
10
|
+
resolveChromeExecutablePath,
|
|
11
|
+
allocateFreePort,
|
|
12
|
+
waitForRemoteDebuggingEndpoint,
|
|
13
|
+
spawnChromeProcess(executablePath, args) {
|
|
14
|
+
return spawn(executablePath, args, {
|
|
15
|
+
detached: true,
|
|
16
|
+
stdio: "ignore"
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
};
|
|
20
|
+
export async function launchManagedChrome(input, deps = defaultDeps) {
|
|
21
|
+
const workspace = await loadWorkspace(input.cwd);
|
|
22
|
+
const vault = getVaultPaths(workspace);
|
|
23
|
+
const name = input.name?.trim() || "research-browser";
|
|
24
|
+
const slug = slugify(name) || "research-browser";
|
|
25
|
+
const profileDirPath = path.join(vault.chromeProfilesDir, slug);
|
|
26
|
+
const targetId = `attach-${slug}`;
|
|
27
|
+
await mkdir(vault.chromeProfilesDir, { recursive: true });
|
|
28
|
+
await mkdir(profileDirPath, { recursive: true });
|
|
29
|
+
const existingTarget = await loadExistingTarget(targetId, workspace.rootDir);
|
|
30
|
+
if (existingTarget && !isSourceLoopManagedLaunchTarget(existingTarget.target) && !input.force) {
|
|
31
|
+
throw new Error(`Attach target ${targetId} already exists and is not a SourceLoop-managed isolated profile. Re-run with --force to replace it.`);
|
|
32
|
+
}
|
|
33
|
+
const reusableTarget = existingTarget && isSourceLoopManagedLaunchTarget(existingTarget.target)
|
|
34
|
+
? existingTarget.target
|
|
35
|
+
: undefined;
|
|
36
|
+
const executablePath = await deps.resolveChromeExecutablePath(input.chromeExecutablePath ?? reusableTarget?.chromeExecutablePath);
|
|
37
|
+
const remoteDebuggingPort = input.remoteDebuggingPort ??
|
|
38
|
+
reusableTarget?.remoteDebuggingPort ??
|
|
39
|
+
(await deps.allocateFreePort());
|
|
40
|
+
const endpoint = `http://127.0.0.1:${remoteDebuggingPort}`;
|
|
41
|
+
if (reusableTarget && !input.force) {
|
|
42
|
+
try {
|
|
43
|
+
await deps.waitForRemoteDebuggingEndpoint(endpoint, 750);
|
|
44
|
+
const persisted = await registerChromeProfileTarget(buildRegisterInput({
|
|
45
|
+
cwd: workspace.rootDir,
|
|
46
|
+
name,
|
|
47
|
+
profileDirPath,
|
|
48
|
+
executablePath,
|
|
49
|
+
remoteDebuggingPort,
|
|
50
|
+
launchArgs: input.launchArgs ?? reusableTarget.launchArgs,
|
|
51
|
+
description: input.description ?? reusableTarget.description,
|
|
52
|
+
createdAt: reusableTarget.createdAt,
|
|
53
|
+
notebooklmReadiness: reusableTarget.notebooklmReadiness,
|
|
54
|
+
notebooklmValidatedAt: reusableTarget.notebooklmValidatedAt,
|
|
55
|
+
force: true
|
|
56
|
+
}));
|
|
57
|
+
return {
|
|
58
|
+
...persisted,
|
|
59
|
+
endpoint,
|
|
60
|
+
profileDirPath,
|
|
61
|
+
launched: false,
|
|
62
|
+
reusedTarget: true
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
// endpoint not already serving this target; continue to launch
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
const launchArgs = [
|
|
70
|
+
`--user-data-dir=${profileDirPath}`,
|
|
71
|
+
`--remote-debugging-port=${remoteDebuggingPort}`,
|
|
72
|
+
"--no-first-run",
|
|
73
|
+
"--no-default-browser-check",
|
|
74
|
+
...(input.launchArgs ?? reusableTarget?.launchArgs ?? []),
|
|
75
|
+
"https://notebooklm.google.com/"
|
|
76
|
+
];
|
|
77
|
+
const processHandle = deps.spawnChromeProcess(executablePath, launchArgs);
|
|
78
|
+
try {
|
|
79
|
+
await deps.waitForRemoteDebuggingEndpoint(endpoint, 15_000);
|
|
80
|
+
}
|
|
81
|
+
catch (error) {
|
|
82
|
+
processHandle.kill("SIGTERM");
|
|
83
|
+
throw error;
|
|
84
|
+
}
|
|
85
|
+
processHandle.unref?.();
|
|
86
|
+
const persisted = await registerChromeProfileTarget(buildRegisterInput({
|
|
87
|
+
cwd: workspace.rootDir,
|
|
88
|
+
name,
|
|
89
|
+
profileDirPath,
|
|
90
|
+
executablePath,
|
|
91
|
+
remoteDebuggingPort,
|
|
92
|
+
launchArgs: input.launchArgs ?? reusableTarget?.launchArgs,
|
|
93
|
+
description: input.description ?? reusableTarget?.description,
|
|
94
|
+
createdAt: reusableTarget?.createdAt,
|
|
95
|
+
notebooklmReadiness: reusableTarget?.notebooklmReadiness,
|
|
96
|
+
notebooklmValidatedAt: reusableTarget?.notebooklmValidatedAt,
|
|
97
|
+
force: Boolean(existingTarget)
|
|
98
|
+
}));
|
|
99
|
+
return {
|
|
100
|
+
...persisted,
|
|
101
|
+
endpoint,
|
|
102
|
+
profileDirPath,
|
|
103
|
+
launched: true,
|
|
104
|
+
reusedTarget: Boolean(reusableTarget)
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
function isSourceLoopManagedLaunchTarget(target) {
|
|
108
|
+
return target.targetType === "profile" && target.ownership === "sourceloop_managed" && target.profileIsolation === "isolated";
|
|
109
|
+
}
|
|
110
|
+
async function loadExistingTarget(targetId, cwd) {
|
|
111
|
+
try {
|
|
112
|
+
const loaded = await loadChromeAttachTarget(targetId, cwd);
|
|
113
|
+
return { target: loaded.target };
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
return undefined;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
function buildRegisterInput(input) {
|
|
120
|
+
return {
|
|
121
|
+
cwd: input.cwd,
|
|
122
|
+
name: input.name,
|
|
123
|
+
profileDirPath: input.profileDirPath,
|
|
124
|
+
ownership: "sourceloop_managed",
|
|
125
|
+
profileIsolation: "isolated",
|
|
126
|
+
chromeExecutablePath: input.executablePath,
|
|
127
|
+
remoteDebuggingPort: input.remoteDebuggingPort,
|
|
128
|
+
...(input.launchArgs ? { launchArgs: input.launchArgs } : {}),
|
|
129
|
+
...(input.description ? { description: input.description } : {}),
|
|
130
|
+
...(input.createdAt ? { createdAt: input.createdAt } : {}),
|
|
131
|
+
...(input.notebooklmReadiness ? { notebooklmReadiness: input.notebooklmReadiness } : {}),
|
|
132
|
+
...(input.notebooklmValidatedAt ? { notebooklmValidatedAt: input.notebooklmValidatedAt } : {}),
|
|
133
|
+
force: input.force
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
//# sourceMappingURL=launch-managed-chrome.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"launch-managed-chrome.js","sourceRoot":"","sources":["../../../src/core/attach/launch-managed-chrome.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,KAAK,EAAqB,MAAM,oBAAoB,CAAC;AAC9D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EACL,sBAAsB,EACtB,2BAA2B,EAE5B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,gBAAgB,EAChB,2BAA2B,EAC3B,8BAA8B,EAC/B,MAAM,gCAAgC,CAAC;AA6BxC,MAAM,WAAW,GAA4B;IAC3C,2BAA2B;IAC3B,gBAAgB;IAChB,8BAA8B;IAC9B,kBAAkB,CAAC,cAAc,EAAE,IAAI;QACrC,OAAO,KAAK,CAAC,cAAc,EAAE,IAAI,EAAE;YACjC,QAAQ,EAAE,IAAI;YACd,KAAK,EAAE,QAAQ;SAChB,CAAC,CAAC;IACL,CAAC;CACF,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,KAA+B,EAC/B,OAAgC,WAAW;IAE3C,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACjD,MAAM,KAAK,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;IACvC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,kBAAkB,CAAC;IACtD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,kBAAkB,CAAC;IACjD,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,UAAU,IAAI,EAAE,CAAC;IAElC,MAAM,KAAK,CAAC,KAAK,CAAC,iBAAiB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,MAAM,KAAK,CAAC,cAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEjD,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,QAAQ,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC;IAC7E,IAAI,cAAc,IAAI,CAAC,+BAA+B,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QAC9F,MAAM,IAAI,KAAK,CACb,iBAAiB,QAAQ,sGAAsG,CAChI,CAAC;IACJ,CAAC;IAED,MAAM,cAAc,GAAG,cAAc,IAAI,+BAA+B,CAAC,cAAc,CAAC,MAAM,CAAC;QAC7F,CAAC,CAAC,cAAc,CAAC,MAAM;QACvB,CAAC,CAAC,SAAS,CAAC;IAEd,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAC3D,KAAK,CAAC,oBAAoB,IAAI,cAAc,EAAE,oBAAoB,CACnE,CAAC;IACF,MAAM,mBAAmB,GACvB,KAAK,CAAC,mBAAmB;QACzB,cAAc,EAAE,mBAAmB;QACnC,CAAC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,oBAAoB,mBAAmB,EAAE,CAAC;IAE3D,IAAI,cAAc,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,8BAA8B,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;YACzD,MAAM,SAAS,GAAG,MAAM,2BAA2B,CACjD,kBAAkB,CAAC;gBACjB,GAAG,EAAE,SAAS,CAAC,OAAO;gBACtB,IAAI;gBACJ,cAAc;gBACd,cAAc;gBACd,mBAAmB;gBACnB,UAAU,EAAE,KAAK,CAAC,UAAU,IAAI,cAAc,CAAC,UAAU;gBACzD,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,cAAc,CAAC,WAAW;gBAC5D,SAAS,EAAE,cAAc,CAAC,SAAS;gBACnC,mBAAmB,EAAE,cAAc,CAAC,mBAAmB;gBACvD,qBAAqB,EAAE,cAAc,CAAC,qBAAqB;gBAC3D,KAAK,EAAE,IAAI;aACZ,CAAC,CACH,CAAC;YAEF,OAAO;gBACL,GAAG,SAAS;gBACZ,QAAQ;gBACR,cAAc;gBACd,QAAQ,EAAE,KAAK;gBACf,YAAY,EAAE,IAAI;aACnB,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,+DAA+D;QACjE,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG;QACjB,mBAAmB,cAAc,EAAE;QACnC,2BAA2B,mBAAmB,EAAE;QAChD,gBAAgB;QAChB,4BAA4B;QAC5B,GAAG,CAAC,KAAK,CAAC,UAAU,IAAI,cAAc,EAAE,UAAU,IAAI,EAAE,CAAC;QACzD,gCAAgC;KACjC,CAAC;IAEF,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAC,cAAc,EAAE,UAAU,CAAC,CAAC;IAE1E,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,8BAA8B,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC9D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC9B,MAAM,KAAK,CAAC;IACd,CAAC;IAED,aAAa,CAAC,KAAK,EAAE,EAAE,CAAC;IAExB,MAAM,SAAS,GAAG,MAAM,2BAA2B,CACjD,kBAAkB,CAAC;QACjB,GAAG,EAAE,SAAS,CAAC,OAAO;QACtB,IAAI;QACJ,cAAc;QACd,cAAc;QACd,mBAAmB;QACnB,UAAU,EAAE,KAAK,CAAC,UAAU,IAAI,cAAc,EAAE,UAAU;QAC1D,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,cAAc,EAAE,WAAW;QAC7D,SAAS,EAAE,cAAc,EAAE,SAAS;QACpC,mBAAmB,EAAE,cAAc,EAAE,mBAAmB;QACxD,qBAAqB,EAAE,cAAc,EAAE,qBAAqB;QAC5D,KAAK,EAAE,OAAO,CAAC,cAAc,CAAC;KAC/B,CAAC,CACH,CAAC;IAEF,OAAO;QACL,GAAG,SAAS;QACZ,QAAQ;QACR,cAAc;QACd,QAAQ,EAAE,IAAI;QACd,YAAY,EAAE,OAAO,CAAC,cAAc,CAAC;KACtC,CAAC;AACJ,CAAC;AAED,SAAS,+BAA+B,CAAC,MAA0B;IACjE,OAAO,MAAM,CAAC,UAAU,KAAK,SAAS,IAAI,MAAM,CAAC,SAAS,KAAK,oBAAoB,IAAI,MAAM,CAAC,gBAAgB,KAAK,UAAU,CAAC;AAChI,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,QAAgB,EAChB,GAAW;IAEX,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAY3B;IACC,OAAO;QACL,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,SAAS,EAAE,oBAA6B;QACxC,gBAAgB,EAAE,UAAmB;QACrC,oBAAoB,EAAE,KAAK,CAAC,cAAc;QAC1C,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;QAC9C,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1D,GAAG,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxF,GAAG,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC,EAAE,qBAAqB,EAAE,KAAK,CAAC,qBAAqB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9F,KAAK,EAAE,KAAK,CAAC,KAAK;KACnB,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { loadWorkspace } from "../workspace/load-workspace.js";
|
|
2
|
+
import { type ChromeAttachTarget, type ChromeAttachOwnership, type ChromeNotebooklmReadiness, type ChromeProfileIsolation } from "../../schemas/attach.js";
|
|
3
|
+
type RegisterChromeProfileTargetInput = {
|
|
4
|
+
name: string;
|
|
5
|
+
profileDirPath: string;
|
|
6
|
+
profileIsolation?: ChromeProfileIsolation;
|
|
7
|
+
ownership?: Exclude<ChromeAttachOwnership, "external">;
|
|
8
|
+
notebooklmReadiness?: ChromeNotebooklmReadiness;
|
|
9
|
+
notebooklmValidatedAt?: string;
|
|
10
|
+
chromeExecutablePath?: string;
|
|
11
|
+
remoteDebuggingPort?: number;
|
|
12
|
+
launchArgs?: string[];
|
|
13
|
+
description?: string;
|
|
14
|
+
createdAt?: string;
|
|
15
|
+
force?: boolean;
|
|
16
|
+
cwd?: string;
|
|
17
|
+
};
|
|
18
|
+
type RegisterChromeEndpointTargetInput = {
|
|
19
|
+
name: string;
|
|
20
|
+
endpoint: string;
|
|
21
|
+
profileIsolation?: ChromeProfileIsolation;
|
|
22
|
+
notebooklmReadiness?: ChromeNotebooklmReadiness;
|
|
23
|
+
notebooklmValidatedAt?: string;
|
|
24
|
+
description?: string;
|
|
25
|
+
force?: boolean;
|
|
26
|
+
cwd?: string;
|
|
27
|
+
};
|
|
28
|
+
export type RegisterChromeAttachTargetResult = {
|
|
29
|
+
target: ChromeAttachTarget;
|
|
30
|
+
markdownPath: string;
|
|
31
|
+
jsonPath: string;
|
|
32
|
+
};
|
|
33
|
+
export declare function registerChromeProfileTarget(input: RegisterChromeProfileTargetInput): Promise<RegisterChromeAttachTargetResult>;
|
|
34
|
+
export declare function registerChromeEndpointTarget(input: RegisterChromeEndpointTargetInput): Promise<RegisterChromeAttachTargetResult>;
|
|
35
|
+
export declare function loadChromeAttachTarget(targetId: string, cwd?: string): Promise<{
|
|
36
|
+
workspace: Awaited<ReturnType<typeof loadWorkspace>>;
|
|
37
|
+
target: ChromeAttachTarget;
|
|
38
|
+
path: string;
|
|
39
|
+
}>;
|
|
40
|
+
export declare function upsertChromeAttachTarget(target: ChromeAttachTarget, input?: {
|
|
41
|
+
cwd?: string;
|
|
42
|
+
force?: boolean;
|
|
43
|
+
}): Promise<RegisterChromeAttachTargetResult>;
|
|
44
|
+
export declare function listChromeAttachTargets(cwd?: string): Promise<ChromeAttachTarget[]>;
|
|
45
|
+
export declare function removeChromeAttachTarget(input: {
|
|
46
|
+
targetId: string;
|
|
47
|
+
cwd?: string;
|
|
48
|
+
}): Promise<void>;
|
|
49
|
+
export {};
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
import { access, mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { loadWorkspace } from "../workspace/load-workspace.js";
|
|
4
|
+
import { chromeAttachTargetSchema } from "../../schemas/attach.js";
|
|
5
|
+
import { slugify } from "../../lib/slugify.js";
|
|
6
|
+
import { getVaultPaths } from "../vault/paths.js";
|
|
7
|
+
import { writeJsonFile } from "../../lib/write-json.js";
|
|
8
|
+
import { makeAliases, makeTags, normalizeObsidianText } from "../../lib/obsidian.js";
|
|
9
|
+
import { toFrontmatterMarkdown } from "../ingest/frontmatter.js";
|
|
10
|
+
import { getChromeTargetNote } from "../vault/notes.js";
|
|
11
|
+
export async function registerChromeProfileTarget(input) {
|
|
12
|
+
return persistChromeAttachTarget(input.cwd, chromeAttachTargetSchema.parse({
|
|
13
|
+
id: `attach-${slugify(input.name)}`,
|
|
14
|
+
type: "chrome_attach_target",
|
|
15
|
+
name: input.name,
|
|
16
|
+
description: input.description,
|
|
17
|
+
profileIsolation: input.profileIsolation ?? "unknown",
|
|
18
|
+
ownership: input.ownership ?? "user_managed",
|
|
19
|
+
notebooklmReadiness: input.notebooklmReadiness ?? "unknown",
|
|
20
|
+
notebooklmValidatedAt: input.notebooklmValidatedAt,
|
|
21
|
+
targetType: "profile",
|
|
22
|
+
profileDirPath: path.resolve(input.profileDirPath),
|
|
23
|
+
chromeExecutablePath: input.chromeExecutablePath ? path.resolve(input.chromeExecutablePath) : undefined,
|
|
24
|
+
remoteDebuggingPort: input.remoteDebuggingPort,
|
|
25
|
+
launchArgs: input.launchArgs ?? [],
|
|
26
|
+
createdAt: input.createdAt ?? new Date().toISOString()
|
|
27
|
+
}), input.force ?? false);
|
|
28
|
+
}
|
|
29
|
+
export async function registerChromeEndpointTarget(input) {
|
|
30
|
+
return persistChromeAttachTarget(input.cwd, chromeAttachTargetSchema.parse({
|
|
31
|
+
id: `attach-${slugify(input.name)}`,
|
|
32
|
+
type: "chrome_attach_target",
|
|
33
|
+
name: input.name,
|
|
34
|
+
description: input.description,
|
|
35
|
+
profileIsolation: input.profileIsolation ?? "unknown",
|
|
36
|
+
ownership: "external",
|
|
37
|
+
notebooklmReadiness: input.notebooklmReadiness ?? "unknown",
|
|
38
|
+
notebooklmValidatedAt: input.notebooklmValidatedAt,
|
|
39
|
+
targetType: "remote_debugging_endpoint",
|
|
40
|
+
endpoint: input.endpoint,
|
|
41
|
+
createdAt: new Date().toISOString()
|
|
42
|
+
}), input.force ?? false);
|
|
43
|
+
}
|
|
44
|
+
export async function loadChromeAttachTarget(targetId, cwd) {
|
|
45
|
+
const workspace = await loadWorkspace(cwd);
|
|
46
|
+
const vault = getVaultPaths(workspace);
|
|
47
|
+
const targetPath = path.join(vault.chromeTargetsDir, `${targetId}.json`);
|
|
48
|
+
const raw = await readFile(targetPath, "utf8");
|
|
49
|
+
const target = chromeAttachTargetSchema.parse(JSON.parse(raw));
|
|
50
|
+
return { workspace, target, path: targetPath };
|
|
51
|
+
}
|
|
52
|
+
export async function upsertChromeAttachTarget(target, input) {
|
|
53
|
+
return persistChromeAttachTarget(input?.cwd, target, input?.force ?? true);
|
|
54
|
+
}
|
|
55
|
+
export async function listChromeAttachTargets(cwd) {
|
|
56
|
+
const workspace = await loadWorkspace(cwd);
|
|
57
|
+
const vault = getVaultPaths(workspace);
|
|
58
|
+
try {
|
|
59
|
+
const entries = await readdir(vault.chromeTargetsDir);
|
|
60
|
+
const jsonFiles = entries.filter((entry) => entry.endsWith(".json"));
|
|
61
|
+
const raw = await Promise.all(jsonFiles.map((entry) => readFile(path.join(vault.chromeTargetsDir, entry), "utf8")));
|
|
62
|
+
return raw.map((value) => chromeAttachTargetSchema.parse(JSON.parse(value))).sort((left, right) => left.name.localeCompare(right.name));
|
|
63
|
+
}
|
|
64
|
+
catch (error) {
|
|
65
|
+
if (isMissingDirectoryError(error)) {
|
|
66
|
+
return [];
|
|
67
|
+
}
|
|
68
|
+
throw error;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
export async function removeChromeAttachTarget(input) {
|
|
72
|
+
const workspace = await loadWorkspace(input.cwd);
|
|
73
|
+
const vault = getVaultPaths(workspace);
|
|
74
|
+
const basePath = path.join(vault.chromeTargetsDir, input.targetId);
|
|
75
|
+
let notePath = `${basePath}.md`;
|
|
76
|
+
try {
|
|
77
|
+
const raw = await readFile(`${basePath}.json`, "utf8");
|
|
78
|
+
const target = chromeAttachTargetSchema.parse(JSON.parse(raw));
|
|
79
|
+
notePath = getChromeTargetNote(workspace, target).absolutePath;
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
// fall back to the legacy id-first markdown path
|
|
83
|
+
}
|
|
84
|
+
await Promise.all([rm(`${basePath}.json`, { force: true }), rm(notePath, { force: true }), rm(`${basePath}.md`, { force: true })]);
|
|
85
|
+
}
|
|
86
|
+
async function persistChromeAttachTarget(cwd, target, force) {
|
|
87
|
+
const workspace = await loadWorkspace(cwd);
|
|
88
|
+
const vault = getVaultPaths(workspace);
|
|
89
|
+
await mkdir(vault.chromeTargetsDir, { recursive: true });
|
|
90
|
+
const basePath = path.join(vault.chromeTargetsDir, target.id);
|
|
91
|
+
const jsonPath = `${basePath}.json`;
|
|
92
|
+
const markdownPath = getChromeTargetNote(workspace, target).absolutePath;
|
|
93
|
+
if (!force && (await fileExists(jsonPath))) {
|
|
94
|
+
throw new Error(`Chrome attach target ${target.id} already exists. Re-run with --force to overwrite it.`);
|
|
95
|
+
}
|
|
96
|
+
await writeJsonFile(jsonPath, target);
|
|
97
|
+
await writeFile(markdownPath, buildChromeAttachMarkdown(target), "utf8");
|
|
98
|
+
return {
|
|
99
|
+
target,
|
|
100
|
+
markdownPath,
|
|
101
|
+
jsonPath
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
function buildChromeAttachMarkdown(target) {
|
|
105
|
+
const title = normalizeObsidianText(target.name, target.id);
|
|
106
|
+
const frontmatter = target.targetType === "profile"
|
|
107
|
+
? {
|
|
108
|
+
type: "chrome-target",
|
|
109
|
+
title,
|
|
110
|
+
aliases: makeAliases(target.id),
|
|
111
|
+
tags: makeTags("sourceloop", "chrome-target", target.targetType),
|
|
112
|
+
mode: target.targetType,
|
|
113
|
+
...(target.description ? { description: target.description } : {}),
|
|
114
|
+
created: target.createdAt,
|
|
115
|
+
updated: target.createdAt
|
|
116
|
+
}
|
|
117
|
+
: {
|
|
118
|
+
type: "chrome-target",
|
|
119
|
+
title,
|
|
120
|
+
aliases: makeAliases(target.id),
|
|
121
|
+
tags: makeTags("sourceloop", "chrome-target", target.targetType),
|
|
122
|
+
mode: target.targetType,
|
|
123
|
+
...(target.description ? { description: target.description } : {}),
|
|
124
|
+
created: target.createdAt,
|
|
125
|
+
updated: target.createdAt
|
|
126
|
+
};
|
|
127
|
+
return toFrontmatterMarkdown(frontmatter, buildChromeAttachBody(target));
|
|
128
|
+
}
|
|
129
|
+
function buildChromeAttachBody(target) {
|
|
130
|
+
const lines = [
|
|
131
|
+
`# ${normalizeObsidianText(target.name, target.id)}`,
|
|
132
|
+
"",
|
|
133
|
+
`- Target Type: ${target.targetType}`,
|
|
134
|
+
`- Profile Isolation: ${target.profileIsolation}`,
|
|
135
|
+
`- Ownership: ${target.ownership}`,
|
|
136
|
+
`- NotebookLM Readiness: ${target.notebooklmReadiness}`
|
|
137
|
+
];
|
|
138
|
+
if (target.notebooklmValidatedAt) {
|
|
139
|
+
lines.push(`- NotebookLM Validated At: ${target.notebooklmValidatedAt}`);
|
|
140
|
+
}
|
|
141
|
+
if (target.targetType === "profile") {
|
|
142
|
+
appendProfileBody(lines, target);
|
|
143
|
+
}
|
|
144
|
+
else {
|
|
145
|
+
appendEndpointBody(lines, target);
|
|
146
|
+
}
|
|
147
|
+
if (target.description) {
|
|
148
|
+
lines.push("", "## Description", target.description);
|
|
149
|
+
}
|
|
150
|
+
return lines.join("\n");
|
|
151
|
+
}
|
|
152
|
+
function appendProfileBody(lines, target) {
|
|
153
|
+
lines.push(`- Profile Directory: ${target.profileDirPath}`);
|
|
154
|
+
if (target.chromeExecutablePath) {
|
|
155
|
+
lines.push(`- Chrome Executable: ${target.chromeExecutablePath}`);
|
|
156
|
+
}
|
|
157
|
+
if (target.remoteDebuggingPort) {
|
|
158
|
+
lines.push(`- Preferred Remote Debugging Port: ${target.remoteDebuggingPort}`);
|
|
159
|
+
}
|
|
160
|
+
if (target.launchArgs.length > 0) {
|
|
161
|
+
lines.push("", "## Launch Args", ...target.launchArgs.map((arg) => `- ${arg}`));
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
function appendEndpointBody(lines, target) {
|
|
165
|
+
lines.push(`- Endpoint: ${target.endpoint}`);
|
|
166
|
+
}
|
|
167
|
+
async function fileExists(filePath) {
|
|
168
|
+
try {
|
|
169
|
+
await access(filePath);
|
|
170
|
+
return true;
|
|
171
|
+
}
|
|
172
|
+
catch {
|
|
173
|
+
return false;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
function isMissingDirectoryError(error) {
|
|
177
|
+
return Boolean(error && typeof error === "object" && "code" in error && error.code === "ENOENT");
|
|
178
|
+
}
|
|
179
|
+
//# sourceMappingURL=manage-targets.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"manage-targets.js","sourceRoot":"","sources":["../../../src/core/attach/manage-targets.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACnF,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EACL,wBAAwB,EAOzB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAmCxD,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,KAAuC;IAEvC,OAAO,yBAAyB,CAC9B,KAAK,CAAC,GAAG,EACT,wBAAwB,CAAC,KAAK,CAAC;QAC7B,EAAE,EAAE,UAAU,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;QACnC,IAAI,EAAE,sBAAsB;QAC5B,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,IAAI,SAAS;QACrD,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,cAAc;QAC5C,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,IAAI,SAAS;QAC3D,qBAAqB,EAAE,KAAK,CAAC,qBAAqB;QAClD,UAAU,EAAE,SAAS;QACrB,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC;QAClD,oBAAoB,EAAE,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,SAAS;QACvG,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;QAC9C,UAAU,EAAE,KAAK,CAAC,UAAU,IAAI,EAAE;QAClC,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACvD,CAAC,EACF,KAAK,CAAC,KAAK,IAAI,KAAK,CACrB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,KAAwC;IAExC,OAAO,yBAAyB,CAC9B,KAAK,CAAC,GAAG,EACT,wBAAwB,CAAC,KAAK,CAAC;QAC7B,EAAE,EAAE,UAAU,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;QACnC,IAAI,EAAE,sBAAsB;QAC5B,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,IAAI,SAAS;QACrD,SAAS,EAAE,UAAU;QACrB,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,IAAI,SAAS;QAC3D,qBAAqB,EAAE,KAAK,CAAC,qBAAqB;QAClD,UAAU,EAAE,2BAA2B;QACvC,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACpC,CAAC,EACF,KAAK,CAAC,KAAK,IAAI,KAAK,CACrB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,QAAgB,EAChB,GAAY;IAEZ,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;IACvC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,GAAG,QAAQ,OAAO,CAAC,CAAC;IACzE,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,wBAAwB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/D,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;AACjD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,MAA0B,EAC1B,KAAyC;IAEzC,OAAO,yBAAyB,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,IAAI,IAAI,CAAC,CAAC;AAC7E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,GAAY;IACxD,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACtD,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QACrE,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;QACpH,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,wBAAwB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAChG,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CACpC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,uBAAuB,CAAC,KAAK,CAAC,EAAE,CAAC;YACnC,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,KAAyC;IACtF,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACjD,MAAM,KAAK,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACnE,IAAI,QAAQ,GAAG,GAAG,QAAQ,KAAK,CAAC;IAChC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,GAAG,QAAQ,OAAO,EAAE,MAAM,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,wBAAwB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/D,QAAQ,GAAG,mBAAmB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,YAAY,CAAC;IACjE,CAAC;IAAC,MAAM,CAAC;QACP,iDAAiD;IACnD,CAAC;IAED,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,QAAQ,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,QAAQ,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACrI,CAAC;AAED,KAAK,UAAU,yBAAyB,CACtC,GAAuB,EACvB,MAA0B,EAC1B,KAAc;IAEd,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;IACvC,MAAM,KAAK,CAAC,KAAK,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;IAC9D,MAAM,QAAQ,GAAG,GAAG,QAAQ,OAAO,CAAC;IACpC,MAAM,YAAY,GAAG,mBAAmB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,YAAY,CAAC;IAEzE,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,wBAAwB,MAAM,CAAC,EAAE,uDAAuD,CAAC,CAAC;IAC5G,CAAC;IAED,MAAM,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACtC,MAAM,SAAS,CAAC,YAAY,EAAE,yBAAyB,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;IAEzE,OAAO;QACL,MAAM;QACN,YAAY;QACZ,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,SAAS,yBAAyB,CAAC,MAA0B;IACzD,MAAM,KAAK,GAAG,qBAAqB,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;IAC5D,MAAM,WAAW,GACf,MAAM,CAAC,UAAU,KAAK,SAAS;QAC/B,CAAC,CAAC;YACE,IAAI,EAAE,eAAe;YACrB,KAAK;YACL,OAAO,EAAE,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/B,IAAI,EAAE,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE,MAAM,CAAC,UAAU,CAAC;YAChE,IAAI,EAAE,MAAM,CAAC,UAAU;YACvB,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClE,OAAO,EAAE,MAAM,CAAC,SAAS;YACzB,OAAO,EAAE,MAAM,CAAC,SAAS;SAC1B;QACH,CAAC,CAAC;YACE,IAAI,EAAE,eAAe;YACrB,KAAK;YACL,OAAO,EAAE,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/B,IAAI,EAAE,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE,MAAM,CAAC,UAAU,CAAC;YAChE,IAAI,EAAE,MAAM,CAAC,UAAU;YACvB,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClE,OAAO,EAAE,MAAM,CAAC,SAAS;YACzB,OAAO,EAAE,MAAM,CAAC,SAAS;SAC1B,CAAC;IAER,OAAO,qBAAqB,CAAC,WAAW,EAAE,qBAAqB,CAAC,MAAM,CAAC,CAAC,CAAC;AAC3E,CAAC;AAED,SAAS,qBAAqB,CAAC,MAA0B;IACvD,MAAM,KAAK,GAAG;QACZ,KAAK,qBAAqB,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE;QACpD,EAAE;QACF,kBAAkB,MAAM,CAAC,UAAU,EAAE;QACrC,wBAAwB,MAAM,CAAC,gBAAgB,EAAE;QACjD,gBAAgB,MAAM,CAAC,SAAS,EAAE;QAClC,2BAA2B,MAAM,CAAC,mBAAmB,EAAE;KACxD,CAAC;IAEF,IAAI,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,8BAA8B,MAAM,CAAC,qBAAqB,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QACpC,iBAAiB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,kBAAkB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACpC,CAAC;IAED,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,gBAAgB,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;IACvD,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAe,EAAE,MAAiC;IAC3E,KAAK,CAAC,IAAI,CAAC,wBAAwB,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC;IAC5D,IAAI,MAAM,CAAC,oBAAoB,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,wBAAwB,MAAM,CAAC,oBAAoB,EAAE,CAAC,CAAC;IACpE,CAAC;IACD,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,sCAAsC,MAAM,CAAC,mBAAmB,EAAE,CAAC,CAAC;IACjF,CAAC;IACD,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,gBAAgB,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC;IAClF,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAe,EAAE,MAAkC;IAC7E,KAAK,CAAC,IAAI,CAAC,eAAe,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;AAC/C,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,QAAgB;IACxC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;QACvB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,KAAc;IAC7C,OAAO,OAAO,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;AACnG,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export function toFrontmatterMarkdown(metadata, body) {
|
|
2
|
+
const lines = ["---"];
|
|
3
|
+
for (const [key, value] of Object.entries(metadata)) {
|
|
4
|
+
if (value === undefined) {
|
|
5
|
+
continue;
|
|
6
|
+
}
|
|
7
|
+
if (Array.isArray(value)) {
|
|
8
|
+
lines.push(`${key}:`);
|
|
9
|
+
if (value.length === 0) {
|
|
10
|
+
lines.push(" []");
|
|
11
|
+
}
|
|
12
|
+
else {
|
|
13
|
+
for (const entry of value) {
|
|
14
|
+
lines.push(` - ${escapeScalar(entry)}`);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
continue;
|
|
18
|
+
}
|
|
19
|
+
lines.push(`${key}: ${escapeScalar(value)}`);
|
|
20
|
+
}
|
|
21
|
+
lines.push("---", "", body.trimEnd(), "");
|
|
22
|
+
return lines.join("\n");
|
|
23
|
+
}
|
|
24
|
+
function escapeScalar(value) {
|
|
25
|
+
if (/^[A-Za-z0-9._:/-]+$/.test(value)) {
|
|
26
|
+
return value;
|
|
27
|
+
}
|
|
28
|
+
return JSON.stringify(value);
|
|
29
|
+
}
|
|
30
|
+
//# sourceMappingURL=frontmatter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"frontmatter.js","sourceRoot":"","sources":["../../../src/core/ingest/frontmatter.ts"],"names":[],"mappings":"AAIA,MAAM,UAAU,qBAAqB,CAAC,QAA2B,EAAE,IAAY;IAC7E,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC;IAEtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,SAAS;QACX,CAAC;QAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;YAEtB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;oBAC1B,KAAK,CAAC,IAAI,CAAC,OAAO,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC3C,CAAC;YACH,CAAC;YAED,SAAS;QACX,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,KAAK,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,EAAE,IAAI,CAAC,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;IAC1C,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,YAAY,CAAC,KAAa;IACjC,IAAI,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACtC,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;AAC/B,CAAC"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
export function htmlToMarkdown(html) {
|
|
2
|
+
const title = extractTitle(html) ?? "Untitled";
|
|
3
|
+
const language = extractLanguage(html);
|
|
4
|
+
const stripped = stripHtmlToText(html);
|
|
5
|
+
const body = `# ${title}\n\n${stripped}`.trim();
|
|
6
|
+
return language
|
|
7
|
+
? {
|
|
8
|
+
title,
|
|
9
|
+
body,
|
|
10
|
+
language
|
|
11
|
+
}
|
|
12
|
+
: {
|
|
13
|
+
title,
|
|
14
|
+
body
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
function extractTitle(html) {
|
|
18
|
+
const match = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
|
|
19
|
+
if (!match) {
|
|
20
|
+
return undefined;
|
|
21
|
+
}
|
|
22
|
+
return decodeEntities(match[1] ?? "").trim();
|
|
23
|
+
}
|
|
24
|
+
function extractLanguage(html) {
|
|
25
|
+
const match = html.match(/<html[^>]*\slang=["']([^"']+)["']/i);
|
|
26
|
+
return match?.[1]?.trim() || undefined;
|
|
27
|
+
}
|
|
28
|
+
function stripHtmlToText(html) {
|
|
29
|
+
const withoutScripts = html
|
|
30
|
+
.replace(/<script[\s\S]*?<\/script>/gi, " ")
|
|
31
|
+
.replace(/<style[\s\S]*?<\/style>/gi, " ");
|
|
32
|
+
const blockSeparated = withoutScripts
|
|
33
|
+
.replace(/<\/(p|div|section|article|li|h1|h2|h3|h4|h5|h6|br)>/gi, "\n")
|
|
34
|
+
.replace(/<(ul|ol)>/gi, "\n")
|
|
35
|
+
.replace(/<(li)>/gi, "- ");
|
|
36
|
+
const noTags = blockSeparated.replace(/<[^>]+>/g, " ");
|
|
37
|
+
const decoded = decodeEntities(noTags);
|
|
38
|
+
return decoded
|
|
39
|
+
.split("\n")
|
|
40
|
+
.map((line) => line.replace(/[ \t]+/g, " ").trim())
|
|
41
|
+
.filter(Boolean)
|
|
42
|
+
.join("\n\n");
|
|
43
|
+
}
|
|
44
|
+
function decodeEntities(value) {
|
|
45
|
+
return value
|
|
46
|
+
.replace(/ /g, " ")
|
|
47
|
+
.replace(/&/g, "&")
|
|
48
|
+
.replace(/</g, "<")
|
|
49
|
+
.replace(/>/g, ">")
|
|
50
|
+
.replace(/"/g, '"')
|
|
51
|
+
.replace(/'/g, "'");
|
|
52
|
+
}
|
|
53
|
+
//# sourceMappingURL=html-to-markdown.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"html-to-markdown.js","sourceRoot":"","sources":["../../../src/core/ingest/html-to-markdown.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC;IAC/C,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,IAAI,GAAG,KAAK,KAAK,OAAO,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;IAEhD,OAAO,QAAQ;QACb,CAAC,CAAC;YACE,KAAK;YACL,IAAI;YACJ,QAAQ;SACT;QACH,CAAC,CAAC;YACE,KAAK;YACL,IAAI;SACL,CAAC;AACR,CAAC;AAED,SAAS,YAAY,CAAC,IAAY;IAChC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAE7D,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AAC/C,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACnC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;IAE/D,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,SAAS,CAAC;AACzC,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACnC,MAAM,cAAc,GAAG,IAAI;SACxB,OAAO,CAAC,6BAA6B,EAAE,GAAG,CAAC;SAC3C,OAAO,CAAC,2BAA2B,EAAE,GAAG,CAAC,CAAC;IAE7C,MAAM,cAAc,GAAG,cAAc;SAClC,OAAO,CAAC,uDAAuD,EAAE,IAAI,CAAC;SACtE,OAAO,CAAC,aAAa,EAAE,IAAI,CAAC;SAC5B,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IAE7B,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;IACvD,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IAEvC,OAAO,OAAO;SACX,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;SAClD,MAAM,CAAC,OAAO,CAAC;SACf,IAAI,CAAC,MAAM,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,cAAc,CAAC,KAAa;IACnC,OAAO,KAAK;SACT,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;AAC5B,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { type SourceDocument } from "../../schemas/source.js";
|
|
2
|
+
export type IngestSourceInput = {
|
|
3
|
+
input: string;
|
|
4
|
+
topicId?: string;
|
|
5
|
+
cwd?: string;
|
|
6
|
+
};
|
|
7
|
+
export type IngestSourceResult = {
|
|
8
|
+
source: SourceDocument;
|
|
9
|
+
outputPath: string;
|
|
10
|
+
};
|
|
11
|
+
export declare function ingestSource(input: IngestSourceInput): Promise<IngestSourceResult>;
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { randomUUID } from "node:crypto";
|
|
4
|
+
import { loadWorkspace } from "../workspace/load-workspace.js";
|
|
5
|
+
import { sourceDocumentSchema } from "../../schemas/source.js";
|
|
6
|
+
import { htmlToMarkdown } from "./html-to-markdown.js";
|
|
7
|
+
import { toFrontmatterMarkdown } from "./frontmatter.js";
|
|
8
|
+
import { loadTopic, refreshTopicArtifacts } from "../topics/manage-topics.js";
|
|
9
|
+
import { writeJsonFile } from "../../lib/write-json.js";
|
|
10
|
+
import { makeAliases, makeTags, normalizeObsidianText } from "../../lib/obsidian.js";
|
|
11
|
+
import { getSourceNote } from "../vault/notes.js";
|
|
12
|
+
export async function ingestSource(input) {
|
|
13
|
+
const cwd = input.cwd ?? process.cwd();
|
|
14
|
+
const workspace = await loadWorkspace(cwd);
|
|
15
|
+
const now = new Date().toISOString();
|
|
16
|
+
if (input.topicId) {
|
|
17
|
+
await loadTopic(input.topicId, cwd);
|
|
18
|
+
}
|
|
19
|
+
if (isUrl(input.input)) {
|
|
20
|
+
const source = await ingestUrl(input.input, now, input.topicId);
|
|
21
|
+
const outputPath = await writeSourceArtifact(workspace.rootDir, source);
|
|
22
|
+
if (input.topicId) {
|
|
23
|
+
await refreshTopicArtifacts(input.topicId, cwd);
|
|
24
|
+
}
|
|
25
|
+
return { source, outputPath };
|
|
26
|
+
}
|
|
27
|
+
const source = await ingestFile(input.input, cwd, now, input.topicId);
|
|
28
|
+
const outputPath = await writeSourceArtifact(workspace.rootDir, source);
|
|
29
|
+
if (input.topicId) {
|
|
30
|
+
await refreshTopicArtifacts(input.topicId, cwd);
|
|
31
|
+
}
|
|
32
|
+
return { source, outputPath };
|
|
33
|
+
}
|
|
34
|
+
async function ingestFile(inputPath, cwd, capturedAt, topicId) {
|
|
35
|
+
const absolutePath = path.resolve(cwd, inputPath);
|
|
36
|
+
const content = await readFile(absolutePath, "utf8");
|
|
37
|
+
const title = normalizeObsidianText(path.basename(absolutePath, path.extname(absolutePath)));
|
|
38
|
+
return {
|
|
39
|
+
id: createSourceId(),
|
|
40
|
+
type: "file",
|
|
41
|
+
sourceUri: absolutePath,
|
|
42
|
+
title,
|
|
43
|
+
topicId,
|
|
44
|
+
capturedAt,
|
|
45
|
+
topicTags: [],
|
|
46
|
+
body: `# ${title}\n\n${content.trim()}`
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
async function ingestUrl(inputUrl, capturedAt, topicId) {
|
|
50
|
+
const response = await fetch(inputUrl);
|
|
51
|
+
if (!response.ok) {
|
|
52
|
+
throw new Error(`Failed to fetch ${inputUrl}: ${response.status} ${response.statusText}`);
|
|
53
|
+
}
|
|
54
|
+
const html = await response.text();
|
|
55
|
+
const converted = htmlToMarkdown(html);
|
|
56
|
+
return {
|
|
57
|
+
id: createSourceId(),
|
|
58
|
+
type: "url",
|
|
59
|
+
sourceUri: inputUrl,
|
|
60
|
+
title: normalizeObsidianText(converted.title, inputUrl),
|
|
61
|
+
topicId,
|
|
62
|
+
capturedAt,
|
|
63
|
+
topicTags: [],
|
|
64
|
+
language: converted.language,
|
|
65
|
+
body: converted.body
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
async function writeSourceArtifact(workspaceRoot, draft) {
|
|
69
|
+
const topicRecord = draft.topicId ? await loadTopic(draft.topicId, workspaceRoot) : undefined;
|
|
70
|
+
const source = sourceDocumentSchema.parse({
|
|
71
|
+
id: draft.id,
|
|
72
|
+
type: draft.type,
|
|
73
|
+
sourceUri: draft.sourceUri,
|
|
74
|
+
title: draft.title,
|
|
75
|
+
topicId: draft.topicId,
|
|
76
|
+
author: draft.author,
|
|
77
|
+
capturedAt: draft.capturedAt,
|
|
78
|
+
topicTags: draft.topicTags,
|
|
79
|
+
language: draft.language
|
|
80
|
+
});
|
|
81
|
+
const markdown = toFrontmatterMarkdown({
|
|
82
|
+
type: "source",
|
|
83
|
+
title: normalizeObsidianText(source.title, source.id),
|
|
84
|
+
aliases: makeAliases(source.id),
|
|
85
|
+
tags: makeTags("sourceloop", "source", source.type),
|
|
86
|
+
...(topicRecord ? { topic: normalizeObsidianText(topicRecord.topic.name, topicRecord.topic.id) } : {}),
|
|
87
|
+
source_kind: source.type,
|
|
88
|
+
...(source.author ? { author: source.author } : {}),
|
|
89
|
+
...(source.language ? { language: source.language } : {}),
|
|
90
|
+
created: source.capturedAt,
|
|
91
|
+
updated: source.capturedAt
|
|
92
|
+
}, draft.body);
|
|
93
|
+
const workspace = await loadWorkspace(workspaceRoot);
|
|
94
|
+
const note = getSourceNote(workspace, source);
|
|
95
|
+
const sourcesDir = path.join(workspaceRoot, "vault/sources");
|
|
96
|
+
await mkdir(sourcesDir, { recursive: true });
|
|
97
|
+
const markdownPath = note.absolutePath;
|
|
98
|
+
const jsonPath = path.join(sourcesDir, `${source.id}.json`);
|
|
99
|
+
await writeFile(markdownPath, markdown, "utf8");
|
|
100
|
+
await writeJsonFile(jsonPath, source);
|
|
101
|
+
return markdownPath;
|
|
102
|
+
}
|
|
103
|
+
function isUrl(value) {
|
|
104
|
+
try {
|
|
105
|
+
const parsed = new URL(value);
|
|
106
|
+
return parsed.protocol === "http:" || parsed.protocol === "https:";
|
|
107
|
+
}
|
|
108
|
+
catch {
|
|
109
|
+
return false;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
function createSourceId() {
|
|
113
|
+
return `src_${randomUUID().replace(/-/g, "").slice(0, 12)}`;
|
|
114
|
+
}
|
|
115
|
+
//# sourceMappingURL=ingest-source.js.map
|