reposets 0.4.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +87 -75
- package/bin/reposets.js +68 -17
- package/cli/commands/credentials.js +170 -49
- package/cli/commands/doctor.js +364 -91
- package/cli/commands/drift.js +48 -0
- package/cli/commands/history.js +203 -0
- package/cli/commands/init.js +110 -104
- package/cli/commands/list.js +60 -39
- package/cli/commands/nuke.js +127 -0
- package/cli/commands/sync.js +219 -59
- package/cli/commands/validate.js +57 -41
- package/cli/flags.js +36 -0
- package/cli/logger.js +48 -0
- package/index.d.ts +471 -1499
- package/index.js +3 -18
- package/lib/config-refs.js +76 -0
- package/lib/credential-labels.js +0 -0
- package/lib/fingerprint.js +52 -0
- package/lib/org-only.js +61 -0
- package/lib/schema-issues.js +50 -0
- package/package.json +11 -10
- package/schemas/annotations.js +81 -0
- package/schemas/common.js +83 -48
- package/schemas/config.js +214 -212
- package/schemas/credentials.js +190 -54
- package/schemas/environment.js +27 -21
- package/schemas/ruleset.js +235 -139
- package/services/ConfigFiles.js +126 -104
- package/services/CredentialResolver.js +97 -33
- package/services/OnePasswordClient.js +88 -16
- package/services/SyncLogger.js +107 -76
- package/store/AppliedState.js +0 -0
- package/store/RepoCache.js +86 -0
- package/store/SyncJournal.js +92 -0
- package/store/migrations.js +86 -0
- package/sync/SyncEngine.js +156 -0
- package/sync/decide.js +56 -0
- package/sync/phase.js +55 -0
- package/sync/phases/cleanup.js +220 -0
- package/sync/phases/code-scanning.js +187 -0
- package/sync/phases/environments.js +106 -0
- package/sync/phases/index.js +39 -0
- package/sync/phases/resource.js +149 -0
- package/sync/phases/rulesets.js +186 -0
- package/sync/phases/secrets.js +138 -0
- package/sync/phases/security.js +129 -0
- package/sync/phases/settings.js +274 -0
- package/sync/phases/variables.js +132 -0
- package/tsdoc-metadata.json +1 -1
- package/bin/reposets.d.ts +0 -1
- package/errors.js +0 -12
- package/lib/crypto.js +0 -27
- package/services/GitHubClient.js +0 -875
- package/services/SyncEngine.js +0 -580
package/services/SyncLogger.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Context, Effect, Layer, Ref } from "effect";
|
|
2
2
|
|
|
3
3
|
//#region src/services/SyncLogger.ts
|
|
4
|
-
|
|
4
|
+
/** Plural forms the naive `+ "s"` gets wrong. */
|
|
5
5
|
function pluralize(resource, count) {
|
|
6
6
|
if (count === 1) return resource;
|
|
7
7
|
if (resource === "ruleset") return "rulesets";
|
|
@@ -9,95 +9,126 @@ function pluralize(resource, count) {
|
|
|
9
9
|
if (resource === "code scanning") return "code scanning";
|
|
10
10
|
return `${resource}s`;
|
|
11
11
|
}
|
|
12
|
+
/**
|
|
13
|
+
* Dry-run-aware output for the sync pipeline.
|
|
14
|
+
*
|
|
15
|
+
* @remarks
|
|
16
|
+
* Every line a run prints goes through here, so `SyncEngine` describes what
|
|
17
|
+
* happened and this service decides how to say it.
|
|
18
|
+
*
|
|
19
|
+
* **There are no verbosity tiers.** There were four — `silent`, `info`,
|
|
20
|
+
* `verbose`, `debug` — and enumerating what they actually gated is what
|
|
21
|
+
* retired them: `verbose` guarded exactly one call site, the per-resource
|
|
22
|
+
* operation line, which is the entire content of a sync. So the boundary did
|
|
23
|
+
* not fall between summary and detail; it fell between *settings* and
|
|
24
|
+
* everything else, and a dry run at the default tier reported a change count
|
|
25
|
+
* with four fifths of the list suppressed.
|
|
26
|
+
*
|
|
27
|
+
* `silent` was worse: it suppressed errors too, so a failing run printed
|
|
28
|
+
* nothing at all and the exit code was the only signal. Redirecting stdout does
|
|
29
|
+
* that job better, because errors go to stderr and survive it.
|
|
30
|
+
*
|
|
31
|
+
* What remains is one output, plus `debug` for two diagnostic suffixes.
|
|
32
|
+
*
|
|
33
|
+
* **Drift outranks everything else a run reports.** `synced 3 secrets` says
|
|
34
|
+
* reposets did its job; a drift line says *someone else* changed the repository
|
|
35
|
+
* and reposets has just overwritten them. That is the one thing a human needs to
|
|
36
|
+
* see without asking for it, so it is emitted at `info` alongside the summaries
|
|
37
|
+
* rather than at `verbose` with the per-resource operations, and it is worded as
|
|
38
|
+
* an observation about a person rather than as an action by the tool.
|
|
39
|
+
*
|
|
40
|
+
* Drift is reported even when nothing was written — the case where an
|
|
41
|
+
* out-of-band edit happens to match the config. The tool has no work to do; the
|
|
42
|
+
* human still changed something, and staying quiet would hide it.
|
|
43
|
+
*
|
|
44
|
+
* Lines are emitted with `Effect.log`, never `Console.log`: the CLI entrypoint
|
|
45
|
+
* installs a logger that routes Effect's output, and a direct console write
|
|
46
|
+
* would bypass both it and the filtering above.
|
|
47
|
+
*
|
|
48
|
+
* @public
|
|
49
|
+
*/
|
|
50
|
+
var SyncLogger = class extends Context.Service()("reposets/SyncLogger") {};
|
|
51
|
+
/**
|
|
52
|
+
* Build a live logger for one run.
|
|
53
|
+
*
|
|
54
|
+
* @remarks
|
|
55
|
+
* A factory rather than a bare layer because both settings are per-invocation:
|
|
56
|
+
* they come from the `--dry-run` and `--debug` flags, which are only known once
|
|
57
|
+
* the command has parsed.
|
|
58
|
+
*
|
|
59
|
+
* @public
|
|
60
|
+
*/
|
|
12
61
|
function SyncLoggerLive(config) {
|
|
13
|
-
const { dryRun,
|
|
62
|
+
const { dryRun, debug } = config;
|
|
14
63
|
return Layer.effect(SyncLogger, Effect.gen(function* () {
|
|
15
64
|
const errors = yield* Ref.make([]);
|
|
16
65
|
const currentRepo = yield* Ref.make("");
|
|
17
|
-
|
|
18
|
-
if (output) return Ref.update(output, (lines) => [...lines, line]);
|
|
19
|
-
return Effect.sync(() => {
|
|
20
|
-
process.stdout.write(`${line}\n`);
|
|
21
|
-
});
|
|
22
|
-
}
|
|
23
|
-
function isVisible(tier) {
|
|
24
|
-
if (logLevel === "silent") return false;
|
|
25
|
-
const levels = [
|
|
26
|
-
"info",
|
|
27
|
-
"verbose",
|
|
28
|
-
"debug"
|
|
29
|
-
];
|
|
30
|
-
return levels.indexOf(logLevel) >= levels.indexOf(tier);
|
|
31
|
-
}
|
|
66
|
+
const emit = (line) => Effect.log(line);
|
|
32
67
|
/**
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
68
|
+
* Failures, on the error channel.
|
|
69
|
+
*
|
|
70
|
+
* @remarks
|
|
71
|
+
* v3 put these on stdout with everything else. They move to stderr —
|
|
72
|
+
* where the CLI entrypoint routes `logError` — so that
|
|
73
|
+
* `reposets sync > log.txt` still shows failures on the terminal while
|
|
74
|
+
* the log captures progress. A deliberate change to piping behavior,
|
|
75
|
+
* permitted because CLI output compatibility is not a goal of the
|
|
76
|
+
* rebuild.
|
|
36
77
|
*/
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
78
|
+
const emitError = (line) => Effect.logError(line);
|
|
79
|
+
/**
|
|
80
|
+
* Pad a verb so the content after it lines up. Past-tense verbs pad to
|
|
81
|
+
* 8; the dry-run `would <present>` forms pad to 14. The result is
|
|
82
|
+
* concatenated directly, with no separating space.
|
|
83
|
+
*/
|
|
84
|
+
const formatVerb = (pastTense, presentTense) => dryRun ? `would ${presentTense}`.padEnd(14) : pastTense.padEnd(8);
|
|
41
85
|
return {
|
|
42
|
-
groupStart(name,
|
|
43
|
-
|
|
44
|
-
return emit(`group: ${name} (${
|
|
86
|
+
groupStart: (name, selected, declared) => {
|
|
87
|
+
const scope = selected === declared ? `${selected} ${selected === 1 ? "repo" : "repos"}` : `${selected} of ${declared} ${declared === 1 ? "repo" : "repos"}`;
|
|
88
|
+
return emit(`group: ${name} (${scope})`);
|
|
45
89
|
},
|
|
46
|
-
repoStart(owner, repo) {
|
|
90
|
+
repoStart: (owner, repo) => {
|
|
47
91
|
const repoSlug = `${owner}/${repo}`;
|
|
48
|
-
|
|
49
|
-
return Effect.gen(function* () {
|
|
50
|
-
yield* Ref.set(currentRepo, repoSlug);
|
|
51
|
-
yield* emit(` repo: ${repoSlug}`);
|
|
52
|
-
});
|
|
53
|
-
},
|
|
54
|
-
repoSkip(owner, repo, reason) {
|
|
55
|
-
if (!isVisible("info")) return Effect.void;
|
|
56
|
-
return Effect.gen(function* () {
|
|
57
|
-
yield* emit(` repo: ${owner}/${repo}`);
|
|
58
|
-
yield* emit(` skip ${reason}`);
|
|
59
|
-
});
|
|
60
|
-
},
|
|
61
|
-
syncSummary(resource, count, detail) {
|
|
62
|
-
if (!isVisible("info")) return Effect.void;
|
|
63
|
-
return emit(` ${formatVerb("synced", "sync")}${count} ${pluralize(resource, count)}${detail ? ` (${detail})` : ""}`);
|
|
92
|
+
return Ref.set(currentRepo, repoSlug).pipe(Effect.andThen(emit(` repo: ${repoSlug}`)));
|
|
64
93
|
},
|
|
65
|
-
settingsApplied() {
|
|
66
|
-
|
|
67
|
-
|
|
94
|
+
settingsApplied: (fields) => {
|
|
95
|
+
const named = [...fields].sort();
|
|
96
|
+
const detail = named.length === 0 ? "" : named.length <= 6 ? ` (${named.join(", ")})` : ` (${named.length} fields: ${named.slice(0, 5).join(", ")}, …)`;
|
|
97
|
+
return emit(` ${formatVerb("applied", "apply")}settings${detail}`);
|
|
68
98
|
},
|
|
69
|
-
cleanupSummary(resource, count, names) {
|
|
70
|
-
|
|
71
|
-
return emit(` ${formatVerb("deleted", "delete")}${count} ${pluralize(resource, count)}${
|
|
99
|
+
cleanupSummary: (resource, count, names) => {
|
|
100
|
+
const suffix = names.length > 0 ? ` (${names.join(", ")})` : "";
|
|
101
|
+
return emit(` ${formatVerb("deleted", "delete")}${count} ${pluralize(resource, count)}${suffix}`);
|
|
72
102
|
},
|
|
73
|
-
syncOperation(verb, resource, name, detail, source) {
|
|
74
|
-
|
|
75
|
-
|
|
103
|
+
syncOperation: (verb, resource, name, detail, source) => {
|
|
104
|
+
const nameStr = name ? ` ${name}` : "";
|
|
105
|
+
const suffix = detail ? ` ${detail}` : "";
|
|
106
|
+
const sourceSuffix = source && debug ? ` <- ${source}` : "";
|
|
107
|
+
return emit(` ${formatVerb(verb, verb)}${resource}${nameStr}${suffix}${sourceSuffix}`);
|
|
76
108
|
},
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
yield* Ref.update(errors, (errs) => [...errs, {
|
|
82
|
-
repo,
|
|
83
|
-
context,
|
|
84
|
-
message
|
|
85
|
-
}]);
|
|
86
|
-
yield* emit(` error ${context}: ${message}`);
|
|
87
|
-
});
|
|
109
|
+
driftDetected: (resource, name, drift) => {
|
|
110
|
+
const consequence = !drift.needsApply ? "already matches config, nothing written" : dryRun ? "would overwrite" : "overwritten";
|
|
111
|
+
const fingerprints = debug ? ` <- applied ${drift.applied} live ${drift.live}` : "";
|
|
112
|
+
return emit(` ${"drift".padEnd(8)}${resource} ${name} changed outside reposets — ${consequence}${fingerprints}`);
|
|
88
113
|
},
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
114
|
+
syncError: (context, message) => Effect.gen(function* () {
|
|
115
|
+
const repo = yield* Ref.get(currentRepo);
|
|
116
|
+
yield* Ref.update(errors, (errs) => [...errs, {
|
|
117
|
+
repo,
|
|
118
|
+
context,
|
|
119
|
+
message
|
|
120
|
+
}]);
|
|
121
|
+
yield* emitError(` error ${context}: ${message}`);
|
|
122
|
+
}),
|
|
123
|
+
finish: () => Effect.gen(function* () {
|
|
124
|
+
const errs = yield* Ref.get(errors);
|
|
125
|
+
if (errs.length === 0) {
|
|
126
|
+
yield* emit("Sync complete!");
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
yield* emitError(`Sync complete with ${errs.length} ${errs.length === 1 ? "error" : "errors"}:`);
|
|
130
|
+
for (const err of errs) yield* emitError(err.repo === "" ? ` ${err.context} — ${err.message}` : ` ${err.repo}: ${err.context} — ${err.message}`);
|
|
131
|
+
})
|
|
101
132
|
};
|
|
102
133
|
}));
|
|
103
134
|
}
|
|
Binary file
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { Context, Duration, Effect, Layer, Schema } from "effect";
|
|
2
|
+
import { Cache } from "@effected/store";
|
|
3
|
+
|
|
4
|
+
//#region src/store/RepoCache.ts
|
|
5
|
+
/**
|
|
6
|
+
* Team ids effectively never change: a team keeps its id across renames, and a
|
|
7
|
+
* deleted team's id is never reissued.
|
|
8
|
+
*/
|
|
9
|
+
const TEAM_TTL = Duration.days(30);
|
|
10
|
+
/**
|
|
11
|
+
* An account's type changes only when a user converts to an organization — a
|
|
12
|
+
* one-way, deliberate, rare act.
|
|
13
|
+
*/
|
|
14
|
+
const OWNER_TTL = Duration.days(30);
|
|
15
|
+
/**
|
|
16
|
+
* Short by design. The language breakdown gates which CodeQL languages get
|
|
17
|
+
* configured, so a stale list silently under-configures code scanning on a repo
|
|
18
|
+
* that just gained a language. Ten minutes covers a multi-repo run without
|
|
19
|
+
* outliving it.
|
|
20
|
+
*/
|
|
21
|
+
const LANGUAGES_TTL = Duration.minutes(10);
|
|
22
|
+
const TeamId = Schema.fromJsonString(Schema.Int);
|
|
23
|
+
const Owner = Schema.fromJsonString(Schema.Literals(["User", "Organization"]));
|
|
24
|
+
const Languages = Schema.fromJsonString(Schema.Array(Schema.String));
|
|
25
|
+
const Private = Schema.fromJsonString(Schema.Boolean);
|
|
26
|
+
const Count = Schema.fromJsonString(Schema.Number);
|
|
27
|
+
/**
|
|
28
|
+
* The three GitHub lookups worth not repeating, held on disk between runs.
|
|
29
|
+
*
|
|
30
|
+
* @remarks
|
|
31
|
+
* A sync run asks the same questions once per repo — what is this owner, what
|
|
32
|
+
* is that team's id, what languages does this repo have — and the answers are
|
|
33
|
+
* either immutable or slow-moving. Caching them on disk turns a group of twenty
|
|
34
|
+
* repos from twenty round trips into one, and turns the second run of the day
|
|
35
|
+
* into none.
|
|
36
|
+
*
|
|
37
|
+
* The read-through, the byte encoding and the decode-drift policy all live in
|
|
38
|
+
* `@effected/store`'s `Cache.through` now. It takes a `Codec<A, string>` and
|
|
39
|
+
* handles the last step to bytes itself, so nothing here touches a
|
|
40
|
+
* `TextEncoder`. A stale entry that no longer decodes is treated as a miss and
|
|
41
|
+
* overwritten — the package's documented contract, not a local choice.
|
|
42
|
+
*
|
|
43
|
+
* `CacheError` stays in the failure channel rather than being swallowed. A
|
|
44
|
+
* cache whose database is unreadable is a real, reportable condition, and a
|
|
45
|
+
* caller that would rather push through can say so with `Effect.catchTag`.
|
|
46
|
+
*
|
|
47
|
+
* TTLs are policy, not configuration: team ids and owner types are held for 30
|
|
48
|
+
* days, repository languages for 10 minutes.
|
|
49
|
+
*
|
|
50
|
+
* @public
|
|
51
|
+
*/
|
|
52
|
+
var RepoCache = class extends Context.Service()("reposets/RepoCache", { make: Effect.gen(function* () {
|
|
53
|
+
const cache = yield* Cache;
|
|
54
|
+
const withCache = (effect) => Effect.provideService(effect, Cache, cache);
|
|
55
|
+
return {
|
|
56
|
+
teamId: (org, slug, onMiss) => withCache(Cache.through(`team:${org}/${slug}`, TeamId, {
|
|
57
|
+
ttl: TEAM_TTL,
|
|
58
|
+
tags: ["team"]
|
|
59
|
+
})(onMiss)),
|
|
60
|
+
ownerType: (owner, onMiss) => withCache(Cache.through(`owner:${owner}`, Owner, {
|
|
61
|
+
ttl: OWNER_TTL,
|
|
62
|
+
tags: ["owner"]
|
|
63
|
+
})(onMiss)),
|
|
64
|
+
repoLanguages: (owner, repo, onMiss) => withCache(Cache.through(`langs:${owner}/${repo}`, Languages, {
|
|
65
|
+
ttl: LANGUAGES_TTL,
|
|
66
|
+
tags: ["langs"]
|
|
67
|
+
})(onMiss)),
|
|
68
|
+
repoWorkflows: (owner, repo, onMiss) => withCache(Cache.through(`workflows:${owner}/${repo}`, Count, {
|
|
69
|
+
ttl: LANGUAGES_TTL,
|
|
70
|
+
tags: ["repo"]
|
|
71
|
+
})(onMiss)),
|
|
72
|
+
repoPrivate: (owner, repo, onMiss) => withCache(Cache.through(`private:${owner}/${repo}`, Private, {
|
|
73
|
+
ttl: LANGUAGES_TTL,
|
|
74
|
+
tags: ["repo"]
|
|
75
|
+
})(Effect.map(onMiss, (repository) => repository.private)))
|
|
76
|
+
};
|
|
77
|
+
}) }) {};
|
|
78
|
+
/**
|
|
79
|
+
* Live cache, over the ambient {@link Cache}.
|
|
80
|
+
*
|
|
81
|
+
* @public
|
|
82
|
+
*/
|
|
83
|
+
const RepoCacheLive = Layer.effect(RepoCache, RepoCache.make);
|
|
84
|
+
|
|
85
|
+
//#endregion
|
|
86
|
+
export { RepoCache, RepoCacheLive };
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { Context, Crypto, DateTime, Effect, Layer } from "effect";
|
|
2
|
+
import { Store } from "@effected/store";
|
|
3
|
+
|
|
4
|
+
//#region src/store/SyncJournal.ts
|
|
5
|
+
/**
|
|
6
|
+
* Append-only record of what each sync run did.
|
|
7
|
+
*
|
|
8
|
+
* @remarks
|
|
9
|
+
* The journal answers "what did that last sync actually do", after the fact and
|
|
10
|
+
* after the process has exited. It is written on every run, including dry runs —
|
|
11
|
+
* a dry run's record is the useful one when someone asks what *would* have
|
|
12
|
+
* happened.
|
|
13
|
+
*
|
|
14
|
+
* @public
|
|
15
|
+
*/
|
|
16
|
+
var SyncJournal = class extends Context.Service()("reposets/SyncJournal", { make: Effect.gen(function* () {
|
|
17
|
+
const store = yield* Store;
|
|
18
|
+
const crypto = yield* Crypto.Crypto;
|
|
19
|
+
const sql = store.client;
|
|
20
|
+
return {
|
|
21
|
+
startRun: (options) => Effect.gen(function* () {
|
|
22
|
+
const id = yield* crypto.randomUUIDv7;
|
|
23
|
+
const startedAt = yield* DateTime.now;
|
|
24
|
+
yield* sql`
|
|
25
|
+
INSERT INTO sync_run (id, started_at, group_name, dry_run)
|
|
26
|
+
VALUES (${id}, ${DateTime.formatIso(startedAt)}, ${options.group ?? null}, ${options.dryRun ? 1 : 0})
|
|
27
|
+
`;
|
|
28
|
+
return id;
|
|
29
|
+
}),
|
|
30
|
+
recordChange: (runId, change) => sql`
|
|
31
|
+
INSERT INTO sync_change (run_id, repo, kind, name, action, detail)
|
|
32
|
+
VALUES (${runId}, ${change.repo}, ${change.kind}, ${change.name}, ${change.action}, ${change.detail ?? null})
|
|
33
|
+
`.pipe(Effect.asVoid),
|
|
34
|
+
finishRun: (runId, outcome, error) => Effect.gen(function* () {
|
|
35
|
+
const finishedAt = yield* DateTime.now;
|
|
36
|
+
yield* sql`
|
|
37
|
+
UPDATE sync_run
|
|
38
|
+
SET finished_at = ${DateTime.formatIso(finishedAt)}, outcome = ${outcome}, error = ${error ?? null}
|
|
39
|
+
WHERE id = ${runId}
|
|
40
|
+
`;
|
|
41
|
+
}).pipe(Effect.asVoid),
|
|
42
|
+
history: (options) => options.repo === void 0 ? sql`
|
|
43
|
+
SELECT r.id, r.started_at AS startedAt, r.finished_at AS finishedAt,
|
|
44
|
+
r.group_name AS "group", r.dry_run AS dryRun, r.outcome, r.error,
|
|
45
|
+
COUNT(c.run_id) AS changes
|
|
46
|
+
FROM sync_run r
|
|
47
|
+
LEFT JOIN sync_change c ON c.run_id = r.id
|
|
48
|
+
GROUP BY r.id
|
|
49
|
+
ORDER BY r.rowid DESC
|
|
50
|
+
LIMIT ${options.limit}
|
|
51
|
+
` : sql`
|
|
52
|
+
SELECT r.id, r.started_at AS startedAt, r.finished_at AS finishedAt,
|
|
53
|
+
r.group_name AS "group", r.dry_run AS dryRun, r.outcome, r.error,
|
|
54
|
+
COUNT(c.run_id) AS changes
|
|
55
|
+
FROM sync_run r
|
|
56
|
+
JOIN sync_change c ON c.run_id = r.id
|
|
57
|
+
WHERE c.repo = ${options.repo}
|
|
58
|
+
GROUP BY r.id
|
|
59
|
+
ORDER BY r.rowid DESC
|
|
60
|
+
LIMIT ${options.limit}
|
|
61
|
+
`,
|
|
62
|
+
changesFor: (runId) => sql`
|
|
63
|
+
SELECT repo, kind, name, action, detail
|
|
64
|
+
FROM sync_change
|
|
65
|
+
WHERE run_id = ${runId}
|
|
66
|
+
ORDER BY rowid
|
|
67
|
+
`,
|
|
68
|
+
prune: (keep) => Effect.gen(function* () {
|
|
69
|
+
const doomed = yield* sql`
|
|
70
|
+
SELECT id FROM sync_run
|
|
71
|
+
WHERE rowid NOT IN (SELECT rowid FROM sync_run ORDER BY rowid DESC LIMIT ${keep})
|
|
72
|
+
`;
|
|
73
|
+
if (doomed.length === 0) return 0;
|
|
74
|
+
yield* sql`DELETE FROM sync_run WHERE id IN ${sql.in(doomed.map((row) => row.id))}`;
|
|
75
|
+
return doomed.length;
|
|
76
|
+
}),
|
|
77
|
+
clear: () => Effect.gen(function* () {
|
|
78
|
+
const counted = yield* sql`SELECT COUNT(*) AS n FROM sync_run`;
|
|
79
|
+
yield* sql`DELETE FROM sync_run`;
|
|
80
|
+
return counted[0]?.n ?? 0;
|
|
81
|
+
})
|
|
82
|
+
};
|
|
83
|
+
}) }) {};
|
|
84
|
+
/**
|
|
85
|
+
* Live journal, over the ambient {@link Store}.
|
|
86
|
+
*
|
|
87
|
+
* @public
|
|
88
|
+
*/
|
|
89
|
+
const SyncJournalLive = Layer.effect(SyncJournal, SyncJournal.make);
|
|
90
|
+
|
|
91
|
+
//#endregion
|
|
92
|
+
export { SyncJournal, SyncJournalLive };
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
|
|
3
|
+
//#region src/store/migrations.ts
|
|
4
|
+
/**
|
|
5
|
+
* The sync journal: one row per run, plus one row per resource the run changed.
|
|
6
|
+
*
|
|
7
|
+
* @remarks
|
|
8
|
+
* Both tables are one migration because they are one unit — rolling back a
|
|
9
|
+
* journal that kept its child table would leave `sync_change` orphaned with a
|
|
10
|
+
* dangling foreign key.
|
|
11
|
+
*
|
|
12
|
+
* `group_name`, not `group` — `GROUP` is a SQL reserved word.
|
|
13
|
+
*
|
|
14
|
+
* `sync_run.id` is an application-generated identifier rather than an
|
|
15
|
+
* autoincrementing integer, so a run's child rows can be written without a round
|
|
16
|
+
* trip to read back the parent's rowid.
|
|
17
|
+
*/
|
|
18
|
+
const journal = {
|
|
19
|
+
id: 1,
|
|
20
|
+
name: "journal",
|
|
21
|
+
up: (sql) => Effect.gen(function* () {
|
|
22
|
+
yield* sql`
|
|
23
|
+
CREATE TABLE sync_run (
|
|
24
|
+
id TEXT PRIMARY KEY,
|
|
25
|
+
started_at TEXT NOT NULL,
|
|
26
|
+
finished_at TEXT,
|
|
27
|
+
group_name TEXT,
|
|
28
|
+
dry_run INTEGER NOT NULL DEFAULT 0,
|
|
29
|
+
outcome TEXT,
|
|
30
|
+
error TEXT
|
|
31
|
+
)
|
|
32
|
+
`;
|
|
33
|
+
yield* sql`
|
|
34
|
+
CREATE TABLE sync_change (
|
|
35
|
+
run_id TEXT NOT NULL REFERENCES sync_run(id) ON DELETE CASCADE,
|
|
36
|
+
repo TEXT NOT NULL,
|
|
37
|
+
kind TEXT NOT NULL,
|
|
38
|
+
name TEXT NOT NULL,
|
|
39
|
+
action TEXT NOT NULL,
|
|
40
|
+
detail TEXT
|
|
41
|
+
)
|
|
42
|
+
`;
|
|
43
|
+
yield* sql`CREATE INDEX sync_change_run ON sync_change(run_id)`;
|
|
44
|
+
}),
|
|
45
|
+
down: (sql) => Effect.gen(function* () {
|
|
46
|
+
yield* sql`DROP TABLE sync_change`;
|
|
47
|
+
yield* sql`DROP TABLE sync_run`;
|
|
48
|
+
})
|
|
49
|
+
};
|
|
50
|
+
/**
|
|
51
|
+
* Last-applied fingerprints, the basis of drift detection.
|
|
52
|
+
*
|
|
53
|
+
* @remarks
|
|
54
|
+
* Keyed `(repo, kind, name)` so drift is tracked per resource per repo — two
|
|
55
|
+
* repos in one group drift independently, which is the whole reason to record it.
|
|
56
|
+
*/
|
|
57
|
+
const drift = {
|
|
58
|
+
id: 2,
|
|
59
|
+
name: "drift",
|
|
60
|
+
up: (sql) => sql`
|
|
61
|
+
CREATE TABLE applied_state (
|
|
62
|
+
repo TEXT NOT NULL,
|
|
63
|
+
kind TEXT NOT NULL,
|
|
64
|
+
name TEXT NOT NULL,
|
|
65
|
+
fingerprint TEXT NOT NULL,
|
|
66
|
+
applied_at TEXT NOT NULL,
|
|
67
|
+
run_id TEXT NOT NULL,
|
|
68
|
+
PRIMARY KEY (repo, kind, name)
|
|
69
|
+
)
|
|
70
|
+
`,
|
|
71
|
+
down: (sql) => sql`DROP TABLE applied_state`
|
|
72
|
+
};
|
|
73
|
+
/**
|
|
74
|
+
* Every migration reposets applies, in ascending `id` order.
|
|
75
|
+
*
|
|
76
|
+
* @remarks
|
|
77
|
+
* **Every migration defines `down`.** `rollback` skips a migration that has none
|
|
78
|
+
* while still removing its ledger row, which would leave the schema change in
|
|
79
|
+
* place and let a later `migrate` re-run its `up` against an existing table.
|
|
80
|
+
*
|
|
81
|
+
* @public
|
|
82
|
+
*/
|
|
83
|
+
const migrations = [journal, drift];
|
|
84
|
+
|
|
85
|
+
//#endregion
|
|
86
|
+
export { migrations };
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import { profileOwner } from "../schemas/credentials.js";
|
|
2
|
+
import { SyncJournal } from "../store/SyncJournal.js";
|
|
3
|
+
import { CredentialResolver } from "../services/CredentialResolver.js";
|
|
4
|
+
import { SyncLogger } from "../services/SyncLogger.js";
|
|
5
|
+
import { RepoCache } from "../store/RepoCache.js";
|
|
6
|
+
import { emptyResult, mergeResults, selectPhases } from "./phase.js";
|
|
7
|
+
import { Context, Effect, Layer } from "effect";
|
|
8
|
+
import { GitHubRepository, Repo, RepoRef } from "@effected/github";
|
|
9
|
+
|
|
10
|
+
//#region src/sync/SyncEngine.ts
|
|
11
|
+
/**
|
|
12
|
+
* Runs the configured phases over every selected repository.
|
|
13
|
+
*
|
|
14
|
+
* @remarks
|
|
15
|
+
* The engine owns sequencing and reporting; it knows nothing about what any
|
|
16
|
+
* phase does. Phases are a list it walks — which is what lets `--only` and
|
|
17
|
+
* `--skip` select a subset without the engine growing a branch per phase.
|
|
18
|
+
*
|
|
19
|
+
* It records changes into a run but does **not** open or close one: the run
|
|
20
|
+
* boundary is the command's, since one `sync` may drive several engines when
|
|
21
|
+
* its groups authenticate as different profiles.
|
|
22
|
+
*
|
|
23
|
+
* **A run never aborts on one repository's failure.** Phases return their errors
|
|
24
|
+
* rather than raising them, so a rejected ruleset on the third repo does not
|
|
25
|
+
* cost the remaining seventeen. Failures are logged inline, accumulated in the
|
|
26
|
+
* journal, and summarised at the end.
|
|
27
|
+
*
|
|
28
|
+
* @public
|
|
29
|
+
*/
|
|
30
|
+
var SyncEngine = class extends Context.Service()("reposets/SyncEngine") {};
|
|
31
|
+
/**
|
|
32
|
+
* Build the engine from a phase list.
|
|
33
|
+
*
|
|
34
|
+
* @remarks
|
|
35
|
+
* The phases are a parameter rather than an import so the engine can be tested
|
|
36
|
+
* against fakes, and so a phase's dependencies stay in the phase rather than
|
|
37
|
+
* accumulating in the engine's `R`.
|
|
38
|
+
*
|
|
39
|
+
* @public
|
|
40
|
+
*/
|
|
41
|
+
function SyncEngineLive(makePhases) {
|
|
42
|
+
return Layer.effect(SyncEngine, Effect.gen(function* () {
|
|
43
|
+
const phases = yield* makePhases;
|
|
44
|
+
const journal = yield* SyncJournal;
|
|
45
|
+
const logger = yield* SyncLogger;
|
|
46
|
+
const resolver = yield* CredentialResolver;
|
|
47
|
+
const cache = yield* RepoCache;
|
|
48
|
+
const repository = yield* GitHubRepository;
|
|
49
|
+
const syncAll = (config, credentials, options) => Effect.gen(function* () {
|
|
50
|
+
const runId = options.runId;
|
|
51
|
+
const selected = selectPhases(phases, {
|
|
52
|
+
...options.only === void 0 ? {} : { only: options.only },
|
|
53
|
+
...options.skip === void 0 ? {} : { skip: options.skip }
|
|
54
|
+
});
|
|
55
|
+
let repos = 0;
|
|
56
|
+
let drifted = 0;
|
|
57
|
+
const results = [];
|
|
58
|
+
for (const [groupName, group] of Object.entries(config.groups)) {
|
|
59
|
+
if (options.group !== void 0 && groupName !== options.group) continue;
|
|
60
|
+
const profile = credentials.profiles[group.credentials];
|
|
61
|
+
const declared = profile === void 0 ? void 0 : profileOwner(profile);
|
|
62
|
+
const owner = declared?.owner ?? "";
|
|
63
|
+
const declaredType = declared?.ownerType ?? "Organization";
|
|
64
|
+
const targeted = options.repo === void 0 ? group.repos : group.repos.filter((repo) => repo === options.repo);
|
|
65
|
+
yield* logger.groupStart(groupName, targeted.length, group.repos.length);
|
|
66
|
+
if (options.repo !== void 0 && targeted.length === 0) continue;
|
|
67
|
+
if (profile === void 0) {
|
|
68
|
+
const known = Object.keys(credentials.profiles);
|
|
69
|
+
const message = `group '${groupName}' names credential profile '${group.credentials}', which does not exist (has: ${known.length === 0 ? "none" : known.join(", ")})`;
|
|
70
|
+
results.push({
|
|
71
|
+
changes: [],
|
|
72
|
+
errors: [{
|
|
73
|
+
context: `group ${groupName}`,
|
|
74
|
+
message
|
|
75
|
+
}]
|
|
76
|
+
});
|
|
77
|
+
yield* logger.syncError(`group ${groupName}`, message);
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
const resolved = yield* resolver.resolveAll(profile, options.configDir).pipe(Effect.orElseSucceed(() => /* @__PURE__ */ new Map()));
|
|
81
|
+
for (const repo of group.repos) {
|
|
82
|
+
if (options.repo !== void 0 && repo !== options.repo) continue;
|
|
83
|
+
repos += 1;
|
|
84
|
+
yield* logger.repoStart(owner, repo);
|
|
85
|
+
const ref = RepoRef.make({
|
|
86
|
+
owner,
|
|
87
|
+
repo
|
|
88
|
+
});
|
|
89
|
+
const observed = yield* cache.ownerType(owner, repository.ownerType.pipe(Repo.provide(ref))).pipe(Effect.orElseSucceed(() => declaredType));
|
|
90
|
+
if (observed !== declaredType) {
|
|
91
|
+
const message = `profile '${group.credentials}' declares ${declaredType === "User" ? `username = "${owner}"` : `org = "${owner}"`}, but GitHub reports ${owner} is ${observed === "User" ? "a user" : "an organization"}`;
|
|
92
|
+
results.push({
|
|
93
|
+
changes: [],
|
|
94
|
+
errors: [{
|
|
95
|
+
context: `${owner}/${repo}`,
|
|
96
|
+
message
|
|
97
|
+
}]
|
|
98
|
+
});
|
|
99
|
+
yield* logger.syncError(`${owner}/${repo}`, message);
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
const ownerType = declaredType;
|
|
103
|
+
const ctx = {
|
|
104
|
+
owner,
|
|
105
|
+
repo,
|
|
106
|
+
slug: `${owner}/${repo}`,
|
|
107
|
+
group: groupName,
|
|
108
|
+
ownerType,
|
|
109
|
+
config,
|
|
110
|
+
credentials: resolved,
|
|
111
|
+
runId,
|
|
112
|
+
dryRun: options.dryRun,
|
|
113
|
+
noCleanup: options.noCleanup,
|
|
114
|
+
configDir: options.configDir
|
|
115
|
+
};
|
|
116
|
+
for (const phase of selected) {
|
|
117
|
+
if (!phase.appliesTo(ctx)) continue;
|
|
118
|
+
const result = yield* phase.run(ctx).pipe(Repo.provide(ref), Effect.orElseSucceed(() => emptyResult));
|
|
119
|
+
results.push(result);
|
|
120
|
+
for (const change of result.changes) {
|
|
121
|
+
if (change.action === "drift-overwritten") drifted += 1;
|
|
122
|
+
yield* journal.recordChange(runId, change).pipe(Effect.ignore);
|
|
123
|
+
}
|
|
124
|
+
for (const error of result.errors) yield* logger.syncError(error.context, error.message);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
if (options.repo !== void 0 && repos === 0) {
|
|
129
|
+
const known = Object.entries(config.groups).filter(([name]) => options.group === void 0 || name === options.group).flatMap(([, group]) => group.repos);
|
|
130
|
+
const message = `no repository named '${options.repo}' in ${options.group === void 0 ? "any configured group" : `group '${options.group}'`} (has: ${known.join(", ") || "none"})`;
|
|
131
|
+
results.push({
|
|
132
|
+
changes: [],
|
|
133
|
+
errors: [{
|
|
134
|
+
context: `--repo ${options.repo}`,
|
|
135
|
+
message
|
|
136
|
+
}]
|
|
137
|
+
});
|
|
138
|
+
yield* logger.syncError(`--repo ${options.repo}`, message);
|
|
139
|
+
}
|
|
140
|
+
const merged = mergeResults(results);
|
|
141
|
+
const errorSummary = merged.errors.length === 0 ? void 0 : merged.errors.length === 1 ? `${merged.errors[0]?.context}: ${merged.errors[0]?.message}` : `${merged.errors.length} errors, first: ${merged.errors[0]?.context}: ${merged.errors[0]?.message}`;
|
|
142
|
+
return {
|
|
143
|
+
runId,
|
|
144
|
+
repos,
|
|
145
|
+
changes: merged.changes.length,
|
|
146
|
+
drifted,
|
|
147
|
+
errors: merged.errors.length,
|
|
148
|
+
errorSummary
|
|
149
|
+
};
|
|
150
|
+
});
|
|
151
|
+
return { syncAll };
|
|
152
|
+
}));
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
//#endregion
|
|
156
|
+
export { SyncEngine, SyncEngineLive };
|