signalk-backup 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/package.json +5 -4
  2. package/plugin/config/schema.d.ts +7 -6
  3. package/plugin/config/schema.d.ts.map +1 -1
  4. package/plugin/config/schema.js +12 -39
  5. package/plugin/config/schema.js.map +1 -1
  6. package/plugin/database-export/iso-week.d.ts +11 -0
  7. package/plugin/database-export/iso-week.d.ts.map +1 -0
  8. package/plugin/database-export/iso-week.js +48 -0
  9. package/plugin/database-export/iso-week.js.map +1 -0
  10. package/plugin/database-export/manifest.d.ts +19 -0
  11. package/plugin/database-export/manifest.d.ts.map +1 -0
  12. package/plugin/database-export/manifest.js +62 -0
  13. package/plugin/database-export/manifest.js.map +1 -0
  14. package/plugin/database-export/questdb.d.ts +25 -13
  15. package/plugin/database-export/questdb.d.ts.map +1 -1
  16. package/plugin/database-export/questdb.js +143 -38
  17. package/plugin/database-export/questdb.js.map +1 -1
  18. package/plugin/database-export/types.d.ts +13 -6
  19. package/plugin/database-export/types.d.ts.map +1 -1
  20. package/plugin/database-export/types.js +6 -3
  21. package/plugin/database-export/types.js.map +1 -1
  22. package/plugin/index.d.ts.map +1 -1
  23. package/plugin/index.js +143 -6
  24. package/plugin/index.js.map +1 -1
  25. package/plugin/proxy.d.ts.map +1 -1
  26. package/plugin/proxy.js +17 -2
  27. package/plugin/proxy.js.map +1 -1
  28. package/plugin/smb-discovery.d.ts +9 -0
  29. package/plugin/smb-discovery.d.ts.map +1 -0
  30. package/plugin/smb-discovery.js +42 -0
  31. package/plugin/smb-discovery.js.map +1 -0
  32. package/plugin/types.d.ts +43 -2
  33. package/plugin/types.d.ts.map +1 -1
  34. package/public/assets/{index-B7P78qlq.js → index-DMT6pdi_.js} +2 -2
  35. package/public/assets/index-DMT6pdi_.js.map +1 -0
  36. package/public/assets/main-BZ1xX0IR.js +11 -0
  37. package/public/assets/main-BZ1xX0IR.js.map +1 -0
  38. package/public/icon.svg +12 -7
  39. package/public/index.html +1 -1
  40. package/public/assets/index-B7P78qlq.js.map +0 -1
  41. package/public/assets/main-DQRPJ7oa.js +0 -11
  42. package/public/assets/main-DQRPJ7oa.js.map +0 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "signalk-backup",
3
- "version": "0.2.0",
3
+ "version": "0.4.0",
4
4
  "description": "Scheduled deduplicated backups of SignalK config to local storage and Google Drive (via Kopia + rclone)",
5
5
  "main": "plugin/index.js",
6
6
  "type": "module",
@@ -28,7 +28,7 @@
28
28
  ],
29
29
  "signalk": {
30
30
  "displayName": "Backup",
31
- "appIcon": "assets/icon.png",
31
+ "appIcon": "icon.svg",
32
32
  "webapp": {
33
33
  "name": "Backup",
34
34
  "description": "Scheduled deduplicated backups to local storage and Google Drive"
@@ -48,10 +48,11 @@
48
48
  "signalk": ">=2.24.0"
49
49
  },
50
50
  "dependencies": {
51
- "@sinclair/typebox": "^0.34.0"
51
+ "@sinclair/typebox": "^0.34.0",
52
+ "bonjour-service": "^1.3.0"
52
53
  },
53
54
  "peerDependencies": {
54
- "signalk-container": ">=0.1.6"
55
+ "signalk-container": ">=1.6.0"
55
56
  },
56
57
  "peerDependenciesMeta": {
57
58
  "signalk-container": {
@@ -1,17 +1,18 @@
1
1
  import { Static } from '@sinclair/typebox';
2
2
  /**
3
- * Schema for the SignalK Admin UI plugin config form.
3
+ * Schema for the SignalK Admin UI plugin config form. Only the
4
+ * per-deployment knobs the SignalK admin needs to set up — backup
5
+ * schedule, retention, cloud sync, exclusions, password, database
6
+ * export are all in the webapp's Settings view at /signalk-backup/.
4
7
  *
5
- * Most user-facing settings (schedule, retention, cloud sync, exclusions)
6
- * live inside the backup-server container's own UI — open it via the
7
- * "Open Backup Console" link from /plugins/signalk-backup/. The fields
8
- * below are only the per-deployment knobs the SignalK admin needs.
8
+ * For LOG_LEVEL: defaults to "info"; power users can override via
9
+ * signalk-container's per-container env override
10
+ * (containerOverrides.signalk-backup-server.env.LOG_LEVEL).
9
11
  */
10
12
  export declare const ConfigSchema: import("@sinclair/typebox").TObject<{
11
13
  managedContainer: import("@sinclair/typebox").TBoolean;
12
14
  imageTag: import("@sinclair/typebox").TString;
13
15
  externalUrl: import("@sinclair/typebox").TString;
14
- logLevel: import("@sinclair/typebox").TUnion<[import("@sinclair/typebox").TLiteral<"trace">, import("@sinclair/typebox").TLiteral<"debug">, import("@sinclair/typebox").TLiteral<"info">, import("@sinclair/typebox").TLiteral<"warn">, import("@sinclair/typebox").TLiteral<"error">, import("@sinclair/typebox").TLiteral<"fatal">]>;
15
16
  databaseExport: import("@sinclair/typebox").TObject<{
16
17
  questdb: import("@sinclair/typebox").TBoolean;
17
18
  intervalMinutes: import("@sinclair/typebox").TNumber;
@@ -1 +1 @@
1
- {"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/config/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAQ,MAAM,EAAE,MAAM,mBAAmB,CAAA;AAEhD;;;;;;;GAOG;AACH,eAAO,MAAM,YAAY;;;;;;;;;EA8DvB,CAAA;AAEF,MAAM,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,YAAY,CAAC,CAAA;AAEhD;;;;;;;;;GASG;AACH,eAAO,MAAM,eAAe,EAAE,MAS7B,CAAA"}
1
+ {"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/config/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAQ,MAAM,EAAE,MAAM,mBAAmB,CAAA;AAEhD;;;;;;;;;GASG;AACH,eAAO,MAAM,YAAY;;;;;;;;EA6BvB,CAAA;AAEF,MAAM,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,YAAY,CAAC,CAAA;AAEhD;;;;;;;;;GASG;AACH,eAAO,MAAM,eAAe,EAAE,MAQ7B,CAAA"}
@@ -1,11 +1,13 @@
1
1
  import { Type } from '@sinclair/typebox';
2
2
  /**
3
- * Schema for the SignalK Admin UI plugin config form.
3
+ * Schema for the SignalK Admin UI plugin config form. Only the
4
+ * per-deployment knobs the SignalK admin needs to set up — backup
5
+ * schedule, retention, cloud sync, exclusions, password, database
6
+ * export are all in the webapp's Settings view at /signalk-backup/.
4
7
  *
5
- * Most user-facing settings (schedule, retention, cloud sync, exclusions)
6
- * live inside the backup-server container's own UI — open it via the
7
- * "Open Backup Console" link from /plugins/signalk-backup/. The fields
8
- * below are only the per-deployment knobs the SignalK admin needs.
8
+ * For LOG_LEVEL: defaults to "info"; power users can override via
9
+ * signalk-container's per-container env override
10
+ * (containerOverrides.signalk-backup-server.env.LOG_LEVEL).
9
11
  */
10
12
  export const ConfigSchema = Type.Object({
11
13
  managedContainer: Type.Boolean({
@@ -25,40 +27,12 @@ export const ConfigSchema = Type.Object({
25
27
  description: 'Used only when managedContainer is disabled. e.g. http://192.168.1.50:3010. ' +
26
28
  'Leave blank when managing the container.'
27
29
  }),
28
- logLevel: Type.Union([
29
- Type.Literal('trace'),
30
- Type.Literal('debug'),
31
- Type.Literal('info'),
32
- Type.Literal('warn'),
33
- Type.Literal('error'),
34
- Type.Literal('fatal')
35
- ], {
36
- default: 'info',
37
- title: 'Log level',
38
- description: 'Forwarded to the backup-server container as LOG_LEVEL.'
39
- }),
30
+ // Must exist in the schema so savePluginOptions round-trips it; the
31
+ // form hides it via uiSchema in src/index.ts (webapp owns the UI).
40
32
  databaseExport: Type.Object({
41
- questdb: Type.Boolean({
42
- default: false,
43
- title: 'Export QuestDB to backup',
44
- description: 'When enabled, the plugin periodically writes QuestDB tables to Parquet files ' +
45
- 'inside the backup data dir. The next snapshot then captures them as part of ' +
46
- 'the regular backup. Filesystem-level QuestDB files are still excluded — only ' +
47
- 'the safe COPY-out exports travel.'
48
- }),
49
- intervalMinutes: Type.Number({
50
- default: 60,
51
- minimum: 5,
52
- maximum: 1440,
53
- title: 'Export interval (minutes)',
54
- description: 'How often the plugin runs database exports. The freshness of DB data inside ' +
55
- 'a backup is bounded by max(this interval, the backup-server snapshot interval). ' +
56
- 'Default 60.'
57
- })
58
- }, {
59
- default: { questdb: false, intervalMinutes: 60 },
60
- title: 'Database export'
61
- })
33
+ questdb: Type.Boolean({ default: false }),
34
+ intervalMinutes: Type.Number({ default: 60, minimum: 5, maximum: 1440 })
35
+ }, { default: { questdb: false, intervalMinutes: 60 } })
62
36
  });
63
37
  /**
64
38
  * Materialised defaults — Signal K only uses the schema's `default` fields
@@ -74,7 +48,6 @@ export const SCHEMA_DEFAULTS = {
74
48
  managedContainer: true,
75
49
  imageTag: 'latest',
76
50
  externalUrl: '',
77
- logLevel: 'info',
78
51
  databaseExport: {
79
52
  questdb: false,
80
53
  intervalMinutes: 60
@@ -1 +1 @@
1
- {"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/config/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAU,MAAM,mBAAmB,CAAA;AAEhD;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC;IACtC,gBAAgB,EAAE,IAAI,CAAC,OAAO,CAAC;QAC7B,OAAO,EAAE,IAAI;QACb,KAAK,EAAE,+CAA+C;QACtD,WAAW,EACT,0FAA0F;YAC1F,4EAA4E;KAC/E,CAAC;IACF,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;QACpB,OAAO,EAAE,QAAQ;QACjB,KAAK,EAAE,qBAAqB;QAC5B,WAAW,EAAE,iFAAiF;KAC/F,CAAC;IACF,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC;QACvB,OAAO,EAAE,EAAE;QACX,KAAK,EAAE,4BAA4B;QACnC,WAAW,EACT,8EAA8E;YAC9E,0CAA0C;KAC7C,CAAC;IACF,QAAQ,EAAE,IAAI,CAAC,KAAK,CAClB;QACE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;QACrB,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;QACrB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QACpB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QACpB,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;QACrB,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;KACtB,EACD;QACE,OAAO,EAAE,MAAM;QACf,KAAK,EAAE,WAAW;QAClB,WAAW,EAAE,wDAAwD;KACtE,CACF;IACD,cAAc,EAAE,IAAI,CAAC,MAAM,CACzB;QACE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC;YACpB,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,0BAA0B;YACjC,WAAW,EACT,+EAA+E;gBAC/E,8EAA8E;gBAC9E,+EAA+E;gBAC/E,mCAAmC;SACtC,CAAC;QACF,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC;YAC3B,OAAO,EAAE,EAAE;YACX,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,2BAA2B;YAClC,WAAW,EACT,8EAA8E;gBAC9E,kFAAkF;gBAClF,aAAa;SAChB,CAAC;KACH,EACD;QACE,OAAO,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,eAAe,EAAE,EAAE,EAAE;QAChD,KAAK,EAAE,iBAAiB;KACzB,CACF;CACF,CAAC,CAAA;AAIF;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,eAAe,GAAW;IACrC,gBAAgB,EAAE,IAAI;IACtB,QAAQ,EAAE,QAAQ;IAClB,WAAW,EAAE,EAAE;IACf,QAAQ,EAAE,MAAM;IAChB,cAAc,EAAE;QACd,OAAO,EAAE,KAAK;QACd,eAAe,EAAE,EAAE;KACpB;CACF,CAAA"}
1
+ {"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/config/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAU,MAAM,mBAAmB,CAAA;AAEhD;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC;IACtC,gBAAgB,EAAE,IAAI,CAAC,OAAO,CAAC;QAC7B,OAAO,EAAE,IAAI;QACb,KAAK,EAAE,+CAA+C;QACtD,WAAW,EACT,0FAA0F;YAC1F,4EAA4E;KAC/E,CAAC;IACF,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;QACpB,OAAO,EAAE,QAAQ;QACjB,KAAK,EAAE,qBAAqB;QAC5B,WAAW,EAAE,iFAAiF;KAC/F,CAAC;IACF,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC;QACvB,OAAO,EAAE,EAAE;QACX,KAAK,EAAE,4BAA4B;QACnC,WAAW,EACT,8EAA8E;YAC9E,0CAA0C;KAC7C,CAAC;IACF,oEAAoE;IACpE,mEAAmE;IACnE,cAAc,EAAE,IAAI,CAAC,MAAM,CACzB;QACE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QACzC,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;KACzE,EACD,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,eAAe,EAAE,EAAE,EAAE,EAAE,CACrD;CACF,CAAC,CAAA;AAIF;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,eAAe,GAAW;IACrC,gBAAgB,EAAE,IAAI;IACtB,QAAQ,EAAE,QAAQ;IAClB,WAAW,EAAE,EAAE;IACf,cAAc,EAAE;QACd,OAAO,EAAE,KAAK;QACd,eAAe,EAAE,EAAE;KACpB;CACF,CAAA"}
@@ -0,0 +1,11 @@
1
+ export interface IsoWeek {
2
+ year: number;
3
+ week: number;
4
+ }
5
+ export declare function weekStartUtc(iw: IsoWeek): Date;
6
+ export declare function weekEndUtc(iw: IsoWeek): Date;
7
+ export declare function isoWeekOf(date: Date): IsoWeek;
8
+ export declare function compareIsoWeek(a: IsoWeek, b: IsoWeek): number;
9
+ export declare function weeksBetween(from: IsoWeek, to: IsoWeek): IsoWeek[];
10
+ export declare function formatIsoWeek(iw: IsoWeek): string;
11
+ //# sourceMappingURL=iso-week.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"iso-week.d.ts","sourceRoot":"","sources":["../../src/database-export/iso-week.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;CACb;AAED,wBAAgB,YAAY,CAAC,EAAE,EAAE,OAAO,GAAG,IAAI,CAS9C;AAED,wBAAgB,UAAU,CAAC,EAAE,EAAE,OAAO,GAAG,IAAI,CAK5C;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAS7C;AAED,wBAAgB,cAAc,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,GAAG,MAAM,CAG7D;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,GAAG,OAAO,EAAE,CAUlE;AAED,wBAAgB,aAAa,CAAC,EAAE,EAAE,OAAO,GAAG,MAAM,CAEjD"}
@@ -0,0 +1,48 @@
1
+ // UTC ISO-week math — shard filenames depend on this matching exactly across processes.
2
+ export function weekStartUtc(iw) {
3
+ // Jan 4 is always in ISO week 1 — anchor backwards from the Monday of that week.
4
+ const jan4 = new Date(Date.UTC(iw.year, 0, 4));
5
+ const mondayOffset = (jan4.getUTCDay() + 6) % 7;
6
+ const week1Monday = new Date(jan4);
7
+ week1Monday.setUTCDate(jan4.getUTCDate() - mondayOffset);
8
+ const start = new Date(week1Monday);
9
+ start.setUTCDate(week1Monday.getUTCDate() + (iw.week - 1) * 7);
10
+ return start;
11
+ }
12
+ export function weekEndUtc(iw) {
13
+ const start = weekStartUtc(iw);
14
+ const end = new Date(start);
15
+ end.setUTCDate(start.getUTCDate() + 7);
16
+ return end;
17
+ }
18
+ export function isoWeekOf(date) {
19
+ const d = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
20
+ // ISO weeks belong to the year of their Thursday — shift to the Thursday of this week first.
21
+ const isoDow = d.getUTCDay() || 7;
22
+ d.setUTCDate(d.getUTCDate() + (4 - isoDow));
23
+ const year = d.getUTCFullYear();
24
+ const yearStart = new Date(Date.UTC(year, 0, 1));
25
+ const week = Math.ceil(((d.getTime() - yearStart.getTime()) / 86_400_000 + 1) / 7);
26
+ return { year, week };
27
+ }
28
+ export function compareIsoWeek(a, b) {
29
+ if (a.year !== b.year)
30
+ return a.year - b.year;
31
+ return a.week - b.week;
32
+ }
33
+ export function weeksBetween(from, to) {
34
+ if (compareIsoWeek(from, to) > 0)
35
+ return [];
36
+ const out = [];
37
+ let cursor = weekStartUtc(from);
38
+ const stop = weekStartUtc(to);
39
+ while (cursor.getTime() <= stop.getTime()) {
40
+ out.push(isoWeekOf(cursor));
41
+ cursor = new Date(cursor.getTime() + 7 * 86_400_000);
42
+ }
43
+ return out;
44
+ }
45
+ export function formatIsoWeek(iw) {
46
+ return `${iw.year}-W${String(iw.week).padStart(2, '0')}`;
47
+ }
48
+ //# sourceMappingURL=iso-week.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"iso-week.js","sourceRoot":"","sources":["../../src/database-export/iso-week.ts"],"names":[],"mappings":"AAAA,wFAAwF;AAOxF,MAAM,UAAU,YAAY,CAAC,EAAW;IACtC,iFAAiF;IACjF,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAC9C,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAA;IAC/C,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,CAAA;IAClC,WAAW,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,YAAY,CAAC,CAAA;IACxD,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,WAAW,CAAC,CAAA;IACnC,KAAK,CAAC,UAAU,CAAC,WAAW,CAAC,UAAU,EAAE,GAAG,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IAC9D,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,EAAW;IACpC,MAAM,KAAK,GAAG,YAAY,CAAC,EAAE,CAAC,CAAA;IAC9B,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,CAAA;IAC3B,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAA;IACtC,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAU;IAClC,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,IAAI,CAAC,WAAW,EAAE,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAA;IAC1F,6FAA6F;IAC7F,MAAM,MAAM,GAAG,CAAC,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IACjC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAA;IAC3C,MAAM,IAAI,GAAG,CAAC,CAAC,cAAc,EAAE,CAAA;IAC/B,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAChD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC,GAAG,UAAU,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IAClF,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;AACvB,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,CAAU,EAAE,CAAU;IACnD,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;IAC7C,OAAO,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;AACxB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAa,EAAE,EAAW;IACrD,IAAI,cAAc,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC;QAAE,OAAO,EAAE,CAAA;IAC3C,MAAM,GAAG,GAAc,EAAE,CAAA;IACzB,IAAI,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;IAC/B,MAAM,IAAI,GAAG,YAAY,CAAC,EAAE,CAAC,CAAA;IAC7B,OAAO,MAAM,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QAC1C,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;QAC3B,MAAM,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,UAAU,CAAC,CAAA;IACtD,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,EAAW;IACvC,OAAO,GAAG,EAAE,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAA;AAC1D,CAAC"}
@@ -0,0 +1,19 @@
1
+ export declare const MANIFEST_FILENAME = "MANIFEST.json";
2
+ export declare const MANIFEST_SCHEMA_VERSION = 1;
3
+ export interface ShardEntry {
4
+ file: string;
5
+ /** ISO 8601 UTC midnight Monday — start of this shard's ISO week. */
6
+ weekStart: string;
7
+ bytes: number;
8
+ exportedAt: string;
9
+ }
10
+ export interface Manifest {
11
+ tableName: string;
12
+ schemaVersion: number;
13
+ shards: ShardEntry[];
14
+ openShard?: string;
15
+ lastExportRun: string;
16
+ }
17
+ export declare function readManifest(tableDir: string): Promise<Manifest | null>;
18
+ export declare function writeManifest(tableDir: string, manifest: Manifest): Promise<void>;
19
+ //# sourceMappingURL=manifest.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../src/database-export/manifest.ts"],"names":[],"mappings":"AAKA,eAAO,MAAM,iBAAiB,kBAAkB,CAAA;AAChD,eAAO,MAAM,uBAAuB,IAAI,CAAA;AAExC,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,qEAAqE;IACrE,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,MAAM,CAAA;IACb,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,MAAM,CAAA;IACjB,aAAa,EAAE,MAAM,CAAA;IACrB,MAAM,EAAE,UAAU,EAAE,CAAA;IACpB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,aAAa,EAAE,MAAM,CAAA;CACtB;AAED,wBAAsB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,CAW7E;AAGD,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAKvF"}
@@ -0,0 +1,62 @@
1
+ // MANIFEST.json read/write for the per-table partitioned export dir.
2
+ import { readFile, writeFile, rename } from 'node:fs/promises';
3
+ import { join } from 'node:path';
4
+ export const MANIFEST_FILENAME = 'MANIFEST.json';
5
+ export const MANIFEST_SCHEMA_VERSION = 1;
6
+ export async function readManifest(tableDir) {
7
+ try {
8
+ const raw = await readFile(join(tableDir, MANIFEST_FILENAME), 'utf-8');
9
+ const parsed = JSON.parse(raw);
10
+ if (!isManifest(parsed))
11
+ return null;
12
+ return parsed;
13
+ }
14
+ catch {
15
+ // ENOENT (first run) or any read/parse error — treat as missing so the
16
+ // exporter rebuilds from scratch idempotently.
17
+ return null;
18
+ }
19
+ }
20
+ // Temp + rename so a half-written manifest can never confuse subsequent runs.
21
+ export async function writeManifest(tableDir, manifest) {
22
+ const finalPath = join(tableDir, MANIFEST_FILENAME);
23
+ const tempPath = `${finalPath}.partial`;
24
+ await writeFile(tempPath, JSON.stringify(manifest, null, 2), 'utf-8');
25
+ await rename(tempPath, finalPath);
26
+ }
27
+ function isValidIsoString(v) {
28
+ return typeof v === 'string' && !Number.isNaN(Date.parse(v));
29
+ }
30
+ function isManifest(v) {
31
+ if (typeof v !== 'object' || v === null)
32
+ return false;
33
+ const r = v;
34
+ if (typeof r.tableName !== 'string')
35
+ return false;
36
+ // Pin the schema version — accept only what this build wrote. A future
37
+ // build with a bumped version should be treated as 'unreadable manifest'
38
+ // and trigger a clean rebuild.
39
+ if (r.schemaVersion !== MANIFEST_SCHEMA_VERSION)
40
+ return false;
41
+ if (!Array.isArray(r.shards))
42
+ return false;
43
+ for (const s of r.shards) {
44
+ if (typeof s !== 'object' || s === null)
45
+ return false;
46
+ const sr = s;
47
+ if (typeof sr.file !== 'string' || sr.file.length === 0)
48
+ return false;
49
+ if (!isValidIsoString(sr.weekStart))
50
+ return false;
51
+ if (typeof sr.bytes !== 'number' || !Number.isFinite(sr.bytes) || sr.bytes < 0)
52
+ return false;
53
+ if (!isValidIsoString(sr.exportedAt))
54
+ return false;
55
+ }
56
+ if (r.openShard !== undefined && typeof r.openShard !== 'string')
57
+ return false;
58
+ if (!isValidIsoString(r.lastExportRun))
59
+ return false;
60
+ return true;
61
+ }
62
+ //# sourceMappingURL=manifest.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../src/database-export/manifest.ts"],"names":[],"mappings":"AAAA,qEAAqE;AAErE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AAC9D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,MAAM,CAAC,MAAM,iBAAiB,GAAG,eAAe,CAAA;AAChD,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAA;AAkBxC,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAgB;IACjD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,iBAAiB,CAAC,EAAE,OAAO,CAAC,CAAA;QACtE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAY,CAAA;QACzC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,OAAO,IAAI,CAAA;QACpC,OAAO,MAAM,CAAA;IACf,CAAC;IAAC,MAAM,CAAC;QACP,uEAAuE;QACvE,+CAA+C;QAC/C,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,QAAgB,EAAE,QAAkB;IACtE,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAA;IACnD,MAAM,QAAQ,GAAG,GAAG,SAAS,UAAU,CAAA;IACvC,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;IACrE,MAAM,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAA;AACnC,CAAC;AAED,SAAS,gBAAgB,CAAC,CAAU;IAClC,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;AAC9D,CAAC;AAED,SAAS,UAAU,CAAC,CAAU;IAC5B,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,KAAK,CAAA;IACrD,MAAM,CAAC,GAAG,CAA4B,CAAA;IACtC,IAAI,OAAO,CAAC,CAAC,SAAS,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAA;IACjD,uEAAuE;IACvE,yEAAyE;IACzE,+BAA+B;IAC/B,IAAI,CAAC,CAAC,aAAa,KAAK,uBAAuB;QAAE,OAAO,KAAK,CAAA;IAC7D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAA;IAC1C,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;QACzB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI;YAAE,OAAO,KAAK,CAAA;QACrD,MAAM,EAAE,GAAG,CAA4B,CAAA;QACvC,IAAI,OAAO,EAAE,CAAC,IAAI,KAAK,QAAQ,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,KAAK,CAAA;QACrE,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,SAAS,CAAC;YAAE,OAAO,KAAK,CAAA;QACjD,IAAI,OAAO,EAAE,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,KAAK,GAAG,CAAC;YAAE,OAAO,KAAK,CAAA;QAC5F,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,UAAU,CAAC;YAAE,OAAO,KAAK,CAAA;IACpD,CAAC;IACD,IAAI,CAAC,CAAC,SAAS,KAAK,SAAS,IAAI,OAAO,CAAC,CAAC,SAAS,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAA;IAC9E,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,aAAa,CAAC;QAAE,OAAO,KAAK,CAAA;IACpD,OAAO,IAAI,CAAA;AACb,CAAC"}
@@ -3,39 +3,51 @@
3
3
  *
4
4
  * The signalk-questdb plugin exposes:
5
5
  * GET /plugins/signalk-questdb/api/full-export/tables → { tables: [...] }
6
- * GET /plugins/signalk-questdb/api/full-export/<table> → parquet stream
6
+ * GET /plugins/signalk-questdb/api/full-export/<table>?from=&to= → parquet stream
7
7
  *
8
8
  * Both are served by SignalK in-process, so we reach them via plain HTTP
9
- * over loopback. No container exec, no shared filesystem ownership
10
- * issues, no copy-completion polling.
9
+ * over loopback. No container exec, no shared filesystem ownership issues.
11
10
  *
12
- * Streams the response body directly to a temp file in the staging dir,
13
- * then atomically renames into place — so a snapshot mid-export never
14
- * sees a half-written parquet.
11
+ * For each table we partition the export by ISO week into one parquet
12
+ * file per week. Closed weeks become byte-identical across export cycles
13
+ * and dedup perfectly in kopia. The current ("open") week plus a small
14
+ * rolling churn window get re-exported each cycle to absorb late arrivals.
15
+ *
16
+ * Streams the response body directly to a temp file in the shard's
17
+ * directory, then atomically renames into place — so a snapshot
18
+ * mid-export never sees a half-written parquet.
15
19
  */
16
20
  import type { DatabaseExporter, ExportResult } from './types.js';
17
21
  export interface QuestDBExporterOptions {
18
- /** SignalK server base URL — typically http://127.0.0.1:3000 */
19
22
  signalkBaseUrl?: string;
20
- /** Optional debug logger. */
21
23
  log?: (msg: string) => void;
22
24
  /** Override fetch (tests). */
23
25
  fetch?: typeof fetch;
26
+ /** Override "now" (tests). */
27
+ now?: () => Date;
24
28
  }
25
29
  export declare class QuestDBExporter implements DatabaseExporter {
26
30
  readonly pluginId = "signalk-questdb";
27
31
  private readonly baseUrl;
28
32
  private readonly fetchImpl;
33
+ private readonly nowImpl;
29
34
  constructor(opts?: QuestDBExporterOptions);
30
35
  private readonly log;
31
- /**
32
- * Detect: hit the tables endpoint. Returns true on HTTP 200 with at
33
- * least one table; false on any error or 503/404 (plugin disabled or
34
- * not loaded).
35
- */
36
36
  detect(): Promise<boolean>;
37
37
  exportAll(stagingDir: string): Promise<ExportResult>;
38
38
  private listTables;
39
39
  private exportTable;
40
+ /**
41
+ * If the staging dir has a `<table>.parquet` directly (pre-partitioning
42
+ * layout), move it aside to `<table>.parquet.legacy`. Idempotent.
43
+ */
44
+ private migrateLegacyFlatFile;
45
+ /**
46
+ * Earliest week recorded in a manifest, or null if no shards exist.
47
+ * Pure helper, no I/O — manifest is already read by the caller.
48
+ */
49
+ private earliestWeekFromManifest;
50
+ private churnCutoffWeek;
51
+ private exportWeekToShard;
40
52
  }
41
53
  //# sourceMappingURL=questdb.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"questdb.d.ts","sourceRoot":"","sources":["../../src/database-export/questdb.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAOH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAe,MAAM,YAAY,CAAA;AAW7E,MAAM,WAAW,sBAAsB;IACrC,gEAAgE;IAChE,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,6BAA6B;IAC7B,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAA;IAC3B,8BAA8B;IAC9B,KAAK,CAAC,EAAE,OAAO,KAAK,CAAA;CACrB;AAED,qBAAa,eAAgB,YAAW,gBAAgB;IACtD,QAAQ,CAAC,QAAQ,qBAAoB;IAErC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAQ;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAc;gBAE5B,IAAI,GAAE,sBAA2B;IAM7C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAuB;IAE3C;;;;OAIG;IACG,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;IAU1B,SAAS,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;YAiC5C,UAAU;YAmBV,WAAW;CAuC1B"}
1
+ {"version":3,"file":"questdb.d.ts","sourceRoot":"","sources":["../../src/database-export/questdb.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAOH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAe,MAAM,YAAY,CAAA;AA0B7E,MAAM,WAAW,sBAAsB;IACrC,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAA;IAC3B,8BAA8B;IAC9B,KAAK,CAAC,EAAE,OAAO,KAAK,CAAA;IACpB,8BAA8B;IAC9B,GAAG,CAAC,EAAE,MAAM,IAAI,CAAA;CACjB;AAED,qBAAa,eAAgB,YAAW,gBAAgB;IACtD,QAAQ,CAAC,QAAQ,qBAAoB;IAErC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAQ;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAc;IACxC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAY;gBAExB,IAAI,GAAE,sBAA2B;IAO7C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAuB;IAErC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;IAU1B,SAAS,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;YA8B5C,UAAU;YAiBV,WAAW;IAmFzB;;;OAGG;YACW,qBAAqB;IAgBnC;;;OAGG;IACH,OAAO,CAAC,wBAAwB;IAQhC,OAAO,CAAC,eAAe;YAOT,iBAAiB;CA8ChC"}
@@ -3,44 +3,45 @@
3
3
  *
4
4
  * The signalk-questdb plugin exposes:
5
5
  * GET /plugins/signalk-questdb/api/full-export/tables → { tables: [...] }
6
- * GET /plugins/signalk-questdb/api/full-export/<table> → parquet stream
6
+ * GET /plugins/signalk-questdb/api/full-export/<table>?from=&to= → parquet stream
7
7
  *
8
8
  * Both are served by SignalK in-process, so we reach them via plain HTTP
9
- * over loopback. No container exec, no shared filesystem ownership
10
- * issues, no copy-completion polling.
9
+ * over loopback. No container exec, no shared filesystem ownership issues.
11
10
  *
12
- * Streams the response body directly to a temp file in the staging dir,
13
- * then atomically renames into place — so a snapshot mid-export never
14
- * sees a half-written parquet.
11
+ * For each table we partition the export by ISO week into one parquet
12
+ * file per week. Closed weeks become byte-identical across export cycles
13
+ * and dedup perfectly in kopia. The current ("open") week plus a small
14
+ * rolling churn window get re-exported each cycle to absorb late arrivals.
15
+ *
16
+ * Streams the response body directly to a temp file in the shard's
17
+ * directory, then atomically renames into place — so a snapshot
18
+ * mid-export never sees a half-written parquet.
15
19
  */
16
20
  import { mkdir, rename, unlink } from 'node:fs/promises';
17
- import { createWriteStream } from 'node:fs';
21
+ import { createWriteStream, existsSync } from 'node:fs';
18
22
  import { pipeline } from 'node:stream/promises';
19
23
  import { Readable } from 'node:stream';
20
24
  import { join } from 'node:path';
25
+ import { weekStartUtc, weekEndUtc, isoWeekOf, formatIsoWeek, weeksBetween, compareIsoWeek } from './iso-week.js';
26
+ import { readManifest, writeManifest, MANIFEST_SCHEMA_VERSION } from './manifest.js';
21
27
  const QUESTDB_PLUGIN_ID = 'signalk-questdb';
22
- /** Default base URL — overridable for tests. SignalK normally listens here. */
23
28
  const DEFAULT_SIGNALK_BASE = 'http://127.0.0.1:3000';
24
- /** Per-request timeout. A full-table export of ~500k rows runs in <1s on
25
- * a Pi over the loopback HTTP path, but pipe between pi-host-network +
26
- * pasta containers can stall — bound at 10 minutes. */
27
29
  const FETCH_TIMEOUT_MS = 600_000;
28
30
  const SAFE_TABLE_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
31
+ /** Closed weeks within this many cycles still get re-exported (late-arrival absorption). */
32
+ const ROLLING_CHURN_WEEKS = 4;
29
33
  export class QuestDBExporter {
30
34
  pluginId = QUESTDB_PLUGIN_ID;
31
35
  baseUrl;
32
36
  fetchImpl;
37
+ nowImpl;
33
38
  constructor(opts = {}) {
34
39
  this.baseUrl = (opts.signalkBaseUrl ?? DEFAULT_SIGNALK_BASE).replace(/\/$/, '');
35
40
  this.fetchImpl = opts.fetch ?? fetch;
41
+ this.nowImpl = opts.now ?? (() => new Date());
36
42
  this.log = (msg) => opts.log?.(`[questdb-export] ${msg}`);
37
43
  }
38
44
  log;
39
- /**
40
- * Detect: hit the tables endpoint. Returns true on HTTP 200 with at
41
- * least one table; false on any error or 503/404 (plugin disabled or
42
- * not loaded).
43
- */
44
45
  async detect() {
45
46
  try {
46
47
  const tables = await this.listTables();
@@ -57,8 +58,6 @@ export class QuestDBExporter {
57
58
  const tables = await this.listTables();
58
59
  const exports = [];
59
60
  for (const table of tables) {
60
- // Defence in depth — the route should reject these too, but a
61
- // malformed name in the request URL is worth catching here.
62
61
  if (!SAFE_TABLE_NAME.test(table)) {
63
62
  this.log(`refusing unsafe table identifier: ${table}`);
64
63
  continue;
@@ -67,7 +66,6 @@ export class QuestDBExporter {
67
66
  exports.push(await this.exportTable(table, stagingDir));
68
67
  }
69
68
  catch (err) {
70
- // Partial coverage > none. Log and keep going.
71
69
  this.log(`export failed for ${table}: ${errMsg(err)}`);
72
70
  }
73
71
  }
@@ -83,9 +81,7 @@ export class QuestDBExporter {
83
81
  // ---------------------------------------------------------------------
84
82
  async listTables() {
85
83
  const url = `${this.baseUrl}/plugins/${this.pluginId}/api/full-export/tables`;
86
- const res = await this.fetchImpl(url, {
87
- signal: AbortSignal.timeout(10_000)
88
- });
84
+ const res = await this.fetchImpl(url, { signal: AbortSignal.timeout(10_000) });
89
85
  if (!res.ok) {
90
86
  throw new Error(`tables HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`);
91
87
  }
@@ -101,23 +97,129 @@ export class QuestDBExporter {
101
97
  return out;
102
98
  }
103
99
  async exportTable(table, stagingDir) {
104
- const url = `${this.baseUrl}/plugins/${this.pluginId}/api/full-export/${table}`;
105
- const finalPath = join(stagingDir, `${table}.parquet`);
100
+ const tableDir = join(stagingDir, table);
101
+ await mkdir(tableDir, { recursive: true });
102
+ // One-shot migration of any pre-partitioning flat file. Move-aside,
103
+ // never delete — user may want to re-import it.
104
+ await this.migrateLegacyFlatFile(stagingDir, table);
105
+ const existing = await readManifest(tableDir);
106
+ const nowIso = this.nowImpl();
107
+ const currentWeek = isoWeekOf(nowIso);
108
+ // Range to materialise: from whatever the manifest already covers,
109
+ // back through (at minimum) the rolling churn window, up to the
110
+ // current week. If no manifest exists yet, start from the churn
111
+ // cutoff — that catches data from the last ROLLING_CHURN_WEEKS
112
+ // weeks on first run. Deeper history pre-dates this code path and
113
+ // is not auto-recovered (would need an admin-triggered backfill,
114
+ // out of scope here).
115
+ const churnCutoff = this.churnCutoffWeek(currentWeek);
116
+ const earliestKnown = this.earliestWeekFromManifest(existing);
117
+ const earliestWeek = earliestKnown !== null && compareIsoWeek(earliestKnown, churnCutoff) < 0
118
+ ? earliestKnown
119
+ : churnCutoff;
120
+ const allWeeks = weeksBetween(earliestWeek, currentWeek);
121
+ const existingShards = new Map();
122
+ for (const s of existing?.shards ?? [])
123
+ existingShards.set(s.file, s);
124
+ const newShards = [];
125
+ let shardsWritten = 0;
126
+ let shardsSkipped = 0;
127
+ let bytesWritten = 0;
128
+ for (const week of allWeeks) {
129
+ const file = `${table}_${formatIsoWeek(week)}.parquet`;
130
+ const isOpen = compareIsoWeek(week, currentWeek) === 0;
131
+ const inChurnWindow = compareIsoWeek(week, churnCutoff) >= 0;
132
+ const known = existingShards.get(file);
133
+ const onDisk = existsSync(join(tableDir, file));
134
+ const mustWrite = isOpen || inChurnWindow || !known || !onDisk;
135
+ if (mustWrite) {
136
+ const { bytes } = await this.exportWeekToShard(table, week, tableDir, file);
137
+ const entry = {
138
+ file,
139
+ weekStart: weekStartUtc(week).toISOString(),
140
+ bytes,
141
+ exportedAt: nowIso.toISOString()
142
+ };
143
+ newShards.push(entry);
144
+ shardsWritten++;
145
+ bytesWritten += bytes;
146
+ }
147
+ else {
148
+ // Closed shard, outside churn window, already present. Reuse the manifest entry.
149
+ newShards.push(known);
150
+ shardsSkipped++;
151
+ }
152
+ }
153
+ const openFile = `${table}_${formatIsoWeek(currentWeek)}.parquet`;
154
+ const manifest = {
155
+ tableName: table,
156
+ schemaVersion: MANIFEST_SCHEMA_VERSION,
157
+ shards: newShards,
158
+ openShard: openFile,
159
+ lastExportRun: nowIso.toISOString()
160
+ };
161
+ await writeManifest(tableDir, manifest);
162
+ this.log(`${table}: ${shardsWritten} shards written, ${shardsSkipped} unchanged, ${bytesWritten} bytes`);
163
+ return {
164
+ table,
165
+ tableDir,
166
+ shardsWritten,
167
+ shardsSkipped,
168
+ bytes: bytesWritten
169
+ };
170
+ }
171
+ /**
172
+ * If the staging dir has a `<table>.parquet` directly (pre-partitioning
173
+ * layout), move it aside to `<table>.parquet.legacy`. Idempotent.
174
+ */
175
+ async migrateLegacyFlatFile(stagingDir, table) {
176
+ const flat = join(stagingDir, `${table}.parquet`);
177
+ const legacy = `${flat}.legacy`;
178
+ if (existsSync(flat) && !existsSync(legacy)) {
179
+ try {
180
+ await rename(flat, legacy);
181
+ this.log(`migrated legacy ${table}.parquet → ${table}.parquet.legacy; partitioned export starting fresh`);
182
+ }
183
+ catch (err) {
184
+ // Non-fatal: leave the flat file in place. Next cycle retries.
185
+ this.log(`legacy-file migration failed for ${table}: ${errMsg(err)}`);
186
+ }
187
+ }
188
+ }
189
+ /**
190
+ * Earliest week recorded in a manifest, or null if no shards exist.
191
+ * Pure helper, no I/O — manifest is already read by the caller.
192
+ */
193
+ earliestWeekFromManifest(manifest) {
194
+ if (!manifest || manifest.shards.length === 0)
195
+ return null;
196
+ const oldest = manifest.shards
197
+ .map((s) => new Date(s.weekStart))
198
+ .reduce((a, b) => (a.getTime() < b.getTime() ? a : b));
199
+ return isoWeekOf(oldest);
200
+ }
201
+ churnCutoffWeek(current) {
202
+ // Walk back ROLLING_CHURN_WEEKS Mondays from the current week's Monday.
203
+ const start = weekStartUtc(current);
204
+ const earlier = new Date(start.getTime() - ROLLING_CHURN_WEEKS * 7 * 86_400_000);
205
+ return isoWeekOf(earlier);
206
+ }
207
+ async exportWeekToShard(table, week, tableDir, filename) {
208
+ const from = weekStartUtc(week).toISOString();
209
+ const to = weekEndUtc(week).toISOString();
210
+ const url = new URL(`${this.baseUrl}/plugins/${this.pluginId}/api/full-export/${table}`);
211
+ url.searchParams.set('from', from);
212
+ url.searchParams.set('to', to);
213
+ const finalPath = join(tableDir, filename);
106
214
  const tempPath = `${finalPath}.partial`;
107
- this.log(`exporting ${table}`);
108
- const res = await this.fetchImpl(url, {
215
+ const res = await this.fetchImpl(url.toString(), {
109
216
  signal: AbortSignal.timeout(FETCH_TIMEOUT_MS)
110
217
  });
111
218
  if (!res.ok || !res.body) {
112
- throw new Error(`HTTP ${res.status}: ${(await res.text().catch(() => '')).slice(0, 200)}`);
219
+ throw new Error(`HTTP ${res.status} for ${filename}: ${(await res.text().catch(() => '')).slice(0, 200)}`);
113
220
  }
114
- // Stream the response body to a temp file, then atomic-rename. This
115
- // means a kopia snapshot that races us mid-export sees either the
116
- // previous .parquet or no entry — never a torn write.
117
221
  const out = createWriteStream(tempPath);
118
222
  let bytes = 0;
119
- out.on('drain', () => undefined);
120
- // Count bytes via a tap — pipeline doesn't expose them otherwise.
121
223
  const reader = Readable.fromWeb(res.body);
122
224
  reader.on('data', (chunk) => {
123
225
  bytes += chunk.length;
@@ -126,15 +228,18 @@ export class QuestDBExporter {
126
228
  await pipeline(reader, out);
127
229
  }
128
230
  catch (err) {
129
- // Best-effort cleanup of partial file.
130
231
  await unlink(tempPath).catch(() => undefined);
131
232
  throw err;
132
233
  }
133
- await rename(tempPath, finalPath);
134
- // rowCount is unknown without a separate query; leaving 0 keeps the
135
- // shape stable. Bytes carries the meaningful "what got captured".
136
- this.log(`exported ${table}: ${bytes} bytes`);
137
- return { table, parquetPath: finalPath, rowCount: 0, bytes };
234
+ try {
235
+ await rename(tempPath, finalPath);
236
+ }
237
+ catch (err) {
238
+ // Rename can fail (cross-device, permissions) — clean up the orphan so it doesn't accumulate.
239
+ await unlink(tempPath).catch(() => undefined);
240
+ throw err;
241
+ }
242
+ return { bytes };
138
243
  }
139
244
  }
140
245
  function errMsg(err) {