revisium 2.0.1 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +32 -0
- package/README.md +23 -3
- package/dist/e2e/setup/global-setup.js +2 -3
- package/dist/e2e/setup/global-setup.js.map +1 -1
- package/dist/e2e/setup/global-teardown.js +0 -2
- package/dist/e2e/setup/global-teardown.js.map +1 -1
- package/dist/e2e/utils/constants.d.ts +0 -2
- package/dist/e2e/utils/constants.js +0 -2
- package/dist/e2e/utils/constants.js.map +1 -1
- package/dist/e2e/utils/docker-helper.d.ts +0 -2
- package/dist/e2e/utils/docker-helper.js +0 -13
- package/dist/e2e/utils/docker-helper.js.map +1 -1
- package/dist/package.json +27 -14
- package/dist/src/commands/migration/apply-migrations.command.d.ts +1 -1
- package/dist/src/commands/migration/apply-migrations.command.js +4 -5
- package/dist/src/commands/migration/apply-migrations.command.js.map +1 -1
- package/dist/src/commands/migration/save-migrations.command.d.ts +1 -1
- package/dist/src/commands/migration/save-migrations.command.js +6 -6
- package/dist/src/commands/migration/save-migrations.command.js.map +1 -1
- package/dist/src/commands/rows/save-rows.command.d.ts +1 -1
- package/dist/src/commands/rows/save-rows.command.js +12 -10
- package/dist/src/commands/rows/save-rows.command.js.map +1 -1
- package/dist/src/commands/rows/upload-rows.command.d.ts +1 -1
- package/dist/src/commands/rows/upload-rows.command.js +12 -15
- package/dist/src/commands/rows/upload-rows.command.js.map +1 -1
- package/dist/src/commands/schema/create-migrations.command.js.map +1 -1
- package/dist/src/commands/schema/save-schema.command.d.ts +1 -1
- package/dist/src/commands/schema/save-schema.command.js +7 -7
- package/dist/src/commands/schema/save-schema.command.js.map +1 -1
- package/dist/src/config/meta-schema.d.ts +3 -0
- package/dist/src/config/meta-schema.js +43 -1
- package/dist/src/config/meta-schema.js.map +1 -1
- package/dist/src/services/connection/api-client-adapter.d.ts +2 -4
- package/dist/src/services/connection/api-client-adapter.js +40 -36
- package/dist/src/services/connection/api-client-adapter.js.map +1 -1
- package/dist/src/services/connection/api-client.d.ts +3 -4
- package/dist/src/services/connection/api-client.js +11 -33
- package/dist/src/services/connection/api-client.js.map +1 -1
- package/dist/src/services/connection/connection-factory.service.d.ts +4 -6
- package/dist/src/services/connection/connection-factory.service.js +18 -39
- package/dist/src/services/connection/connection-factory.service.js.map +1 -1
- package/dist/src/services/connection/connection.service.d.ts +2 -73
- package/dist/src/services/connection/connection.service.js +2 -11
- package/dist/src/services/connection/connection.service.js.map +1 -1
- package/dist/src/services/sync/commit-revision.service.js +6 -28
- package/dist/src/services/sync/commit-revision.service.js.map +1 -1
- package/dist/src/services/sync/row-sync.service.d.ts +5 -5
- package/dist/src/services/sync/row-sync.service.js +10 -10
- package/dist/src/services/sync/row-sync.service.js.map +1 -1
- package/dist/src/services/sync/sync-data.service.d.ts +1 -0
- package/dist/src/services/sync/sync-data.service.js +21 -21
- package/dist/src/services/sync/sync-data.service.js.map +1 -1
- package/dist/src/services/sync/sync-schema.service.d.ts +1 -0
- package/dist/src/services/sync/sync-schema.service.js +11 -10
- package/dist/src/services/sync/sync-schema.service.js.map +1 -1
- package/dist/src/services/url/auth-prompt.service.js +1 -1
- package/dist/src/services/url/auth-prompt.service.js.map +1 -1
- package/dist/src/types/migration.types.d.ts +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/docs/authentication.md +3 -3
- package/docs/configuration.md +58 -9
- package/docs/docker-deployment.md +48 -13
- package/docs/migrate-commands.md +35 -10
- package/docs/rows-commands.md +30 -7
- package/docs/schema-commands.md +21 -5
- package/docs/sync-commands.md +44 -12
- package/docs/url-format.md +2 -2
- package/e2e/setup/global-setup.ts +3 -9
- package/e2e/setup/global-teardown.ts +0 -6
- package/e2e/utils/constants.ts +0 -2
- package/e2e/utils/docker-helper.ts +0 -23
- package/package.json +27 -14
- package/src/commands/migration/apply-migrations.command.ts +5 -6
- package/src/commands/migration/save-migrations.command.ts +7 -6
- package/src/commands/rows/save-rows.command.ts +14 -28
- package/src/commands/rows/upload-rows.command.ts +7 -15
- package/src/commands/schema/create-migrations.command.ts +1 -1
- package/src/commands/schema/save-schema.command.ts +9 -14
- package/src/config/meta-schema.ts +47 -0
- package/src/services/connection/__tests__/connection-factory.service.spec.ts +117 -0
- package/src/services/connection/__tests__/connection.service.spec.ts +27 -117
- package/src/services/connection/api-client-adapter.ts +41 -45
- package/src/services/connection/api-client.ts +11 -50
- package/src/services/connection/connection-factory.service.ts +35 -65
- package/src/services/connection/connection.service.ts +3 -14
- package/src/services/sync/__tests__/row-sync.service.spec.ts +3 -6
- package/src/services/sync/commit-revision.service.ts +7 -51
- package/src/services/sync/row-sync.service.ts +4 -18
- package/src/services/sync/sync-data.service.ts +32 -45
- package/src/services/sync/sync-schema.service.ts +14 -22
- package/src/services/url/auth-prompt.service.ts +1 -1
- package/src/types/migration.types.ts +2 -2
- package/dist/src/__generated__/api.d.ts +0 -688
- package/dist/src/__generated__/api.js +0 -698
- package/dist/src/__generated__/api.js.map +0 -1
- package/e2e/docker-compose.e2e.yml +0 -31
- package/src/__generated__/api.ts +0 -2598
|
@@ -29,7 +29,6 @@ export interface OrderBy {
|
|
|
29
29
|
|
|
30
30
|
export interface ApiClient {
|
|
31
31
|
rows(
|
|
32
|
-
revisionId: string,
|
|
33
32
|
tableId: string,
|
|
34
33
|
options: { first: number; after?: string; orderBy?: OrderBy[] },
|
|
35
34
|
): Promise<{
|
|
@@ -41,13 +40,11 @@ export interface ApiClient {
|
|
|
41
40
|
}>;
|
|
42
41
|
|
|
43
42
|
createRows(
|
|
44
|
-
revisionId: string,
|
|
45
43
|
tableId: string,
|
|
46
44
|
data: { rows: { rowId: string; data: object }[]; isRestore?: boolean },
|
|
47
45
|
): Promise<{ data?: unknown; error?: unknown }>;
|
|
48
46
|
|
|
49
47
|
updateRows(
|
|
50
|
-
revisionId: string,
|
|
51
48
|
tableId: string,
|
|
52
49
|
data: { rows: { rowId: string; data: object }[]; isRestore?: boolean },
|
|
53
50
|
): Promise<{ data?: unknown; error?: unknown }>;
|
|
@@ -65,7 +62,6 @@ const DEFAULT_BATCH_SIZE = 100;
|
|
|
65
62
|
export class RowSyncService {
|
|
66
63
|
async syncTableRows(
|
|
67
64
|
api: ApiClient,
|
|
68
|
-
targetRevisionId: string,
|
|
69
65
|
tableId: string,
|
|
70
66
|
sourceRows: RowData[],
|
|
71
67
|
batchSize: number = DEFAULT_BATCH_SIZE,
|
|
@@ -80,12 +76,7 @@ export class RowSyncService {
|
|
|
80
76
|
updateErrors: 0,
|
|
81
77
|
};
|
|
82
78
|
|
|
83
|
-
const existingRows = await this.getExistingRows(
|
|
84
|
-
api,
|
|
85
|
-
targetRevisionId,
|
|
86
|
-
tableId,
|
|
87
|
-
onProgress,
|
|
88
|
-
);
|
|
79
|
+
const existingRows = await this.getExistingRows(api, tableId, onProgress);
|
|
89
80
|
|
|
90
81
|
const { rowsToCreate, rowsToUpdate, skippedCount } = this.categorizeRows(
|
|
91
82
|
sourceRows,
|
|
@@ -96,7 +87,6 @@ export class RowSyncService {
|
|
|
96
87
|
if (rowsToCreate.length > 0) {
|
|
97
88
|
const createResult = await this.processBatchCreate(
|
|
98
89
|
api,
|
|
99
|
-
targetRevisionId,
|
|
100
90
|
tableId,
|
|
101
91
|
rowsToCreate,
|
|
102
92
|
batchSize,
|
|
@@ -109,7 +99,6 @@ export class RowSyncService {
|
|
|
109
99
|
if (rowsToUpdate.length > 0) {
|
|
110
100
|
const updateResult = await this.processBatchUpdate(
|
|
111
101
|
api,
|
|
112
|
-
targetRevisionId,
|
|
113
102
|
tableId,
|
|
114
103
|
rowsToUpdate,
|
|
115
104
|
batchSize,
|
|
@@ -124,7 +113,6 @@ export class RowSyncService {
|
|
|
124
113
|
|
|
125
114
|
async getExistingRows(
|
|
126
115
|
api: ApiClient,
|
|
127
|
-
revisionId: string,
|
|
128
116
|
tableId: string,
|
|
129
117
|
onProgress?: ProgressCallback,
|
|
130
118
|
): Promise<Map<string, JsonValue>> {
|
|
@@ -133,7 +121,7 @@ export class RowSyncService {
|
|
|
133
121
|
let hasMore = true;
|
|
134
122
|
|
|
135
123
|
while (hasMore) {
|
|
136
|
-
const result = await api.rows(
|
|
124
|
+
const result = await api.rows(tableId, {
|
|
137
125
|
first: 100,
|
|
138
126
|
after,
|
|
139
127
|
orderBy: [{ field: 'id', direction: 'asc' }],
|
|
@@ -194,7 +182,6 @@ export class RowSyncService {
|
|
|
194
182
|
|
|
195
183
|
private async processBatchCreate(
|
|
196
184
|
api: ApiClient,
|
|
197
|
-
revisionId: string,
|
|
198
185
|
tableId: string,
|
|
199
186
|
rows: RowData[],
|
|
200
187
|
batchSize: number,
|
|
@@ -207,7 +194,7 @@ export class RowSyncService {
|
|
|
207
194
|
operation: 'create',
|
|
208
195
|
onProgress,
|
|
209
196
|
executeBatch: (batch) =>
|
|
210
|
-
api.createRows(
|
|
197
|
+
api.createRows(tableId, {
|
|
211
198
|
rows: batch.map((r) => ({ rowId: r.id, data: r.data as object })),
|
|
212
199
|
isRestore: true,
|
|
213
200
|
}),
|
|
@@ -217,7 +204,6 @@ export class RowSyncService {
|
|
|
217
204
|
|
|
218
205
|
private async processBatchUpdate(
|
|
219
206
|
api: ApiClient,
|
|
220
|
-
revisionId: string,
|
|
221
207
|
tableId: string,
|
|
222
208
|
rows: RowData[],
|
|
223
209
|
batchSize: number,
|
|
@@ -230,7 +216,7 @@ export class RowSyncService {
|
|
|
230
216
|
operation: 'update',
|
|
231
217
|
onProgress,
|
|
232
218
|
executeBatch: (batch) =>
|
|
233
|
-
api.updateRows(
|
|
219
|
+
api.updateRows(tableId, {
|
|
234
220
|
rows: batch.map((r) => ({ rowId: r.id, data: r.data as object })),
|
|
235
221
|
isRestore: true,
|
|
236
222
|
}),
|
|
@@ -1,12 +1,9 @@
|
|
|
1
1
|
import { Injectable } from '@nestjs/common';
|
|
2
|
+
import { RevisionScope } from '@revisium/client';
|
|
2
3
|
import { SyncApiService, ConnectionInfo } from './sync-api.service';
|
|
3
4
|
import { TableDependencyService } from './table-dependency.service';
|
|
4
|
-
import {
|
|
5
|
-
|
|
6
|
-
RowData,
|
|
7
|
-
ApiClient,
|
|
8
|
-
RowSyncError,
|
|
9
|
-
} from './row-sync.service';
|
|
5
|
+
import { RowSyncService, RowData, RowSyncError } from './row-sync.service';
|
|
6
|
+
import { createApiClientAdapter } from '../connection';
|
|
10
7
|
import { LoggerService } from '../common';
|
|
11
8
|
import {
|
|
12
9
|
DataSyncResult,
|
|
@@ -60,17 +57,12 @@ export class SyncDataService {
|
|
|
60
57
|
tables: string[],
|
|
61
58
|
): Promise<Record<string, JsonSchema>> {
|
|
62
59
|
const schemas: Record<string, JsonSchema> = {};
|
|
60
|
+
const revisionScope = this.getRevisionScope(connection);
|
|
63
61
|
|
|
64
62
|
for (const tableId of tables) {
|
|
65
63
|
try {
|
|
66
|
-
const
|
|
67
|
-
|
|
68
|
-
tableId,
|
|
69
|
-
);
|
|
70
|
-
|
|
71
|
-
if (result.data) {
|
|
72
|
-
schemas[tableId] = result.data as JsonSchema;
|
|
73
|
-
}
|
|
64
|
+
const schema = await revisionScope.getTableSchema(tableId);
|
|
65
|
+
schemas[tableId] = schema as JsonSchema;
|
|
74
66
|
} catch (error) {
|
|
75
67
|
this.logger.indentWarn(
|
|
76
68
|
`Could not fetch schema for table ${tableId}: ${error instanceof Error ? error.message : String(error)}`,
|
|
@@ -105,30 +97,24 @@ export class SyncDataService {
|
|
|
105
97
|
tableFilter?: string[],
|
|
106
98
|
): Promise<string[]> {
|
|
107
99
|
const tables: string[] = [];
|
|
100
|
+
const revisionScope = this.getRevisionScope(connection);
|
|
108
101
|
let cursor: string | undefined;
|
|
109
102
|
|
|
110
103
|
do {
|
|
111
|
-
const result = await
|
|
112
|
-
revisionId: connection.revisionId,
|
|
104
|
+
const result = await revisionScope.getTables({
|
|
113
105
|
first: 100,
|
|
114
106
|
after: cursor,
|
|
115
107
|
});
|
|
116
108
|
|
|
117
|
-
|
|
118
|
-
throw new Error(
|
|
119
|
-
`Failed to get tables: ${JSON.stringify(result.error)}`,
|
|
120
|
-
);
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
for (const edge of result.data.edges) {
|
|
109
|
+
for (const edge of result.edges) {
|
|
124
110
|
const tableId = edge.node.id;
|
|
125
111
|
if (!tableFilter || tableFilter.includes(tableId)) {
|
|
126
112
|
tables.push(tableId);
|
|
127
113
|
}
|
|
128
114
|
}
|
|
129
115
|
|
|
130
|
-
cursor = result.
|
|
131
|
-
? result.
|
|
116
|
+
cursor = result.pageInfo.hasNextPage
|
|
117
|
+
? (result.pageInfo.endCursor ?? undefined)
|
|
132
118
|
: undefined;
|
|
133
119
|
} while (cursor);
|
|
134
120
|
|
|
@@ -140,24 +126,17 @@ export class SyncDataService {
|
|
|
140
126
|
tableId: string,
|
|
141
127
|
): Promise<RowData[]> {
|
|
142
128
|
const rows: RowData[] = [];
|
|
129
|
+
const revisionScope = this.getRevisionScope(connection);
|
|
143
130
|
let cursor: string | undefined;
|
|
144
131
|
|
|
145
132
|
do {
|
|
146
|
-
const result = await
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
{
|
|
150
|
-
|
|
151
|
-
after: cursor,
|
|
152
|
-
orderBy: [{ field: 'id', direction: 'asc' }],
|
|
153
|
-
},
|
|
154
|
-
);
|
|
155
|
-
|
|
156
|
-
if (result.error) {
|
|
157
|
-
throw new Error(`Failed to get rows: ${JSON.stringify(result.error)}`);
|
|
158
|
-
}
|
|
133
|
+
const result = await revisionScope.getRows(tableId, {
|
|
134
|
+
first: 100,
|
|
135
|
+
after: cursor,
|
|
136
|
+
orderBy: [{ field: 'id', direction: 'asc' }],
|
|
137
|
+
});
|
|
159
138
|
|
|
160
|
-
for (const edge of result.
|
|
139
|
+
for (const edge of result.edges) {
|
|
161
140
|
rows.push({
|
|
162
141
|
id: edge.node.id,
|
|
163
142
|
data: edge.node.data as Record<string, unknown>,
|
|
@@ -169,8 +148,8 @@ export class SyncDataService {
|
|
|
169
148
|
{ labels: { fetch: 'Loading from source' }, indent: ' ' },
|
|
170
149
|
);
|
|
171
150
|
|
|
172
|
-
cursor = result.
|
|
173
|
-
? result.
|
|
151
|
+
cursor = result.pageInfo.hasNextPage
|
|
152
|
+
? (result.pageInfo.endCursor ?? undefined)
|
|
174
153
|
: undefined;
|
|
175
154
|
} while (cursor);
|
|
176
155
|
|
|
@@ -195,9 +174,11 @@ export class SyncDataService {
|
|
|
195
174
|
|
|
196
175
|
let existingRows: Map<string, JsonValue>;
|
|
197
176
|
try {
|
|
177
|
+
const targetApiClient = createApiClientAdapter(
|
|
178
|
+
this.getRevisionScope(target),
|
|
179
|
+
);
|
|
198
180
|
existingRows = await this.rowSync.getExistingRows(
|
|
199
|
-
|
|
200
|
-
target.draftRevisionId,
|
|
181
|
+
targetApiClient,
|
|
201
182
|
tableId,
|
|
202
183
|
);
|
|
203
184
|
} catch {
|
|
@@ -310,9 +291,11 @@ export class SyncDataService {
|
|
|
310
291
|
const sourceRows = await this.getSourceRows(source, tableId);
|
|
311
292
|
this.logger.syncFound(sourceRows.length, 'rows in source');
|
|
312
293
|
|
|
294
|
+
const targetApiClient = createApiClientAdapter(
|
|
295
|
+
this.getRevisionScope(target),
|
|
296
|
+
);
|
|
313
297
|
const stats = await this.rowSync.syncTableRows(
|
|
314
|
-
|
|
315
|
-
target.draftRevisionId,
|
|
298
|
+
targetApiClient,
|
|
316
299
|
tableId,
|
|
317
300
|
sourceRows,
|
|
318
301
|
batchSize,
|
|
@@ -334,6 +317,10 @@ export class SyncDataService {
|
|
|
334
317
|
};
|
|
335
318
|
}
|
|
336
319
|
|
|
320
|
+
private getRevisionScope(connection: ConnectionInfo): RevisionScope {
|
|
321
|
+
return connection.revisionScope;
|
|
322
|
+
}
|
|
323
|
+
|
|
337
324
|
private createEmptyResult(): DataSyncResult {
|
|
338
325
|
return {
|
|
339
326
|
tables: [],
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Injectable } from '@nestjs/common';
|
|
2
|
+
import { RevisionScope } from '@revisium/client';
|
|
2
3
|
import { SyncApiService, ConnectionInfo } from './sync-api.service';
|
|
3
4
|
import { LoggerService } from '../common';
|
|
4
5
|
import { SchemaSyncResult } from 'src/types/sync.types';
|
|
@@ -41,17 +42,9 @@ export class SyncSchemaService {
|
|
|
41
42
|
private async getMigrations(
|
|
42
43
|
connection: ConnectionInfo,
|
|
43
44
|
): Promise<Migration[]> {
|
|
44
|
-
const
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
if (result.error) {
|
|
49
|
-
throw new Error(
|
|
50
|
-
`Failed to get migrations: ${JSON.stringify(result.error)}`,
|
|
51
|
-
);
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
return result.data as Migration[];
|
|
45
|
+
const revisionScope = this.getRevisionScope(connection);
|
|
46
|
+
const migrations = await revisionScope.getMigrations();
|
|
47
|
+
return migrations as Migration[];
|
|
55
48
|
}
|
|
56
49
|
|
|
57
50
|
private analyzeMigrations(migrations: Migration[]): SchemaSyncResult {
|
|
@@ -99,19 +92,14 @@ export class SyncSchemaService {
|
|
|
99
92
|
const tablesRemoved: string[] = [];
|
|
100
93
|
let migrationsApplied = 0;
|
|
101
94
|
|
|
102
|
-
|
|
103
|
-
const result = await connection.client.api.applyMigrations(
|
|
104
|
-
connection.draftRevisionId,
|
|
105
|
-
[migration],
|
|
106
|
-
);
|
|
95
|
+
const revisionScope = this.getRevisionScope(connection);
|
|
107
96
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
}
|
|
97
|
+
for (const migration of migrations) {
|
|
98
|
+
const results = await revisionScope.applyMigrationsWithStatus([
|
|
99
|
+
migration,
|
|
100
|
+
]);
|
|
113
101
|
|
|
114
|
-
const response =
|
|
102
|
+
const response = results[0];
|
|
115
103
|
|
|
116
104
|
if (response.status === 'failed') {
|
|
117
105
|
throw new Error(
|
|
@@ -152,4 +140,8 @@ export class SyncSchemaService {
|
|
|
152
140
|
tablesRemoved,
|
|
153
141
|
};
|
|
154
142
|
}
|
|
143
|
+
|
|
144
|
+
private getRevisionScope(connection: ConnectionInfo): RevisionScope {
|
|
145
|
+
return connection.revisionScope;
|
|
146
|
+
}
|
|
155
147
|
}
|