syncables 0.10.0

package/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

package/README.md

@@ -0,0 +1,115 @@
+# Syncables
+
+This repository contains a sync engine that can be used to download (and in the future also update) a collection of objects from an API. It can be configured declaratively by extending the OpenAPI spec.
+
+Find the path that can be used to fetch a (paged) collection of items. This will typically look something like this:
+
+```yaml
+paths:
+ /widgets:
+    get:
+      parameters:
+        - description: Token specifying which result page to return. Optional.
+          in: query
+          name: pageToken
+          schema:
+            type: string
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/CalendarList"
+components:
+  schemas:
+    CalendarList:
+      properties:
+        results:
+          items:
+            properties:
+              backgroundColor:
+                type: string
+          type: array
+        nextPageToken:
+          description: Token used to access the next page of this result. Omitted if no further results are available.
+          type: string
+      type: object
+```
+
+Under `paths['/widgets']['get']['responses']['200']['content']['application/json']`, add an object `syncable`, in which you can specify:
+* `name`: a `string` descriptor of the collection, e.g. `"widgets"`
+* `pagingStrategy`: one of `pageNumber`, `offset`, `pageToken`, `dateRange` or (experimental) `rangeHeader`
+* `query`: an object containing query parameters to add in addition to the paging-related ones
+* `itemsPathInResponse`: path within the response body schema, as an array of strings, that contains the array of items (default: `[]` for the response body root)
+* `defaultPageSize`: tell Syncable how many items per page (max) to expect by default
+* `forcePageSize`: if possible, let Syncable tell the API which page size to use (not applicable to paging strategy `dateRange`)
+* when using `forcePageSize`, you can add `forcePageSizeParamInQuery` if it's not `pageSize` (not applicable to paging strategies `dateRange` and `rangeHeader`).
+* for a `pageNumber` paging strategy, you can add `pageNumberParamInQuery` if it's not `page`.
+* for an `offset` paging strategy, you can add `offsetParamInQuery` if it's not `offset`.
+* for a `pageToken` paging strategy, you can add `pageTokenParamInQuery` if it's not `pageToken` and `pageTokenPathInResponse` if it's not `['nextPageToken']`
+* for a `dateRange` paging strategy, you can add `startDateParamInQuery` if it's not `startDate`, `endDateParamInQuery` if it's not `endDate`, `startDate` if it's not `'20000101000000'`, and `endDate` if it's not `'99990101000000'`
+
+## Usage
+### Create the OAD
+You start with an OAD, for instance one from [here](https://github.com/tubsproject/syncables/tree/main/openapi/oad).
+Probably the API you want to use is not specifying syncables yet, so you'll need to add those yourself using an overlay, something like the ones you see [here](https://github.com/tubsproject/syncables/tree/main/openapi/overlay).
+To compute the effect of the overlay you can run a command like:
+```sh
+./node_modules/.bin/overlayjs --openapi ./openapi/oad/acube-peppol.yaml --overlay ./openapi/overlay/acube-peppol-overlay.yaml > openapi/generated/acube.yaml
+```
+### Generate the type
+Assuming you're working in TypeScript, you can benefit from generating types from the OAD, like so:
+```ts
+npx openapi-typescript openapi/generated/acube.yaml -o ./src/types/acube.d.ts
+```
+### Install syncables
+Depending on your preferred package manager, you can run something like this to install [syncables from npm](https://www.npmjs.com/package/syncables):
+```sh
+pnpm install syncables
+```
+
+### Write your code
+Now you have the AOD with the definition of the syncable, and the type for the items you will sync, you can write code like [this](./src/readme-example.ts):
+```ts
+import { readFileSync } from 'fs';
+import { components } from './src/types/google-calendar.js';
+import { Syncable } from 'syncable';
+
+type Entry = components['schemas']['CalendarListEntry'];
+const specStr = readFileSync('./openapi/generated/google-calendar.yaml').toString();
+
+const syncable = new Syncable<Entry>({
+  specStr,
+  syncableName: 'widgets',
+  authHeaders: {
+    Authorization: `Bearer ${process.env.GOOGLE_BEARER_TOKEN}`
+  },
+  dbConn: 'postgresql://syncables:syncables@localhost:5432/db_unit_tests'
+});
+await syncable.fullFetch();
+```
+
+You can use the [`showcase-google-calendar` branch of this repo]() to run a simple OAuth client that can obtain a value for the `GOOGLE_BEARER_TOKEN` environment variable.
+
+## Development
+```sh
+git clone https://github.com/tubsproject/syncables
+cd syncables
+pnpm install
+pnpm generate
+pnpm build
+pnpm test
+pnpm prettier
+pnpm login
+pnpm publish
+```
+
+## Usage
+```sh
+docker compose up -d
+# run your code that calls syncable.fullFetch();
+docker exec -it db psql postgresql://syncables:syncables@localhost:5432/db_unit_tests -c "\d+"
+```
+
+## Acknowledgements
+The `__tests__/integration/mock-server' folder is a copy of [`@scalar/mock-server`](https://github.com/scalar/scalar/blob/main/packages/mock-server/src) - thanks [Scalar](https://scalar.com)!

package/build/src/db.js

@@ -0,0 +1,73 @@
+/* eslint @typescript-eslint/no-explicit-any: 0 */
+import { Client } from 'pg';
+export { Client } from 'pg';
+export async function getPostgresClient() {
+    if (!process.env.DATABASE_URL) {
+        throw new Error('DATABASE_URL not set');
+    }
+    const client = new Client({
+        connectionString: process.env.DATABASE_URL,
+        ssl: {
+            rejectUnauthorized: process.env.NODE_ENV === 'production',
+        },
+    });
+    await client.connect();
+    return client;
+}
+export function getFields(openApiSpec, endPoint, rowsFrom) {
+    const successResponseProperties = openApiSpec.paths[endPoint]?.get?.responses?.['200']?.content;
+    // console.log(openApiSpec.paths, endPoint);
+    const schema = successResponseProperties?.['application/ld+json']?.schema ||
+        successResponseProperties?.['application/json']?.schema;
+    // console.log(`Schema for ${endPoint}:`, JSON.stringify(schema, null, 2));
+    // if (typeof schema['$ref'] !== 'undefined') {
+    //   schema = resolveRef(openApiSpec, schema['$ref']);
+    // }
+    // const whatWeWant = schema?.properties?.[rowsFrom].items?.properties;
+    const whatWeWant = typeof rowsFrom === 'string'
+        ? schema?.properties?.[rowsFrom]?.items?.properties
+        : schema?.items?.properties;
+    // console.log(
+    // `What we want (getFields ${endPoint} ${rowsFrom}):`,
+    // JSON.stringify(whatWeWant, null, 2),
+    // );
+    return whatWeWant;
+}
+export async function createSqlTable(client, tableName, whatWeWant) {
+    const rowSpecs = [];
+    // console.log(
+    //   `What we want (createSqlTable ${tableName}):`,
+    //   JSON.stringify(whatWeWant, null, 2),
+    // );
+    if (!whatWeWant) {
+        throw new Error(`No fields found for table ${tableName}`);
+    }
+    Object.entries(whatWeWant).forEach(([key, value]) => {
+        const type = value.type;
+        if (type === 'string') {
+            rowSpecs.push(`"S${key}" TEXT`);
+        }
+        else if (type === 'boolean') {
+            rowSpecs.push(`"S${key}" BOOLEAN`);
+        }
+        else if (type === 'number') {
+            rowSpecs.push(`"S${key}" INTEGER`);
+        }
+    });
+    const createTableQuery = `
+CREATE TABLE IF NOT EXISTS ${tableName.replace('-', '_')} (
+  ${rowSpecs.join(',\n  ')}\n
+);
+`;
+    // console.log(createTableQuery);
+    await client.query(createTableQuery);
+}
+export async function insertData(client, tableName, items, fields) {
+    // console.log(`Fetched data:`, items);
+    await Promise.all(items.map((item) => {
+        const insertQuery = `INSERT INTO ${tableName.replace('-', '_')} (${fields.map((x) => `"S${x}"`).join(', ')}) VALUES (${fields.map((field) => `'${item[field]}'`).join(', ')})`;
+        // console.log(`Executing insert query: ${insertQuery}`);
+        return client.query(insertQuery);
+    }));
+}
+//# sourceMappingURL=db.js.map

package/build/src/db.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"db.js","sourceRoot":"","sources":["../../src/db.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAE5B,MAAM,CAAC,KAAK,UAAU,iBAAiB;IACrC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC;QACxB,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY;QAC1C,GAAG,EAAE;YACH,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY;SAC1D;KACF,CAAC,CAAC;IACH,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IACvB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,WAAgB,EAChB,QAAgB,EAChB,QAA4B;IAE5B,MAAM,yBAAyB,GAC7B,WAAW,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IAChE,4CAA4C;IAC5C,MAAM,MAAM,GACV,yBAAyB,EAAE,CAAC,qBAAqB,CAAC,EAAE,MAAM;QAC1D,yBAAyB,EAAE,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC1D,2EAA2E;IAC3E,+CAA+C;IAC/C,sDAAsD;IACtD,IAAI;IACJ,uEAAuE;IACvE,MAAM,UAAU,GACd,OAAO,QAAQ,KAAK,QAAQ;QAC1B,CAAC,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,UAAU;QACnD,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,UAAU,CAAC;IAChC,eAAe;IACf,uDAAuD;IACvD,uCAAuC;IACvC,KAAK;IACL,OAAO,UAAU,CAAC;AACpB,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAc,EACd,SAAiB,EACjB,UAA+C;IAE/C,MAAM,QAAQ,GAAG,EAAE,CAAC;IACpB,eAAe;IACf,mDAAmD;IACnD,yCAAyC;IACzC,KAAK;IACL,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,6BAA6B,SAAS,EAAE,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QAClD,MAAM,IAAI,GAAI,KAA0B,CAAC,IAAI,CAAC;QAC9C,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtB,QAAQ,CAAC,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAC;QAClC,CAAC;aAAM,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YAC9B,QAAQ,CAAC,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,CAAC;QACrC,CAAC;aAAM,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,QAAQ,CAAC,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,CAAC;QACrC,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,gBAAgB,GAAG;6BACE,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC;IACpD,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC;;CAEzB,CAAC;IACA,iCAAiC;IACjC,MAAM,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;AACvC,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,MAAc,EACd,SAAiB,EACjB,KAAY,EACZ,MAAgB;IAEhB,uCAAuC;IACvC,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE;QACtB,MAAM,WAAW,GAAG,eAAe,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;QAC/K,yDAAyD;QACzD,OAAO,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IACnC,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}

package/build/src/readme-example.js

@@ -0,0 +1,16 @@
+import { readFileSync } from 'fs';
+import { Syncable } from './syncable.js';
+const specFilename = './openapi/generated/google-calendar.yaml';
+const specStr = readFileSync(specFilename).toString();
+const syncable = new Syncable({
+    specStr,
+    specFilename,
+    syncableName: 'widgets',
+    authHeaders: {
+        Authorization: `Bearer ${process.env.GOOGLE_BEARER_TOKEN}`,
+    },
+    dbConn: 'postgresql://syncables:syncables@localhost:5432/db_unit_tests',
+});
+const data = await syncable.fullFetch();
+console.log(data);
+//# sourceMappingURL=readme-example.js.map

package/build/src/readme-example.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"readme-example.js","sourceRoot":"","sources":["../../src/readme-example.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAElC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAGzC,MAAM,YAAY,GAAG,0CAA0C,CAAC;AAChE,MAAM,OAAO,GAAG,YAAY,CAAC,YAAY,CAAC,CAAC,QAAQ,EAAE,CAAC;AAEtD,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAQ;IACnC,OAAO;IACP,YAAY;IACZ,YAAY,EAAE,SAAS;IACvB,WAAW,EAAE;QACX,aAAa,EAAE,UAAU,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE;KAC3D;IACD,MAAM,EAAE,+DAA+D;CACxE,CAAC,CAAC;AACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,CAAC;AACxC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC"}

package/build/src/syncable.js

@@ -0,0 +1,275 @@
+import { EventEmitter } from 'events';
+import { default as urljoin } from 'url-join';
+import { default as createDebug } from 'debug';
+import { Client, getFields, createSqlTable, insertData } from './db.js';
+import { dereference } from '@readme/openapi-parser';
+import { parse } from 'yaml';
+import { getObjectPath } from '../__tests__/integration/mock-server/apply-pagination.js';
+const debug = createDebug('syncable');
+export class Syncable extends EventEmitter {
+    fetchFunction;
+    config;
+    specStr;
+    specFilename;
+    syncableName;
+    spec;
+    authHeaders = {};
+    client = null;
+    constructor({ specStr, specFilename, syncableName, authHeaders = {}, fetchFunction = fetch, dbConn, }) {
+        super();
+        this.specStr = specStr;
+        this.specFilename = specFilename;
+        this.syncableName = syncableName;
+        this.authHeaders = authHeaders;
+        this.fetchFunction = fetchFunction;
+        if (dbConn) {
+            this.client = new Client({
+                connectionString: dbConn,
+                ssl: {
+                    rejectUnauthorized: process.env.NODE_ENV === 'production',
+                },
+            });
+        }
+    }
+    async parseSpec() {
+        let specObj;
+        if (this.specFilename.endsWith('.json')) {
+            specObj = JSON.parse(this.specStr);
+        }
+        else {
+            specObj = parse(this.specStr);
+        }
+        const schema = await dereference(specObj);
+        for (const path of Object.keys(schema.paths)) {
+            const pathItem = schema.paths[path];
+            if (pathItem.get && pathItem.get.responses['200']) {
+                const response = 
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                pathItem.get.responses['200'].content['application/json'];
+                if (response.syncable && response.syncable.name === this.syncableName) {
+                    const config = {
+                        baseUrl: 
+                        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                        schema.servers && schema.servers.length > 0
+                            ? // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                                schema.servers[0].url
+                            : '',
+                        urlPath: path,
+                        name: response.syncable.name,
+                        pagingStrategy: response.syncable.pagingStrategy,
+                        query: response.syncable.query || {},
+                        itemsPathInResponse: response.syncable.itemsPathInResponse || [],
+                        defaultPageSize: response.syncable.defaultPageSize,
+                        forcePageSize: response.syncable.forcePageSize,
+                        forcePageSizeParamInQuery: response.syncable.forcePageSizeParamInQuery,
+                    };
+                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                    console.log('baseUrl:', config.baseUrl, schema.servers);
+                    if (response.syncable.pagingStrategy === 'pageNumber') {
+                        config.pageNumberParamInQuery =
+                            response.syncable.pageNumberParamInQuery || 'page';
+                    }
+                    else if (response.syncable.pagingStrategy === 'offset') {
+                        config.offsetParamInQuery =
+                            response.syncable.offsetParamInQuery || 'offset';
+                    }
+                    else if (response.syncable.pagingStrategy === 'pageToken') {
+                        config.pageTokenParamInQuery =
+                            response.syncable.pageTokenParamInQuery || 'pageToken';
+                        config.nextPageTokenPathInResponse = response.syncable
+                            .nextPageTokenPathInResponse || ['nextPageToken'];
+                    }
+                    else if (response.syncable.pagingStrategy === 'dateRange') {
+                        config.startDateParamInQuery =
+                            response.syncable.startDateParamInQuery || 'startDate';
+                        config.endDateParamInQuery =
+                            response.syncable.endDateParamInQuery || 'endDate';
+                        config.startDate = response.syncable.startDate || '20000101000000';
+                        config.endDate = response.syncable.endDate || '99990101000000';
+                    }
+                    this.config = config;
+                    return schema;
+                }
+            }
+        }
+        throw new Error(`Syncable with name "${this.syncableName}" not found in spec`);
+    }
+    async doFetch(url, headers = {}, minNumItemsToExpect = 1) {
+        debug('Fetching', url, headers);
+        const response = await this.fetchFunction(url, {
+            headers: Object.assign({}, this.authHeaders, headers),
+        });
+        if (!response.ok) {
+            throw new Error(`Fetch error: ${response.status} ${response.statusText} for URL ${url} (${await response.text()})`);
+        }
+        const responseData = await response.json();
+        // console.log('responseData nextPageToken', Object.keys(responseData), this.config.pageTokenParamInQuery, responseData[this.config.pageTokenParamInQuery]);
+        let items = responseData;
+        for (let i = 0; i < this.config.itemsPathInResponse.length; i++) {
+            const pathPart = this.config.itemsPathInResponse[i];
+            if (typeof items !== 'object' || items === null || !(pathPart in items)) {
+                throw new Error(`Invalid itemsPathInResponse: could not find path part "${pathPart}" in response ${url}`);
+            }
+            items = items[pathPart];
+        }
+        // console.log('parsed responseData', responseData, items, this.config.itemsPathInResponse);
+        let nextPageToken = undefined;
+        try {
+            nextPageToken = getObjectPath(responseData, this.config.nextPageTokenPathInResponse || ['nextPageToken']);
+        }
+        catch (err) {
+            // ignore
+            void err;
+        }
+        return {
+            items,
+            hasMore: items.length >= minNumItemsToExpect,
+            nextPageToken,
+        };
+    }
+    async pageNumberFetch() {
+        let allData = [];
+        let page = 1;
+        let hasMore = true;
+        while (hasMore) {
+            const url = this.getUrl();
+            Object.entries(this.config.query || {}).forEach(([key, value]) => {
+                url.searchParams.append(key, value);
+            });
+            url.searchParams.append(this.config.pageNumberParamInQuery, page.toString());
+            if (this.config.forcePageSize) {
+                const param = this.config.forcePageSizeParamInQuery || 'pageSize';
+                url.searchParams.append(param, this.config.forcePageSize.toString());
+            }
+            const data = await this.doFetch(url.toString(), {}, this.config.forcePageSize || this.config.defaultPageSize || 1);
+            allData = allData.concat(data.items);
+            hasMore = data.hasMore;
+            page += 1;
+        }
+        return allData;
+    }
+    async offsetFetch() {
+        let allData = [];
+        let offset = 0;
+        let hasMore = true;
+        while (hasMore) {
+            const url = this.getUrl();
+            Object.entries(this.config.query || {}).forEach(([key, value]) => {
+                url.searchParams.append(key, value);
+            });
+            url.searchParams.append(this.config.offsetParamInQuery, offset.toString());
+            if (this.config.forcePageSize) {
+                const param = this.config.forcePageSizeParamInQuery || 'pageSize';
+                url.searchParams.append(param, this.config.forcePageSize.toString());
+            }
+            const data = await this.doFetch(url.toString(), {}, this.config.forcePageSize || this.config.defaultPageSize || 1);
+            allData = allData.concat(data.items);
+            hasMore = data.hasMore;
+            offset += data.items.length;
+        }
+        return allData;
+    }
+    async pageTokenFetch() {
+        let allData = [];
+        let nextPageToken = null;
+        do {
+            const url = this.getUrl();
+            Object.entries(this.config.query || {}).forEach(([key, value]) => {
+                url.searchParams.append(key, value);
+            });
+            if (this.config.forcePageSize) {
+                const param = this.config.forcePageSizeParamInQuery || 'pageSize';
+                url.searchParams.append(param, this.config.forcePageSize.toString());
+            }
+            if (nextPageToken) {
+                url.searchParams.append(this.config.pageTokenParamInQuery, nextPageToken);
+            }
+            const data = await this.doFetch(url.toString(), {}, this.config.forcePageSize || this.config.defaultPageSize || 1);
+            // console.log('fetched', data);
+            allData = allData.concat(data.items);
+            nextPageToken = data.nextPageToken || null;
+        } while (nextPageToken);
+        return allData;
+    }
+    getUrl() {
+        const joined = urljoin(this.config.baseUrl, this.config.urlPath);
+        return new URL(joined);
+    }
+    async dateRangeFetch() {
+        let allData = [];
+        let startDate = this.config.startDate || null;
+        const endDate = this.config.endDate || null;
+        while (true) {
+            const url = this.getUrl();
+            Object.entries(this.config.query || {}).forEach(([key, value]) => {
+                url.searchParams.append(key, value);
+            });
+            if (startDate) {
+                url.searchParams.append('startDate', startDate);
+            }
+            if (endDate) {
+                url.searchParams.append('endDate', endDate);
+            }
+            const data = await this.doFetch(url.toString());
+            allData = allData.concat(data.items);
+            if (data.items.length === 0 || !data.hasMore) {
+                break;
+            }
+            // Assuming items are sorted by date ascending
+            startDate = data.items[data.items.length - 1].date; // FIXME
+        }
+        return allData;
+    }
+    async rangeHeaderFetch() {
+        let allData = [];
+        const numItemsPerPage = this.config.forcePageSize || 20;
+        let rangeHeader = `id ..; max=${numItemsPerPage}`;
+        while (true) {
+            const url = this.getUrl();
+            Object.entries(this.config.query || {}).forEach(([key, value]) => {
+                url.searchParams.append(key, value);
+            });
+            const data = await this.doFetch(url.toString(), {
+                Range: rangeHeader,
+            }, numItemsPerPage);
+            allData = allData.concat(data.items);
+            const lastItemId = data.items.length > 0
+                ? data.items[data.items.length - 1].id
+                : null;
+            rangeHeader = `id ]${lastItemId}..; max=${numItemsPerPage}`;
+            if (data.items.length < numItemsPerPage || !data.hasMore) {
+                break;
+            }
+        }
+        return allData;
+    }
+    async doFullFetch() {
+        switch (this.config['pagingStrategy']) {
+            case 'pageNumber':
+                return this.pageNumberFetch();
+            case 'offset':
+                return this.offsetFetch();
+            case 'pageToken':
+                return this.pageTokenFetch();
+            case 'dateRange':
+                return this.dateRangeFetch();
+            case 'rangeHeader':
+                return this.rangeHeaderFetch();
+            default:
+                throw new Error(`Unknown paging strategy: ${this.config['pagingStrategy']}`);
+        }
+    }
+    async fullFetch() {
+        const schema = await this.parseSpec();
+        const data = await this.doFullFetch();
+        if (this.client) {
+            await this.client.connect();
+            const fields = getFields(schema, this.config.urlPath, this.config.itemsPathInResponse.join('.'));
+            await createSqlTable(this.client, this.config.name, fields);
+            await insertData(this.client, this.config.name, data, Object.keys(fields).filter((f) => ['string'].indexOf(fields[f].type) !== -1));
+            await this.client.end();
+        }
+        return data;
+    }
+}
+//# sourceMappingURL=syncable.js.map

package/build/src/syncable.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"syncable.js","sourceRoot":"","sources":["../../src/syncable.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,OAAO,IAAI,OAAO,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,OAAO,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACxE,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,0DAA0D,CAAC;AAEzF,MAAM,KAAK,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;AA2BtC,MAAM,OAAO,QAAY,SAAQ,YAAY;IAC3C,aAAa,CAAe;IAC5B,MAAM,CAAiB;IACvB,OAAO,CAAS;IAChB,YAAY,CAAS;IACrB,YAAY,CAAS;IACrB,IAAI,CAAgD;IACpD,WAAW,GAA8B,EAAE,CAAC;IAC5C,MAAM,GAAkB,IAAI,CAAC;IAC7B,YAAY,EACV,OAAO,EACP,YAAY,EACZ,YAAY,EACZ,WAAW,GAAG,EAAE,EAChB,aAAa,GAAG,KAAK,EACrB,MAAM,GAQP;QACC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC;gBACvB,gBAAgB,EAAE,MAAM;gBACxB,GAAG,EAAE;oBACH,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY;iBAC1D;aACF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAS;QACb,IAAI,OAAO,CAAC;QACZ,IAAI,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YACxC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACrC,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAChC,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC;QAC1C,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7C,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,QAAQ,CAAC,GAAG,IAAI,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;gBAClD,MAAM,QAAQ;gBACZ,8DAA8D;gBAC7D,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAS,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;gBACrE,IAAI,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,KAAK,IAAI,CAAC,YAAY,EAAE,CAAC;oBACtE,MAAM,MAAM,GAAmB;wBAC7B,OAAO;wBACL,8DAA8D;wBAC7D,MAAc,CAAC,OAAO,IAAK,MAAc,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;4BAC3D,CAAC,CAAC,8DAA8D;gCAC7D,MAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG;4BAChC,CAAC,CAAC,EAAE;wBACR,OAAO,EAAE,IAAI;wBACb,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,IAAI;wBAC5B,cAAc,EAAE,QAAQ,CAAC,QAAQ,CAAC,cAAc;wBAChD,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE;wBACpC,mBAAmB,EAAE,QAAQ,CAAC,QAAQ,CAAC,mBAAmB,IAAI,EAAE;wBAChE,eAAe,EAAE,QAAQ,CAAC,QAAQ,CAAC,eAAe;wBAClD,aAAa,EAAE,QAAQ,CAAC,QAAQ,CAAC,aAAa;wBAC9C,yBAAyB,EACvB,QAAQ,CAAC,QAAQ,CAAC,yBAAyB;qBAC9C,CAAC;oBACF,8DAA8D;oBAC9D,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,OAAO,EAAG,MAAc,CAAC,OAAO,CAAC,CAAC;oBACjE,IAAI,QAAQ,CAAC,QAAQ,CAAC,cAAc,KAAK,YAAY,EAAE,CAAC;wBACtD,MAAM,CAAC,sBAAsB;4BAC3B,QAAQ,CAAC,QAAQ,CAAC,sBAAsB,IAAI,MAAM,CAAC;oBACvD,CAAC;yBAAM,IAAI,QAAQ,CAAC,QAAQ,CAAC,cAAc,KAAK,QAAQ,EAAE,CAAC;wBACzD,MAAM,CAAC,kBAAkB;4BACvB,QAAQ,CAAC,QAAQ,CAAC,kBAAkB,IAAI,QAAQ,CAAC;oBACrD,CAAC;yBAAM,IAAI,QAAQ,CAAC,QAAQ,CAAC,cAAc,KAAK,WAAW,EAAE,CAAC;wBAC5D,MAAM,CAAC,qBAAqB;4BAC1B,QAAQ,CAAC,QAAQ,CAAC,qBAAqB,IAAI,WAAW,CAAC;wBACzD,MAAM,CAAC,2BAA2B,GAAG,QAAQ,CAAC,QAAQ;6BACnD,2BAA2B,IAAI,CAAC,eAAe,CAAC,CAAC;oBACtD,CAAC;yBAAM,IAAI,QAAQ,CAAC,QAAQ,CAAC,cAAc,KAAK,WAAW,EAAE,CAAC;wBAC5D,MAAM,CAAC,qBAAqB;4BAC1B,QAAQ,CAAC,QAAQ,CAAC,qBAAqB,IAAI,WAAW,CAAC;wBACzD,MAAM,CAAC,mBAAmB;4BACxB,QAAQ,CAAC,QAAQ,CAAC,mBAAmB,IAAI,SAAS,CAAC;wBACrD,MAAM,CAAC,SAAS,GAAG,QAAQ,CAAC,QAAQ,CAAC,SAAS,IAAI,gBAAgB,CAAC;wBACnE,MAAM,CAAC,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,OAAO,IAAI,gBAAgB,CAAC;oBACjE,CAAC;oBACD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;oBACrB,OAAO,MAAM,CAAC;gBAChB,CAAC;YACH,CAAC;QACH,CAAC;QACD,MAAM,IAAI,KAAK,CACb,uBAAuB,IAAI,CAAC,YAAY,qBAAqB,CAC9D,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,GAAW,EACX,UAAqC,EAAE,EACvC,sBAA8B,CAAC;QAE/B,KAAK,CAAC,UAAU,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;QAChC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE;YAC7C,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC;SACtD,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CACb,gBAAgB,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,YAAY,GAAG,KAAK,MAAM,QAAQ,CAAC,IAAI,EAAE,GAAG,CACnG,CAAC;QACJ,CAAC;QAED,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC3C,4JAA4J;QAC5J,IAAI,KAAK,GAAG,YAAY,CAAC;QACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChE,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;YACpD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,QAAQ,IAAI,KAAK,CAAC,EAAE,CAAC;gBACxE,MAAM,IAAI,KAAK,CACb,0DAA0D,QAAQ,iBAAiB,GAAG,EAAE,CACzF,CAAC;YACJ,CAAC;YACD,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC1B,CAAC;QACD,4FAA4F;QAC5F,IAAI,aAAa,GAAuB,SAAS,CAAC;QAClD,IAAI,CAAC;YACH,aAAa,GAAG,aAAa,CAC3B,YAAY,EACZ,IAAI,CAAC,MAAM,CAAC,2BAA2B,IAAI,CAAC,eAAe,CAAC,CAC7D,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS;YACT,KAAK,GAAG,CAAC;QACX,CAAC;QACD,OAAO;YACL,KAAK;YACL,OAAO,EAAE,KAAK,CAAC,MAAM,IAAI,mBAAmB;YAC5C,aAAa;SACd,CAAC;IACJ,CAAC;IACO,KAAK,CAAC,eAAe;QAC3B,IAAI,OAAO,GAAQ,EAAE,CAAC;QACtB,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,OAAO,GAAG,IAAI,CAAC;QAEnB,OAAO,OAAO,EAAE,CAAC;YACf,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;gBAC/D,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACtC,CAAC,CAAC,CAAC;YACH,GAAG,CAAC,YAAY,CAAC,MAAM,CACrB,IAAI,CAAC,MAAM,CAAC,sBAAsB,EAClC,IAAI,CAAC,QAAQ,EAAE,CAChB,CAAC;YACF,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;gBAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,yBAAyB,IAAI,UAAU,CAAC;gBAClE,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;YACvE,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAC7B,GAAG,CAAC,QAAQ,EAAE,EACd,EAAE,EACF,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,IAAI,CAAC,CAC9D,CAAC;YACF,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;YACvB,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,WAAW;QACvB,IAAI,OAAO,GAAQ,EAAE,CAAC;QACtB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,OAAO,GAAG,IAAI,CAAC;QAEnB,OAAO,OAAO,EAAE,CAAC;YACf,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;gBAC/D,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACtC,CAAC,CAAC,CAAC;YACH,GAAG,CAAC,YAAY,CAAC,MAAM,CACrB,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAC9B,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;YACF,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;gBAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,yBAAyB,IAAI,UAAU,CAAC;gBAClE,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;YACvE,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAC7B,GAAG,CAAC,QAAQ,EAAE,EACd,EAAE,EACF,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,IAAI,CAAC,CAC9D,CAAC;YACF,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;YACvB,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAC9B,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,IAAI,OAAO,GAAQ,EAAE,CAAC;QACtB,IAAI,aAAa,GAAkB,IAAI,CAAC;QAExC,GAAG,CAAC;YACF,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;gBAC/D,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACtC,CAAC,CAAC,CAAC;YACH,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;gBAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,yBAAyB,IAAI,UAAU,CAAC;gBAClE,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;YACvE,CAAC;YACD,IAAI,aAAa,EAAE,CAAC;gBAClB,GAAG,CAAC,YAAY,CAAC,MAAM,CACrB,IAAI,CAAC,MAAM,CAAC,qBAAqB,EACjC,aAAa,CACd,CAAC;YACJ,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAC7B,GAAG,CAAC,QAAQ,EAAE,EACd,EAAE,EACF,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,IAAI,CAAC,CAC9D,CAAC;YACF,gCAAgC;YAChC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrC,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC;QAC7C,CAAC,QAAQ,aAAa,EAAE;QAExB,OAAO,OAAO,CAAC;IACjB,CAAC;IACO,MAAM;QACZ,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACjE,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACzB,CAAC;IACO,KAAK,CAAC,cAAc;QAC1B,IAAI,OAAO,GAAQ,EAAE,CAAC;QACtB,IAAI,SAAS,GAAkB,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC;QAC7D,MAAM,OAAO,GAAkB,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC;QAE3D,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;gBAC/D,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACtC,CAAC,CAAC,CAAC;YACH,IAAI,SAAS,EAAE,CAAC;gBACd,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAClD,CAAC;YACD,IAAI,OAAO,EAAE,CAAC;gBACZ,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YAC9C,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;YAChD,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC7C,MAAM;YACR,CAAC;YACD,8CAA8C;YAC9C,SAAS,GACP,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CACjC,CAAC,IAAI,CAAC,CAAC,QAAQ;QAClB,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC5B,IAAI,OAAO,GAAQ,EAAE,CAAC;QACtB,MAAM,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,EAAE,CAAC;QACxD,IAAI,WAAW,GAAG,cAAc,eAAe,EAAE,CAAC;QAElD,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;gBAC/D,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACtC,CAAC,CAAC,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAC7B,GAAG,CAAC,QAAQ,EAAE,EACd;gBACE,KAAK,EAAE,WAAW;aACnB,EACD,eAAe,CAChB,CAAC;YACF,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,UAAU,GACd,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;gBACnB,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAA+B,CAAC,EAAE;gBACrE,CAAC,CAAC,IAAI,CAAC;YACX,WAAW,GAAG,OAAO,UAAU,WAAW,eAAe,EAAE,CAAC;YAE5D,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,eAAe,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACzD,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,WAAW;QACvB,QAAQ,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACtC,KAAK,YAAY;gBACf,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC;YAChC,KAAK,QAAQ;gBACX,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;YAC5B,KAAK,WAAW;gBACd,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC;YAC/B,KAAK,WAAW;gBACd,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC;YAC/B,KAAK,aAAa;gBAChB,OAAO,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACjC;gBACE,MAAM,IAAI,KAAK,CACb,4BAA4B,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAC5D,CAAC;QACN,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAS;QACb,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QAEtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,SAAS,CACtB,MAAM,EACN,IAAI,CAAC,MAAM,CAAC,OAAO,EACnB,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,IAAI,CAAC,GAAG,CAAC,CAC1C,CAAC;YACF,MAAM,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC5D,MAAM,UAAU,CACd,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,MAAM,CAAC,IAAI,EAChB,IAAI,EACJ,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,CACxB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CACjD,CACF,CAAC;YACF,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QAC1B,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}

package/compose.yaml

@@ -0,0 +1,15 @@
+services:
+  db:
+    build:
+      context: ./docker/db
+    container_name: db
+    restart: unless-stopped
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: postgres
+      POSTGRES_APP_DB: db_unit_tests
+      POSTGRES_APP_USER: syncables
+      POSTGRES_APP_PASSWORD: syncables

package/package.json

@@ -0,0 +1,76 @@
+{
+  "name": "syncables",
+  "version": "0.10.0",
+  "description": "Generate sync engine code from OpenAPI specifications",
+  "main": "build/src/syncable.js",
+  "type": "module",
+  "engines": {
+    "node": ">= 22.11 < 23"
+  },
+  "devDependencies": {
+    "@eslint/js": "~9.17",
+    "@faker-js/faker": "^10.2.0",
+    "@hono/node-server": "^1.19.9",
+    "@scalar/helpers": "^0.2.10",
+    "@scalar/json-magic": "^0.9.5",
+    "@scalar/openapi-parser": "^0.24.6",
+    "@scalar/openapi-types": "^0.5.3",
+    "@scalar/openapi-upgrader": "^0.1.8",
+    "@scalar/workspace-store": "^0.28.1",
+    "@stoplight/yaml": "^4.3.0",
+    "@types/eslint__js": "~8.42",
+    "@types/node": "~20",
+    "@typescript-eslint/parser": "~8.19",
+    "@typescript-eslint/utils": "^8.46.4",
+    "@vitest/coverage-v8": "^3.0.8",
+    "@vitest/eslint-plugin": "~1.1",
+    "build-seed-context": "link:../../scalar/scalar/utils/build-seed-context",
+    "eslint": "~9.17",
+    "eslint-config-prettier": "~9.1",
+    "globals": "~15.14",
+    "hono": "^4.11.7",
+    "prettier": "~3.4",
+    "rimraf": "~6.0",
+    "ts-api-utils": "~2.0",
+    "types": "link:../../scalar/scalar/types",
+    "typescript": "~5.7",
+    "typescript-eslint": "~8.19",
+    "vitest": "^3.0.8"
+  },
+  "author": "Michiel de Jong <michiel@unhosted.org>",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@readme/openapi-parser": "^5.4.0",
+    "debug": "^4.4.3",
+    "express": "^5.1.0",
+    "express-session": "^1.18.2",
+    "fast-xml-parser": "^5.3.0",
+    "openapi-overlays-js": "^0.2.0",
+    "passport": "^0.7.0",
+    "passport-oauth": "^1.0.0",
+    "pg": "^8.16.3",
+    "url-join": "^5.0.0",
+    "utils": "link:@scalar/types/utils",
+    "yaml": "^2.8.0"
+  },
+  "volta": {
+    "node": "22.12.0"
+  },
+  "scripts": {
+    "start": "node build/src/cron.js",
+    "clean": "rimraf coverage build tmp",
+    "prebuild": "npm run lint",
+    "generate": "sh ./ts-gen.sh",
+    "build": "tsc -p tsconfig.json",
+    "build:watch": "tsc -w -p tsconfig.json",
+    "build:release": "npm run clean && tsc -p tsconfig.release.json",
+    "lint": "eslint .",
+    "test": "npm run test:unit && npm run test:integration",
+    "test:unit": "vitest run unit --config __tests__/vitest.config.ts",
+    "test:integration": "vitest run integration --config __tests__/vitest.config.ts",
+    "test:coverage": "vitest run unit --config __tests__/vitest.config.ts --coverage.enabled --coverage.all",
+    "prettier": "prettier \"{src,__tests__}/**/*.{ts,mts}\" --config .prettierrc --write",
+    "prettier:check": "prettier \"{src,__tests__}/**/*.{ts,mts}\" --config .prettierrc --check",
+    "test:watch": "vitest unit"
+  }
+}