ts2workflows 0.1.0

package/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright © 2024 Antti Ajanki, <antti.ajanki@iki.fi>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+“Software”), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,82 @@
+# Typescript to GCP Workflows transpiler
+
+ts2workflows converts Typescript code to a [GCP Workflows](https://cloud.google.com/workflows/docs/apis) program.
+
+Only a subset of Typescript features are supported. The [language reference](language_reference.md) describes the supported Typescript features.
+
+See the [samples](samples) directory for code examples.
+
+Project status: It's possible to write Workflows programs, but the transpiler has not been extensively tested. Expect some rough edges!
+
+## Command for transpiling a Typescript file to Workflows syntax
+
+Converting Typescript code in a file samples/sample1.ts into GCP Workflows YAML syntax on stdout:
+
+```sh
+npx ts2workflows samples/sample1.ts
+```
+
+When developing ts2workflows, you can run the transpiler directly from the source directory:
+
+```sh
+npx tsx src/cli.ts samples/sample1.ts
+```
+
+## Type checking
+
+One benefit of writing the workflow programs in Typescript is that the sources can be type checked.
+
+This example command shows how to type check source files in the [samples](samples) directory using the standard Typescript compiler `tsc`. The command prints typing errors or finishes silently, if there are no errors.
+
+```sh
+npx tsc --project samples/tsconfig.workflows.json
+```
+
+The file [samples/tsconfig.workflows.json](samples/tsconfig.workflows.json) contains a sample configuration for type checking workflow sources.
+
+Type annotations for [Workflows standard library functions and expression helpers](https://cloud.google.com/workflows/docs/reference/stdlib/overview) and for some [connectors](https://cloud.google.com/workflows/docs/reference/googleapis) are provided in [types/workflowslib.d.ts](types/workflowslib.d.ts). They are also included in the published npm module. To import type annotations in a project that has ts2workflows module as a dependency, use the following import command:
+
+```javascript
+import { http, retry_policy } from 'ts2workflows/types/workflowslib'
+```
+
+Type checking step is completely separate from the transpiling. The ts2workflows command ignores type annotations.
+
+## Build
+
+```
+npm install
+npm run build
+```
+
+## Run unit tests
+
+```
+npm run test
+```
+
+## Architecture
+
+A transpilation using ts2workflows consists of five phases:
+
+- parsing Typescript source code
+- converting to intermediate representation (IR)
+- transforming and optimizing the IR
+- assigning names for the Workflows steps
+- outputting in the program in Workflows YAML format
+
+The main function implementing these phases is `transpile()` in [src/transpiler/index.ts](src/transpiler/index.ts).
+
+The transpiler parses a Typescript source file using [@typescript-eslint/typescript-estree](https://www.npmjs.com/package/@typescript-eslint/typescript-estree) module by the [typescript-eslint](https://typescript-eslint.io/) project. The result of the parsing is an [ESTree](https://github.com/estree/estree/blob/master/README.md)-compatible abstract syntax tree (AST) of the source code. [typescript-eslint playgroud](https://typescript-eslint.io/play/) is an useful tool for exploring the Typescript parsing.
+
+Next, ts2workflows converts the AST to a custom intermediate representation (IR). The IR format brings the representation of the program closer the GCP Workflows language than Typescript code. The types for IR are defined in the [src/ast](src/ast) directory. Particularly, see [src/ast/steps.ts](src/ast/steps.ts) for Workflow step types and [src/ast/expressions.ts](src/ast/expressions.ts) for expression types.
+
+The transpiler applies a few transformations to the IR to make it a valid and optimized Workflows program. These transformations, for example, merge consecutive assignments into a single assign step and ensure that blocking function are invoked by call steps (as required by Workflows). The transformations are implemented in [src/transpiler/transformations.ts](src/transpiler/transformations.ts).
+
+To spare the programmer from labeling each and every Workflows step manually, ts2workflows automatically assigns names to the steps. The automatically choosen step names consist of the step type and a sequential number. The implementation is in [src/ast/stepnames.ts](src/ast/stepnames.ts).
+
+Finally, the transpiler converts the result into a YAML document.
+
+## License
+
+[The MIT License](LICENSE)

package/dist/ast/expressions.d.ts

@@ -0,0 +1,57 @@
+export type VariableName = string;
+export type BinaryOperator = '+' | '-' | '*' | '/' | '//' | '%' | '==' | '!=' | '>' | '>=' | '<' | '<=' | 'in' | 'and' | 'or';
+export type UnaryOperator = '-' | '+' | 'not';
+export type Primitive = null | string | number | boolean | (Primitive | Expression)[] | {
+    [key: string]: Primitive | Expression;
+};
+export type LiteralValueOrLiteralExpression = null | string | number | boolean | LiteralValueOrLiteralExpression[] | {
+    [key: string]: LiteralValueOrLiteralExpression;
+};
+export type Expression = PrimitiveExpression | BinaryExpression | VariableReferenceExpression | FunctionInvocationExpression | MemberExpression | UnaryExpression;
+export declare class PrimitiveExpression {
+    readonly expressionType = "primitive";
+    readonly value: Primitive;
+    constructor(value: Primitive);
+    toString(): string;
+}
+export declare class BinaryExpression {
+    readonly expressionType = "binary";
+    readonly binaryOperator: BinaryOperator;
+    readonly left: Expression;
+    readonly right: Expression;
+    constructor(left: Expression, binaryOperator: BinaryOperator, right: Expression);
+    toString(): string;
+}
+export declare class VariableReferenceExpression {
+    readonly expressionType = "variableReference";
+    readonly variableName: VariableName;
+    constructor(variableName: VariableName);
+    toString(): string;
+}
+export declare class FunctionInvocationExpression {
+    readonly expressionType = "functionInvocation";
+    readonly functionName: string;
+    readonly arguments: Expression[];
+    constructor(functionName: string, argumentExpressions: Expression[]);
+    toString(): string;
+}
+export declare class MemberExpression {
+    readonly expressionType = "member";
+    readonly object: Expression;
+    readonly property: Expression;
+    readonly computed: boolean;
+    constructor(object: Expression, property: Expression, computed: boolean);
+    toString(): string;
+}
+export declare class UnaryExpression {
+    readonly expressionType = "unary";
+    readonly operator: UnaryOperator;
+    readonly value: Expression;
+    constructor(operator: UnaryOperator, value: Expression);
+    toString(): string;
+}
+export declare function isExpression(val: Primitive | Expression): val is Expression;
+export declare function expressionToLiteralValueOrLiteralExpression(ex: Expression): LiteralValueOrLiteralExpression;
+export declare function isLiteral(ex: Expression): boolean;
+export declare function isFullyQualifiedName(ex: Expression): boolean;
+//# sourceMappingURL=expressions.d.ts.map

package/dist/ast/expressions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"expressions.d.ts","sourceRoot":"","sources":["../../src/ast/expressions.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,YAAY,GAAG,MAAM,CAAA;AACjC,MAAM,MAAM,cAAc,GACtB,GAAG,GACH,GAAG,GACH,GAAG,GACH,GAAG,GACH,IAAI,GACJ,GAAG,GACH,IAAI,GACJ,IAAI,GACJ,GAAG,GACH,IAAI,GACJ,GAAG,GACH,IAAI,GACJ,IAAI,GACJ,KAAK,GACL,IAAI,CAAA;AACR,MAAM,MAAM,aAAa,GAAG,GAAG,GAAG,GAAG,GAAG,KAAK,CAAA;AA2B7C,MAAM,MAAM,SAAS,GACjB,IAAI,GACJ,MAAM,GACN,MAAM,GACN,OAAO,GACP,CAAC,SAAS,GAAG,UAAU,CAAC,EAAE,GAC1B;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,GAAG,UAAU,CAAA;CAAE,CAAA;AAE7C,MAAM,MAAM,+BAA+B,GACvC,IAAI,GACJ,MAAM,GACN,MAAM,GACN,OAAO,GACP,+BAA+B,EAAE,GACjC;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,+BAA+B,CAAA;CAAE,CAAA;AAEtD,MAAM,MAAM,UAAU,GAClB,mBAAmB,GACnB,gBAAgB,GAChB,2BAA2B,GAC3B,4BAA4B,GAC5B,gBAAgB,GAChB,eAAe,CAAA;AAInB,qBAAa,mBAAmB;IAC9B,QAAQ,CAAC,cAAc,eAAc;IACrC,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAA;gBAEb,KAAK,EAAE,SAAS;IAM5B,QAAQ,IAAI,MAAM;CAgBnB;AAGD,qBAAa,gBAAgB;IAC3B,QAAQ,CAAC,cAAc,YAAW;IAClC,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAA;IACvC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAA;IACzB,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAA;gBAGxB,IAAI,EAAE,UAAU,EAChB,cAAc,EAAE,cAAc,EAC9B,KAAK,EAAE,UAAU;IAOnB,QAAQ,IAAI,MAAM;CAwBnB;AAID,qBAAa,2BAA2B;IACtC,QAAQ,CAAC,cAAc,uBAAsB;IAC7C,QAAQ,CAAC,YAAY,EAAE,YAAY,CAAA;gBAEvB,YAAY,EAAE,YAAY;IAItC,QAAQ,IAAI,MAAM;CAGnB;AAGD,qBAAa,4BAA4B;IACvC,QAAQ,CAAC,cAAc,wBAAuB;IAC9C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,SAAS,EAAE,UAAU,EAAE,CAAA;gBAEpB,YAAY,EAAE,MAAM,EAAE,mBAAmB,EAAE,UAAU,EAAE;IAKnE,QAAQ,IAAI,MAAM;CAInB;AAGD,qBAAa,gBAAgB;IAC3B,QAAQ,CAAC,cAAc,YAAW;IAClC,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAA;IAC3B,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAA;IAC7B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAA;gBAEd,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO;IAMvE,QAAQ,IAAI,MAAM;CAOnB;AAED,qBAAa,eAAe;IAC1B,QAAQ,CAAC,cAAc,WAAU;IACjC,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAA;IAChC,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAA;gBAEd,QAAQ,EAAE,aAAa,EAAE,KAAK,EAAE,UAAU;IAKtD,QAAQ,IAAI,MAAM;CAQnB;AAuBD,wBAAgB,YAAY,CAAC,GAAG,EAAE,SAAS,GAAG,UAAU,GAAG,GAAG,IAAI,UAAU,CAE3E;AAGD,wBAAgB,2CAA2C,CACzD,EAAE,EAAE,UAAU,GACb,+BAA+B,CA2BjC;AA+DD,wBAAgB,SAAS,CAAC,EAAE,EAAE,UAAU,GAAG,OAAO,CAcjD;AAqBD,wBAAgB,oBAAoB,CAAC,EAAE,EAAE,UAAU,GAAG,OAAO,CAmB5D"}

package/dist/ast/expressions.js

@@ -0,0 +1,300 @@
+import { isRecord } from '../utils.js';
+// Operator precendence for unary and binary operators in the Workflows language.
+// Note that these differ somewhat from Javascript.
+// https://cloud.google.com/workflows/docs/reference/syntax/datatypes
+const operatorPrecedenceValue = new Map([
+    ['not', 7],
+    // ['-', 6], // unary minus, commented out to avoid confusion with binary minus
+    ['*', 5],
+    ['/', 5],
+    ['//', 5],
+    ['%', 5],
+    ['+', 4],
+    ['-', 4],
+    ['<=', 3],
+    ['>=', 3],
+    ['<', 3],
+    ['>', 3],
+    ['==', 3],
+    ['===', 3],
+    ['!=', 3],
+    ['!==', 3],
+    ['in', 3],
+    ['and', 2],
+    ['or', 1],
+]);
+// A primitive (string, number, list, etc) value
+// TODO: Is this needed? Use unboxed Primitive instead?
+export class PrimitiveExpression {
+    expressionType = 'primitive';
+    value;
+    constructor(value) {
+        this.value = value;
+    }
+    // Return the string representation of this expression.
+    // Not enclosed in ${}.
+    toString() {
+        const val = this.value;
+        if (Array.isArray(val)) {
+            const elements = val.map((v) => {
+                return isExpression(v) ? v.toString() : primitiveToString(v);
+            });
+            return `[${elements.join(', ')}]`;
+        }
+        else if (isRecord(val)) {
+            const elements = Object.entries(val).map(([k, v]) => {
+                return `"${k}": ${isExpression(v) ? v.toString() : primitiveToString(v)}`;
+            });
+            return `{${elements.join(', ')}}`;
+        }
+        else {
+            return `${JSON.stringify(val)}`;
+        }
+    }
+}
+// expr OPERATOR expr
+export class BinaryExpression {
+    expressionType = 'binary';
+    binaryOperator;
+    left;
+    right;
+    constructor(left, binaryOperator, right) {
+        this.binaryOperator = binaryOperator;
+        this.left = left;
+        this.right = right;
+    }
+    toString() {
+        let leftString = this.left.toString();
+        let rightString = this.right.toString();
+        if (this.left.expressionType === 'binary') {
+            const leftOpValue = operatorPrecedenceValue.get(this.left.binaryOperator) ?? 0;
+            const thisOpValue = operatorPrecedenceValue.get(this.binaryOperator) ?? 0;
+            if (leftOpValue < thisOpValue) {
+                leftString = `(${leftString})`;
+            }
+        }
+        if (this.right.expressionType === 'binary') {
+            const rightOpValue = operatorPrecedenceValue.get(this.right.binaryOperator) ?? 0;
+            const thisOpValue = operatorPrecedenceValue.get(this.binaryOperator) ?? 0;
+            if (rightOpValue < thisOpValue) {
+                rightString = `(${rightString})`;
+            }
+        }
+        return `${leftString} ${this.binaryOperator} ${rightString}`;
+    }
+}
+// Variable name: a plain identifier (y, year) or a list
+// element accessor (names[3])
+export class VariableReferenceExpression {
+    expressionType = 'variableReference';
+    variableName;
+    constructor(variableName) {
+        this.variableName = variableName;
+    }
+    toString() {
+        return this.variableName;
+    }
+}
+// Function invocation with anonymous parameters: sys.get_env("GOOGLE_CLOUD_PROJECT_ID")
+export class FunctionInvocationExpression {
+    expressionType = 'functionInvocation';
+    functionName;
+    arguments;
+    constructor(functionName, argumentExpressions) {
+        this.functionName = functionName;
+        this.arguments = argumentExpressions;
+    }
+    toString() {
+        const argumentStrings = this.arguments.map((x) => x.toString());
+        return `${this.functionName}(${argumentStrings.join(', ')})`;
+    }
+}
+// object.property or object[property]
+export class MemberExpression {
+    expressionType = 'member';
+    object;
+    property;
+    computed;
+    constructor(object, property, computed) {
+        this.object = object;
+        this.property = property;
+        this.computed = computed;
+    }
+    toString() {
+        if (this.computed) {
+            return `${this.object.toString()}[${this.property.toString()}]`;
+        }
+        else {
+            return `${this.object.toString()}.${this.property.toString()}`;
+        }
+    }
+}
+export class UnaryExpression {
+    expressionType = 'unary';
+    operator;
+    value;
+    constructor(operator, value) {
+        this.operator = operator;
+        this.value = value;
+    }
+    toString() {
+        const separator = this.operator === 'not' ? ' ' : '';
+        let valueString = this.value.toString();
+        if (this.value.expressionType === 'binary') {
+            valueString = `(${valueString})`;
+        }
+        return `${this.operator}${separator}${valueString}`;
+    }
+}
+// Convert a Primitive to a string representation.
+// Does not add ${}.
+function primitiveToString(val) {
+    if (Array.isArray(val)) {
+        const elements = val.map((x) => {
+            return isExpression(x) ? x.toString() : primitiveToString(x);
+        });
+        return `[${elements.join(', ')}]`;
+    }
+    else if (val !== null && typeof val === 'object') {
+        const items = Object.entries(val).map(([k, v]) => {
+            return [k, isExpression(v) ? v.toString() : primitiveToString(v)];
+        });
+        const elements = items.map(([k, v]) => `"${k}":${v}`);
+        return `{${elements.join(',')}}`;
+    }
+    else {
+        return JSON.stringify(val);
+    }
+}
+export function isExpression(val) {
+    return val instanceof Object && 'expressionType' in val && !isRecord(val);
+}
+// Returns a literal for simple terms and a literal expression enclosed in ${} for complex terms.
+export function expressionToLiteralValueOrLiteralExpression(ex) {
+    switch (ex.expressionType) {
+        case 'primitive':
+            return primitiveExpressionToLiteralValueOrLiteralExpression(ex);
+        case 'binary':
+        case 'variableReference':
+        case 'functionInvocation':
+        case 'member':
+            return `\${${ex.toString()}}`;
+        case 'unary':
+            if (ex.value.expressionType === 'primitive' &&
+                typeof ex.value.value === 'number') {
+                if (ex.operator === '+') {
+                    return ex.value.value;
+                }
+                else if (ex.operator === '-') {
+                    return -ex.value.value;
+                }
+                else {
+                    return `\${${ex.toString()}}`;
+                }
+            }
+            else {
+                return `\${${ex.toString()}}`;
+            }
+    }
+}
+function primitiveExpressionToLiteralValueOrLiteralExpression(ex) {
+    if (typeof ex.value === 'number' ||
+        typeof ex.value === 'string' ||
+        typeof ex.value === 'boolean' ||
+        ex.value === null) {
+        return ex.value;
+    }
+    else if (Array.isArray(ex.value)) {
+        return ex.value.map((x) => {
+            if (isExpression(x)) {
+                return expressionToLiteralValueOrLiteralExpression(x);
+            }
+            else if (Array.isArray(x) || isRecord(x)) {
+                return primitiveExpressionToLiteralValueOrLiteralExpression(new PrimitiveExpression(x));
+            }
+            else if (x === null ||
+                typeof x === 'string' ||
+                typeof x === 'number' ||
+                typeof x === 'boolean') {
+                return x;
+            }
+            else {
+                return `\${${primitiveToString(x)}}`;
+            }
+        });
+    }
+    else if (isRecord(ex.value)) {
+        return Object.fromEntries(Object.entries(ex.value).map(([k, v]) => {
+            if (isExpression(v)) {
+                return [k, expressionToLiteralValueOrLiteralExpression(v)];
+            }
+            else if (Array.isArray(v) || isRecord(v)) {
+                return [
+                    k,
+                    primitiveExpressionToLiteralValueOrLiteralExpression(new PrimitiveExpression(v)),
+                ];
+            }
+            else if (v === null ||
+                typeof v === 'string' ||
+                typeof v === 'number' ||
+                typeof v === 'boolean') {
+                return [k, v];
+            }
+            else {
+                return [k, `\${${primitiveToString(v)}}`];
+            }
+        }));
+    }
+    else {
+        return `\${${ex.toString()}}`;
+    }
+}
+// Returns true if expression is a literal value.
+// Examples of literals: number, string, array of numbers or strings, etc.
+// Examples of non-literals: array that contains complex expressions.
+export function isLiteral(ex) {
+    switch (ex.expressionType) {
+        case 'primitive':
+            return primitiveIsLiteral(ex.value);
+        case 'unary':
+            return isLiteral(ex.value);
+        case 'binary':
+        case 'variableReference':
+        case 'functionInvocation':
+        case 'member':
+            return false;
+    }
+}
+function primitiveIsLiteral(value) {
+    if (Array.isArray(value)) {
+        return value.every((x) => {
+            return isExpression(x) ? isLiteral(x) : primitiveIsLiteral(x);
+        });
+    }
+    else if (isRecord(value)) {
+        return Object.values(value).every((x) => {
+            return isExpression(x) ? isLiteral(x) : primitiveIsLiteral(x);
+        });
+    }
+    else {
+        return (typeof value === 'string' ||
+            typeof value === 'number' ||
+            typeof value === 'boolean' ||
+            value === null);
+    }
+}
+export function isFullyQualifiedName(ex) {
+    switch (ex.expressionType) {
+        case 'primitive':
+        case 'binary':
+        case 'functionInvocation':
+            return false;
+        case 'variableReference':
+            return true;
+        case 'member':
+            return (isFullyQualifiedName(ex.object) &&
+                (ex.computed || isFullyQualifiedName(ex.property)));
+        case 'unary':
+            return isFullyQualifiedName(ex.value);
+    }
+}

package/dist/ast/stepnames.d.ts

@@ -0,0 +1,9 @@
+import { WorkflowAST } from './steps.js';
+import { WorkflowApp } from './workflows.js';
+export declare class StepNameGenerator {
+    private counters;
+    constructor();
+    generate(prefix: string): string;
+}
+export declare function generateStepNames(ast: WorkflowAST): WorkflowApp;
+//# sourceMappingURL=stepnames.d.ts.map

package/dist/ast/stepnames.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stepnames.d.ts","sourceRoot":"","sources":["../../src/ast/stepnames.ts"],"names":[],"mappings":"AAAA,OAAO,EASL,WAAW,EAGZ,MAAM,YAAY,CAAA;AACnB,OAAO,EAAe,WAAW,EAAE,MAAM,gBAAgB,CAAA;AAQzD,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAqB;;IAMrC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;CAMjC;AAED,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,WAAW,GAAG,WAAW,CAS/D"}