tmql-orchestration 0.4.1

package/LICENSE

@@ -0,0 +1,57 @@
+## Elastic License 2.0 (ELv2)
+
+### Acceptance
+
+By using the software, you agree to all of the terms and conditions below.
+
+### Copyright License
+
+The licensor grants you a non-exclusive, royalty-free, worldwide, non-sublicensable, non-transferable license to use, copy, distribute, make available, and prepare derivative works of the software, in each case subject to the limitations and conditions below.
+
+### Limitations
+
+You may not provide the software to third parties as a hosted or managed service, where the service provides users with access to any substantial set of the features or functionality of the software.
+
+You may not move, change, disable, or circumvent the license key functionality in the software, and you may not remove or obscure any functionality in the software that is protected by the license key.
+
+You may not alter, remove, or obscure any licensing, copyright, or other notices of the licensor in the software. Any use of the licensor's trademarks is subject to applicable law.
+
+### Patents
+
+The licensor grants you a license, under any patent claims the licensor can license, or becomes able to license, to make, have made, use, sell, offer for sale, import and have imported the software, in each case subject to the limitations and conditions in this license. This license does not cover any patent claims that you cause to be infringed by modifications or additions to the software. If you or your company make any written claim that the software infringes or contributes to infringement of any patent, your patent license for the software granted under these terms ends immediately. If your company makes such a claim, your patent license ends immediately for work on behalf of your company.
+
+### Notices
+
+You must ensure that anyone who gets a copy of any part of the software from you also gets a copy of these terms.
+
+If you modify the software, you must include in any modified copies of the software prominent notices stating that you have modified the software.
+
+### No Other Rights
+
+These terms do not imply any licenses other than those expressly granted in these terms.
+
+### Termination
+
+If you use the software in violation of these terms, such use is not licensed, and your licenses will automatically terminate. If the licensor provides you with a notice of your violation, and you cease all violation of this license no later than 30 days after you receive that notice, your licenses will be reinstated retroactively. However, if you violate these terms after such reinstatement, any additional violation of these terms will cause your licenses to terminate automatically and permanently.
+
+### No Liability
+
+_As far as the law allows, the software comes as is, without any warranty or condition, and the licensor will not be liable to you for any damages arising out of these terms or the use or nature of the software, under any kind of legal claim._
+
+### Definitions
+
+The **licensor** is the entity offering these terms, and the **software** is the software the licensor makes available under these terms, including any portion of it.
+
+**you** refers to the individual or entity agreeing to these terms.
+
+**your company** is any legal entity, sole proprietorship, or other kind of organization that you work for, plus all organizations that have control over, are under the control of, or are under common control with that organization. **control** means ownership of substantially all the assets of an entity, or the power to direct its management and policies by vote, contract, or otherwise. Control can be direct or indirect.
+
+**your licenses** are all the licenses granted to you for the software under these terms.
+
+**use** means anything you do with the software requiring one of your licenses.
+
+**trademark** means trademarks, service marks, and similar rights.
+
+---
+
+Copyright (c) 2026 Ryan Peggs & Tim Vyas. All rights reserved.

package/dist/index.d.ts

@@ -0,0 +1,12 @@
+/**
+ * tmql-orchestration
+ *
+ * DAG composition and orchestration layer for tmql.
+ * Provides TMModel and TMProject for building data pipelines.
+ */
+export { TMModel, isTMModel } from "./model/TMModel";
+export { TMProject } from "./project/TMProject";
+export type { InferModelOutput } from "./model/TMModel";
+export type { MaterializeConfig, MergeOptions, CollectionMode, TypedTimeSeriesOptions, ModelConfig, } from "./model/TMModel";
+export type { ProjectConfig, RunOptions, ModelRunStats, ProjectRunResult, ExecutionPlan, ValidationResult, ValidationError, ValidationWarning, } from "./project/TMProject";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAGhD,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,YAAY,EACV,iBAAiB,EACjB,YAAY,EACZ,cAAc,EACd,sBAAsB,EACtB,WAAW,GACZ,MAAM,iBAAiB,CAAC;AACzB,YAAY,EACV,aAAa,EACb,UAAU,EACV,aAAa,EACb,gBAAgB,EAChB,aAAa,EACb,gBAAgB,EAChB,eAAe,EACf,iBAAiB,GAClB,MAAM,qBAAqB,CAAC"}

package/dist/index.js

@@ -0,0 +1,9 @@
+/**
+ * tmql-orchestration
+ *
+ * DAG composition and orchestration layer for tmql.
+ * Provides TMModel and TMProject for building data pipelines.
+ */
+// DAG Composition
+export { TMModel, isTMModel } from "./model/TMModel";
+export { TMProject } from "./project/TMProject";

package/dist/model/TMModel.d.ts

@@ -0,0 +1,162 @@
+/**
+ * TMModel - DAG Pipeline Composition
+ *
+ * A model represents a named, materializable pipeline with typed input/output.
+ * Models can depend on collections or other models, forming a DAG.
+ */
+import { TimeSeriesCollectionOptions } from "mongodb";
+import { Document, TMPipeline, TMSource, InferSourceType, FieldSelector, FieldSelectorsThatInferTo, TopLevelField } from "tmql";
+/**
+ * Type-safe extension of MongoDB's TimeSeriesCollectionOptions.
+ * Constrains timeField to Date fields and metaField to string keys.
+ */
+export type TypedTimeSeriesOptions<TDoc extends Document> = Omit<TimeSeriesCollectionOptions, "timeField" | "metaField"> & {
+    /** Must be a field of type Date in TDoc */
+    timeField: TopLevelField<FieldSelectorsThatInferTo<TDoc, Date>>;
+    /** Must be a string key in TDoc */
+    metaField?: TopLevelField<FieldSelector<TDoc>>;
+    /** TTL - auto-expire documents after N seconds */
+    expireAfterSeconds?: number;
+};
+/**
+ * Type-safe $merge stage options.
+ */
+type TopLevelFieldOf<T extends Document> = TopLevelField<FieldSelector<T>>;
+export type MergeOptions<TOutput extends Document> = {
+    /** Field(s) to match on - must exist in output document */
+    on: TopLevelFieldOf<TOutput> | TopLevelFieldOf<TOutput>[];
+    /** Action when document matches existing */
+    whenMatched?: "replace" | "merge" | "keepExisting" | "fail";
+    /** Action when document doesn't match existing */
+    whenNotMatched?: "insert" | "discard" | "fail";
+};
+export type CollectionMode<TOutput extends Document> = {
+    $out: Record<string, never>;
+} | {
+    $merge: MergeOptions<TOutput>;
+};
+/**
+ * Materialization configuration.
+ */
+export type MaterializeConfig<TOutput extends Document = Document> = {
+    type: "view";
+    db?: string;
+} | {
+    type: "collection";
+    db?: string;
+    mode: CollectionMode<TOutput>;
+    timeseries?: TypedTimeSeriesOptions<TOutput>;
+};
+/**
+ * Model configuration - source can be a collection OR another model.
+ * When source is a model, this creates an implicit DAG edge.
+ *
+ * Pipelines inside models use `mode: "model"` which allows lookup/unionWith
+ * from other TMModels (not just TMCollections).
+ */
+export type ModelConfig<TName extends string, TSource extends TMSource<Document>, TOutput extends Document, TMaterializeConfig extends MaterializeConfig<TOutput>> = {
+    name: TName;
+    /** Source collection or upstream model */
+    from: TSource;
+    /** Pipeline function - receives a "model" mode pipeline that can lookup from other models */
+    pipeline: (p: TMPipeline<InferSourceType<TSource>, InferSourceType<TSource>, "model">) => TMPipeline<InferSourceType<TSource>, TOutput, "model">;
+    /** Materialization configuration - required */
+    materialize: TMaterializeConfig;
+};
+/**
+ * Extract the output document type from a TMModel.
+ */
+export type InferModelOutput<T> = T extends TMModel<any, any, infer O, any> ? O : never;
+/**
+ * Type predicate to check if a value is a TMModel.
+ */
+export declare function isTMModel(value: unknown): value is TMModel;
+/**
+ * A named, materializable pipeline with typed input/output.
+ * Models form a DAG through their `from` property.
+ */
+export declare class TMModel<TName extends string = string, TInput extends Document = Document, TOutput extends Document = Document, TMaterializeConfig extends MaterializeConfig<TOutput> = MaterializeConfig<TOutput>> implements TMSource<TOutput> {
+    /** Preset modes for common materialization patterns */
+    static readonly Mode: {
+        /** Replace entire collection using $out */
+        readonly Replace: {
+            readonly $out: {};
+        };
+        /** Upsert by _id using $merge */
+        readonly Upsert: {
+            readonly $merge: {
+                readonly on: "_id";
+                readonly whenMatched: "replace";
+                readonly whenNotMatched: "insert";
+            };
+        };
+        /** Append only (fail on match) using $merge */
+        readonly Append: {
+            readonly $merge: {
+                readonly on: "_id";
+                readonly whenMatched: "fail";
+                readonly whenNotMatched: "insert";
+            };
+        };
+    };
+    /** TMSource discriminator */
+    readonly sourceType: "model";
+    readonly name: TName;
+    readonly materialize: TMaterializeConfig;
+    readonly __inputType: TInput;
+    readonly __outputType: TOutput;
+    private readonly _from;
+    private readonly _pipelineFn;
+    constructor(config: ModelConfig<TName, TMSource<TInput>, TOutput, TMaterializeConfig>);
+    /**
+     * Get the source (collection or upstream model).
+     */
+    getSource(): TMSource<TInput>;
+    /**
+     * Check if the source is a model (vs a collection).
+     */
+    sourceIsModel(): boolean;
+    /**
+     * Get the upstream model if source is a model, otherwise undefined.
+     */
+    getUpstreamModel(): TMModel<string, any, TInput, any> | undefined;
+    /**
+     * Get the source collection name.
+     */
+    getSourceCollectionName(): string;
+    /**
+     * Get the source database name.
+     */
+    getSourceDatabase(): string | undefined;
+    /**
+     * Get the output collection name (where this model materializes).
+     */
+    getOutputCollectionName(): string;
+    /**
+     * Get the database name for output (if specified).
+     */
+    getOutputDatabase(): string | undefined;
+    /**
+     * Build the pipeline stages for this model.
+     */
+    getPipelineStages(): Document[];
+    /**
+     * Get ancestor sources from lookup/unionWith stages.
+     * These are sources referenced in the pipeline but not via the `from` property.
+     */
+    getAncestorsFromStages(): TMSource<any>[];
+    /**
+     * Internal: build the pipeline and return the TMPipeline instance.
+     */
+    private _buildPipeline;
+    /**
+     * Build the complete aggregation pipeline including output stage.
+     */
+    buildPipeline(): Document[];
+    /**
+     * Build the $out or $merge stage based on materialization config.
+     */
+    private buildOutputStage;
+}
+export {};
+//# sourceMappingURL=TMModel.d.ts.map

package/dist/model/TMModel.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TMModel.d.ts","sourceRoot":"","sources":["../../src/model/TMModel.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,2BAA2B,EAAE,MAAM,SAAS,CAAC;AACtD,OAAO,EACL,QAAQ,EACR,UAAU,EACV,QAAQ,EACR,eAAe,EACf,aAAa,EACb,yBAAyB,EACzB,aAAa,EACd,MAAM,MAAM,CAAC;AAMd;;;GAGG;AACH,MAAM,MAAM,sBAAsB,CAAC,IAAI,SAAS,QAAQ,IAAI,IAAI,CAC9D,2BAA2B,EAC3B,WAAW,GAAG,WAAW,CAC1B,GAAG;IACF,2CAA2C;IAC3C,SAAS,EAAE,aAAa,CAAC,yBAAyB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IAChE,mCAAmC;IACnC,SAAS,CAAC,EAAE,aAAa,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/C,kDAAkD;IAClD,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B,CAAC;AAMF;;GAEG;AACH,KAAK,eAAe,CAAC,CAAC,SAAS,QAAQ,IAAI,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;AAE3E,MAAM,MAAM,YAAY,CAAC,OAAO,SAAS,QAAQ,IAAI;IACnD,2DAA2D;IAC3D,EAAE,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;IAC1D,4CAA4C;IAC5C,WAAW,CAAC,EAAE,SAAS,GAAG,OAAO,GAAG,cAAc,GAAG,MAAM,CAAC;IAC5D,kDAAkD;IAClD,cAAc,CAAC,EAAE,QAAQ,GAAG,SAAS,GAAG,MAAM,CAAC;CAChD,CAAC;AAEF,MAAM,MAAM,cAAc,CAAC,OAAO,SAAS,QAAQ,IAC/C;IAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;CAAE,GAC/B;IAAE,MAAM,EAAE,YAAY,CAAC,OAAO,CAAC,CAAA;CAAE,CAAC;AAMtC;;GAEG;AACH,MAAM,MAAM,iBAAiB,CAAC,OAAO,SAAS,QAAQ,GAAG,QAAQ,IAC7D;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,CAAC,EAAE,MAAM,CAAA;CAAE,GAC7B;IACE,IAAI,EAAE,YAAY,CAAC;IACnB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,sBAAsB,CAAC,OAAO,CAAC,CAAC;CAC9C,CAAC;AAMN;;;;;;GAMG;AACH,MAAM,MAAM,WAAW,CACrB,KAAK,SAAS,MAAM,EACpB,OAAO,SAAS,QAAQ,CAAC,QAAQ,CAAC,EAClC,OAAO,SAAS,QAAQ,EACxB,kBAAkB,SAAS,iBAAiB,CAAC,OAAO,CAAC,IACnD;IACF,IAAI,EAAE,KAAK,CAAC;IACZ,0CAA0C;IAC1C,IAAI,EAAE,OAAO,CAAC;IACd,6FAA6F;IAC7F,QAAQ,EAAE,CACR,CAAC,EAAE,UAAU,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,eAAe,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,KACvE,UAAU,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC5D,+CAA+C;IAC/C,WAAW,EAAE,kBAAkB,CAAC;CACjC,CAAC;AAMF;;GAEG;AACH,MAAM,MAAM,gBAAgB,CAAC,CAAC,IAC5B,CAAC,SAAS,OAAO,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;AAExD;;GAEG;AACH,wBAAgB,SAAS,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,OAAO,CAO1D;AAED;;;GAGG;AACH,qBAAa,OAAO,CAClB,KAAK,SAAS,MAAM,GAAG,MAAM,EAC7B,MAAM,SAAS,QAAQ,GAAG,QAAQ,EAClC,OAAO,SAAS,QAAQ,GAAG,QAAQ,EACnC,kBAAkB,SAChB,iBAAiB,CAAC,OAAO,CAAC,GAAG,iBAAiB,CAAC,OAAO,CAAC,CACzD,YAAW,QAAQ,CAAC,OAAO,CAAC;IAE5B,uDAAuD;IACvD,MAAM,CAAC,QAAQ,CAAC,IAAI;QAClB,2CAA2C;;;;QAE3C,iCAAiC;;;;;;;;QAIjC,+CAA+C;;;;;;;;MAItC;IAEX,6BAA6B;IAC7B,QAAQ,CAAC,UAAU,EAAG,OAAO,CAAU;IAEvC,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;IACrB,QAAQ,CAAC,WAAW,EAAE,kBAAkB,CAAC;IAGzC,QAAQ,CAAC,WAAW,EAAG,MAAM,CAAC;IAC9B,QAAQ,CAAC,YAAY,EAAG,OAAO,CAAC;IAEhC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAmB;IACzC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAEc;gBAGxC,MAAM,EAAE,WAAW,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,kBAAkB,CAAC;IAU3E;;OAEG;IACH,SAAS,IAAI,QAAQ,CAAC,MAAM,CAAC;IAI7B;;OAEG;IACH,aAAa,IAAI,OAAO;IAIxB;;OAEG;IACH,gBAAgB,IAAI,OAAO,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAOjE;;OAEG;IACH,uBAAuB,IAAI,MAAM;IAIjC;;OAEG;IACH,iBAAiB,IAAI,MAAM,GAAG,SAAS;IAIvC;;OAEG;IACH,uBAAuB,IAAI,MAAM;IAIjC;;OAEG;IACH,iBAAiB,IAAI,MAAM,GAAG,SAAS;IAOvC;;OAEG;IACH,iBAAiB,IAAI,QAAQ,EAAE;IAI/B;;;OAGG;IACH,sBAAsB,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE;IAIzC;;OAEG;IACH,OAAO,CAAC,cAAc;IAUtB;;OAEG;IACH,aAAa,IAAI,QAAQ,EAAE;IAY3B;;OAEG;IACH,OAAO,CAAC,gBAAgB;CAqCzB"}

package/dist/model/TMModel.js

@@ -0,0 +1,167 @@
+/**
+ * TMModel - DAG Pipeline Composition
+ *
+ * A model represents a named, materializable pipeline with typed input/output.
+ * Models can depend on collections or other models, forming a DAG.
+ */
+import { TMPipeline, } from "tmql";
+/**
+ * Type predicate to check if a value is a TMModel.
+ */
+export function isTMModel(value) {
+    return (typeof value === "object" &&
+        value !== null &&
+        "sourceType" in value &&
+        value.sourceType === "model");
+}
+/**
+ * A named, materializable pipeline with typed input/output.
+ * Models form a DAG through their `from` property.
+ */
+export class TMModel {
+    /** Preset modes for common materialization patterns */
+    static Mode = {
+        /** Replace entire collection using $out */
+        Replace: { $out: {} },
+        /** Upsert by _id using $merge */
+        Upsert: {
+            $merge: { on: "_id", whenMatched: "replace", whenNotMatched: "insert" },
+        },
+        /** Append only (fail on match) using $merge */
+        Append: {
+            $merge: { on: "_id", whenMatched: "fail", whenNotMatched: "insert" },
+        },
+    };
+    /** TMSource discriminator */
+    sourceType = "model";
+    name;
+    materialize;
+    // Phantom types for inference (not used at runtime)
+    __inputType;
+    __outputType;
+    _from;
+    _pipelineFn;
+    constructor(config) {
+        this.name = config.name;
+        this._from = config.from;
+        this._pipelineFn = config.pipeline;
+        this.materialize = config.materialize;
+    }
+    /**
+     * Get the source (collection or upstream model).
+     */
+    getSource() {
+        return this._from;
+    }
+    /**
+     * Check if the source is a model (vs a collection).
+     */
+    sourceIsModel() {
+        return isTMModel(this._from);
+    }
+    /**
+     * Get the upstream model if source is a model, otherwise undefined.
+     */
+    getUpstreamModel() {
+        if (this.sourceIsModel()) {
+            return this._from;
+        }
+        return undefined;
+    }
+    /**
+     * Get the source collection name.
+     */
+    getSourceCollectionName() {
+        return this._from.getOutputCollectionName();
+    }
+    /**
+     * Get the source database name.
+     */
+    getSourceDatabase() {
+        return this._from.getOutputDatabase();
+    }
+    /**
+     * Get the output collection name (where this model materializes).
+     */
+    getOutputCollectionName() {
+        return this.name;
+    }
+    /**
+     * Get the database name for output (if specified).
+     */
+    getOutputDatabase() {
+        if ("db" in this.materialize) {
+            return this.materialize.db;
+        }
+        return undefined;
+    }
+    /**
+     * Build the pipeline stages for this model.
+     */
+    getPipelineStages() {
+        return this._buildPipeline().getPipeline();
+    }
+    /**
+     * Get ancestor sources from lookup/unionWith stages.
+     * These are sources referenced in the pipeline but not via the `from` property.
+     */
+    getAncestorsFromStages() {
+        return this._buildPipeline().getAncestorsFromStages();
+    }
+    /**
+     * Internal: build the pipeline and return the TMPipeline instance.
+     */
+    _buildPipeline() {
+        // Start with an empty "model" mode pipeline (allows lookup from other models)
+        const startPipeline = new TMPipeline({
+            pipeline: [],
+        });
+        // Execute pipeline function
+        return this._pipelineFn(startPipeline);
+    }
+    /**
+     * Build the complete aggregation pipeline including output stage.
+     */
+    buildPipeline() {
+        const stages = this.getPipelineStages();
+        // Add output stage based on materialization
+        const outputStage = this.buildOutputStage();
+        if (outputStage) {
+            return [...stages, outputStage];
+        }
+        return stages;
+    }
+    /**
+     * Build the $out or $merge stage based on materialization config.
+     */
+    buildOutputStage() {
+        const outputCollection = this.getOutputCollectionName();
+        const outputDb = this.getOutputDatabase();
+        if (this.materialize.type === "view") {
+            // Views are created separately, not via pipeline
+            return null;
+        }
+        if (this.materialize.type === "collection") {
+            const mode = this.materialize.mode;
+            if ("$out" in mode) {
+                return outputDb ?
+                    { $out: { db: outputDb, coll: outputCollection } }
+                    : { $out: outputCollection };
+            }
+            if ("$merge" in mode) {
+                const into = outputDb ?
+                    { db: outputDb, coll: outputCollection }
+                    : outputCollection;
+                return {
+                    $merge: {
+                        into,
+                        on: mode.$merge.on,
+                        whenMatched: mode.$merge.whenMatched ?? "replace",
+                        whenNotMatched: mode.$merge.whenNotMatched ?? "insert",
+                    },
+                };
+            }
+        }
+        return null;
+    }
+}

package/dist/project/TMProject.d.ts

@@ -0,0 +1,141 @@
+/**
+ * TMProject - DAG Orchestrator for TMModels
+ *
+ * Manages a collection of models, resolves dependencies,
+ * and executes them in topological order.
+ */
+import { MongoClient } from "mongodb";
+import { TMModel } from "../model/TMModel";
+export type ProjectConfig = {
+    name: string;
+    models: TMModel<any, any, any, any>[];
+    defaultDatabase?: string;
+};
+export type RunOptions = {
+    /** Specific models to run (default: all) */
+    targets?: string[];
+    /** Models to exclude */
+    exclude?: string[];
+    /** Log plan without executing */
+    dryRun?: boolean;
+    /** MongoDB client (falls back to tmql singleton) */
+    client?: MongoClient;
+    /** Database name (falls back to project default) */
+    databaseName?: string;
+    /** Callbacks */
+    onModelStart?: (name: string) => void;
+    onModelComplete?: (name: string, stats: ModelRunStats) => void;
+    onModelError?: (name: string, error: Error) => void;
+};
+export type ModelRunStats = {
+    durationMs: number;
+};
+export type ProjectRunResult = {
+    success: boolean;
+    modelsRun: string[];
+    modelsFailed: string[];
+    stats: Record<string, ModelRunStats>;
+    totalDurationMs: number;
+};
+export type ExecutionPlan = {
+    /** Models grouped by parallel execution stage */
+    stages: string[][];
+    /** Total models in plan */
+    totalModels: number;
+    /** Render as Mermaid diagram */
+    toMermaid(): string;
+    /** Render as text */
+    toString(): string;
+};
+export type ValidationResult = {
+    valid: boolean;
+    errors: ValidationError[];
+    warnings: ValidationWarning[];
+};
+export type ValidationError = {
+    type: "cycle" | "missing_ref" | "duplicate_name";
+    message: string;
+    models: string[];
+};
+export type ValidationWarning = {
+    type: "orphan";
+    message: string;
+    models: string[];
+};
+/**
+ * TMProject - DAG orchestrator for TMModels.
+ *
+ * Models are provided at construction time and validated immediately.
+ * The project is immutable after creation.
+ *
+ * @example
+ * ```typescript
+ * import { stgEvents, dailyMetrics } from "./models/analytics";
+ *
+ * const analyticsProject = new TMProject({
+ *   name: "analytics",
+ *   models: [stgEvents, dailyMetrics],
+ * });
+ *
+ * // See execution plan
+ * console.log(analyticsProject.toMermaid());
+ *
+ * // Run all models
+ * await analyticsProject.run();
+ * ```
+ */
+export declare class TMProject {
+    readonly name: string;
+    private readonly defaultDatabase;
+    private readonly models;
+    constructor(config: ProjectConfig);
+    /**
+     * Get a model by name.
+     */
+    getModel<TName extends string>(name: TName): TMModel<TName, any, any, any> | undefined;
+    /**
+     * Get all models.
+     */
+    getModels(): TMModel<any, any, any, any>[];
+    /**
+     * Build an execution plan (topological sort).
+     */
+    plan(options?: Pick<RunOptions, "targets" | "exclude">): ExecutionPlan;
+    /**
+     * Validate the DAG (check for cycles, missing refs, etc.).
+     */
+    validate(): ValidationResult;
+    /**
+     * Execute the DAG (or subset of models).
+     */
+    run(options?: RunOptions): Promise<ProjectRunResult>;
+    /**
+     * Execute a single model.
+     */
+    private executeModel;
+    /**
+     * Render the DAG as a Mermaid diagram.
+     */
+    toMermaid(): string;
+    /**
+     * Get models to run based on targets and exclusions.
+     */
+    private getModelsToRun;
+    /**
+     * Build dependency graph: { modelName: [dependencies] }
+     */
+    private buildDependencyGraph;
+    /**
+     * Topological sort into parallel stages.
+     */
+    private topologicalSort;
+    /**
+     * Detect cycles in the dependency graph.
+     */
+    private detectCycle;
+    /**
+     * Render dependency graph as Mermaid.
+     */
+    private toMermaidFromDeps;
+}
+//# sourceMappingURL=TMProject.d.ts.map

package/dist/project/TMProject.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TMProject.d.ts","sourceRoot":"","sources":["../../src/project/TMProject.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEtC,OAAO,EAAE,OAAO,EAAa,MAAM,kBAAkB,CAAC;AAMtD,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;IACtC,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,4CAA4C;IAC5C,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,wBAAwB;IACxB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,iCAAiC;IACjC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,oDAAoD;IACpD,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,oDAAoD;IACpD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB;IAChB,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACtC,eAAe,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;IAC/D,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;CACrD,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IACrC,eAAe,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,iDAAiD;IACjD,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;IACnB,2BAA2B;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,SAAS,IAAI,MAAM,CAAC;IACpB,qBAAqB;IACrB,QAAQ,IAAI,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CAC/B,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,OAAO,GAAG,aAAa,GAAG,gBAAgB,CAAC;IACjD,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAMF;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,SAAS;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAqB;IACrD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2C;gBAEtD,MAAM,EAAE,aAAa;IAsCjC;;OAEG;IACH,QAAQ,CAAC,KAAK,SAAS,MAAM,EAC3B,IAAI,EAAE,KAAK,GACV,OAAO,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,SAAS;IAI5C;;OAEG;IACH,SAAS,IAAI,OAAO,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IAI1C;;OAEG;IACH,IAAI,CAAC,OAAO,GAAE,IAAI,CAAC,UAAU,EAAE,SAAS,GAAG,SAAS,CAAM,GAAG,aAAa;IAuB1E;;OAEG;IACH,QAAQ,IAAI,gBAAgB;IAmE5B;;OAEG;IACG,GAAG,CAAC,OAAO,GAAE,UAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA6F9D;;OAEG;YACW,YAAY;IAqE1B;;OAEG;IACH,SAAS,IAAI,MAAM;IAOnB;;OAEG;IACH,OAAO,CAAC,cAAc;IA4BtB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAkB5B;;OAEG;IACH,OAAO,CAAC,eAAe;IAiCvB;;OAEG;IACH,OAAO,CAAC,WAAW;IAwCnB;;OAEG;IACH,OAAO,CAAC,iBAAiB;CAmB1B"}

package/dist/project/TMProject.js

@@ -0,0 +1,432 @@
+/**
+ * TMProject - DAG Orchestrator for TMModels
+ *
+ * Manages a collection of models, resolves dependencies,
+ * and executes them in topological order.
+ */
+import { tmql } from "tmql";
+import { isTMModel } from "../model/TMModel";
+// ============================================================================
+// TMProject Class
+// ============================================================================
+/**
+ * TMProject - DAG orchestrator for TMModels.
+ *
+ * Models are provided at construction time and validated immediately.
+ * The project is immutable after creation.
+ *
+ * @example
+ * ```typescript
+ * import { stgEvents, dailyMetrics } from "./models/analytics";
+ *
+ * const analyticsProject = new TMProject({
+ *   name: "analytics",
+ *   models: [stgEvents, dailyMetrics],
+ * });
+ *
+ * // See execution plan
+ * console.log(analyticsProject.toMermaid());
+ *
+ * // Run all models
+ * await analyticsProject.run();
+ * ```
+ */
+export class TMProject {
+    name;
+    defaultDatabase;
+    models;
+    constructor(config) {
+        this.name = config.name;
+        this.defaultDatabase =
+            config.defaultDatabase !== undefined ? config.defaultDatabase : undefined;
+        // Build models map, auto-discovering all ancestors (including lookup and unionWith)
+        this.models = new Map();
+        const addModelWithAncestors = (model) => {
+            if (this.models.has(model.name))
+                return;
+            // First add upstream ancestor (from property)
+            const upstream = model.getUpstreamModel();
+            if (upstream) {
+                addModelWithAncestors(upstream);
+            }
+            // Then add ancestors from lookup/unionWith stages (filter to models only)
+            for (const ancestor of model.getAncestorsFromStages().filter(isTMModel)) {
+                addModelWithAncestors(ancestor);
+            }
+            // Finally add this model
+            this.models.set(model.name, model);
+        };
+        for (const model of config.models) {
+            addModelWithAncestors(model);
+        }
+        // Validate immediately - fail fast
+        const validation = this.validate();
+        if (!validation.valid) {
+            const errorMessages = validation.errors
+                .map((e) => `  - ${e.message}`)
+                .join("\n");
+            throw new Error(`Project "${this.name}" has validation errors:\n${errorMessages}`);
+        }
+    }
+    /**
+     * Get a model by name.
+     */
+    getModel(name) {
+        return this.models.get(name);
+    }
+    /**
+     * Get all models.
+     */
+    getModels() {
+        return Array.from(this.models.values());
+    }
+    /**
+     * Build an execution plan (topological sort).
+     */
+    plan(options = {}) {
+        const { targets, exclude = [] } = options;
+        // Get models to run
+        const modelsToRun = this.getModelsToRun(targets, exclude);
+        // Build dependency graph
+        const deps = this.buildDependencyGraph(modelsToRun);
+        // Topological sort into stages (parallel batches)
+        const stages = this.topologicalSort(deps);
+        return {
+            stages,
+            totalModels: stages.flat().length,
+            toMermaid: () => this.toMermaidFromDeps(deps),
+            toString: () => stages
+                .map((stage, i) => `Stage ${i + 1}: ${stage.join(", ")}`)
+                .join("\n"),
+        };
+    }
+    /**
+     * Validate the DAG (check for cycles, missing refs, etc.).
+     */
+    validate() {
+        const errors = [];
+        const warnings = [];
+        // Check for duplicate names
+        const names = new Set();
+        for (const model of this.models.values()) {
+            if (names.has(model.name)) {
+                errors.push({
+                    type: "duplicate_name",
+                    message: `Duplicate model name: "${model.name}"`,
+                    models: [model.name],
+                });
+            }
+            names.add(model.name);
+        }
+        // Check for missing dependencies
+        for (const model of this.models.values()) {
+            const upstream = model.getUpstreamModel();
+            if (upstream && !this.models.has(upstream.name)) {
+                errors.push({
+                    type: "missing_ref",
+                    message: `Model "${model.name}" depends on "${upstream.name}" which is not in this project`,
+                    models: [model.name, upstream.name],
+                });
+            }
+        }
+        // Check for cycles
+        const cycle = this.detectCycle();
+        if (cycle) {
+            errors.push({
+                type: "cycle",
+                message: `Circular dependency detected: ${cycle.join(" -> ")}`,
+                models: cycle,
+            });
+        }
+        // Check for orphan models (no downstream dependencies)
+        const hasDownstream = new Set();
+        for (const model of this.models.values()) {
+            const upstream = model.getUpstreamModel();
+            if (upstream) {
+                hasDownstream.add(upstream.name);
+            }
+        }
+        const orphans = Array.from(this.models.values())
+            .filter((m) => !hasDownstream.has(m.name))
+            .map((m) => m.name);
+        // Only warn if there are multiple orphans (leaf nodes are expected)
+        if (orphans.length > 1) {
+            warnings.push({
+                type: "orphan",
+                message: `Multiple leaf models with no downstream dependencies: ${orphans.join(", ")}`,
+                models: orphans,
+            });
+        }
+        return {
+            valid: errors.length === 0,
+            errors,
+            warnings,
+        };
+    }
+    /**
+     * Execute the DAG (or subset of models).
+     */
+    async run(options = {}) {
+        const startTime = Date.now();
+        const { targets, exclude, dryRun = false, client, databaseName, onModelStart, onModelComplete, onModelError, } = options;
+        const planOptions = {};
+        if (targets !== undefined)
+            planOptions.targets = targets;
+        if (exclude !== undefined)
+            planOptions.exclude = exclude;
+        const executionPlan = this.plan(planOptions);
+        const modelsRun = [];
+        const modelsFailed = [];
+        const stats = {};
+        if (dryRun) {
+            console.log("Dry run - execution plan:");
+            console.log(executionPlan.toString());
+            return {
+                success: true,
+                modelsRun: [],
+                modelsFailed: [],
+                stats: {},
+                totalDurationMs: Date.now() - startTime,
+            };
+        }
+        // Get MongoDB client
+        const mongoClient = client ?? tmql.client;
+        if (!mongoClient) {
+            throw new Error("No MongoDB client available. Either pass one via options.client or call tmql.connect() first.");
+        }
+        const dbName = databaseName ?? this.defaultDatabase;
+        // Execute stages sequentially, models within a stage in parallel
+        for (const stage of executionPlan.stages) {
+            const stageResults = await Promise.allSettled(stage.map(async (modelName) => {
+                const model = this.models.get(modelName);
+                const modelStart = Date.now();
+                onModelStart?.(modelName);
+                try {
+                    await this.executeModel(model, mongoClient, dbName);
+                    const modelStats = {
+                        durationMs: Date.now() - modelStart,
+                    };
+                    stats[modelName] = modelStats;
+                    modelsRun.push(modelName);
+                    onModelComplete?.(modelName, modelStats);
+                    return { success: true, modelName };
+                }
+                catch (error) {
+                    const err = error instanceof Error ? error : new Error(String(error));
+                    modelsFailed.push(modelName);
+                    onModelError?.(modelName, err);
+                    throw err;
+                }
+            }));
+            // Check for failures
+            const failures = stageResults.filter((r) => r.status === "rejected");
+            if (failures.length > 0) {
+                // Stop execution on failure
+                break;
+            }
+        }
+        return {
+            success: modelsFailed.length === 0,
+            modelsRun,
+            modelsFailed,
+            stats,
+            totalDurationMs: Date.now() - startTime,
+        };
+    }
+    /**
+     * Execute a single model.
+     */
+    async executeModel(model, client, dbName) {
+        // Build pipeline with output stage
+        const pipeline = model.buildPipeline();
+        // Get source collection
+        const sourceCollection = model.getSourceCollectionName();
+        const outputDb = model.getOutputDatabase() ?? dbName;
+        // Handle view creation separately
+        if (model.materialize.type === "view") {
+            const viewName = model.getOutputCollectionName();
+            const viewDb = client.db(outputDb);
+            // Drop existing view if exists
+            try {
+                await viewDb.dropCollection(viewName);
+            }
+            catch {
+                // View might not exist
+            }
+            // Create view
+            await viewDb.createCollection(viewName, {
+                viewOn: sourceCollection,
+                pipeline: model.getPipelineStages(),
+            });
+            return;
+        }
+        // Handle time-series collection creation
+        if (model.materialize.type === "collection" &&
+            "timeseries" in model.materialize &&
+            model.materialize.timeseries) {
+            const collName = model.getOutputCollectionName();
+            const db = client.db(outputDb);
+            // Check if collection exists
+            const collections = await db
+                .listCollections({ name: collName })
+                .toArray();
+            if (collections.length === 0) {
+                // Create time-series collection
+                await db.createCollection(collName, {
+                    timeseries: {
+                        timeField: model.materialize.timeseries.timeField,
+                        metaField: model.materialize.timeseries.metaField,
+                        granularity: model.materialize.timeseries.granularity,
+                    },
+                    expireAfterSeconds: model.materialize.timeseries.expireAfterSeconds,
+                });
+            }
+        }
+        // Execute aggregation
+        // Use source database for reading, output stage handles writing to correct db
+        const sourceDb = model.getSourceDatabase() ?? dbName;
+        const db = client.db(sourceDb);
+        const collection = db.collection(sourceCollection);
+        const cursor = collection.aggregate(pipeline);
+        await cursor.toArray();
+    }
+    /**
+     * Render the DAG as a Mermaid diagram.
+     */
+    toMermaid() {
+        const deps = this.buildDependencyGraph(Array.from(this.models.values()).map((m) => m.name));
+        return this.toMermaidFromDeps(deps);
+    }
+    /**
+     * Get models to run based on targets and exclusions.
+     */
+    getModelsToRun(targets, exclude = []) {
+        const excludeSet = new Set(exclude);
+        if (targets && targets.length > 0) {
+            // Run specified targets and their dependencies
+            const toRun = new Set();
+            const addWithDeps = (name) => {
+                if (toRun.has(name) || excludeSet.has(name))
+                    return;
+                const model = this.models.get(name);
+                if (!model) {
+                    throw new Error(`Target model "${name}" not found in project`);
+                }
+                toRun.add(name);
+                const upstream = model.getUpstreamModel();
+                if (upstream) {
+                    addWithDeps(upstream.name);
+                }
+            };
+            targets.forEach(addWithDeps);
+            return Array.from(toRun);
+        }
+        // Run all models except excluded
+        return Array.from(this.models.keys()).filter((name) => !excludeSet.has(name));
+    }
+    /**
+     * Build dependency graph: { modelName: [dependencies] }
+     */
+    buildDependencyGraph(modelNames) {
+        const deps = new Map();
+        for (const name of modelNames) {
+            const model = this.models.get(name);
+            if (!model)
+                continue;
+            const modelDeps = [];
+            const upstream = model.getUpstreamModel();
+            if (upstream && modelNames.includes(upstream.name)) {
+                modelDeps.push(upstream.name);
+            }
+            deps.set(name, modelDeps);
+        }
+        return deps;
+    }
+    /**
+     * Topological sort into parallel stages.
+     */
+    topologicalSort(deps) {
+        const stages = [];
+        const remaining = new Map(deps);
+        const completed = new Set();
+        while (remaining.size > 0) {
+            // Find models with all dependencies satisfied
+            const ready = [];
+            for (const [name, modelDeps] of remaining) {
+                if (modelDeps.every((dep) => completed.has(dep))) {
+                    ready.push(name);
+                }
+            }
+            if (ready.length === 0 && remaining.size > 0) {
+                throw new Error(`Circular dependency detected among: ${Array.from(remaining.keys()).join(", ")}`);
+            }
+            // Add to current stage
+            stages.push(ready);
+            // Mark as completed
+            for (const name of ready) {
+                completed.add(name);
+                remaining.delete(name);
+            }
+        }
+        return stages;
+    }
+    /**
+     * Detect cycles in the dependency graph.
+     */
+    detectCycle() {
+        const visited = new Set();
+        const recStack = new Set();
+        const path = [];
+        const dfs = (name) => {
+            visited.add(name);
+            recStack.add(name);
+            path.push(name);
+            const model = this.models.get(name);
+            if (model) {
+                const upstream = model.getUpstreamModel();
+                if (upstream && this.models.has(upstream.name)) {
+                    if (!visited.has(upstream.name)) {
+                        const cycle = dfs(upstream.name);
+                        if (cycle)
+                            return cycle;
+                    }
+                    else if (recStack.has(upstream.name)) {
+                        // Found cycle
+                        const cycleStart = path.indexOf(upstream.name);
+                        return [...path.slice(cycleStart), upstream.name];
+                    }
+                }
+            }
+            path.pop();
+            recStack.delete(name);
+            return null;
+        };
+        for (const name of this.models.keys()) {
+            if (!visited.has(name)) {
+                const cycle = dfs(name);
+                if (cycle)
+                    return cycle;
+            }
+        }
+        return null;
+    }
+    /**
+     * Render dependency graph as Mermaid.
+     */
+    toMermaidFromDeps(deps) {
+        const lines = ["graph TD"];
+        // Define all nodes with labels first
+        for (const [name] of deps) {
+            const model = this.models.get(name);
+            const matType = model?.materialize.type ?? "unknown";
+            lines.push(`  ${name}["${name}<br/>(${matType})"]`);
+        }
+        // Then add edges
+        for (const [name, modelDeps] of deps) {
+            for (const dep of modelDeps) {
+                lines.push(`  ${dep} --> ${name}`);
+            }
+        }
+        return lines.join("\n");
+    }
+}

package/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "tmql-orchestration",
+  "version": "0.4.1",
+  "description": "DAG composition and orchestration layer for tmql - Build data pipelines with TMModel and TMProject",
+  "author": "Ryan Peggs & Tim Vyas",
+  "license": "Elastic-2.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/covahltd/tmql.git",
+    "directory": "packages/tmql-orchestration"
+  },
+  "keywords": [
+    "mongodb",
+    "dag",
+    "orchestration",
+    "data-pipeline",
+    "etl",
+    "tmql",
+    "materialization"
+  ],
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "build:watch": "tsc --watch",
+    "typecheck": "tsc --noEmit"
+  },
+  "peerDependencies": {
+    "tmql": ">=0.4.0 <1.0.0"
+  },
+  "dependencies": {
+    "mongodb": "^6.17.0"
+  },
+  "devDependencies": {
+    "tmql": "workspace:*",
+    "typescript": "^5.9.3"
+  }
+}