pytrilogy 0.3.142__cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

trilogy/scripts/dependency/README.md

@@ -0,0 +1,131 @@
+# PreQL Import Resolver
+
+A Rust-based CLI tool and Python library for parsing PreQL (Trilogy) files and resolving import dependencies with ETL-aware dependency ordering.
+
+## Features
+
+- Parse PreQL files to extract imports, datasource declarations, and persist statements
+- Resolve import dependencies transitively
+- Build dependency graphs with ETL-aware ordering:
+  - Files that persist (write) to a datasource run before files that declare it, even if they import it.
+  - Standard import dependencies (imported files run before importing files)
+
+Exit codes:
+- `0`: Success
+- `1`: Error (parse error, file not found, circular dependency, etc.)
+
+## CLI Usage
+
+### Parse a single file
+
+```bash
+preql-import-resolver parse path/to/file.preql --format pretty
+```
+
+### Parse a directory
+
+```bash
+preql-import-resolver parse path/to/directory --recursive --format json
+```
+
+### Resolve dependencies
+
+```bash
+preql-import-resolver resolve path/to/file.preql --format pretty
+```
+
+### Analyze datasources
+
+```bash
+preql-import-resolver datasources path/to/directory --recursive
+```
+
+## Python Integration
+
+The Rust resolver is integrated into the Python package via PyO3 and maturin.
+
+### Building the Python Extension
+
+```bash
+cd trilogy/scripts/dependency
+maturin develop  # For development
+# or
+maturin build --release  # For production
+pip install target/wheels/*.whl
+```
+
+### Using in Python
+
+```python
+from trilogy.scripts.dependency import DependencyResolver, ETLDependencyStrategy, create_script_nodes
+from pathlib import Path
+
+# Create script nodes from files
+files = [Path("model1.preql"), Path("model2.preql")]
+nodes = create_script_nodes(files)
+
+# Use the ETL dependency strategy (backed by Rust)
+resolver = DependencyResolver(strategy=ETLDependencyStrategy())
+graph = resolver.build_graph(nodes)
+
+# Get execution order
+import networkx as nx
+execution_order = list(nx.topological_sort(graph))
+```
+
+The `ETLDependencyStrategy` uses the Rust-based resolver under the hood for fast, accurate dependency analysis based on:
+- Import statements
+- Datasource declarations
+- Persist statements (append/overwrite/persist)
+
+## Development
+
+### Running Rust Tests
+
+```bash
+cd trilogy/scripts/dependency
+cargo test
+```
+
+All tests include:
+- Unit tests for parser (17 tests covering imports, datasources, persist statements)
+- Unit tests for resolver (5 tests covering dependency resolution logic)
+- Integration tests for CLI (12 tests covering all CLI commands)
+
+### Building the CLI
+
+```bash
+cargo build --release
+```
+
+The binary will be at `target/release/preql-import-resolver` (or `.exe` on Windows).
+
+### Building for Python
+
+Run maturin from the base of the pytrilogy repo. [not this directory.]
+
+```bash
+# Development mode (installs in current Python environment)
+maturin develop
+
+# Production build
+maturin build --release
+
+# The wheel will be in target/wheels/
+```
+
+## Dependency Ordering Rules
+
+The resolver implements three key dependency rules:
+
+1. **Import Dependencies** : Imported files should run before importing files
+2. **Persist-Before-Declare**: Files that persist to a datasource must run before files that declare it, even if they import that file. 
+
+### Edge Cases
+
+- Case 1: file A imports from file B → B must run before A for all datasources in B
+- Case 2: file A imports from file B, then updates datasource from file B → update takes precedence, so A runs before B.
+
+## Grammar Limitations
+
+The grammar is currently focused on dependency-relevant constructs (imports, datasources, persist statements). It does not parse all Trilogy syntax. It can be extended in the future.

trilogy/scripts/dependency/build.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Build script for preql-import-resolver
+
+set -e
+
+echo "Building preql-import-resolver..."
+
+# Check for Rust
+if ! command -v cargo &> /dev/null; then
+    echo "Rust is not installed. Install from https://rustup.rs/"
+    exit 1
+fi
+
+# Build release binary
+cargo build --release
+
+echo ""
+echo "Build successful!"
+echo "Binary location: ./target/release/preql-import-resolver"
+echo ""
+echo "To install globally, run:"
+echo "  cargo install --path ."
+echo ""
+echo "To test:"
+echo "  ./target/release/preql-import-resolver --help"

trilogy/scripts/dependency/src/directory_resolver.rs

@@ -0,0 +1,177 @@
+use crate::parser::parse_file;
+use std::collections::{HashMap, HashSet};
+use std::fs;
+use std::path::PathBuf;
+
+#[derive(Debug, Clone)]
+pub struct FileInfo {
+    pub path: PathBuf,
+    pub datasources: Vec<String>,
+    pub persists: Vec<String>,
+}
+
+#[derive(Debug, Clone)]
+pub struct DirectoryGraph {
+    pub files: HashMap<PathBuf, FileInfo>,
+    pub imports: HashMap<PathBuf, Vec<PathBuf>>,
+    pub warnings: Vec<String>,
+}
+
+#[derive(Debug, Clone)]
+pub struct Edge {
+    pub from: PathBuf,
+    pub to: PathBuf,
+    pub reason: EdgeReason,
+}
+
+#[derive(Debug, Clone)]
+pub enum EdgeReason {
+    Import,
+    PersistBeforeDeclare { datasource: String },
+}
+
+/// Process files in a directory, discovering transitive imports
+pub fn process_directory_with_imports(
+    initial_files: Vec<PathBuf>,
+) -> Result<DirectoryGraph, String> {
+    let mut all_imports: HashMap<PathBuf, Vec<PathBuf>> = HashMap::new();
+    let mut files_info: HashMap<PathBuf, FileInfo> = HashMap::new();
+    let mut files_to_process = initial_files;
+    let mut processed_files: HashSet<PathBuf> = HashSet::new();
+    let mut warnings = Vec::new();
+
+    while let Some(file) = files_to_process.pop() {
+        let canonical = match fs::canonicalize(&file) {
+            Ok(c) => c,
+            Err(e) => {
+                warnings.push(format!("Failed to canonicalize {}: {}", file.display(), e));
+                continue;
+            }
+        };
+
+        if processed_files.contains(&canonical) {
+            continue;
+        }
+        processed_files.insert(canonical.clone());
+
+        let content = match fs::read_to_string(&file) {
+            Ok(c) => c,
+            Err(e) => {
+                warnings.push(format!("Failed to read {}: {}", file.display(), e));
+                continue;
+            }
+        };
+
+        let parsed = match parse_file(&content) {
+            Ok(p) => p,
+            Err(e) => {
+                warnings.push(format!("Failed to parse {}: {}", file.display(), e));
+                continue;
+            }
+        };
+
+        let mut resolved_imports = Vec::new();
+        let file_dir = file.parent().unwrap_or(std::path::Path::new("."));
+
+        for import in &parsed.imports {
+            if import.is_stdlib {
+                continue;
+            }
+            if let Some(resolved) = import.resolve(file_dir) {
+                if resolved.exists() {
+                    if let Ok(resolved_canonical) = fs::canonicalize(&resolved) {
+                        resolved_imports.push(resolved_canonical.clone());
+                        if !processed_files.contains(&resolved_canonical) {
+                            files_to_process.push(resolved_canonical);
+                        }
+                    }
+                }
+            }
+        }
+
+        let datasources: Vec<String> = parsed.datasources.iter().map(|d| d.name.clone()).collect();
+        let persists: Vec<String> = parsed.persists.iter().map(|p| p.target_datasource.clone()).collect();
+
+        all_imports.insert(canonical.clone(), resolved_imports);
+        files_info.insert(
+            canonical.clone(),
+            FileInfo {
+                path: canonical,
+                datasources,
+                persists,
+            },
+        );
+    }
+
+    Ok(DirectoryGraph {
+        files: files_info,
+        imports: all_imports,
+        warnings,
+    })
+}
+
+/// Build edges from a directory graph
+pub fn build_edges(graph: &DirectoryGraph) -> Vec<Edge> {
+    let mut edges = Vec::new();
+    let known_files: HashSet<PathBuf> = graph.files.keys().cloned().collect();
+
+    // Rule 1: Import dependencies (imported files run before importing files)
+    for (file, imports) in &graph.imports {
+        for resolved_path in imports {
+            if !known_files.contains(resolved_path) {
+                continue;
+            }
+            edges.push(Edge {
+                from: resolved_path.clone(),
+                to: file.clone(),
+                reason: EdgeReason::Import,
+            });
+        }
+    }
+
+    // Rule 2: Persist-before-declare
+    // Files that persist to a datasource must run BEFORE files that declare that datasource.
+    // This takes precedence over import edges - if the updater imports the declarer,
+    // we need to remove that import edge and add the persist-before-declare edge instead.
+    for (declarer_path, declarer_info) in &graph.files {
+        for ds_name in &declarer_info.datasources {
+            for (updater_path, updater_info) in &graph.files {
+                if updater_path == declarer_path {
+                    continue;
+                }
+
+                if updater_info.persists.contains(ds_name) {
+                    edges.push(Edge {
+                        from: updater_path.clone(),
+                        to: declarer_path.clone(),
+                        reason: EdgeReason::PersistBeforeDeclare {
+                            datasource: ds_name.clone(),
+                        },
+                    });
+                }
+            }
+        }
+    }
+
+    // Rule 3: Remove import edges that conflict with persist-before-declare
+    // If file A imports file B, but A also persists to a datasource declared by B,
+    // then the import edge (B -> A) conflicts with persist-before-declare (A -> B).
+    // In this case, persist-before-declare takes precedence.
+    let persist_edges: HashSet<(PathBuf, PathBuf)> = edges
+        .iter()
+        .filter(|e| matches!(e.reason, EdgeReason::PersistBeforeDeclare { .. }))
+        .map(|e| (e.from.clone(), e.to.clone()))
+        .collect();
+
+    edges.retain(|edge| {
+        if matches!(edge.reason, EdgeReason::Import) {
+            // Check if there's a conflicting persist-before-declare edge in the opposite direction
+            let reverse = (edge.to.clone(), edge.from.clone());
+            !persist_edges.contains(&reverse)
+        } else {
+            true
+        }
+    });
+
+    edges
+}

trilogy/scripts/dependency/src/lib.rs

@@ -0,0 +1,16 @@
+mod parser;
+mod resolver;
+mod directory_resolver;
+pub mod python_bindings;
+
+pub use parser::{
+    parse_file, parse_imports, DatasourceDeclaration, ImportStatement, ParseError, ParsedFile,
+    PersistMode, PersistStatement,
+};
+pub use resolver::{
+    DatasourceInfo, DependencyGraph, FileNode, ImportInfo, ImportResolver, PersistInfo,
+    ResolveError,
+};
+pub use directory_resolver::{
+    process_directory_with_imports, build_edges, DirectoryGraph, Edge, EdgeReason, FileInfo,
+};