data-flow-control 0.1.7__tar.gz → 0.1.8__tar.gz

{data_flow_control-0.1.7 → data_flow_control-0.1.8}/Cargo.lock

@@ -1593,7 +1593,7 @@ dependencies = [
 
 [[package]]
 name = "passant-cli"
-version = "0.1.7"
+version = "0.1.8"
 dependencies = [
  "anyhow",
  "assert_cmd",
@@ -1605,7 +1605,7 @@ dependencies = [
 
 [[package]]
 name = "passant-core"
-version = "0.1.7"
+version = "0.1.8"
 dependencies = [
  "anyhow",
  "criterion",
@@ -1623,7 +1623,7 @@ dependencies = [
 
 [[package]]
 name = "passant-py"
-version = "0.1.7"
+version = "0.1.8"
 dependencies = [
  "passant-core",
  "pyo3",
@@ -2411,6 +2411,18 @@ checksum = "05a528114c392209b3264855ad491fcce534b94a38771b0a0b97a79379275ce8"
 dependencies = [
  "log",
  "serde",
+ "sqlparser_derive",
+]
+
+[[package]]
+name = "sqlparser_derive"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
 ]
 
 [[package]]

{data_flow_control-0.1.7 → data_flow_control-0.1.8}/Cargo.toml

@@ -3,7 +3,7 @@ members = ["passant-core", "passant-py"]
 resolver = "2"
 
 [workspace.package]
-version = "0.1.7"
+version = "0.1.8"
 edition = "2024"
 license = "MIT"
 authors = ["Charlie Summers"]
@@ -14,7 +14,7 @@ clap = { version = "4.5", features = ["derive"] }
 pyo3 = { version = "0.23", features = ["abi3-py39"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
-sqlparser = { version = "0.53", features = ["serde"] }
+sqlparser = { version = "0.53", features = ["serde", "visitor"] }
 thiserror = "2.0"
 smallvec = "1.14"
 assert_cmd = "2.0"

{data_flow_control-0.1.7 → data_flow_control-0.1.8}/PKG-INFO

@@ -1,10 +1,7 @@
 Metadata-Version: 2.4
 Name: data-flow-control
-Version: 0.1.7
-Requires-Dist: boto3>=1.26.0
+Version: 0.1.8
 Requires-Dist: duckdb>=1.3.0
-Requires-Dist: langchain>=1.3,<2
-Requires-Dist: langchain-core>=1.3,<2
 Requires-Dist: numpy>=1.24.4
 Requires-Dist: sqlglot==28.3.0
 Requires-Dist: clickhouse-connect>=0.7.0 ; extra == 'clickhouse'
@@ -14,10 +11,13 @@ Requires-Dist: maturin>=1.7.0 ; extra == 'dev'
 Requires-Dist: pre-commit>=4.0.0 ; extra == 'dev'
 Requires-Dist: pytest>=8.4.2 ; extra == 'dev'
 Requires-Dist: ruff>=0.11.0 ; extra == 'dev'
+Requires-Dist: langchain>=1.3,<2 ; extra == 'langchain'
+Requires-Dist: langchain-core>=1.3,<2 ; extra == 'langchain'
 Requires-Dist: psycopg[binary]>=3.2.0 ; extra == 'postgres'
 Provides-Extra: clickhouse
 Provides-Extra: datafusion
 Provides-Extra: dev
+Provides-Extra: langchain
 Provides-Extra: postgres
 License-File: LICENSE
 Summary: Rust-backed Data Flow Control rewrite engine

{data_flow_control-0.1.7 → data_flow_control-0.1.8}/passant-core/src/rewriter/derived_policy.rs

@@ -257,7 +257,9 @@ fn inject_hidden_projections_in_factor(
 
     let inner_scope = TableScope::from_select(inner_select);
 
-    for index in prep.deferred_indices.iter().copied() {
+    let mut deferred_indices = prep.deferred_indices.iter().copied().collect::<Vec<_>>();
+    deferred_indices.sort_unstable();
+    for index in deferred_indices {
         let Some(policy) = store.policy(index) else {
             continue;
         };
@@ -422,10 +424,8 @@ fn projection_has_equivalent(select: &Select, expr: &Expr, alias: &str) -> bool
     let key = crate::sql::ExprKey::from_expr(expr);
     select.projection.iter().any(|item| {
         if let Some((proj_expr, proj_alias)) = projection_expr_and_name(item) {
-            if proj_alias.is_some_and(|name| name == alias) {
-                return true;
-            }
-            return crate::sql::ExprKey::from_expr(proj_expr) == key;
+            return proj_alias.is_some_and(|name| name == alias)
+                && crate::sql::ExprKey::from_expr(proj_expr) == key;
         }
         false
     })
@@ -441,6 +441,9 @@ fn find_equivalent_projection_alias(select: &Select, metric_expr: &Expr) -> Opti
             continue;
         }
         if let Some(name) = alias {
+            if name.starts_with("__passant_filter_") {
+                continue;
+            }
             return Some(name.to_string());
         }
         if let Some(name) = super::expr::projected_column_name(expr) {

{data_flow_control-0.1.7 → data_flow_control-0.1.8}/passant-core/src/rewriter/exists.rs

@@ -5,6 +5,7 @@ use sqlparser::ast::{
     SelectItem, SetExpr, Statement, TableAlias, TableFactor, TableWithJoins,
 };
 
+use crate::catalog::TableCatalog;
 use crate::identifiers::{TableKey, column_name_from_expr, table_name_from_column_expr};
 use crate::parser::parse_query;
 use crate::partial_push::ExtraDfcFilter;
@@ -43,7 +44,11 @@ fn table_with_joins_contains_base(table: &TableWithJoins, base: &str) -> bool {
     })
 }
 
-fn from_index_for_in_outer_expr(select: &Select, outer_expr: &Expr) -> usize {
+fn from_index_for_in_outer_expr(
+    select: &Select,
+    outer_expr: &Expr,
+    catalog: &TableCatalog,
+) -> usize {
     if let Some(table_name) = table_name_from_column_expr(outer_expr) {
         let target = table_name.as_str();
         for (index, table_with_joins) in select.from.iter().enumerate() {
@@ -52,9 +57,40 @@ fn from_index_for_in_outer_expr(select: &Select, outer_expr: &Expr) -> usize {
             }
         }
     }
+    if let Some(column) = column_name_from_expr(outer_expr) {
+        let matching = select
+            .from
+            .iter()
+            .enumerate()
+            .filter_map(|(index, table_with_joins)| {
+                table_with_joins_contains_column(table_with_joins, column.as_str(), catalog)
+                    .then_some(index)
+            })
+            .collect::<Vec<_>>();
+        if let [index] = matching.as_slice() {
+            return *index;
+        }
+    }
     select.from.len().saturating_sub(1)
 }
 
+fn table_with_joins_contains_column(
+    table: &TableWithJoins,
+    column: &str,
+    catalog: &TableCatalog,
+) -> bool {
+    std::iter::once(&table.relation)
+        .chain(table.joins.iter().map(|join| &join.relation))
+        .filter_map(table_factor_base_and_alias)
+        .any(|(base, _)| {
+            catalog.columns(&base).is_some_and(|columns| {
+                columns
+                    .iter()
+                    .any(|candidate| candidate.eq_ignore_ascii_case(column))
+            })
+        })
+}
+
 fn qualify_unqualified_column_in_expr(expr: &mut Expr, column: &str, table: &str) {
     match expr {
         Expr::Identifier(ident) if ident.value.eq_ignore_ascii_case(column) => {
@@ -413,7 +449,7 @@ impl PassantRewriter {
                     "IN rewrite requires a FROM clause",
                 ));
             }
-            let from_index = from_index_for_in_outer_expr(select, expr.as_ref());
+            let from_index = from_index_for_in_outer_expr(select, expr.as_ref(), &self.catalog);
             select.from[from_index].joins.push(Join {
                 relation: TableFactor::Derived {
                     lateral: false,

{data_flow_control-0.1.7 → data_flow_control-0.1.8}/passant-core/src/rewriter/expr.rs

@@ -1,7 +1,7 @@
 use crate::aggregate_registry::AggregateRegistry;
 use crate::sql::{parse_projection_expr, wrap_table_with_filter};
 use sqlparser::ast::{
-    BinaryOperator, Expr, FunctionArg, FunctionArgExpr, FunctionArguments, SelectItem, TableFactor,
+    Expr, FunctionArg, FunctionArgExpr, FunctionArguments, SelectItem, TableFactor,
 };
 
 use crate::diagnostics::RewriteError;
@@ -89,13 +89,7 @@ pub(crate) fn add_filter(
     Ok(())
 }
 
-pub(crate) fn and_expr(left: Expr, right: Expr) -> Expr {
-    Expr::BinaryOp {
-        left: Box::new(left),
-        op: BinaryOperator::And,
-        right: Box::new(right),
-    }
-}
+pub(crate) use crate::sql::builders::and_expr;
 
 pub(crate) fn parse_expr(sql: &str) -> Result<Expr, RewriteError> {
     parse_projection_expr(sql)