baml-py 0.1.7__tar.gz

baml_py-0.1.7/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.3
+Name: baml-py
+Version: 0.1.7
+License-File: LICENSE
+License-File: LICENSE
+Summary: BAML python bindings (pyproject.toml)
+Author: Boundary <contact@boundaryml.com>
+Author-email: Boundary <contact@boundaryml.com>
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+
+BAML python bindings (readme.md)
+

baml_py-0.1.7/README.md

@@ -0,0 +1 @@
+BAML python bindings (readme.md)

baml_py-0.1.7/baml-lib/baml-core/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+edition = "2021"
+name = "internal-baml-core"
+version.workspace = true
+authors.workspace = true
+description.workspace = true
+license-file.workspace = true
+
+[dependencies]
+internal-baml-jinja.workspace = true
+internal-baml-diagnostics = { path = "../diagnostics" }
+internal-baml-schema-ast = { path = "../schema-ast" }
+internal-baml-parser-database = { path = "../parser-database" }
+internal-baml-prompt-parser = { path = "../prompt-parser" }
+anyhow.workspace = true
+askama.workspace = true
+indexmap.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+enumflags2 = "0.7"
+rayon = "1.8.0"
+handlebars = "4.4.0"
+log = "0.4.20"
+chrono = "0.4.31"
+whoami = "1.4.1"
+textwrap = "0.16.0"
+either.workspace = true
+strsim = "0.10.0"
+semver = "1.0.20"
+regex = "1.10.3"
+shellwords = "1.1.0"
+
+
+[dev-dependencies]
+base64 = "0.13.0"
+dissimilar = "1.0.4"
+expect-test = "1.1.0"
+indoc.workspace = true
+either = "1.8.1"

baml_py-0.1.7/baml-lib/baml-core/askama.toml

@@ -0,0 +1,6 @@
+[general]
+dirs = ["src/generate/generate_ruby_client/templates"]
+# whitespace can be either preserve, suppress, or minimize
+# suppress and minimize are both too aggressive for us
+whitespace = "preserve"
+escape = "none"

baml_py-0.1.7/baml-lib/baml-core/src/common/preview_features.rs

@@ -0,0 +1,88 @@
+use serde::{Serialize, Serializer};
+use std::fmt;
+
+/// A set of preview features.
+pub type PreviewFeatures = enumflags2::BitFlags<PreviewFeature>;
+
+macro_rules! features {
+    ($( $variant:ident $(,)? ),*) => {
+        #[enumflags2::bitflags]
+        #[repr(u64)]
+        #[derive(Debug, Copy, Clone, PartialEq, Eq)]
+        pub enum PreviewFeature {
+            $( $variant,)*
+        }
+
+        impl PreviewFeature {
+            pub fn parse_opt(s: &str) -> Option<Self> {
+                $(
+                    if s.eq_ignore_ascii_case(stringify!($variant)) { return Some(Self::$variant) }
+                )*
+
+                None
+            }
+        }
+
+        impl fmt::Display for PreviewFeature {
+            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+                let variant = match self { $( Self::$variant => stringify!($variant),)* };
+                let mut first_char = variant.chars().next().unwrap();
+                first_char.make_ascii_lowercase();
+                f.write_fmt(format_args!("{first_char}{rest}", rest = &variant[1..]))
+            }
+        }
+    };
+}
+
+// (Usually) Append-only list of features.
+features!(ScalarJson,);
+
+/// Generator preview features
+pub const ALL_PREVIEW_FEATURES: FeatureMap = FeatureMap {
+    active: enumflags2::BitFlags::EMPTY,
+    deprecated: enumflags2::BitFlags::EMPTY,
+    hidden: enumflags2::BitFlags::EMPTY,
+};
+
+#[derive(Debug)]
+pub struct FeatureMap {
+    /// Valid, visible features.
+    active: PreviewFeatures,
+
+    /// Deprecated features.
+    #[allow(dead_code)]
+    deprecated: PreviewFeatures,
+
+    /// Hidden preview features are valid features, but are not propagated into the tooling
+    /// (as autocomplete or similar) or into error messages (eg. showing a list of valid features).
+    hidden: PreviewFeatures,
+}
+
+impl FeatureMap {
+    pub const fn active_features(&self) -> PreviewFeatures {
+        self.active
+    }
+
+    pub const fn hidden_features(&self) -> PreviewFeatures {
+        self.hidden
+    }
+
+    #[allow(dead_code)]
+    pub(crate) fn is_valid(&self, flag: PreviewFeature) -> bool {
+        (self.active | self.hidden).contains(flag)
+    }
+
+    #[allow(dead_code)]
+    pub(crate) fn is_deprecated(&self, flag: PreviewFeature) -> bool {
+        self.deprecated.contains(flag)
+    }
+}
+
+impl Serialize for PreviewFeature {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&self.to_string())
+    }
+}

baml_py-0.1.7/baml-lib/baml-core/src/common.rs

@@ -0,0 +1,5 @@
+//! This module contains shared constants and logic that can be used by engines.
+
+mod preview_features;
+
+pub use self::preview_features::{PreviewFeature, PreviewFeatures, ALL_PREVIEW_FEATURES};

baml_py-0.1.7/baml-lib/baml-core/src/configuration/configuration_struct.rs

@@ -0,0 +1,24 @@
+use super::Generator;
+use crate::{internal_baml_diagnostics::Diagnostics, lockfile::LockFileWrapper, PreviewFeature};
+use enumflags2::BitFlags;
+
+#[derive(Debug)]
+pub struct Configuration {
+    pub generators: Vec<(Generator, LockFileWrapper)>,
+}
+
+impl Configuration {
+    pub fn validate_that_one_datasource_is_provided(&self) -> Result<(), Diagnostics> {
+        Ok(())
+    }
+
+    pub fn max_identifier_length(&self) -> usize {
+        1024
+    }
+
+    pub fn preview_features(&self) -> BitFlags<PreviewFeature> {
+        self.generators
+            .iter()
+            .fold(BitFlags::empty(), |acc, _generator| acc)
+    }
+}

baml_py-0.1.7/baml-lib/baml-core/src/configuration/generator.rs

@@ -0,0 +1,217 @@
+use crate::ast::WithName;
+use anyhow::Result;
+use internal_baml_parser_database::ast::Expression;
+use regex::Regex;
+use serde::Serialize;
+use std::{collections::HashMap, path::PathBuf, process::Command};
+
+#[derive(Debug, Serialize, Clone)]
+#[serde(untagged)]
+pub enum GeneratorConfigValue {
+    String(String),
+    Array(Vec<GeneratorConfigValue>),
+    Map(HashMap<String, GeneratorConfigValue>),
+}
+
+impl From<String> for GeneratorConfigValue {
+    fn from(value: String) -> Self {
+        Self::String(value)
+    }
+}
+
+impl From<&Expression> for GeneratorConfigValue {
+    fn from(expr: &Expression) -> Self {
+        match expr {
+            Expression::BoolValue(val, _) => val.to_string().into(),
+            Expression::NumericValue(val, _) => val.clone().into(),
+            Expression::StringValue(val, _) => val.clone().into(),
+            Expression::Array(elements, _) => {
+                Self::Array(elements.iter().map(From::from).collect())
+            }
+            Expression::Map(elements, _) => Self::Map(
+                elements
+                    .iter()
+                    .map(|(k, v)| (k.to_string(), From::from(v)))
+                    .collect(),
+            ),
+            Expression::Identifier(idn) => idn.name().to_string().into(),
+            Expression::RawStringValue(val) => val.value().to_string().into(),
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Clone)]
+pub enum GeneratorLanguage {
+    #[serde(rename = "python")]
+    Python,
+    #[serde(rename = "typescript")]
+    TypeScript,
+    #[serde(rename = "Ruby")]
+    Ruby,
+}
+
+impl GeneratorLanguage {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            Self::Python => "python",
+            Self::TypeScript => "typescript",
+            Self::Ruby => "ruby",
+        }
+    }
+
+    pub fn package_name(&self) -> &'static str {
+        match self {
+            Self::Python => "baml",
+            Self::TypeScript => "@boundaryml/baml-core",
+            Self::Ruby => "baml",
+        }
+    }
+
+    /// This is redundant with version_command::get_client_version
+    fn client_version(&self, command: &str) -> Option<String> {
+        let cmd = format!("{} {}", command, self.package_name());
+
+        // Run the command and get the version
+        Command::new(cmd)
+            .output()
+            .map(|output| {
+                // If the command fails, return None
+                if !output.status.success() {
+                    return None;
+                }
+
+                // If the command succeeds, return the version
+                String::from_utf8(output.stdout)
+                    .ok()
+                    .map(|v| v.trim().to_string())
+                    .map(|v| self.parse_version(v.as_str()).ok())
+                    .flatten()
+            })
+            .ok()
+            .flatten()
+    }
+
+    /// This is redundant with version_command::extract_client_version
+    pub fn parse_version(&self, output: &str) -> Result<String> {
+        match self {
+            Self::Python => {
+                // Split the string by lines, and take the first line that has "version" in it
+                let version_line_re = Regex::new(r#"(?i)\b(?:version)\b"#)?;
+
+                let Some(version_line) = output
+                    .trim()
+                    .lines()
+                    .find(|line| version_line_re.is_match(line))
+                else {
+                    return Err(anyhow::format_err!(
+                        "Could not infer the version of the client"
+                    ));
+                };
+
+                let version_re = Regex::new("[0-9][^ ]*")?;
+
+                let Some(version) = version_re.find(version_line) else {
+                    return Err(anyhow::format_err!(
+                        "Could not parse the version of the client"
+                    ));
+                };
+
+                Ok(version.as_str().to_string())
+            }
+            Self::TypeScript => {
+                // Look for "<package_name>@<version>"
+                let version_re = Regex::new(&format!(r#"{}@[^ ]+"#, self.package_name()))?;
+
+                let Some(version) = version_re.find(output.trim()) else {
+                    return Err(anyhow::format_err!(
+                        "Could not parse the version of the client:{}\n{}",
+                        self.package_name(),
+                        output
+                    ));
+                };
+
+                Ok(version.as_str().split('@').last().unwrap().to_string())
+            }
+            Self::Ruby => {
+                // Look for "<package_name>@<version>"
+                let version_re = Regex::new(&format!(r#"{} ([^)]+)"#, self.package_name()))?;
+
+                let Some(version) = version_re.captures(output.trim()) else {
+                    return Err(anyhow::format_err!(
+                        "Could not find the version of the client: {}\n{}",
+                        self.package_name(),
+                        output
+                    ));
+                };
+                let Some(version) = version.get(1) else {
+                    return Err(anyhow::format_err!(
+                        "Could not parse the version of the client: {}\n{}",
+                        self.package_name(),
+                        output
+                    ));
+                };
+
+                Ok(version.as_str().into())
+            }
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct Generator {
+    pub name: String,
+    pub language: GeneratorLanguage,
+    pub test_command: String,
+    pub install_command: String,
+    pub package_version_command: String,
+
+    #[serde(skip)]
+    pub used_in_tests: bool,
+
+    // This is generated from the package_version_command
+    pub client_version: Option<String>,
+
+    /// Where the language source code is located (e.g. place where pyproject.toml is located or package.json is located)
+    pub project_root: PathBuf,
+
+    /// Where the generated code should be placed (usually a subdirectory of the project_root)
+    pub output_path: PathBuf,
+
+    #[serde(skip)]
+    pub(crate) span: crate::ast::Span,
+}
+
+impl Generator {
+    pub fn new(
+        name: String,
+        project_root: PathBuf,
+        language: GeneratorLanguage,
+        test_command: String,
+        install_command: String,
+        package_version_command: String,
+        output_path: Option<PathBuf>,
+        used_in_tests: Option<bool>,
+        span: crate::ast::Span,
+    ) -> Result<Self, std::io::Error> {
+        Ok(Self {
+            name,
+            output_path: output_path
+                .as_ref()
+                .unwrap_or(&project_root)
+                .join("baml_client"),
+            project_root,
+            client_version: language.client_version(&package_version_command),
+            language,
+            test_command,
+            install_command,
+            package_version_command,
+            span,
+            used_in_tests: used_in_tests.unwrap_or(false),
+        })
+    }
+
+    pub fn cli_version(&self) -> &'static str {
+        env!("CARGO_PKG_VERSION")
+    }
+}

baml_py-0.1.7/baml-lib/baml-core/src/configuration.rs

@@ -0,0 +1,5 @@
+mod configuration_struct;
+mod generator;
+
+pub use configuration_struct::*;
+pub use generator::*;

baml_py-0.1.7/baml-lib/baml-core/src/generate/dir_writer.rs

@@ -0,0 +1,242 @@
+use std::io::{self, ErrorKind};
+use std::thread::sleep;
+use std::time::Duration;
+use std::{
+    collections::{HashMap, HashSet},
+    path::PathBuf,
+};
+
+use log::info;
+
+#[derive(PartialEq, Eq, Hash)]
+pub(super) struct LibImport {
+    pub lib: String,
+    pub as_name: Option<String>,
+}
+
+#[derive(PartialEq, Eq, Hash)]
+pub(super) struct Import {
+    pub lib: String,
+    pub name: String,
+    pub as_name: Option<String>,
+}
+
+pub(super) struct FileContent {
+    content: Vec<String>,
+    import_libs: HashSet<LibImport>,
+    imports: Vec<Import>,
+    exports: Vec<String>,
+    export_all: bool,
+}
+
+impl FileContent {
+    pub fn add_import_lib(&mut self, lib: impl Into<String>, as_name: Option<&str>) {
+        self.import_libs.insert(LibImport {
+            lib: lib.into(),
+            as_name: as_name.map(|s| s.to_string()),
+        });
+    }
+
+    pub fn add_import(
+        &mut self,
+        lib: impl Into<String>,
+        name: impl AsRef<str>,
+        as_name: Option<&str>,
+        is_export: bool,
+    ) {
+        self.imports.push(Import {
+            lib: lib.into(),
+            name: name.as_ref().to_string(),
+            as_name: as_name.map(|s| s.to_string()),
+        });
+        if is_export || self.export_all {
+            self.exports
+                .push(as_name.unwrap_or(name.as_ref()).to_string());
+        }
+    }
+
+    pub fn add_export(&mut self, name: impl Into<String>) {
+        self.exports.push(name.into());
+    }
+
+    pub fn trim_append(&mut self, content: String) {
+        let content = content.trim();
+        if content.len() > 0 {
+            self.content.push(content.to_string());
+        }
+    }
+
+    #[allow(dead_code)]
+    pub fn append(&mut self, content: String) {
+        if content.len() > 0 {
+            self.content.push(content.to_string());
+        }
+    }
+}
+
+pub(super) struct FileCollector<L: LanguageFeatures> {
+    // map of path to a an object that has the trail File
+    files: HashMap<PathBuf, FileContent>,
+    /// This is used to keep track of the last file that was written to
+    /// Useful for catching bugs to ensure that we don't write to the same file twice.
+    last_file: Option<PathBuf>,
+
+    lang: L,
+}
+
+impl<L: LanguageFeatures> FileCollector<L> {
+    pub(super) fn new(lang: L) -> Self {
+        Self {
+            files: HashMap::new(),
+            last_file: None,
+            lang,
+        }
+    }
+
+    pub(super) fn finish_file(&mut self) {
+        assert!(self.last_file.is_some(), "No file to finish!");
+        self.last_file = None;
+    }
+
+    pub(super) fn start_file<T: AsRef<str>>(
+        &mut self,
+        dir: &str,
+        name: T,
+        export_all: bool,
+    ) -> &mut FileContent {
+        assert!(
+            self.last_file.is_none(),
+            "Cannot start a new file before finishing the last one!"
+        );
+
+        let path = self.lang.to_file_path(dir, name.as_ref());
+        self.last_file = Some(path.clone());
+        self.files.entry(path).or_insert_with(|| FileContent {
+            content: vec![],
+            import_libs: HashSet::new(),
+            imports: vec![],
+            exports: vec![],
+            export_all,
+        })
+    }
+
+    pub(super) fn commit(&self, dir: &PathBuf) -> std::io::Result<()> {
+        let output_path = dir;
+        // info!("Writing files to {}", output_path.to_string_lossy());
+
+        let temp_path = PathBuf::from(format!("{}.tmp", output_path.to_string_lossy().to_string()));
+
+        // if the .tmp dir exists, delete it so we can get back to a working state without user intervention.
+        let delete_attempts = 3; // Number of attempts to delete the directory
+        let attempt_interval = Duration::from_millis(200); // Wait time between attempts
+
+        for attempt in 1..=delete_attempts {
+            if temp_path.exists() {
+                match std::fs::remove_dir_all(&temp_path) {
+                    Ok(_) => {
+                        println!("Temp directory successfully removed.");
+                        break; // Exit loop after successful deletion
+                    }
+                    Err(e) if e.kind() == ErrorKind::Other && attempt < delete_attempts => {
+                        info!(
+                            "Attempt {}: Failed to delete temp directory: {}",
+                            attempt, e
+                        );
+                        sleep(attempt_interval); // Wait before retrying
+                    }
+                    Err(e) => {
+                        // For other errors or if it's the last attempt, fail with an error
+                        return Err(io::Error::new(
+                            e.kind(),
+                            format!("Failed to delete temp directory: {}", e),
+                        ));
+                    }
+                }
+            } else {
+                break;
+            }
+        }
+
+        if temp_path.exists() {
+            // If the directory still exists after the loop, return an error
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                "Failed to delete existing temp directory within the timeout",
+            ));
+        }
+
+        // Sort the files by path so that we always write to the same file
+        let mut files = self.files.iter().collect::<Vec<_>>();
+        files.sort_by(|(a, _), (b, _)| a.cmp(b));
+
+        for (relative_file_path, file) in &files {
+            let full_file_path = temp_path.join(relative_file_path);
+            std::fs::create_dir_all(full_file_path.parent().unwrap())?;
+            std::fs::write(&full_file_path, &self.format_file(file))?;
+        }
+
+        let _ = std::fs::remove_dir_all(dir);
+        let res = std::fs::rename(&temp_path, output_path);
+
+        info!("Wrote {} files to {}", files.len(), dir.display());
+        res
+    }
+
+    fn format_file(&self, content: &FileContent) -> String {
+        let mut result = vec![];
+
+        let content_prefix = self.lang.content_prefix();
+        if content_prefix.len() > 0 {
+            result.push(content_prefix.to_string());
+        }
+
+        if content.imports.len() + content.import_libs.len() > 0 {
+            result.push(
+                self.lang
+                    .format_imports(&content.import_libs, &content.imports),
+            );
+        }
+
+        if content.content.len() > 0 {
+            result.push(content.content.join("\n\n") + "\n");
+        }
+
+        if content.exports.len() > 0 {
+            result.push(self.lang.format_exports(&content.exports));
+        }
+
+        // Ensure that the file ends with newlines.
+        result.push("".into());
+
+        result.join("\n\n")
+    }
+}
+
+// Add a trait per language that can be used to convert an Import into a string
+pub(super) trait LanguageFeatures {
+    fn to_file_path(&self, path: &str, name: &str) -> PathBuf;
+    fn format_imports(&self, libs: &HashSet<LibImport>, import: &Vec<Import>) -> String;
+    fn format_exports(&self, exports: &Vec<String>) -> String;
+    fn content_prefix(&self) -> &'static str;
+}
+
+pub(super) trait WithFileContentTs<L: LanguageFeatures> {
+    fn file_dir(&self) -> &'static str;
+    fn file_name(&self) -> String;
+    fn write(&self, fc: &mut FileCollector<L>);
+}
+
+pub(super) trait WithFileContentRuby<L: LanguageFeatures> {
+    fn file_dir(&self) -> &'static str;
+    fn file_name(&self) -> String;
+    fn write(&self, fc: &mut FileCollector<L>);
+}
+
+// Until rust supports trait specialization, we can't implement a trait for the same type twice, even if it's generic and the generic type is different (e.g. a diff language).
+// See https://users.rust-lang.org/t/multiple-trait-implementations-based-on-generic-type-bound/17064
+// So to fix we have to hack around and actually have a *different* trait for Python
+// pub(super) trait WithFileContentPy<L: LanguageFeatures> {
+//     fn file_dir(&self) -> &'static str;
+//     fn file_name(&self) -> String;
+//     fn write(&self, fc: &mut FileCollector<L>);
+// }

baml_py-0.1.7/baml-lib/baml-core/src/generate/generate_pipeline.rs

@@ -0,0 +1,24 @@
+use internal_baml_parser_database::ParserDatabase;
+
+use crate::{
+    configuration::{Generator, GeneratorLanguage},
+    ir::IntermediateRepr,
+    lockfile::LockFileWrapper,
+};
+
+use super::generate_python_client_old::generate_py;
+use super::generate_ts_client::generate_ts;
+
+
+pub fn generate_pipeline(
+    db: &ParserDatabase,
+    gen: &Generator,
+    ir: &IntermediateRepr,
+    lock: &LockFileWrapper,
+) -> anyhow::Result<()> {
+    match gen.language {
+        GeneratorLanguage::Python => generate_py(db, gen, lock).map_err(anyhow::Error::new),
+        GeneratorLanguage::TypeScript => generate_ts(&ir, gen).map_err(anyhow::Error::new),
+        _ => anyhow::bail!("generator v1 only supports python and TS"),
+    }
+}