maplib 0.18.18__tar.gz → 0.19.5__tar.gz

{maplib-0.18.18 → maplib-0.19.5}/Cargo.lock

@@ -2451,7 +2451,7 @@ dependencies = [
 
 [[package]]
 name = "py_maplib"
-version = "0.18.18"
+version = "0.19.5"
 dependencies = [
  "chrono",
  "cimxml",

{maplib-0.18.18 → maplib-0.19.5}/Cargo.toml

@@ -12,7 +12,7 @@ polars = { version = "0.52.0", features = ["simd", "nightly", "new_streaming", "
 polars-core = { version = "0.52.0" }
 rayon = "1.10.0"
 regex = "1.11.1"
-oxrdf = { version="0.2.4" }
+oxrdf = { version = "0.2.4" }
 oxrdfio = { version = "0.1.7" }
 oxttl = { version="0.1.7" }
 oxiri = "0.2.11"

{maplib-0.18.18 → maplib-0.19.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: maplib
-Version: 0.18.18
+Version: 0.19.5
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3 :: Only
@@ -81,7 +81,7 @@ template = Template(
         Parameter(variable=ings_var, rdf_type=RDFType.Nested(RDFType.IRI()))
     ],
     instances= [
-        Triple(p_var, a(), pi.suf("Pizza")),
+        Triple(p_var, a, pi.suf("Pizza")),
         Triple(p_var, pi.suf("fromCountry"), c_var),
         Triple(
             p_var, 

{maplib-0.18.18 → maplib-0.19.5}/README.md

@@ -55,7 +55,7 @@ template = Template(
         Parameter(variable=ings_var, rdf_type=RDFType.Nested(RDFType.IRI()))
     ],
     instances= [
-        Triple(p_var, a(), pi.suf("Pizza")),
+        Triple(p_var, a, pi.suf("Pizza")),
         Triple(p_var, pi.suf("fromCountry"), c_var),
         Triple(
             p_var, 

{maplib-0.18.18 → maplib-0.19.5}/lib/datalog/src/inference.rs

@@ -11,8 +11,7 @@ pub enum DatalogError {
 }
 
 #[derive(Clone)]
-pub struct InferenceResult {
-}
+pub struct InferenceResult {}
 
 pub fn infer(
     _triplestore: &mut Triplestore,

maplib-0.19.5/lib/datalog/src/python.rs

@@ -0,0 +1,19 @@
+use crate::inference::InferenceResult;
+use pyo3::{pyclass, pymethods, PyResult};
+
+#[derive(Clone)]
+#[pyclass(name = "InferenceResult")]
+pub struct PyInferenceResult {
+    pub inner: InferenceResult,
+}
+
+#[pymethods]
+impl PyInferenceResult {
+    fn __repr__(&self) -> PyResult<String> {
+        Ok("Contact Data Treehouse to try".to_string())
+    }
+
+    fn __str__(&self) -> PyResult<String> {
+        Ok("Contact Data Treehouse to try".to_string())
+    }
+}

{maplib-0.18.18 → maplib-0.19.5}/lib/fts/src/lib.rs

@@ -14,7 +14,7 @@ pub enum FtsError {}
 pub struct FtsIndex {}
 
 impl FtsIndex {
-    pub fn new(_path: &Path, _uuid:&str) -> Result<FtsIndex, FtsError> {
+    pub fn new(_path: &Path, _uuid: &str) -> Result<FtsIndex, FtsError> {
         unimplemented!("Contact Data Treehouse to enable full text search")
     }
 
@@ -31,7 +31,7 @@ impl FtsIndex {
         unimplemented!("Contact Data Treehouse to enable full text search")
     }
 
-    pub fn commit(&mut self, _drop_writer:bool) -> Result<(), FtsError> {
+    pub fn commit(&mut self, _drop_writer: bool) -> Result<(), FtsError> {
         unimplemented!("Contact Data Treehouse to enable full text search")
     }
 

{maplib-0.18.18 → maplib-0.19.5}/lib/maplib/src/lib.rs

@@ -3,4 +3,3 @@ extern crate chrono_tz;
 
 pub mod errors;
 pub mod model;
-mod prefixes;

{maplib-0.18.18 → maplib-0.19.5}/lib/maplib/src/model/constant_terms.rs

@@ -8,13 +8,13 @@ use polars::prelude::{
 };
 use rayon::iter::ParallelIterator;
 use rayon::prelude::IntoParallelIterator;
+use representation::constants::{OTTR_BLANK_NODE, OTTR_IRI};
 use representation::rdf_to_polars::{
     polars_literal_values_to_series, rdf_named_node_to_polars_literal_value,
     rdf_term_to_polars_expr,
 };
 use representation::BaseRDFNodeType;
 use std::ops::Deref;
-use templates::constants::{OTTR_BLANK_NODE, OTTR_IRI};
 
 const BLANK_NODE_SERIES_NAME: &str = "blank_node_series";
 

{maplib-0.18.18 → maplib-0.19.5}/lib/maplib/src/model/default.rs

@@ -1,12 +1,12 @@
 use super::Model;
 use crate::model::errors::MappingError;
 use crate::model::MapOptions;
+use representation::constants::{DEFAULT_PREFIX_IRI, OTTR_IRI, OTTR_TRIPLE};
 use std::collections::HashMap;
 use templates::ast::{
     Argument, ConstantTerm, ConstantTermOrList, Instance, ListExpanderType, PType, Parameter,
     Signature, StottrTerm, Template,
 };
-use templates::constants::{DEFAULT_PREFIX, OTTR_IRI, OTTR_TRIPLE};
 use tracing::warn;
 
 use crate::errors::MaplibError;
@@ -121,7 +121,7 @@ impl Model {
                             list_expand: false,
                             term: StottrTerm::ConstantTerm(ConstantTermOrList::ConstantTerm(
                                 ConstantTerm::Iri(
-                                    NamedNode::new(format!("{DEFAULT_PREFIX}{c}"))
+                                    NamedNode::new(format!("{DEFAULT_PREFIX_IRI}{c}"))
                                         .map_err(MappingError::IriParseError)?,
                                 ),
                             )),
@@ -137,7 +137,7 @@ impl Model {
 
         let template_name = format!(
             "{}default_template_{}",
-            DEFAULT_PREFIX, &self.default_template_counter
+            DEFAULT_PREFIX_IRI, &self.default_template_counter
         );
         self.default_template_counter += 1;
         let template = Template {

{maplib-0.18.18 → maplib-0.19.5}/lib/maplib/src/model/expansion.rs

@@ -12,6 +12,7 @@ use polars::prelude::{
     NamedFrom, Series,
 };
 use rayon::iter::{IndexedParallelIterator, ParallelDrainRange, ParallelIterator};
+use representation::constants::OTTR_TRIPLE;
 use representation::dataset::NamedGraph;
 use representation::multitype::split_df_multicols;
 use representation::rdf_to_polars::rdf_named_node_to_polars_literal_value;
@@ -24,7 +25,6 @@ use std::time::Instant;
 use templates::ast::{
     ConstantTerm, ConstantTermOrList, Instance, ListExpanderType, PType, Signature, StottrTerm,
 };
-use templates::constants::OTTR_TRIPLE;
 use templates::MappingColumnType;
 use tracing::debug;
 use triplestore::TriplesToAdd;
@@ -302,6 +302,7 @@ impl Model {
                     subject_type: subject_type,
                     object_type: object_type,
                     predicate: predicate.clone(),
+                    graph: graph.clone(),
                     subject_cat_state: subject_state,
                     object_cat_state: object_state,
                     predicate_cat_state: predicate_state,
@@ -310,7 +311,7 @@ impl Model {
         }
 
         self.triplestore
-            .add_triples_vec(all_triples_to_add, false, graph)
+            .add_triples_vec(all_triples_to_add, false)
             .map_err(MappingError::TriplestoreError)?;
 
         self.blank_node_counter = new_blank_node_counter;

{maplib-0.18.18 → maplib-0.19.5}/lib/maplib/src/model.rs

@@ -25,9 +25,9 @@ use triplestore::sparql::errors::SparqlError;
 use triplestore::sparql::{QueryResult, QuerySettings, UpdateResult};
 use triplestore::{IndexingOptions, NewTriples, Triplestore};
 
-use crate::prefixes::get_default_prefixes;
 use datalog::ast::DatalogRuleset;
 use representation::dataset::NamedGraph;
+use representation::prefixes::get_default_prefixes;
 use tracing::instrument;
 
 pub struct Model {
@@ -79,7 +79,7 @@ impl Model {
             if let Some(indexing) = indexing {
                 indexing
             } else {
-                IndexingOptions::new(false, None, None)
+                IndexingOptions::default()
             }
         } else {
             indexing.unwrap_or_default()
@@ -184,7 +184,16 @@ impl Model {
             self.truncate_graph(&graph)
         }
         self.triplestore
-            .read_triples_from_path(p, rdf_format, base_iri, transient, parallel, checked, graph)
+            .read_triples_from_path(
+                p,
+                rdf_format,
+                base_iri,
+                transient,
+                parallel,
+                checked,
+                graph,
+                &self.prefixes,
+            )
             .map_err(MaplibError::TriplestoreError)
     }
 
@@ -217,7 +226,16 @@ impl Model {
             self.truncate_graph(&graph)
         }
         self.triplestore
-            .read_triples_from_string(s, rdf_format, base_iri, transient, parallel, checked, graph)
+            .read_triples_from_string(
+                s,
+                rdf_format,
+                base_iri,
+                transient,
+                parallel,
+                checked,
+                graph,
+                &self.prefixes,
+            )
             .map_err(MaplibError::TriplestoreError)
     }
 
@@ -369,6 +387,7 @@ impl Model {
         only_shapes: Option<Vec<NamedNode>>,
         deactivate_shapes: Vec<NamedNode>,
         dry_run: bool,
+        serial: bool,
     ) -> Result<ValidationReport, MaplibError> {
         let res = validate(
             &mut self.triplestore,
@@ -384,6 +403,7 @@ impl Model {
             deactivate_shapes,
             dry_run,
             Some(self.prefixes.clone()),
+            serial,
         );
         res.map_err(|x| x.into())
     }
@@ -465,4 +485,20 @@ impl Model {
         );
         Ok(res.map_err(|x| MaplibError::DatalogError(x))?)
     }
+
+    pub fn detach_graph(
+        &mut self,
+        graph: &NamedGraph,
+        preserve_name: bool,
+    ) -> Result<Model, MaplibError> {
+        let triplestore = self.triplestore.detach_graph(graph, preserve_name)?;
+        Ok(Model {
+            template_dataset: self.template_dataset.clone(),
+            triplestore,
+            blank_node_counter: self.blank_node_counter,
+            default_template_counter: self.default_template_counter,
+            indexing: self.indexing.clone(),
+            prefixes: self.prefixes.clone(),
+        })
+    }
 }

{maplib-0.18.18 → maplib-0.19.5}/lib/query_processing/src/cats.rs

@@ -25,52 +25,55 @@ pub fn create_compatible_cats(
     let mut string_cast = HashSet::new();
 
     for t in base_types {
-        let need_native_cat_cast = check_need_native_cat_cast(&t, &states);
-        if need_native_cat_cast {
-            let locals = find_all_locals(&t, &states);
-            let mut cats = Cats::new_empty(Some(global_cats.read().unwrap().deref()));
-            let renc_local = cats.merge(locals);
-            let renc_local: HashMap<_, _> = renc_local
-                .into_iter()
-                .map(|(uu, map)| {
-                    let mut blank_renc = None;
-                    let mut iri_renc = vec![];
-                    let mut literal_renc_map = HashMap::new();
-                    for (p, renc) in map {
-                        match p {
-                            CatType::IRI => {
-                                iri_renc.push(renc);
-                            }
-                            CatType::Blank => {
-                                blank_renc = Some(renc);
-                            }
-                            CatType::Literal(l) => {
-                                literal_renc_map.insert(l, renc);
-                            }
-                        }
-                    }
-                    let iri_renc = if iri_renc.is_empty() {
-                        None
-                    } else {
-                        let mut renc_map =
-                            HashMap::with_capacity_and_hasher(2, BuildHasherDefault::default());
-                        for renc in iri_renc {
-                            renc_map.extend(renc.cat_map.iter().map(|(x, y)| (*x, *y)))
-                        }
-                        Some(CatReEnc {
-                            cat_map: Arc::new(renc_map),
-                        })
-                    };
-                    (uu, (blank_renc, iri_renc, literal_renc_map))
-                })
-                .collect();
-            let state = BaseCatState::CategoricalNative(false, Some(LockedCats::new(cats)));
-            native_cat_map.insert(t.clone(), (renc_local, state));
-        }
         let need_string_cast = check_need_string_cast(&t, &states);
         if need_string_cast {
             string_cast.insert(t.clone());
         }
+        if !need_string_cast {
+            let need_native_cat_cast = check_need_native_cat_cast(&t, &states);
+            if need_native_cat_cast {
+                let locals = find_all_locals(&t, &states);
+                let mut cats = Cats::new_empty(Some(global_cats.read().unwrap().deref()));
+                // We do not store local cats to disk
+                let renc_local = cats.merge(locals, None);
+                let renc_local: HashMap<_, _> = renc_local
+                    .into_iter()
+                    .map(|(uu, map)| {
+                        let mut blank_renc = None;
+                        let mut iri_renc = vec![];
+                        let mut literal_renc_map = HashMap::new();
+                        for (p, renc) in map {
+                            match p {
+                                CatType::IRI => {
+                                    iri_renc.push(renc);
+                                }
+                                CatType::Blank => {
+                                    blank_renc = Some(renc);
+                                }
+                                CatType::Literal(l) => {
+                                    literal_renc_map.insert(l, renc);
+                                }
+                            }
+                        }
+                        let iri_renc = if iri_renc.is_empty() {
+                            None
+                        } else {
+                            let mut renc_map =
+                                HashMap::with_capacity_and_hasher(2, BuildHasherDefault::default());
+                            for renc in iri_renc {
+                                renc_map.extend(renc.cat_map.iter().map(|(x, y)| (*x, *y)))
+                            }
+                            Some(CatReEnc {
+                                cat_map: Arc::new(renc_map),
+                            })
+                        };
+                        (uu, (blank_renc, iri_renc, literal_renc_map))
+                    })
+                    .collect();
+                let state = BaseCatState::CategoricalNative(false, Some(LockedCats::new(cats)));
+                native_cat_map.insert(t.clone(), (renc_local, state));
+            }
+        }
     }
 
     let mut exploded: Vec<_> = expressions

{maplib-0.18.18 → maplib-0.19.5}/lib/query_processing/src/errors.rs

@@ -16,4 +16,6 @@ pub enum QueryProcessingError {
     BadNumberOfFunctionArguments(Function, usize, String),
     #[error("Maximum estimated rows `{}` in result exceeds configured maximum `{}`. You may have a cross join in your query, please double check. Alternatively, try setting the max_rows parameter to a higher value. Left columns: `{}` Right columns: `{}`", .0, .1, .2, .3)]
     MaxRowsReached(usize, usize, String, String),
+    #[error("The function `{}` is not implemented yet", .0)]
+    UnimplementedFunction(String),
 }

{maplib-0.18.18 → maplib-0.19.5}/lib/query_processing/src/expressions/functions.rs

@@ -935,7 +935,7 @@ pub fn func_expression(
                     .rdf_node_types
                     .insert(outer_context.as_str().to_string(), t_new);
             } else {
-                todo!("Function {nn} is not implemented yet")
+                return Err(QueryProcessingError::UnimplementedFunction(nn.to_string()));
             }
         }
         Function::StrBefore | Function::StrAfter => {

{maplib-0.18.18 → maplib-0.19.5}/lib/query_processing/src/expressions.rs

@@ -32,7 +32,8 @@ pub fn named_node_local_enc(nn: &NamedNode, global_cats: &Cats) -> (Expr, Option
     if let Some(enc) = global_cats.encode_iri_slice(&[nn.as_str()]).pop().unwrap() {
         (lit(enc).cast(DataType::UInt32), None)
     } else {
-        let (enc, local) = Cats::new_singular_iri(nn.as_str(), global_cats.get_iri_counter());
+        // We do not store these local cats to disk
+        let (enc, local) = Cats::new_singular_iri(nn.as_str(), global_cats.get_iri_counter(), None);
         (lit(enc).cast(DataType::UInt32), Some(local))
     }
 }
@@ -46,12 +47,14 @@ pub fn blank_node_local_enc(bl: &BlankNode, global_cats: &Cats) -> (Expr, Option
     if let Some(enc) = global_cats.encode_blanks(&[bl.as_str()]).pop().unwrap() {
         (lit(enc).cast(DataType::UInt32), None)
     } else {
-        let (enc, local) = Cats::new_singular_blank(bl.as_str(), global_cats.get_iri_counter());
+        // Local cats not stored to disk, hence None path
+        let (enc, local) =
+            Cats::new_singular_blank(bl.as_str(), global_cats.get_iri_counter(), None);
         (lit(enc).cast(DataType::UInt32), Some(local))
     }
 }
 
-pub fn maybe_literal_enc(l: &Literal, global_cats: &Cats) -> (Expr, BaseRDFNodeType, BaseCatState) {
+pub fn literal_enc(l: &Literal, global_cats: &Cats) -> (Expr, BaseRDFNodeType, BaseCatState) {
     let bt = BaseRDFNodeType::Literal(l.datatype().into_owned());
     if literal_is_cat(l.datatype()) {
         if let Some(enc) = global_cats
@@ -67,7 +70,8 @@ pub fn maybe_literal_enc(l: &Literal, global_cats: &Cats) -> (Expr, BaseRDFNodeT
         } else {
             let dt = l.datatype().into_owned();
             let offset = global_cats.get_literal_counter(&dt);
-            let (enc, local) = Cats::new_singular_literal(l.value(), dt, offset);
+            // Local cats are not stored to disk, hence None
+            let (enc, local) = Cats::new_singular_literal(l.value(), dt, offset, None);
             (
                 lit(enc).cast(DataType::UInt32),
                 bt,
@@ -106,7 +110,7 @@ pub fn literal(
     context: &Context,
     global_cats: &Cats,
 ) -> Result<SolutionMappings, QueryProcessingError> {
-    let (e, bt, bs) = maybe_literal_enc(lit, global_cats);
+    let (e, bt, bs) = literal_enc(lit, global_cats);
     solution_mappings.mappings = solution_mappings
         .mappings
         .with_column(e.alias(context.as_str()));
@@ -361,23 +365,75 @@ pub fn if_expression(
     middle_context: &Context,
     right_context: &Context,
     outer_context: &Context,
+    global_cats: LockedCats,
 ) -> Result<SolutionMappings, QueryProcessingError> {
+    //
+    let mut exploded: Vec<_> = create_compatible_cats(
+        vec![
+            Some(col(middle_context.as_str())),
+            Some(col(right_context.as_str())),
+        ],
+        vec![
+            Some(
+                solution_mappings
+                    .rdf_node_types
+                    .get(middle_context.as_str())
+                    .unwrap()
+                    .clone(),
+            ),
+            Some(
+                solution_mappings
+                    .rdf_node_types
+                    .get(right_context.as_str())
+                    .unwrap()
+                    .clone(),
+            ),
+        ],
+        global_cats,
+    )
+    .into_iter()
+    .map(|x| x.unwrap())
+    .collect();
+    assert_eq!(exploded.len(), 2);
+
+    let right_exploded = exploded.pop().unwrap();
+    let mid_exploded = exploded.pop().unwrap();
+
+    let mut mid_exprs = vec![];
+    let mut right_exprs = vec![];
+
+    let mut base_type_map = HashMap::new();
+    for (t, (exprs, base_state)) in right_exploded {
+        base_type_map.insert(t, base_state);
+        right_exprs.extend(exprs);
+    }
+    for (_, (exprs, _)) in mid_exploded {
+        mid_exprs.extend(exprs);
+    }
+    let mid_expr = if mid_exprs.len() > 1 {
+        as_struct(mid_exprs)
+    } else {
+        mid_exprs.pop().unwrap()
+    };
+
+    let right_expr = if right_exprs.len() > 1 {
+        as_struct(right_exprs)
+    } else {
+        right_exprs.pop().unwrap()
+    };
+
     solution_mappings.mappings = solution_mappings.mappings.with_column(
         (Expr::Ternary {
             predicate: Arc::new(col(left_context.as_str())),
-            truthy: Arc::new(col(middle_context.as_str())),
-            falsy: Arc::new(col(right_context.as_str())),
+            truthy: Arc::new(mid_expr),
+            falsy: Arc::new(right_expr),
         })
         .alias(outer_context.as_str()),
     );
-    //Todo: generalize..
-    let existing_type = solution_mappings
-        .rdf_node_types
-        .get(middle_context.as_str())
-        .unwrap();
-    solution_mappings
-        .rdf_node_types
-        .insert(outer_context.as_str().to_string(), existing_type.clone());
+    solution_mappings.rdf_node_types.insert(
+        outer_context.as_str().to_string(),
+        RDFNodeState::from_map(base_type_map),
+    );
     solution_mappings = drop_inner_contexts(
         solution_mappings,
         &vec![left_context, middle_context, right_context],

{maplib-0.18.18 → maplib-0.19.5}/lib/representation/src/base_rdf_type.rs

@@ -212,9 +212,11 @@ fn literal_type(
         xsd::UNSIGNED_INT => DataType::UInt32,
         xsd::UNSIGNED_LONG => DataType::UInt64,
         xsd::UNSIGNED_SHORT => DataType::UInt16,
+        xsd::UNSIGNED_BYTE => DataType::UInt8,
         xsd::INTEGER | xsd::LONG => DataType::Int64,
         xsd::INT => DataType::Int32,
         xsd::SHORT => DataType::Int16,
+        xsd::BYTE => DataType::Int8,
         xsd::DOUBLE | xsd::DECIMAL => DataType::Float64,
         xsd::FLOAT => DataType::Float32,
         xsd::BOOLEAN => DataType::Boolean,

{maplib-0.18.18 → maplib-0.19.5}/lib/representation/src/cats/decode.rs

@@ -8,30 +8,20 @@ use polars::frame::column::ScalarColumn;
 use polars::frame::DataFrame;
 use polars::prelude::{col, Column, Expr, IntoColumn, IntoLazy, NamedFrom, SeriesSealed};
 use polars::series::Series;
-use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator};
+use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
 use std::borrow::Cow;
 
 impl CatEncs {
-    pub fn decode(&self, ser: &Series) -> Series {
+    pub fn decode_series(&self, ser: &Series) -> Series {
         let original_name = ser.name().clone();
         let uch: Vec<_> = ser.u32().unwrap().iter().collect();
-        let decoded_vec_iter = uch
-            .into_par_iter()
-            .map(|x| x.map(|x| self.rev_map.get(&x).unwrap()));
-
-        let decoded_vec: Vec<_> = decoded_vec_iter.map(|x| x.map(|x| x.as_str())).collect();
+        let decoded_vec = self.maps.decode_batch(uch.as_ref());
         let new_ser = Series::new(original_name, decoded_vec);
-        //
-        // let mut new_ser = if let CatType::Prefix(pre) = cat_type {
-        //     Series::from_iter(decoded_ser.map(|x| x.map(|x| format!("{}{}", pre.as_str(), x))))
-        // } else {
-        //     Series::from_iter(decoded_ser.map(|x| x.map(|x| x.as_str())))
-        // };
         new_ser
     }
 
     pub fn maybe_decode_string(&self, u: &u32) -> Option<&str> {
-        self.rev_map.get(u).map(|x| x.as_str())
+        self.maps.maybe_decode(u)
     }
 }
 
@@ -68,7 +58,7 @@ impl Cats {
     pub fn decode_of_type(&self, ser: &Series, bt: &BaseRDFNodeType) -> Series {
         let ct = CatType::from_base_rdf_node_type(bt);
         if let Some(enc) = self.cat_map.get(&ct) {
-            enc.decode(ser)
+            enc.decode_series(ser)
         } else {
             unreachable!("Should never be called when type does not exist")
         }
@@ -120,6 +110,9 @@ impl Cats {
         local_cats: Option<LockedCats>,
     ) -> Series {
         let local = local_cats.as_ref().map(|x| x.read().unwrap());
+        if ser.dtype() != &DataType::UInt32 {
+            panic!("Series should have datatype UInt32 {:?}", ser);
+        }
         let u32s = ser.u32().unwrap();
         let us: Vec<_> = u32s.iter().collect();