maplib 0.19.5__tar.gz → 0.19.10__tar.gz

{maplib-0.19.5 → maplib-0.19.10}/Cargo.lock

@@ -32,9 +32,9 @@ dependencies = [
 
 [[package]]
 name = "aho-corasick"
-version = "1.1.3"
+version = "1.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
+checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
 dependencies = [
  "memchr",
 ]
@@ -1414,6 +1414,7 @@ dependencies = [
  "oxrdfio",
  "polars",
  "pyo3",
+ "query_processing",
  "rayon",
  "representation",
  "rstest",
@@ -2451,7 +2452,7 @@ dependencies = [
 
 [[package]]
 name = "py_maplib"
-version = "0.19.5"
+version = "0.19.10"
 dependencies = [
  "chrono",
  "cimxml",
@@ -3171,15 +3172,15 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.145"
+version = "1.0.149"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c"
+checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
 dependencies = [
  "itoa",
  "memchr",
- "ryu",
  "serde",
  "serde_core",
+ "zmij",
 ]
 
 [[package]]
@@ -3793,11 +3794,11 @@ dependencies = [
 name = "triplestore"
 version = "0.5.0"
 dependencies = [
+ "aho-corasick",
  "file_io",
  "fts",
  "itoa",
  "memmap2",
- "nohash-hasher",
  "oxrdf",
  "oxrdfio",
  "oxttl",
@@ -3808,6 +3809,8 @@ dependencies = [
  "rayon",
  "representation",
  "ryu",
+ "serde_json",
+ "simd-json",
  "sparesults",
  "spargebra",
  "sprs",
@@ -4494,6 +4497,12 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2f06ae92f42f5e5c42443fd094f245eb656abf56dd7cce9b8b263236565e00f2"
 
+[[package]]
+name = "zmij"
+version = "1.0.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dfcd145825aace48cff44a8844de64bf75feec3080e0aa5cdbde72961ae51a65"
+
 [[package]]
 name = "zstd"
 version = "0.13.3"

{maplib-0.19.5 → maplib-0.19.10}/Cargo.toml

@@ -32,6 +32,9 @@ fundu = "2.0.1"
 memmap2 = "0.9.5"
 sprs = "0.11.3"
 walkdir = "2.5.0"
+aho-corasick = "1.1.4"
+simd-json = { version = "0.17.0", features = ["serde_json"] }
+serde_json = {version = "1.0.149"}
 
 tracing = { version = "0.1", features = [ "log" ] }
 tracing-subscriber = { version = "0.3.19", features = [ "env-filter" ] }

{maplib-0.19.5 → maplib-0.19.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: maplib
-Version: 0.19.5
+Version: 0.19.10
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3 :: Only

{maplib-0.19.5 → maplib-0.19.10}/lib/maplib/Cargo.toml

@@ -14,6 +14,7 @@ shacl = { path = "../shacl" }
 representation = { path = "../representation" }
 datalog = {path = "../datalog"}
 cimxml = {path = "../cimxml"}
+query_processing = {path ="../query_processing"}
 
 rayon = { workspace = true }
 oxrdf = { workspace = true }

{maplib-0.19.5 → maplib-0.19.10}/lib/maplib/src/model/expansion.rs

@@ -11,6 +11,7 @@ use polars::prelude::{
     by_name, col, lit, Column, DataFrame, DataType, Expr, IntoColumn, IntoLazy, LazyFrame,
     NamedFrom, Series,
 };
+use query_processing::expressions::non_multi_col_is_null_workaround;
 use rayon::iter::{IndexedParallelIterator, ParallelDrainRange, ParallelIterator};
 use representation::constants::OTTR_TRIPLE;
 use representation::dataset::NamedGraph;
@@ -31,7 +32,6 @@ use triplestore::TriplesToAdd;
 
 const LIST_COL: &str = "list";
 const FIRST_COL: &str = "first";
-
 const REST_COL: &str = "rest";
 
 impl Model {
@@ -253,7 +253,9 @@ impl Model {
             predicate,
         } in ok_triples
         {
-            let has_multi = subject_type.is_multi() || object_type.is_multi();
+            let subject_multi = subject_type.is_multi();
+            let object_multi = object_type.is_multi();
+            let has_multi = subject_multi || object_multi;
 
             let mut types = HashMap::from([
                 (SUBJECT_COL_NAME.to_string(), subject_type),
@@ -270,23 +272,40 @@ impl Model {
             } else {
                 vec![(df, types)]
             };
-            for (df, mut types) in dfs {
-                let (subject_type, subject_state) = types
+            for (mut df, mut part_types) in dfs {
+                let (subject_type, subject_state) = part_types
                     .remove(SUBJECT_COL_NAME)
                     .unwrap()
                     .map
                     .into_iter()
                     .next()
                     .unwrap();
-                let (object_type, object_state) = types
+                let (object_type, object_state) = part_types
                     .remove(OBJECT_COL_NAME)
                     .unwrap()
                     .map
                     .into_iter()
                     .next()
                     .unwrap();
+                // Splitting by multi introduces nulls
+                if has_multi {
+                    let mut lf = df.lazy();
+                    if subject_multi {
+                        lf = lf.filter(
+                            non_multi_col_is_null_workaround(col(SUBJECT_COL_NAME), &subject_type)
+                                .not(),
+                        )
+                    }
+                    if object_multi {
+                        lf = lf.filter(
+                            non_multi_col_is_null_workaround(col(OBJECT_COL_NAME), &object_type)
+                                .not(),
+                        )
+                    }
+                    df = lf.collect().unwrap();
+                }
                 let predicate_state = if predicate.is_none() {
-                    let (_, s) = types
+                    let (_, s) = part_types
                         .remove(PREDICATE_COL_NAME)
                         .unwrap()
                         .map

{maplib-0.19.5 → maplib-0.19.10}/lib/maplib/src/model.rs

@@ -15,6 +15,7 @@ use representation::solution_mapping::EagerSolutionMappings;
 use representation::RDFNodeState;
 use shacl::{validate, ValidationReport};
 use std::collections::HashMap;
+use std::fs;
 use std::io::Write;
 use std::path::Path;
 use templates::ast::{ConstantTermOrList, PType, Template};
@@ -29,6 +30,7 @@ use datalog::ast::DatalogRuleset;
 use representation::dataset::NamedGraph;
 use representation::prefixes::get_default_prefixes;
 use tracing::instrument;
+use triplestore::errors::TriplestoreError;
 
 pub struct Model {
     pub template_dataset: TemplateDataset,
@@ -167,6 +169,37 @@ impl Model {
         Ok(return_template_iri)
     }
 
+    #[instrument(skip_all)]
+    pub fn map_json_path(
+        &mut self,
+        path: &Path,
+        prefix: &NamedNode,
+        graph: &NamedGraph,
+        transient: bool,
+    ) -> Result<(), MaplibError> {
+        let mut u8s =
+            fs::read(path).map_err(|x| TriplestoreError::ReadJSONFileError(x.to_string()))?;
+
+        self.triplestore
+            .map_json(&mut u8s, prefix, graph, transient)
+            .map_err(MaplibError::TriplestoreError)
+    }
+
+    #[instrument(skip_all)]
+    pub fn map_json_string(
+        &mut self,
+        mut p: String,
+        prefix: &NamedNode,
+        graph: &NamedGraph,
+        transient: bool,
+    ) -> Result<(), MaplibError> {
+        //Safety: we are never reading this vec back to a string
+        let u8s = unsafe { p.as_mut_vec() };
+        self.triplestore
+            .map_json(u8s, prefix, graph, transient)
+            .map_err(MaplibError::TriplestoreError)
+    }
+
     #[allow(clippy::too_many_arguments)]
     #[instrument(skip_all)]
     pub fn read_triples(
@@ -313,9 +346,14 @@ impl Model {
         buffer: &mut W,
         graph: &NamedGraph,
         rdf_format: RdfFormat,
+        prefixes: Option<&HashMap<String, NamedNode>>,
     ) -> Result<(), MaplibError> {
+        let mut use_prefixes = self.prefixes.clone();
+        if let Some(prefixes) = prefixes {
+            use_prefixes.extend(prefixes.clone());
+        }
         self.triplestore
-            .write_triples(buffer, rdf_format, graph)
+            .write_triples(buffer, rdf_format, graph, &use_prefixes)
             .map_err(MaplibError::TriplestoreError)?;
         Ok(())
     }

{maplib-0.19.5 → maplib-0.19.10}/lib/pydf_io/src/to_python.rs

@@ -104,7 +104,7 @@ pub fn df_to_py_df(
     let pyarrow = PyModule::import(py, "pyarrow")?;
     let polars = PyModule::import(py, "polars")?;
     let py_df = to_py_df(&chunk, names.as_slice(), py, &pyarrow, &polars)?;
-    if include_datatypes || debug_outputs.is_some() {
+    if include_datatypes {
         Py::new(
             py,
             PySolutionMappings {

{maplib-0.19.5 → maplib-0.19.10}/lib/query_processing/src/expressions.rs

@@ -159,15 +159,37 @@ pub fn binary_expression(
         .get(right_context.as_str())
         .unwrap();
     if left_type.is_none() || right_type.is_none() {
-        solution_mappings.mappings = solution_mappings.mappings.with_column(
-            lit(LiteralValue::untyped_null())
-                .cast(BaseRDFNodeType::None.default_input_polars_data_type())
-                .alias(outer_context.as_str()),
-        );
-        solution_mappings.rdf_node_types.insert(
-            outer_context.as_str().to_string(),
-            BaseRDFNodeType::None.into_default_input_rdf_node_state(),
-        );
+        if matches!(
+            expression,
+            Expression::Equal(..)
+                | Expression::Less(..)
+                | Expression::Greater(..)
+                | Expression::GreaterOrEqual(..)
+                | Expression::LessOrEqual(..)
+                | Expression::And(..)
+                | Expression::Or(..)
+        ) {
+            let bool = BaseRDFNodeType::Literal(xsd::BOOLEAN.into_owned());
+            solution_mappings.mappings = solution_mappings.mappings.with_column(
+                lit(LiteralValue::untyped_null())
+                    .cast(bool.default_input_polars_data_type())
+                    .alias(outer_context.as_str()),
+            );
+            solution_mappings.rdf_node_types.insert(
+                outer_context.as_str().to_string(),
+                bool.into_default_input_rdf_node_state(),
+            );
+        } else {
+            solution_mappings.mappings = solution_mappings.mappings.with_column(
+                lit(LiteralValue::untyped_null())
+                    .cast(BaseRDFNodeType::None.default_input_polars_data_type())
+                    .alias(outer_context.as_str()),
+            );
+            solution_mappings.rdf_node_types.insert(
+                outer_context.as_str().to_string(),
+                BaseRDFNodeType::None.into_default_input_rdf_node_state(),
+            );
+        }
         solution_mappings =
             drop_inner_contexts(solution_mappings, &vec![left_context, right_context]);
         return Ok(solution_mappings);
@@ -342,23 +364,27 @@ pub fn bound(
 pub fn expr_is_null_workaround(expr: Expr, rdf_node_type: &RDFNodeState) -> Expr {
     if !rdf_node_type.is_multi() {
         let b = rdf_node_type.get_base_type().unwrap();
-        match b {
-            BaseRDFNodeType::Literal(l) => {
-                if l.as_ref() == rdf::LANG_STRING {
-                    expr.struct_()
-                        .field_by_name(LANG_STRING_VALUE_FIELD)
-                        .is_null()
-                } else {
-                    expr.is_null()
-                }
-            }
-            _ => expr.is_null(),
-        }
+        non_multi_col_is_null_workaround(expr, b)
     } else {
         create_all_types_null_expression(expr, rdf_node_type.get_sorted_types())
     }
 }
 
+pub fn non_multi_col_is_null_workaround(expr: Expr, base_rdf_node_type: &BaseRDFNodeType) -> Expr {
+    match base_rdf_node_type {
+        BaseRDFNodeType::Literal(l) => {
+            if l.as_ref() == rdf::LANG_STRING {
+                expr.struct_()
+                    .field_by_name(LANG_STRING_VALUE_FIELD)
+                    .is_null()
+            } else {
+                expr.is_null()
+            }
+        }
+        _ => expr.is_null(),
+    }
+}
+
 pub fn if_expression(
     mut solution_mappings: SolutionMappings,
     left_context: &Context,

{maplib-0.19.5 → maplib-0.19.10}/lib/representation/src/base_rdf_type.rs

@@ -196,7 +196,11 @@ impl BaseRDFNodeType {
         } else if matches!(self, BaseRDFNodeType::IRI | BaseRDFNodeType::BlankNode) {
             DataType::UInt32
         } else if let BaseRDFNodeType::Literal(l) = self {
-            literal_type(l.as_ref(), &BaseCatState::String, false)
+            literal_type(
+                l.as_ref(),
+                &BaseCatState::CategoricalNative(false, None),
+                false,
+            )
         } else {
             unreachable!("Should never happen")
         }
@@ -225,8 +229,9 @@ fn literal_type(
                 DataType::Struct(polars_lang_fields(base_cat_state))
             } else {
                 match base_cat_state {
-                    BaseCatState::CategoricalNative(_, _) => DataType::UInt32,
-                    BaseCatState::String | BaseCatState::NonString => DataType::String,
+                    BaseCatState::String
+                    | BaseCatState::NonString
+                    | BaseCatState::CategoricalNative(..) => DataType::String,
                 }
             }
         }

{maplib-0.19.5 → maplib-0.19.10}/lib/representation/src/cats/decode.rs

@@ -20,12 +20,21 @@ impl CatEncs {
         new_ser
     }
 
-    pub fn maybe_decode_string(&self, u: &u32) -> Option<&str> {
+    pub fn maybe_decode_string(&self, u: &u32) -> Option<Cow<'_, str>> {
         self.maps.maybe_decode(u)
     }
 }
 
 impl Cats {
+    pub fn maybe_decode_of_type(&self, u: &u32, bt: &BaseRDFNodeType) -> Option<Cow<'_, str>> {
+        let ct = CatType::from_base_rdf_node_type(bt);
+        if let Some(cat_encs) = self.cat_map.get(&ct) {
+            cat_encs.maybe_decode_string(u)
+        } else {
+            None
+        }
+    }
+
     pub fn decode_iri_u32s(&self, us: &[u32], local_cats: Option<LockedCats>) -> Vec<NamedNode> {
         us.iter()
             .map(|x| self.decode_iri_u32(x, local_cats.clone()))
@@ -135,7 +144,7 @@ impl Cats {
                             break;
                         }
                     }
-                    Some(Cow::Borrowed(s.expect("Expect all cats to resolve")))
+                    Some(s.expect("Expect all cats to resolve"))
                 } else {
                     None
                 };

{maplib-0.19.5 → maplib-0.19.10}/lib/representation/src/cats/encode.rs

@@ -10,14 +10,14 @@ use std::collections::HashMap;
 use std::path::Path;
 
 impl CatEncs {
-    pub fn new_empty(path: Option<&Path>) -> CatEncs {
+    pub fn new_empty(path: Option<&Path>, bt: &BaseRDFNodeType) -> CatEncs {
         CatEncs {
-            maps: CatMaps::new_empty(path),
+            maps: CatMaps::new_empty(path, bt),
         }
     }
 
-    pub fn new_singular(value: &str, u: u32, path: Option<&Path>) -> CatEncs {
-        let maps = CatMaps::new_singular(value, u, path);
+    pub fn new_singular(value: &str, u: u32, path: Option<&Path>, bt: &BaseRDFNodeType) -> CatEncs {
+        let maps = CatMaps::new_singular(value, u, path, bt);
         CatEncs { maps }
     }
 
@@ -112,7 +112,7 @@ impl Cats {
         };
 
         let enc = self.get_encs(t).pop();
-        let mut new_enc = CatEncs::new_empty(path);
+        let mut new_enc = CatEncs::new_empty(path, t);
         let strch = series.str().unwrap();
         let encoded_global: Vec<_> = strch
             .iter()

{maplib-0.19.5 → maplib-0.19.10}/lib/representation/src/cats/globalize.rs

@@ -27,7 +27,8 @@ impl Cats {
         for ct in cat_triples {
             let t = CatType::IRI;
             if !self.cat_map.contains_key(&t) {
-                self.cat_map.insert(t.clone(), CatEncs::new_empty(path));
+                self.cat_map
+                    .insert(t.clone(), CatEncs::new_empty(path, &BaseRDFNodeType::IRI));
             }
             let enc = self.cat_map.get_mut(&t).unwrap();
             let pred = ct.predicate.as_str().to_string();

{maplib-0.19.5 → maplib-0.19.10}/lib/representation/src/cats/image.rs

@@ -2,7 +2,7 @@ use super::{reencode_solution_mappings, CatEncs, CatType};
 use super::{CatReEnc, Cats};
 use crate::cats::LockedCats;
 use crate::solution_mapping::{BaseCatState, EagerSolutionMappings};
-use crate::{BaseRDFNodeType, RDFNodeState};
+use crate::{BaseRDFNodeType, RDFNodeState, LANG_STRING_LANG_FIELD, LANG_STRING_VALUE_FIELD};
 use nohash_hasher::NoHashHasher;
 use std::collections::{HashMap, HashSet};
 use std::hash::BuildHasherDefault;
@@ -21,20 +21,44 @@ impl Cats {
                 let t = sm.rdf_node_types.get(c).unwrap();
                 for (bt, bs) in &t.map {
                     if let BaseCatState::CategoricalNative(_, local) = bs {
-                        let ser = if t.is_multi() {
-                            sm.mappings
+                        let mut sers = vec![];
+                        if bt.is_lang_string() {
+                            let ser1 = sm
+                                .mappings
                                 .column(c)
                                 .unwrap()
                                 .struct_()
                                 .unwrap()
-                                .field_by_name(&bt.field_col_name())
+                                .field_by_name(LANG_STRING_VALUE_FIELD)
+                                .unwrap();
+                            sers.push(ser1);
+                            let ser2 = sm
+                                .mappings
+                                .column(c)
+                                .unwrap()
+                                .struct_()
+                                .unwrap()
+                                .field_by_name(LANG_STRING_LANG_FIELD)
+                                .unwrap();
+                            sers.push(ser2);
+                        } else if t.is_multi() {
+                            let ser = sm
+                                .mappings
+                                .column(c)
                                 .unwrap()
+                                .struct_()
+                                .unwrap()
+                                .field_by_name(&bt.field_col_name())
+                                .unwrap();
+                            sers.push(ser);
                         } else {
-                            sm.mappings
+                            let ser = sm
+                                .mappings
                                 .column(c)
                                 .unwrap()
                                 .as_materialized_series()
-                                .clone()
+                                .clone();
+                            sers.push(ser);
                         };
                         if !s.contains_key(bt) {
                             s.insert(bt.clone(), HashSet::new());
@@ -48,16 +72,18 @@ impl Cats {
                             None
                         };
                         let ss = s.get_mut(bt).unwrap();
-                        let new_ss = ser
-                            .u32()
-                            .unwrap()
-                            .into_iter()
-                            .filter(|x| x.is_some())
-                            .map(|x| x.unwrap());
-                        if let Some(local_rev_map) = local_rev_map {
-                            ss.extend(new_ss.filter(|x| !local_rev_map.maps.contains_u32(x)));
-                        } else {
-                            ss.extend(new_ss);
+                        for ser in sers {
+                            let new_ss = ser
+                                .u32()
+                                .unwrap()
+                                .into_iter()
+                                .filter(|x| x.is_some())
+                                .map(|x| x.unwrap());
+                            if let Some(local_rev_map) = local_rev_map {
+                                ss.extend(new_ss.filter(|x| !local_rev_map.maps.contains_u32(x)));
+                            } else {
+                                ss.extend(new_ss);
+                            }
                         }
                     }
                 }