maplib 0.19.9__tar.gz → 0.19.10__tar.gz

{maplib-0.19.9 → maplib-0.19.10}/Cargo.lock

@@ -2452,7 +2452,7 @@ dependencies = [
 
 [[package]]
 name = "py_maplib"
-version = "0.19.9"
+version = "0.19.10"
 dependencies = [
  "chrono",
  "cimxml",
@@ -3172,15 +3172,15 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.145"
+version = "1.0.149"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c"
+checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
 dependencies = [
  "itoa",
  "memchr",
- "ryu",
  "serde",
  "serde_core",
+ "zmij",
 ]
 
 [[package]]
@@ -3809,6 +3809,8 @@ dependencies = [
  "rayon",
  "representation",
  "ryu",
+ "serde_json",
+ "simd-json",
  "sparesults",
  "spargebra",
  "sprs",
@@ -4495,6 +4497,12 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2f06ae92f42f5e5c42443fd094f245eb656abf56dd7cce9b8b263236565e00f2"
 
+[[package]]
+name = "zmij"
+version = "1.0.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dfcd145825aace48cff44a8844de64bf75feec3080e0aa5cdbde72961ae51a65"
+
 [[package]]
 name = "zstd"
 version = "0.13.3"

{maplib-0.19.9 → maplib-0.19.10}/Cargo.toml

@@ -33,6 +33,8 @@ memmap2 = "0.9.5"
 sprs = "0.11.3"
 walkdir = "2.5.0"
 aho-corasick = "1.1.4"
+simd-json = { version = "0.17.0", features = ["serde_json"] }
+serde_json = {version = "1.0.149"}
 
 tracing = { version = "0.1", features = [ "log" ] }
 tracing-subscriber = { version = "0.3.19", features = [ "env-filter" ] }

{maplib-0.19.9 → maplib-0.19.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: maplib
-Version: 0.19.9
+Version: 0.19.10
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3 :: Only

{maplib-0.19.9 → maplib-0.19.10}/lib/maplib/src/model/expansion.rs

@@ -11,6 +11,7 @@ use polars::prelude::{
     by_name, col, lit, Column, DataFrame, DataType, Expr, IntoColumn, IntoLazy, LazyFrame,
     NamedFrom, Series,
 };
+use query_processing::expressions::non_multi_col_is_null_workaround;
 use rayon::iter::{IndexedParallelIterator, ParallelDrainRange, ParallelIterator};
 use representation::constants::OTTR_TRIPLE;
 use representation::dataset::NamedGraph;
@@ -27,7 +28,6 @@ use templates::ast::{
 };
 use templates::MappingColumnType;
 use tracing::debug;
-use query_processing::expressions::non_multi_col_is_null_workaround;
 use triplestore::TriplesToAdd;
 
 const LIST_COL: &str = "list";
@@ -291,10 +291,16 @@ impl Model {
                 if has_multi {
                     let mut lf = df.lazy();
                     if subject_multi {
-                        lf = lf.filter(non_multi_col_is_null_workaround(col(SUBJECT_COL_NAME), &subject_type).not())
+                        lf = lf.filter(
+                            non_multi_col_is_null_workaround(col(SUBJECT_COL_NAME), &subject_type)
+                                .not(),
+                        )
                     }
                     if object_multi {
-                        lf = lf.filter(non_multi_col_is_null_workaround(col(OBJECT_COL_NAME), &object_type).not())
+                        lf = lf.filter(
+                            non_multi_col_is_null_workaround(col(OBJECT_COL_NAME), &object_type)
+                                .not(),
+                        )
                     }
                     df = lf.collect().unwrap();
                 }

{maplib-0.19.9 → maplib-0.19.10}/lib/maplib/src/model.rs

@@ -15,6 +15,7 @@ use representation::solution_mapping::EagerSolutionMappings;
 use representation::RDFNodeState;
 use shacl::{validate, ValidationReport};
 use std::collections::HashMap;
+use std::fs;
 use std::io::Write;
 use std::path::Path;
 use templates::ast::{ConstantTermOrList, PType, Template};
@@ -29,6 +30,7 @@ use datalog::ast::DatalogRuleset;
 use representation::dataset::NamedGraph;
 use representation::prefixes::get_default_prefixes;
 use tracing::instrument;
+use triplestore::errors::TriplestoreError;
 
 pub struct Model {
     pub template_dataset: TemplateDataset,
@@ -167,6 +169,37 @@ impl Model {
         Ok(return_template_iri)
     }
 
+    #[instrument(skip_all)]
+    pub fn map_json_path(
+        &mut self,
+        path: &Path,
+        prefix: &NamedNode,
+        graph: &NamedGraph,
+        transient: bool,
+    ) -> Result<(), MaplibError> {
+        let mut u8s =
+            fs::read(path).map_err(|x| TriplestoreError::ReadJSONFileError(x.to_string()))?;
+
+        self.triplestore
+            .map_json(&mut u8s, prefix, graph, transient)
+            .map_err(MaplibError::TriplestoreError)
+    }
+
+    #[instrument(skip_all)]
+    pub fn map_json_string(
+        &mut self,
+        mut p: String,
+        prefix: &NamedNode,
+        graph: &NamedGraph,
+        transient: bool,
+    ) -> Result<(), MaplibError> {
+        //Safety: we are never reading this vec back to a string
+        let u8s = unsafe { p.as_mut_vec() };
+        self.triplestore
+            .map_json(u8s, prefix, graph, transient)
+            .map_err(MaplibError::TriplestoreError)
+    }
+
     #[allow(clippy::too_many_arguments)]
     #[instrument(skip_all)]
     pub fn read_triples(

{maplib-0.19.9 → maplib-0.19.10}/lib/representation/src/base_rdf_type.rs

@@ -196,7 +196,11 @@ impl BaseRDFNodeType {
         } else if matches!(self, BaseRDFNodeType::IRI | BaseRDFNodeType::BlankNode) {
             DataType::UInt32
         } else if let BaseRDFNodeType::Literal(l) = self {
-            literal_type(l.as_ref(), &BaseCatState::CategoricalNative(false, None), false)
+            literal_type(
+                l.as_ref(),
+                &BaseCatState::CategoricalNative(false, None),
+                false,
+            )
         } else {
             unreachable!("Should never happen")
         }
@@ -225,7 +229,9 @@ fn literal_type(
                 DataType::Struct(polars_lang_fields(base_cat_state))
             } else {
                 match base_cat_state {
-                    BaseCatState::String | BaseCatState::NonString | BaseCatState::CategoricalNative(..) => DataType::String,
+                    BaseCatState::String
+                    | BaseCatState::NonString
+                    | BaseCatState::CategoricalNative(..) => DataType::String,
                 }
             }
         }

{maplib-0.19.9 → maplib-0.19.10}/lib/representation/src/cats/decode.rs

@@ -26,7 +26,7 @@ impl CatEncs {
 }
 
 impl Cats {
-    pub fn maybe_decode_of_type(&self, u:&u32, bt:&BaseRDFNodeType) -> Option<Cow<'_, str>> {
+    pub fn maybe_decode_of_type(&self, u: &u32, bt: &BaseRDFNodeType) -> Option<Cow<'_, str>> {
         let ct = CatType::from_base_rdf_node_type(bt);
         if let Some(cat_encs) = self.cat_map.get(&ct) {
             cat_encs.maybe_decode_string(u)

{maplib-0.19.9 → maplib-0.19.10}/lib/representation/src/cats/image.rs

@@ -23,7 +23,8 @@ impl Cats {
                     if let BaseCatState::CategoricalNative(_, local) = bs {
                         let mut sers = vec![];
                         if bt.is_lang_string() {
-                            let ser1 = sm.mappings
+                            let ser1 = sm
+                                .mappings
                                 .column(c)
                                 .unwrap()
                                 .struct_()
@@ -31,7 +32,8 @@ impl Cats {
                                 .field_by_name(LANG_STRING_VALUE_FIELD)
                                 .unwrap();
                             sers.push(ser1);
-                            let ser2 = sm.mappings
+                            let ser2 = sm
+                                .mappings
                                 .column(c)
                                 .unwrap()
                                 .struct_()
@@ -40,7 +42,8 @@ impl Cats {
                                 .unwrap();
                             sers.push(ser2);
                         } else if t.is_multi() {
-                            let ser = sm.mappings
+                            let ser = sm
+                                .mappings
                                 .column(c)
                                 .unwrap()
                                 .struct_()
@@ -49,7 +52,8 @@ impl Cats {
                                 .unwrap();
                             sers.push(ser);
                         } else {
-                            let ser = sm.mappings
+                            let ser = sm
+                                .mappings
                                 .column(c)
                                 .unwrap()
                                 .as_materialized_series()

{maplib-0.19.9 → maplib-0.19.10}/lib/representation/src/cats/re_encode.rs

@@ -1,7 +1,10 @@
 use super::{CatEncs, CatTriples, CatType, Cats};
 use crate::cats::LockedCats;
 use crate::solution_mapping::{BaseCatState, EagerSolutionMappings};
-use crate::{BaseRDFNodeType, LANG_STRING_LANG_FIELD, LANG_STRING_VALUE_FIELD, OBJECT_COL_NAME, SUBJECT_COL_NAME};
+use crate::{
+    BaseRDFNodeType, LANG_STRING_LANG_FIELD, LANG_STRING_VALUE_FIELD, OBJECT_COL_NAME,
+    SUBJECT_COL_NAME,
+};
 use nohash_hasher::NoHashHasher;
 use polars::datatypes::PlSmallStr;
 use polars::error::PolarsResult;

{maplib-0.19.9 → maplib-0.19.10}/lib/representation/src/constants.rs

@@ -1,3 +1,6 @@
+pub const LANG_LANG_FIELD_ALT_NAME: &str = "language";
+pub const LANG_VALUE_FIELD_ALT_NAME: &str = "value";
+
 pub const OTTR_TRIPLE: &str = "http://ns.ottr.xyz/0.4/Triple";
 pub const OTTR_PREFIX: &str = "ottr";
 pub const OTTR_PREFIX_IRI: &str = "http://ns.ottr.xyz/0.4/";
@@ -23,4 +26,4 @@ pub const SHACL_PREFIX_IRI: &str = "http://www.w3.org/ns/shacl#";
 pub const XSD_ANY_URI: &str = "http://www.w3.org/2001/XMLSchema#anyURI";
 
 pub const FOAF_PREFIX: &str = "foaf";
-pub const FOAF_PREFIX_IRI: &str = "http://xmlns.com/foaf/0.1/";
+pub const FOAF_PREFIX_IRI: &str = "http://xmlns.com/foaf/0.1/";

{maplib-0.19.9 → maplib-0.19.10}/lib/representation/src/prefixes.rs

@@ -1,4 +1,8 @@
-use crate::constants::{DEFAULT_PREFIX, DEFAULT_PREFIX_IRI, FOAF_PREFIX, FOAF_PREFIX_IRI, OTTR_PREFIX, OTTR_PREFIX_IRI, OWL_PREFIX, OWL_PREFIX_IRI, RDFS_PREFIX, RDFS_PREFIX_IRI, RDF_PREFIX, RDF_PREFIX_IRI, SHACL_PREFIX, SHACL_PREFIX_IRI, XSD_PREFIX, XSD_PREFIX_IRI};
+use crate::constants::{
+    DEFAULT_PREFIX, DEFAULT_PREFIX_IRI, FOAF_PREFIX, FOAF_PREFIX_IRI, OTTR_PREFIX, OTTR_PREFIX_IRI,
+    OWL_PREFIX, OWL_PREFIX_IRI, RDFS_PREFIX, RDFS_PREFIX_IRI, RDF_PREFIX, RDF_PREFIX_IRI,
+    SHACL_PREFIX, SHACL_PREFIX_IRI, XSD_PREFIX, XSD_PREFIX_IRI,
+};
 use oxrdf::NamedNode;
 use std::collections::HashMap;
 

{maplib-0.19.9 → maplib-0.19.10}/lib/representation/src/rdf_to_polars.rs

@@ -76,10 +76,14 @@ pub fn rdf_literal_to_polars_expr(l: &Literal) -> Expr {
     }
 }
 
-pub fn rdf_literal_to_polars_literal_value(lit: &Literal) -> LiteralValue {
-    let datatype = lit.datatype();
-    let value = lit.value();
+pub fn rdf_literal_to_polars_literal_value(literal: &Literal) -> LiteralValue {
+    rdf_literal_to_polars_literal_value_impl(literal.value(), literal.datatype())
+}
 
+pub fn rdf_literal_to_polars_literal_value_impl(
+    value: &str,
+    datatype: NamedNodeRef,
+) -> LiteralValue {
     if datatype == xsd::STRING {
         LiteralValue::Scalar(Scalar::from(PlSmallStr::from_string(value.to_string())))
     } else if datatype == rdf::LANG_STRING {

{maplib-0.19.9 → maplib-0.19.10}/lib/triplestore/Cargo.toml

@@ -32,6 +32,8 @@ memmap2.workspace = true
 sparesults.workspace = true
 tracing.workspace = true
 aho-corasick.workspace = true
+simd-json.workspace = true
+serde_json.workspace = true
 
 pyo3 = { workspace = true, optional = true }
 

{maplib-0.19.9 → maplib-0.19.10}/lib/triplestore/src/errors.rs

@@ -47,6 +47,10 @@ pub enum TriplestoreError {
     SparqlQueryError(String),
     #[error("Found bad list: {0}")]
     BadListError(String),
+    #[error("Error reading JSON file: {0}")]
+    ReadJSONFileError(String),
+    #[error("Prefix IRI is invalid: {0}")]
+    InvalidPrefixIRI(String),
 }
 
 impl<T> From<PoisonError<T>> for TriplestoreError {

{maplib-0.19.9 → maplib-0.19.10}/lib/triplestore/src/lib.rs

@@ -4,6 +4,7 @@ extern crate core;
 pub mod cats;
 mod dblf;
 pub mod errors;
+mod map_json;
 pub mod native_parquet_write;
 pub mod query_solutions;
 pub mod rdfs_inferencing;