libjay 0.2.1__tar.gz → 0.3.0__tar.gz

{libjay-0.2.1 → libjay-0.3.0}/Cargo.lock

@@ -494,7 +494,7 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
 
 [[package]]
 name = "libjay"
-version = "0.2.1"
+version = "0.3.0"
 dependencies = [
  "libjay-testkit",
  "multiversion",
@@ -510,22 +510,23 @@ dependencies = [
 
 [[package]]
 name = "libjay-capi"
-version = "0.2.1"
+version = "0.3.0"
 dependencies = [
  "libjay",
 ]
 
 [[package]]
 name = "libjay-devtools"
-version = "0.2.1"
+version = "0.3.0"
 dependencies = [
+ "libjay",
  "libjay-testkit",
  "rayon",
 ]
 
 [[package]]
 name = "libjay-python"
-version = "0.2.1"
+version = "0.3.0"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -537,7 +538,7 @@ dependencies = [
 
 [[package]]
 name = "libjay-testkit"
-version = "0.2.1"
+version = "0.3.0"
 dependencies = [
  "libjay",
 ]

{libjay-0.2.1 → libjay-0.3.0}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["crates/libjay", "crates/libjay-python", "crates/libjay-testkit"]
 
 [workspace.package]
-version = "0.2.1"
+version = "0.3.0"
 edition = "2024"
 rust-version = "1.89"
 license = "MIT"

{libjay-0.2.1 → libjay-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: libjay
-Version: 0.2.1
+Version: 0.3.0
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research

{libjay-0.2.1 → libjay-0.3.0}/crates/libjay/src/array.rs

@@ -793,20 +793,32 @@ pub enum Layout {
 /// honour [`Array::layout`] or take [`Array::to_row_major`] first; the
 /// runtime's rule is that a value reaching a verb has already been made
 /// row-major unless that verb asked for the other one.
+///
+/// A SPARSE array is the one exception to "the buffer holds every element":
+/// `shape` is still the logical shape, but `data` holds only the stored
+/// cells and [`crate::sparse::Sparse`] says where they sit. Only `$.`, the
+/// display and `":` read that form; every other reader takes
+/// [`Array::densified`] first.
 #[derive(Clone, Debug)]
 pub struct Array {
     pub shape: Vec<usize>,
     pub data: Data,
     layout: Layout,
+    sparse: Option<crate::sparse::Handle>,
+    proto: Option<std::sync::Arc<Array>>,
 }
 
 /// Two arrays are equal when they hold the same elements at the same
-/// indices, whatever buffer order each of them keeps.
+/// indices, whatever buffer order — or storage kind — each of them keeps.
 impl PartialEq for Array {
     fn eq(&self, other: &Array) -> bool {
         if self.shape != other.shape {
             return false;
         }
+        if self.sparse.is_some() || other.sparse.is_some() {
+            let (a, b) = (self.densified(), other.densified());
+            return a.to_row_major().data == b.to_row_major().data;
+        }
         if self.layout == other.layout {
             return self.data == other.data;
         }
@@ -817,7 +829,52 @@ impl PartialEq for Array {
 impl Array {
     pub fn new(shape: Vec<usize>, data: Data) -> Array {
         debug_assert_eq!(shape.iter().product::<usize>(), data.len());
-        Array { shape, data, layout: Layout::RowMajor }
+        Array { shape, data, layout: Layout::RowMajor, sparse: None, proto: None }
+    }
+
+    /// A sparse array: the logical `shape`, the stored cells, and the
+    /// description of where they sit. `data` holds `entries` cells and not
+    /// one element per position, so this is the only constructor that does
+    /// not tie the buffer's length to the shape.
+    pub fn sparse(shape: Vec<usize>, data: Data, sparse: crate::sparse::Sparse) -> Array {
+        Array { shape, data, layout: Layout::RowMajor, sparse: Some(std::sync::Arc::new(sparse)), proto: None }
+    }
+
+    /// True while the array holds only its stored cells.
+    pub fn is_sparse(&self) -> bool {
+        self.sparse.is_some()
+    }
+
+    /// The item an array with no items would have held — APL's prototype.
+    ///
+    /// A simple array's type says what its fills look like, so nothing has
+    /// to be remembered; a nested one does, since an empty buffer of boxes
+    /// no longer says whether its items were pairs of numbers or of
+    /// characters. `0⍴⊂2 3⍴9` is such an array, and `↑` of it answers the
+    /// 2 by 3 table of zeros this holds. Only the operations that make an
+    /// empty out of a nested array set it, and only APL reads it.
+    pub fn proto(&self) -> Option<&Array> {
+        self.proto.as_deref()
+    }
+
+    /// The same array, remembering what its items looked like.
+    pub fn with_proto(mut self, proto: Array) -> Array {
+        self.proto = Some(std::sync::Arc::new(proto));
+        self
+    }
+
+    /// How this array is stored sparsely, or None for a dense one.
+    pub fn sparse_parts(&self) -> Option<&crate::sparse::Sparse> {
+        self.sparse.as_deref()
+    }
+
+    /// This array with every position materialised. A dense array is a
+    /// refcount bump; a sparse one is expanded here and nowhere else.
+    pub fn densified(&self) -> Array {
+        match &self.sparse {
+            None => self.clone(),
+            Some(s) => crate::sparse::densify(self, s),
+        }
     }
 
     /// An array whose buffer holds its first axis fastest — the columns of
@@ -825,7 +882,7 @@ impl Array {
     pub fn col_major(shape: Vec<usize>, data: Data) -> Array {
         debug_assert_eq!(shape.iter().product::<usize>(), data.len());
         let layout = if shape.len() < 2 { Layout::RowMajor } else { Layout::ColMajor };
-        Array { shape, data, layout }
+        Array { shape, data, layout, sparse: None, proto: None }
     }
 
     /// The same buffer read the other way round. The caller is asserting
@@ -964,7 +1021,16 @@ impl Array {
     /// Widen the elements. A cast reads and writes the buffer as it lies,
     /// so the layout comes through untouched.
     pub fn cast(&self, to: DType) -> Option<Array> {
-        Some(Array { shape: self.shape.clone(), data: self.data.cast(to)?, layout: self.layout })
+        if self.is_sparse() {
+            return self.densified().cast(to);
+        }
+        Some(Array {
+            shape: self.shape.clone(),
+            data: self.data.cast(to)?,
+            layout: self.layout,
+            sparse: None,
+            proto: self.proto.clone(),
+        })
     }
 
     /// Split into cells: the trailing `cell_rank` axes form the cell shape,
@@ -1062,9 +1128,20 @@ impl Array {
             Data::Ext(v) => Some(v.iter().map(crate::exact::ext_to_f64).collect()),
             Data::Rat(v) => Some(v.iter().map(Rat::to_f64).collect()),
             Data::F64(v) => Some(v.to_vec()),
-            // A complex value is not a real one, even when its imaginary
-            // part is zero: the caller wants a real and must ask for it.
-            Data::Complex(_) | Data::Char(_) | Data::Symbol(_) | Data::Box(_) => None,
+            // A complex value whose imaginary part is zero IS a real one
+            // wherever a real is wanted: J answers `1 <. j. 0` with 0 and
+            // `i. 3j0` with `0 1 2`, while `3!:0 j. 0` still reports the
+            // complex type, so the demotion is at the use and not at the
+            // making.
+            Data::Complex(v) => {
+                v.iter().map(|z| (z[1] == 0.0).then_some(z[0])).collect()
+            }
+            // An EMPTY array carries no value of the wrong type, so it is
+            // acceptable numeric data whatever type it was written at:
+            // `#. ''` is 0 in J and `¯3⊥''` is 0 in GNU APL. An empty BOX
+            // is not: J refuses `2 #. 0$<1` where it answers `2 #. ''`.
+            Data::Char(_) | Data::Symbol(_) if self.count() == 0 => Some(Vec::new()),
+            Data::Char(_) | Data::Symbol(_) | Data::Box(_) => None,
         }
     }
 
@@ -1089,8 +1166,90 @@ impl Array {
                 }
                 Some(out)
             }
-            Data::Complex(_) | Data::Char(_) | Data::Symbol(_) | Data::Box(_) => None,
+            // The same two readings [`Array::to_f64_vec`] gives: a complex
+            // with no imaginary part is the real it displays as, and an
+            // empty of a non-numeric type holds no value to refuse.
+            Data::Complex(v) => v
+                .iter()
+                .map(|z| {
+                    (z[1] == 0.0 && z[0].fract() == 0.0 && z[0].abs() < i64::MAX as f64)
+                        .then_some(z[0] as i64)
+                })
+                .collect(),
+            Data::Char(_) | Data::Symbol(_) if self.count() == 0 => Some(Vec::new()),
+            Data::Char(_) | Data::Symbol(_) | Data::Box(_) => None,
+        }
+    }
+
+    /// Numeric contents as i64 where a COUNT, a LENGTH or an INDEX is
+    /// wanted, admitting a float that is merely near a whole number.
+    ///
+    /// Both references round such a float to the whole number beside it
+    /// rather than refusing it — `⍳2-1E¯14` is `1 2` and `(2-1e_14) {. 1 2 3`
+    /// is `1 2` — and neither admission is the comparison tolerance:
+    /// `⎕CT←0` and `9!:19 (0)` leave both exactly where they are. The two
+    /// admissions differ in shape. [`NearInt::J`] is relative, a value
+    /// within `2^-44` of a whole number's magnitude; [`NearInt::Apl`] is
+    /// absolute, `1e-10`, whatever the magnitude; [`NearInt::Tolerant`] is
+    /// relative and follows the comparison tolerance in force. Everything a
+    /// full integer apart is still a refusal in all three.
+    pub fn to_i64_vec_near(&self, near: NearInt) -> Option<Vec<i64>> {
+        let Data::F64(v) = &self.data else {
+            // Every other type is exact or is refused outright; only a
+            // float can be near a whole number without being one.
+            return self.to_i64_vec();
+        };
+        v.iter().map(|&x| near.round(x)).collect()
+    }
+}
+
+/// The near-integer admission a count, a length or an index position uses.
+///
+/// In J and in GNU APL it is a language constant: neither lets a program
+/// move it, and neither is the comparison tolerance the same program can
+/// set. Dyalog's is the exception — relative and scaled by `⎕CT` — so the
+/// rule is a dialect setting there, and [`NearInt::Tolerant`] carries the
+/// tolerance in force with it.
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub enum NearInt {
+    /// J: `|x - n| ≤ 2^-44 × max(|x|, |n|)`, so the window grows with the
+    /// magnitude and closes completely at zero.
+    J,
+    /// APL: `|x - n| < 1e-10` at every magnitude.
+    Apl,
+    /// Dyalog: the dialect's own tolerant equality against the whole
+    /// number, so the window grows with the magnitude and `⎕CT` moves it.
+    Tolerant(crate::verb::Tol),
+}
+
+impl NearInt {
+    /// J's relative admission, which is also the value J's comparison
+    /// tolerance starts at — the two are separate settings that happen to
+    /// share a number.
+    pub const J_RELATIVE: f64 = 1.0 / 17_592_186_044_416.0;
+    /// APL's absolute admission.
+    pub const APL_ABSOLUTE: f64 = 1e-10;
+
+    /// The rule for a language, at that language's shipped dialect.
+    pub fn of(lang: crate::Lang) -> NearInt {
+        match lang {
+            crate::Lang::J => NearInt::J,
+            crate::Lang::Apl => NearInt::Apl,
+        }
+    }
+
+    /// The whole number `x` stands for, or None when it stands for none.
+    pub fn round(self, x: f64) -> Option<i64> {
+        if x.fract() == 0.0 {
+            return (x.abs() < i64::MAX as f64).then_some(x as i64);
         }
+        let n = x.round();
+        let within = match self {
+            NearInt::J => (x - n).abs() <= Self::J_RELATIVE * x.abs().max(n.abs()),
+            NearInt::Apl => (x - n).abs() < Self::APL_ABSOLUTE,
+            NearInt::Tolerant(tol) => tol.eq(x, n),
+        };
+        (within && n.abs() < i64::MAX as f64).then_some(n as i64)
     }
 }
 

{libjay-0.2.1 → libjay-0.3.0}/crates/libjay/src/complex.rs

@@ -39,9 +39,23 @@ pub fn conj(a: Cx) -> Cx {
     [a[0], -a[1]]
 }
 
+/// A complex product is four real ones, and each of them follows J's rule
+/// that a zero factor wins: `_ * 0j1` is `0j_` and `0j_ * 0j_` is `__`
+/// only when `_ * 0` is 0 rather than a NaN. It is also what gives `j. _`
+/// its value, because `j.` multiplies by the imaginary unit. GNU APL never
+/// reaches the case — it refuses an infinite operand to `×` outright — so
+/// the rule costs nothing there.
+#[inline]
+fn prod(x: f64, y: f64) -> f64 {
+    if (x == 0.0 || y == 0.0) && !(x.is_finite() && y.is_finite()) {
+        return 0.0;
+    }
+    x * y
+}
+
 #[inline]
 pub fn mul(a: Cx, b: Cx) -> Cx {
-    [a[0] * b[0] - a[1] * b[1], a[0] * b[1] + a[1] * b[0]]
+    [prod(a[0], b[0]) - prod(a[1], b[1]), prod(a[0], b[1]) + prod(a[1], b[0])]
 }
 
 /// Division, with J's rule for a zero divisor carried onto both parts:
@@ -156,7 +170,10 @@ pub fn pow(a: Cx, b: Cx) -> Cx {
     if a[1] == 0.0 && a[0] < 0.0 && b[1] == 0.0 {
         let m = (-a[0]).powf(b[0]);
         let (c, s) = cos_sin_pi(b[0]);
-        return [m * c, m * s];
+        // `prod`, not `*`: at a half turn the cosine is an exact zero, and
+        // an infinite magnitude beside it is the zero-factor case again.
+        // `__ ^ 0.5` is `0j_` and `__ ^ 1.5` is `0j__`.
+        return [prod(m, c), prod(m, s)];
     }
     exp(mul(b, ln(a)))
 }

{libjay-0.2.1 → libjay-0.3.0}/crates/libjay/src/device/codegen.rs

@@ -408,9 +408,36 @@ fn helpers(needs: &Needs, tol: Tol, p: Precision) -> String {
         s.push_str("  return x / y;\n}\n");
     }
     if needs.residue {
+        // The quotient is rounded with the dialect's tolerance, as it is on
+        // the host: J takes the tolerant floor and answers an exact zero
+        // when the product is tolerantly the dividend, GNU APL shifts the
+        // quotient by `⎕CT` and reads the remainder against the modulus.
+        let ct = lit(tol.ct, p);
         s.push_str(&format!("fn residue(x: {t}, y: {t}) -> {t} {{\n"));
         s.push_str(&format!("  if (x == {zero}) {{ return y; }}\n"));
-        s.push_str("  return y - x * floor(y / x);\n}\n");
+        if tol.by_smaller {
+            s.push_str("  let q = y / x;\n");
+            s.push_str("  let c = ceil(q);\n");
+            s.push_str(&format!(
+                "  let k = select(floor(q), c, abs(q - c) < {ct} * min(abs(q), abs(c)));\n"
+            ));
+            s.push_str("  let d = x * k;\n");
+            s.push_str(&format!(
+                "  if (abs(y - d) < {ct} * min(abs(y), abs(d))) {{ return {zero}; }}\n"
+            ));
+            s.push_str("  return y - d;\n}\n");
+        } else {
+            s.push_str("  let q = y / x;\n  let c = ceil(q);\n  let gap = c - q;\n");
+            s.push_str(&format!(
+                "  let k = select(floor(q), c, gap <= {ct} || gap < {ct} * max(abs(q), abs(c)));\n"
+            ));
+            s.push_str("  let r = y - x * k;\n");
+            s.push_str(&format!("  if (abs(r) < {ct} * abs(x)) {{ return {zero}; }}\n"));
+            s.push_str(&format!(
+                "  if (r != {zero} && (r < {zero}) != (x < {zero})) {{ return r + x; }}\n"
+            ));
+            s.push_str("  return r;\n}\n");
+        }
     }
     if !s.is_empty() {
         s.push('\n');

{libjay-0.2.1 → libjay-0.3.0}/crates/libjay/src/device/mod.rs

@@ -145,12 +145,14 @@ impl Device {
     pub fn upload(&self, y: &Array) -> Result<Array, DeviceError> {
         let Some(backend) = self.backend() else { return Ok(y.clone()) };
         // What goes to the device is the elements in row-major order; a
-        // column-major argument is laid out once before it leaves.
+        // column-major argument is laid out once before it leaves, and a
+        // sparse one is expanded — a device buffer is one element per
+        // position and nothing else.
         let laid_out;
-        let y = if y.is_row_major() {
+        let y = if y.is_row_major() && !y.is_sparse() {
             y
         } else {
-            laid_out = y.to_row_major();
+            laid_out = y.densified().to_row_major();
             &laid_out
         };
         // A float array is uploaded from its own buffer; anything else is
@@ -307,6 +309,9 @@ pub enum Refusal {
     Unsupported(&'static str),
     /// The kernel itself would decline these inputs, device or no device.
     Declined,
+    /// The shader's answer holds an infinity or a NaN, where the dialect
+    /// has a rule of its own and the shader has only IEEE arithmetic.
+    NonFinite,
     /// Too little data to pay for a dispatch.
     TooSmall,
     /// The device refused: an allocation, a shader, a queue submission.
@@ -323,6 +328,9 @@ impl Refusal {
             }
             Refusal::Unsupported(op) => format!("`{op}` has no shader form here"),
             Refusal::Declined => "the fused kernel declined these inputs".into(),
+            Refusal::NonFinite => {
+                "the answer holds an infinity or a NaN, which the dialect's rules read".into()
+            }
             Refusal::TooSmall => "there is too little data to pay for a dispatch".into(),
             Refusal::Failed(e) => format!("the device refused: {e}"),
         }

{libjay-0.2.1 → libjay-0.3.0}/crates/libjay/src/error.rs

@@ -30,6 +30,11 @@ pub enum ErrorKind {
     Domain,
     Type,
     Value,
+    /// Arithmetic with no value at all: J refuses a NaN its own arithmetic
+    /// made (`_ - _`, `2 | _`), and names the failure this way. A NaN the
+    /// program itself wrote (`_.`) travels through unrefused, so this is
+    /// about the operation, not the operand.
+    Nan,
     /// Present in the language, not implemented yet.
     NotYet,
     /// Absent from the language itself; will never exist.
@@ -52,6 +57,7 @@ impl ErrorKind {
             ErrorKind::Domain => "domain error",
             ErrorKind::Type => "type error",
             ErrorKind::Value => "value error",
+            ErrorKind::Nan => "NaN error",
             ErrorKind::NotYet => "not supported yet",
             ErrorKind::Language => "not in the language",
             ErrorKind::Sandbox => "closed by the sandbox",
@@ -99,6 +105,13 @@ impl Error {
         Self::new(ErrorKind::Domain, msg, Some(span))
     }
 
+    /// Arithmetic whose answer is a NaN nobody asked for. The message names
+    /// the operation and the pair that produced it, in the source language's
+    /// own spelling of the infinities.
+    pub fn nan(msg: impl Into<String>, span: Span) -> Self {
+        Self::new(ErrorKind::Nan, msg, Some(span))
+    }
+
     pub fn internal(msg: impl Into<String>) -> Self {
         Self::new(ErrorKind::Internal, msg, None)
     }

{libjay-0.2.1 → libjay-0.3.0}/crates/libjay/src/explain.rs

@@ -226,6 +226,11 @@ fn control_lines(c: &Control, depth: usize, p: &Program, tr: &Trace, out: &mut S
             let _ = writeln!(out, "{pad}branch →");
             expr_lines(target, depth + 1, p, tr, out);
         }
+        Control::Guard { test, body } => {
+            let _ = writeln!(out, "{pad}guard — the dfn's answer when it holds");
+            block("test", test, out);
+            block("body", body, out);
+        }
         Control::If { arms, otherwise } => {
             let _ = writeln!(out, "{pad}if — {} arm(s)", arms.len());
             for (i, arm) in arms.iter().enumerate() {
@@ -377,6 +382,23 @@ fn source_of(p: &Program, e: &Expr) -> String {
 
 // ------------------------------------------------------------------- verbs
 
+/// One operand of a user-written operator: the function's own tree, or the
+/// one line an array operand needs.
+fn operand_lines(
+    o: &crate::verb::Operand,
+    depth: usize,
+    p: &Program,
+    tr: &Trace,
+    out: &mut String,
+) {
+    match o {
+        crate::verb::Operand::Func(v) => verb_lines(v, depth, p, tr, out),
+        crate::verb::Operand::Value(_) => {
+            let _ = writeln!(out, "{}an array operand", " ".repeat(depth * STEP));
+        }
+    }
+}
+
 fn verb_lines(v: &Verb, depth: usize, p: &Program, tr: &Trace, out: &mut String) {
     let pad = " ".repeat(depth * STEP);
     let head = |out: &mut String, what: &str| {
@@ -395,6 +417,10 @@ fn verb_lines(v: &Verb, depth: usize, p: &Program, tr: &Trace, out: &mut String)
             head(out, "reduce (insert between items)");
             verb_lines(u, depth + 1, p, tr, out);
         }
+        Verb::NWise(u) => {
+            head(out, "reduce (insert between items); n-wise with a left argument");
+            verb_lines(u, depth + 1, p, tr, out);
+        }
         Verb::Fit(u, n) => {
             head(out, &format!("fit !.{n} (comparison tolerance)"));
             verb_lines(u, depth + 1, p, tr, out);
@@ -433,10 +459,12 @@ fn verb_lines(v: &Verb, depth: usize, p: &Program, tr: &Trace, out: &mut String)
         }
         Verb::UserDerived { def, alpha, omega } => {
             head(out, "a user-written operator with its operands");
-            verb_lines(def, depth + 1, p, tr, out);
-            verb_lines(alpha, depth + 1, p, tr, out);
+            if let Ok(body) = def.pick(alpha, omega.as_ref()) {
+                verb_lines(body, depth + 1, p, tr, out);
+            }
+            operand_lines(alpha, depth + 1, p, tr, out);
             if let Some(g) = omega {
-                verb_lines(g, depth + 1, p, tr, out);
+                operand_lines(g, depth + 1, p, tr, out);
             }
         }
         Verb::KeyPairs(u) => {
@@ -571,6 +599,10 @@ fn verb_lines(v: &Verb, depth: usize, p: &Program, tr: &Trace, out: &mut String)
         }
         Verb::SelfRef => head(out, "self-reference (the definition it stands in)"),
         Verb::Named(n) => head(out, &format!("verb named {n}, resolved when it is applied")),
+        Verb::UnderRavel(u) => {
+            head(out, "under ravel (flatten, apply, put the shape back)");
+            verb_lines(u, depth + 1, p, tr, out);
+        }
         Verb::Each(u, kind) => {
             head(
                 out,