safetensors 0.5.1__tar.gz → 0.5.2__tar.gz

{safetensors-0.5.1 → safetensors-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: safetensors
-Version: 0.5.1
+Version: 0.5.2
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Education

{safetensors-0.5.1 → safetensors-0.5.2}/bindings/python/Cargo.lock

@@ -163,7 +163,7 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
 
 [[package]]
 name = "safetensors"
-version = "0.5.1"
+version = "0.5.2"
 dependencies = [
  "serde",
  "serde_json",
@@ -171,7 +171,7 @@ dependencies = [
 
 [[package]]
 name = "safetensors-python"
-version = "0.5.1"
+version = "0.5.2"
 dependencies = [
  "memmap2",
  "pyo3",
@@ -201,9 +201,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.134"
+version = "1.0.135"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d"
+checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9"
 dependencies = [
  "itoa",
  "memchr",
@@ -213,9 +213,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.94"
+version = "2.0.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "987bc0be1cdea8b10216bd06e2ca407d40b9543468fafd3ddfb02f36e77f71f3"
+checksum = "46f71c0377baf4ef1cc3e3402ded576dccc315800fbc62dfc7fe04b009773b4a"
 dependencies = [
  "proc-macro2",
  "quote",

{safetensors-0.5.1 → safetensors-0.5.2}/bindings/python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "safetensors-python"
-version = "0.5.1"
+version = "0.5.2"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

{safetensors-0.5.1 → safetensors-0.5.2/bindings/python}/py_src/safetensors/__init__.pyi

@@ -69,7 +69,7 @@ class safe_open:
             The device on which you want the tensors.
     """
 
-    def __init__(filename, framework, device=...):
+    def __init__(self, filename, framework, device=...):
         pass
     def __enter__(self):
         """

{safetensors-0.5.1 → safetensors-0.5.2}/bindings/python/stub.py

@@ -42,10 +42,7 @@ def fn_predicate(obj):
         return (
             obj.__doc__
             and obj.__text_signature__
-            and (
-                not obj.__name__.startswith("_")
-                or obj.__name__ in {"__enter__", "__exit__"}
-            )
+            and (not obj.__name__.startswith("_") or obj.__name__ in {"__enter__", "__exit__"})
         )
     if inspect.isgetsetdescriptor(obj):
         return obj.__doc__ and not obj.__name__.startswith("_")
@@ -81,15 +78,14 @@ def pyi_file(obj, indent=""):
 
         body = ""
         if obj.__doc__:
-            body += (
-                f'{indent}"""\n{indent}{do_indent(obj.__doc__, indent)}\n{indent}"""\n'
-            )
+            body += f'{indent}"""\n{indent}{do_indent(obj.__doc__, indent)}\n{indent}"""\n'
 
         fns = inspect.getmembers(obj, fn_predicate)
 
         # Init
         if obj.__text_signature__:
-            body += f"{indent}def __init__{obj.__text_signature__}:\n"
+            signature = obj.__text_signature__.replace("(", "(self, ")
+            body += f"{indent}def __init__{signature}:\n"
             body += f"{indent+INDENT}pass\n"
             body += "\n"
 
@@ -146,11 +142,7 @@ def do_black(content, is_pyi):
 
 
 def write(module, directory, origin, check=False):
-    submodules = [
-        (name, member)
-        for name, member in inspect.getmembers(module)
-        if inspect.ismodule(member)
-    ]
+    submodules = [(name, member) for name, member in inspect.getmembers(module) if inspect.ismodule(member)]
 
     filename = os.path.join(directory, "__init__.pyi")
     pyi_content = pyi_file(module)
@@ -159,9 +151,7 @@ def write(module, directory, origin, check=False):
     if check:
         with open(filename, "r") as f:
             data = f.read()
-            assert (
-                data == pyi_content
-            ), f"The content of {filename} seems outdated, please run `python stub.py`"
+            assert data == pyi_content, f"The content of {filename} seems outdated, please run `python stub.py`"
     else:
         with open(filename, "w") as f:
             f.write(pyi_content)
@@ -184,9 +174,7 @@ def write(module, directory, origin, check=False):
         if check:
             with open(filename, "r") as f:
                 data = f.read()
-                assert (
-                    data == py_content
-                ), f"The content of {filename} seems outdated, please run `python stub.py`"
+                assert data == py_content, f"The content of {filename} seems outdated, please run `python stub.py`"
         else:
             with open(filename, "w") as f:
                 f.write(py_content)

{safetensors-0.5.1/bindings/python → safetensors-0.5.2}/py_src/safetensors/__init__.pyi

@@ -69,7 +69,7 @@ class safe_open:
             The device on which you want the tensors.
     """
 
-    def __init__(filename, framework, device=...):
+    def __init__(self, filename, framework, device=...):
         pass
     def __enter__(self):
         """

{safetensors-0.5.1 → safetensors-0.5.2}/safetensors/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "safetensors"
-version = "0.5.1"
+version = "0.5.2"
 edition = "2021"
 homepage = "https://github.com/huggingface/safetensors"
 repository = "https://github.com/huggingface/safetensors"
@@ -21,14 +21,20 @@ exclude = [ "rust-toolchain", "target/*", "Cargo.lock"]
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-serde = {version = "1.0", features = ["derive"]}
-serde_json = "1.0"
+hashbrown = { version = "0.15.2", features = ["serde"], optional = true }
+serde = { version = "1.0", default-features = false, features = ["derive"] }
+serde_json = { version = "1.0", default-features = false }
 
 [dev-dependencies]
 criterion = "0.5"
 memmap2 = "0.9"
 proptest = "1.4"
 
+[features]
+default = ["std"]
+std = ["serde/default", "serde_json/default"]
+alloc = ["serde/alloc", "serde_json/alloc", "hashbrown"]
+
 [[bench]]
 name = "benchmark"
 harness = false

safetensors-0.5.2/safetensors/src/lib.rs

@@ -0,0 +1,43 @@
+#![deny(missing_docs)]
+#![doc = include_str!("../README.md")]
+#![cfg_attr(not(feature = "std"), no_std)]
+pub mod slice;
+pub mod tensor;
+/// serialize_to_file only valid in std
+#[cfg(feature = "std")]
+pub use tensor::serialize_to_file;
+pub use tensor::{serialize, Dtype, SafeTensorError, SafeTensors, View};
+
+#[cfg(feature = "alloc")]
+#[macro_use]
+extern crate alloc;
+
+#[cfg(all(feature = "std", feature = "alloc"))]
+compile_error!("must choose either the `std` or `alloc` feature, but not both.");
+#[cfg(all(not(feature = "std"), not(feature = "alloc")))]
+compile_error!("must choose either the `std` or `alloc` feature");
+
+/// A facade around all the types we need from the `std`, `core`, and `alloc`
+/// crates. This avoids elaborate import wrangling having to happen in every
+/// module.
+mod lib {
+    #[cfg(not(feature = "std"))]
+    mod no_stds {
+        pub use alloc::borrow::Cow;
+        pub use alloc::string::{String, ToString};
+        pub use alloc::vec::Vec;
+        pub use hashbrown::HashMap;
+    }
+    #[cfg(feature = "std")]
+    mod stds {
+        pub use std::borrow::Cow;
+        pub use std::collections::HashMap;
+        pub use std::string::{String, ToString};
+        pub use std::vec::Vec;
+    }
+    /// choose std or no_std to export by feature flag
+    #[cfg(not(feature = "std"))]
+    pub use no_stds::*;
+    #[cfg(feature = "std")]
+    pub use stds::*;
+}

{safetensors-0.5.1 → safetensors-0.5.2}/safetensors/src/slice.rs

@@ -1,7 +1,7 @@
 //! Module handling lazy loading via iterating on slices on the original buffer.
+use crate::lib::{String, ToString, Vec};
 use crate::tensor::TensorView;
-use std::fmt;
-use std::ops::{
+use core::ops::{
     Bound, Range, RangeBounds, RangeFrom, RangeFull, RangeInclusive, RangeTo, RangeToInclusive,
 };
 
@@ -40,8 +40,8 @@ fn display_bound(bound: &Bound<usize>) -> String {
 }
 
 /// Intended for Python users mostly or at least for its conventions
-impl fmt::Display for TensorIndexer {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+impl core::fmt::Display for TensorIndexer {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
         match self {
             TensorIndexer::Select(n) => {
                 write!(f, "{n}")
@@ -77,7 +77,7 @@ macro_rules! impl_from_range {
     ($range_type:ty) => {
         impl From<$range_type> for TensorIndexer {
             fn from(range: $range_type) -> Self {
-                use std::ops::Bound::*;
+                use core::ops::Bound::*;
 
                 let start = match range.start_bound() {
                     Included(idx) => Included(*idx),

{safetensors-0.5.1 → safetensors-0.5.2}/safetensors/src/tensor.rs

@@ -1,11 +1,9 @@
 //! Module Containing the most important structures
+use crate::lib::{Cow, HashMap, String, ToString, Vec};
 use crate::slice::{InvalidSlice, SliceIterator, TensorIndexer};
 use serde::{ser::SerializeMap, Deserialize, Deserializer, Serialize, Serializer};
-use std::borrow::Cow;
-use std::collections::HashMap;
-use std::fs::File;
-use std::io::{BufWriter, Write};
-use std::path::Path;
+#[cfg(feature = "std")]
+use std::io::Write;
 
 const MAX_HEADER_SIZE: usize = 100_000_000;
 
@@ -32,6 +30,7 @@ pub enum SafeTensorError {
     /// The offsets declared for tensor with name `String` in the header are invalid
     InvalidOffset(String),
     /// IoError
+    #[cfg(feature = "std")]
     IoError(std::io::Error),
     /// JSON error
     JsonError(serde_json::Error),
@@ -46,6 +45,7 @@ pub enum SafeTensorError {
     ValidationOverflow,
 }
 
+#[cfg(feature = "std")]
 impl From<std::io::Error> for SafeTensorError {
     fn from(error: std::io::Error) -> SafeTensorError {
         SafeTensorError::IoError(error)
@@ -58,13 +58,13 @@ impl From<serde_json::Error> for SafeTensorError {
     }
 }
 
-impl std::fmt::Display for SafeTensorError {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+impl core::fmt::Display for SafeTensorError {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
         write!(f, "{self:?}")
     }
 }
 
-impl std::error::Error for SafeTensorError {}
+impl core::error::Error for SafeTensorError {}
 
 struct PreparedData {
     n: u64,
@@ -164,7 +164,7 @@ pub trait View {
     fn data_len(&self) -> usize;
 }
 
-fn prepare<S: AsRef<str> + Ord + std::fmt::Display, V: View, I: IntoIterator<Item = (S, V)>>(
+fn prepare<S: AsRef<str> + Ord + core::fmt::Display, V: View, I: IntoIterator<Item = (S, V)>>(
     data: I,
     data_info: &Option<HashMap<String, String>>,
     // ) -> Result<(Metadata, Vec<&'hash TensorView<'data>>, usize), SafeTensorError> {
@@ -212,7 +212,7 @@ fn prepare<S: AsRef<str> + Ord + std::fmt::Display, V: View, I: IntoIterator<Ite
 
 /// Serialize to an owned byte buffer the dictionnary of tensors.
 pub fn serialize<
-    S: AsRef<str> + Ord + std::fmt::Display,
+    S: AsRef<str> + Ord + core::fmt::Display,
     V: View,
     I: IntoIterator<Item = (S, V)>,
 >(
@@ -240,14 +240,15 @@ pub fn serialize<
 /// Serialize to a regular file the dictionnary of tensors.
 /// Writing directly to file reduces the need to allocate the whole amount to
 /// memory.
+#[cfg(feature = "std")]
 pub fn serialize_to_file<
-    S: AsRef<str> + Ord + std::fmt::Display,
+    S: AsRef<str> + Ord + core::fmt::Display,
     V: View,
     I: IntoIterator<Item = (S, V)>,
 >(
     data: I,
     data_info: &Option<HashMap<String, String>>,
-    filename: &Path,
+    filename: &std::path::Path,
 ) -> Result<(), SafeTensorError> {
     let (
         PreparedData {
@@ -255,7 +256,7 @@ pub fn serialize_to_file<
         },
         tensors,
     ) = prepare(data, data_info)?;
-    let mut f = BufWriter::new(File::create(filename)?);
+    let mut f = std::io::BufWriter::new(std::fs::File::create(filename)?);
     f.write_all(n.to_le_bytes().as_ref())?;
     f.write_all(&header_bytes)?;
     for tensor in tensors {
@@ -303,7 +304,7 @@ impl<'data> SafeTensors<'data> {
             return Err(SafeTensorError::InvalidHeaderLength);
         }
         let string =
-            std::str::from_utf8(&buffer[8..stop]).map_err(|_| SafeTensorError::InvalidHeader)?;
+            core::str::from_utf8(&buffer[8..stop]).map_err(|_| SafeTensorError::InvalidHeader)?;
         // Assert the string starts with {
         // NOTE: Add when we move to 0.4.0
         // if !string.starts_with('{') {
@@ -719,6 +720,9 @@ mod tests {
     use super::*;
     use crate::slice::IndexOp;
     use proptest::prelude::*;
+    #[cfg(not(feature = "std"))]
+    extern crate std;
+    use std::io::Write;
 
     const MAX_DIMENSION: usize = 8;
     const MAX_SIZE: usize = 8;
@@ -1021,10 +1025,13 @@ mod tests {
         std::fs::remove_file(&filename).unwrap();
 
         // File api
-        serialize_to_file(&metadata, &None, Path::new(&filename)).unwrap();
-        let raw = std::fs::read(&filename).unwrap();
-        let _deserialized = SafeTensors::deserialize(&raw).unwrap();
-        std::fs::remove_file(&filename).unwrap();
+        #[cfg(feature = "std")]
+        {
+            serialize_to_file(&metadata, &None, std::path::Path::new(&filename)).unwrap();
+            let raw = std::fs::read(&filename).unwrap();
+            let _deserialized = SafeTensors::deserialize(&raw).unwrap();
+            std::fs::remove_file(&filename).unwrap();
+        }
     }
 
     #[test]
@@ -1097,7 +1104,7 @@ mod tests {
         let n = serialized.len();
 
         let filename = "out.safetensors";
-        let mut f = BufWriter::new(File::create(filename).unwrap());
+        let mut f = std::io::BufWriter::new(std::fs::File::create(filename).unwrap());
         f.write_all(n.to_le_bytes().as_ref()).unwrap();
         f.write_all(serialized).unwrap();
         f.write_all(b"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0").unwrap();

safetensors-0.5.1/safetensors/src/lib.rs

@@ -1,5 +0,0 @@
-#![deny(missing_docs)]
-#![doc = include_str!("../README.md")]
-pub mod slice;
-pub mod tensor;
-pub use tensor::{serialize, serialize_to_file, Dtype, SafeTensorError, SafeTensors, View};