osc-data 0.2.2__tar.gz → 0.2.5__tar.gz

{osc_data-0.2.2 → osc_data-0.2.5}/.gitignore

@@ -175,5 +175,5 @@ target/
 .venv
 *.ipynb
 *.svg
-cargo.lock
+Cargo.lock
 uv.lock

{osc_data-0.2.2 → osc_data-0.2.5}/Cargo.lock

@@ -11,11 +11,17 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "anyhow"
+version = "1.0.99"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100"
+
 [[package]]
 name = "autocfg"
-version = "1.4.0"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
+checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
 
 [[package]]
 name = "cblas-sys"
@@ -71,18 +77,24 @@ checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd"
 
 [[package]]
 name = "inventory"
-version = "0.3.20"
+version = "0.3.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab08d7cd2c5897f2c949e5383ea7c7db03fb19130ffcfbf7eda795137ae3cb83"
+checksum = "bc61209c082fbeb19919bee74b176221b27223e27b65d781eb91af24eb1fb46e"
 dependencies = [
  "rustversion",
 ]
 
+[[package]]
+name = "itoa"
+version = "1.0.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
+
 [[package]]
 name = "libc"
-version = "0.2.172"
+version = "0.2.175"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"
+checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543"
 
 [[package]]
 name = "matrixmultiply"
@@ -180,17 +192,20 @@ checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
 name = "osc-data"
 version = "0.1.0"
 dependencies = [
+ "anyhow",
  "ndarray",
  "numpy",
  "pyo3",
  "regex",
+ "serde",
+ "serde_json",
 ]
 
 [[package]]
 name = "portable-atomic"
-version = "1.11.0"
+version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"
+checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
 
 [[package]]
 name = "portable-atomic-util"
@@ -203,18 +218,18 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.95"
+version = "1.0.101"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
+checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de"
 dependencies = [
  "unicode-ident",
 ]
 
 [[package]]
 name = "pyo3"
-version = "0.25.0"
+version = "0.25.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f239d656363bcee73afef85277f1b281e8ac6212a1d42aa90e55b90ed43c47a4"
+checksum = "8970a78afe0628a3e3430376fc5fd76b6b45c4d43360ffd6cdd40bdde72b682a"
 dependencies = [
  "indoc",
  "inventory",
@@ -230,9 +245,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-build-config"
-version = "0.25.0"
+version = "0.25.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "755ea671a1c34044fa165247aaf6f419ca39caa6003aee791a0df2713d8f1b6d"
+checksum = "458eb0c55e7ece017adeba38f2248ff3ac615e53660d7c71a238d7d2a01c7598"
 dependencies = [
  "once_cell",
  "target-lexicon",
@@ -240,9 +255,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-ffi"
-version = "0.25.0"
+version = "0.25.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc95a2e67091e44791d4ea300ff744be5293f394f1bafd9f78c080814d35956e"
+checksum = "7114fe5457c61b276ab77c5055f206295b812608083644a5c5b2640c3102565c"
 dependencies = [
  "libc",
  "pyo3-build-config",
@@ -250,9 +265,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros"
-version = "0.25.0"
+version = "0.25.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a179641d1b93920829a62f15e87c0ed791b6c8db2271ba0fd7c2686090510214"
+checksum = "a8725c0a622b374d6cb051d11a0983786448f7785336139c3c94f5aa6bef7e50"
 dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
@@ -262,9 +277,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros-backend"
-version = "0.25.0"
+version = "0.25.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9dff85ebcaab8c441b0e3f7ae40a6963ecea8a9f5e74f647e33fcf5ec9a1e89e"
+checksum = "4109984c22491085343c05b0dbc54ddc405c3cf7b4374fc533f5c3313a572ccc"
 dependencies = [
  "heck",
  "proc-macro2",
@@ -290,9 +305,9 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
 
 [[package]]
 name = "rayon"
-version = "1.10.0"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
+checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f"
 dependencies = [
  "either",
  "rayon-core",
@@ -300,9 +315,9 @@ dependencies = [
 
 [[package]]
 name = "rayon-core"
-version = "1.12.1"
+version = "1.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
 dependencies = [
  "crossbeam-deque",
  "crossbeam-utils",
@@ -310,9 +325,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.11.1"
+version = "1.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
+checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -322,9 +337,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.4.9"
+version = "0.4.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
+checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -333,9 +348,9 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.8.5"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
+checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001"
 
 [[package]]
 name = "rustc-hash"
@@ -345,15 +360,64 @@ checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
 
 [[package]]
 name = "rustversion"
+version = "1.0.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
+
+[[package]]
+name = "ryu"
 version = "1.0.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2"
+checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
+
+[[package]]
+name = "serde"
+version = "1.0.224"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6aaeb1e94f53b16384af593c71e20b095e958dab1d26939c1b70645c5cfbcc0b"
+dependencies = [
+ "serde_core",
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.224"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32f39390fa6346e24defbcdd3d9544ba8a19985d0af74df8501fbfe9a64341ab"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.224"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87ff78ab5e8561c9a675bfc1785cb07ae721f0ee53329a595cefd8c04c2ac4e0"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.145"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c"
+dependencies = [
+ "itoa",
+ "memchr",
+ "ryu",
+ "serde",
+ "serde_core",
+]
 
 [[package]]
 name = "syn"
-version = "2.0.101"
+version = "2.0.106"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf"
+checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -362,15 +426,15 @@ dependencies = [
 
 [[package]]
 name = "target-lexicon"
-version = "0.13.2"
+version = "0.13.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a"
+checksum = "df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.18"
+version = "1.0.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
+checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d"
 
 [[package]]
 name = "unindent"

{osc_data-0.2.2 → osc_data-0.2.5}/Cargo.toml

@@ -2,6 +2,7 @@
 name = "osc-data"
 version = "0.1.0"
 edition = "2021"
+readme = "README.md"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [lib]
@@ -13,3 +14,6 @@ pyo3 = { version = "0.25.0", features = ["extension-module", "multiple-pymethods
 ndarray = { version = "0.16.1", features = ["blas", "rayon"] }
 numpy = "0.25.0"
 regex = "1.11.1"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+anyhow = "1"

{osc_data-0.2.2 → osc_data-0.2.5}/PKG-INFO

@@ -1,8 +1,12 @@
 Metadata-Version: 2.4
 Name: osc-data
-Version: 0.2.2
+Version: 0.2.5
 Requires-Dist: kaldifst>=1.7.14
 Requires-Dist: pydantic>=2.11.7
+Requires-Dist: docarray
+Requires-Dist: requests>=2.32.5
+Requires-Dist: librosa>=0.11.0
+Requires-Dist: av>=10.0.0
 Summary: data represent, processing
 Author-email: wangmengdi <790990241@qq.com>
 Requires-Python: >=3.9

osc_data-0.2.5/osc_data/audio.py

@@ -0,0 +1,68 @@
+from typing import Union
+from pathlib import Path
+from io import BytesIO
+import requests
+
+from docarray import BaseDoc
+from docarray.typing import NdArray
+from pydantic import Field, ConfigDict
+import librosa
+
+
+class Audio(BaseDoc):
+    """Audio object that represents an audio file.
+
+    Args:
+        uri (Union[str, Path]): URL of the audio file or local path.
+        sample_rate (Optional[int], optional): Sample rate of the audio file. Defaults to None.
+        data (Optional[NdArray], optional): Data of the audio file. Defaults to None.
+    """
+
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True, validate_assignment=True, extra="ignore"
+    )
+    uri: Union[str, Path] | None = Field(
+        None, description="URL of the audio file or local path."
+    )
+    sample_rate: int | None = Field(None, description="Sample rate of the audio file.")
+    data: NdArray = Field(None, description="Data of the audio file.")
+
+    def load(self, sample_rate: int | None = None, mono: bool | None = None) -> "Audio":
+        """Load the audio file from the URL."""
+        try:
+            if Path(self.uri).exists():
+                data, sample_rate = librosa.load(self.uri, sr=sample_rate, mono=mono)
+            else:
+                bytes_ = requests.get(self.uri).content
+                data, sample_rate = librosa.load(
+                    BytesIO(bytes_), sr=sample_rate, mono=mono
+                )
+            self.data = data
+            self.sample_rate = sample_rate
+        except Exception as e:
+            raise ValueError(f"Failed to load audio from {self.uri}. {e}")
+        return self
+
+    def load_example(self) -> "Audio":
+        example_path = Path(__file__).parent / "assets" / "audio" / "example.wav"
+        self.uri = example_path
+        return self.load()
+
+    def display(self):
+        """Display the audio file."""
+        from IPython.display import Audio as IPAudio
+
+        return IPAudio(
+            data=self.data,
+            rate=self.sample_rate,
+        )
+
+    @property
+    def duration_s(self):
+        """Get the duration of the audio file in seconds."""
+        return librosa.get_duration(y=self.data, sr=self.sample_rate)
+
+    @property
+    def duration_ms(self):
+        """Get the duration of the audio file in milliseconds."""
+        return self.duration_s * 1000

{osc_data-0.2.2 → osc_data-0.2.5}/osc_data/text.py

@@ -4,6 +4,8 @@ import string
 from kaldifst import TextNormalizer as KaldiTextNormalizer
 from pydantic import BaseModel, model_validator, ConfigDict
 
+from ._core import text as core_text
+
 
 EOS = "<EOS>"
 TN_ORDERS = {
@@ -167,3 +169,15 @@ class TextNormalizer(BaseModel):
         text = self.reorder.reorder(text)
         text = self.verbalizer(text)
         return text
+
+
+class TextCleaner(BaseModel):
+    remove_emoji: bool = False
+    to_half_width: bool = False
+
+    def clean(self, text: str) -> str:
+        if self.remove_emoji:
+            text = core_text.remove_emojis(text)
+        if self.to_half_width:
+            text = core_text.to_half_width(text)
+        return text

osc_data-0.2.5/osc_data/video.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import av
+import numpy as np
+from docarray import BaseDoc
+from docarray.typing import VideoNdArray, VideoUrl
+from pydantic import Field, ConfigDict
+
+
+class Video(BaseDoc):
+    """Video object that represents a video file.
+
+    Args:
+        uri (Union[str, Path]): URL of the video file or local path.
+        data (NdArray): Frames data (N, H, W, 3) uint8 RGB.
+        fps (float): Frames per second.
+        duration (float): Duration in seconds.
+    """
+
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True, validate_assignment=True, extra="ignore"
+    )
+
+    uri: VideoUrl | None = Field(
+        None, description="URL of the video file or local path."
+    )
+    data: VideoNdArray = Field(None, description="Frames data (N, H, W, 3) uint8 RGB.")
+    key_frames: list[int] | None = Field(None, description="Key frames")
+    fps: int | None = Field(None, description="Frames per second")
+    duration: float | None = Field(None, description="Duration in seconds")
+
+    def load(self) -> "Video":
+        """Load the video from local path or URL using av library."""
+        if self.uri is None:
+            raise ValueError("Video URI is not set")
+
+        try:
+            with av.open(str(self.uri)) as container:
+                video_stream = container.streams.video[0]
+
+                # 获取视频基本信息
+                self.fps = round(video_stream.average_rate)
+                self.duration = round(float(video_stream.duration * video_stream.time_base), 2)
+
+                # 提取所有帧和关键帧
+                frames = []
+                key_frame_indices = []
+
+                for i, frame in enumerate(container.decode(video=0)):
+                    # 转换为RGB格式的numpy数组
+                    frame_rgb = frame.to_ndarray(format="rgb24")
+                    frames.append(frame_rgb)
+
+                    # 检查是否为关键帧
+                    if frame.key_frame:
+                        key_frame_indices.append(i)
+
+                # 将帧数据组合成VideoNdArray格式 (N, H, W, 3)
+                if frames:
+                    self.data = np.array(frames, dtype=np.uint8)
+                    self.key_frames = key_frame_indices
+                else:
+                    self.data = None
+                    self.key_frames = []
+
+        except Exception as e:
+            # 如果加载失败，重置所有属性
+            self.data = None
+            self.key_frames = []
+            self.fps = None
+            self.duration = None
+            raise RuntimeError(f"Failed to load video: {e}")
+
+        return self
+
+    def split_by_key_frames(self) -> list[VideoNdArray]:
+        """Split the video by key frames."""
+        if self.key_frames is None:
+            raise ValueError("Key frames are not set")
+        if len(self.key_frames) == 0 or len(self.key_frames) == 1:
+            return [self.data]
+        videos = []
+        start = 0
+        for i in self.key_frames[1:]:
+            videos.append(self.data[start:i])
+            start = i
+        videos.append(self.data[start:])
+        return videos
+
+    def display(self):
+        """Display the video data."""
+        if hasattr(self.data, "display"):
+            self.data.display()
+        else:
+            print(
+                f"Video data shape: {self.data.shape if self.data is not None else 'None'}"
+            )

{osc_data-0.2.2 → osc_data-0.2.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "osc-data"
-version = "0.2.2"
+version = "0.2.5"
 description = "data represent, processing"
 readme = "README.md"
 authors = [
@@ -10,6 +10,10 @@ requires-python = ">=3.9"
 dependencies = [
     "kaldifst>=1.7.14",
     "pydantic>=2.11.7",
+    "docarray",
+    "requests>=2.32.5",
+    "librosa>=0.11.0",
+    "av>=10.0.0",
 ]
 
 [tool.maturin]
@@ -23,7 +27,7 @@ build-backend = "maturin"
 [dependency-groups]
 dev = [
     "ipykernel>=6.29.5",
-    "maturin>=1.8.6",
+    "maturin>=1.8.7",
     "numpy>=2.0.2",
     "pip>=25.1.1",
 ]

{osc_data-0.2.2 → osc_data-0.2.5}/src/lib.rs

@@ -1,4 +1,5 @@
 mod audio;
+mod text;
 mod text_stream;
 use pyo3::prelude::*;
 
@@ -7,5 +8,6 @@ use pyo3::prelude::*;
 fn rust_ext<'py>(_py: Python<'py>, m: &Bound<'py, PyModule>) -> PyResult<()> {
     let _ = audio::register_module(m)?;
     let _ = text_stream::register_module(m)?;
+    let _ = text::register_module(m)?;
     Ok(())
 }

osc_data-0.2.5/src/text.rs

@@ -0,0 +1,40 @@
+use pyo3::prelude::*;
+use regex::Regex;
+
+#[pyfunction]
+fn remove_emojis(input: &str) -> String {
+    // 匹配常见 emoji 的 Unicode 范围
+    let emoji_regex = Regex::new(
+        r"[\u{1F600}-\u{1F64F}\
+          \u{1F300}-\u{1F5FF}\
+          \u{1F680}-\u{1F6FF}\
+          \u{1F1E6}-\u{1F1FF}\
+          \u{2600}-\u{26FF}\
+          \u{2700}-\u{27BF}]",
+    )
+    .unwrap();
+    emoji_regex.replace_all(input, "").to_string()
+}
+
+#[pyfunction]
+fn to_half_width(input: String) -> String {
+    input
+        .chars()
+        .map(|c| match c {
+            '\u{3000}' => ' ', // 全角空格
+            '\u{FF01}'..='\u{FF5E}' => {
+                // 常见全角字符范围（！～）
+                std::char::from_u32((c as u32) - 0xFEE0).unwrap_or(c)
+            }
+            _ => c,
+        })
+        .collect()
+}
+
+pub fn register_module(core_module: &Bound<'_, PyModule>) -> PyResult<()> {
+    let text_module = PyModule::new(core_module.py(), "text")?;
+    text_module.add_function(wrap_pyfunction!(remove_emojis, &text_module)?)?;
+    text_module.add_function(wrap_pyfunction!(to_half_width, &text_module)?)?;
+    core_module.add_submodule(&text_module)?;
+    Ok(())
+}

{osc_data-0.2.2 → osc_data-0.2.5}/src/text_stream.rs

@@ -198,8 +198,8 @@ fn remove_emojis(input: &str) -> String {
 }
 
 pub fn register_module(core_module: &Bound<'_, PyModule>) -> PyResult<()> {
-    let audio_module = PyModule::new(core_module.py(), "text_stream")?;
-    audio_module.add_class::<TextStreamSentencizer>()?;
-    core_module.add_submodule(&audio_module)?;
+    let text_stream_module = PyModule::new(core_module.py(), "text_stream")?;
+    text_stream_module.add_class::<TextStreamSentencizer>()?;
+    core_module.add_submodule(&text_stream_module)?;
     Ok(())
 }

osc_data-0.2.2/osc_data/audio.py

@@ -1,23 +0,0 @@
-import numpy as np
-
-from ._core import audio
-
-
-def low_frame_rate(frames: np.ndarray, m: int, n: int) -> np.ndarray:
-    """apply low frame rate
-    Args:
-        frames: input frames. shape: (batch_size, n_frames, n_features)
-        m: number of frames to combine
-        n: number of frames to skip
-    """
-
-    return audio.low_frame_rate(frames, m, n)
-
-
-def compute_decibel(frames: np.ndarray) -> np.ndarray:
-    """compute decibel
-    Args:
-        frames: input frames. shape: (n_frames, n_features)
-    """
-
-    return audio.compute_decibel(frames)