raqeem 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- raqeem-0.2.1/PKG-INFO +72 -0
- raqeem-0.2.1/README.md +49 -0
- raqeem-0.2.1/pyproject.toml +34 -0
- raqeem-0.2.1/raqeem-core/Cargo.toml +20 -0
- raqeem-0.2.1/raqeem-core/src/arabic.rs +94 -0
- raqeem-0.2.1/raqeem-core/src/endpoint.rs +52 -0
- raqeem-0.2.1/raqeem-core/src/error.rs +22 -0
- raqeem-0.2.1/raqeem-core/src/lib.rs +150 -0
- raqeem-0.2.1/raqeem-core/src/output.rs +24 -0
- raqeem-0.2.1/raqeem-core/src/provider.rs +25 -0
- raqeem-0.2.1/raqeem-core/tests/endpoint_mock.rs +68 -0
- raqeem-0.2.1/raqeem-python/Cargo.lock +1387 -0
- raqeem-0.2.1/raqeem-python/Cargo.toml +21 -0
- raqeem-0.2.1/raqeem-python/README.md +49 -0
- raqeem-0.2.1/raqeem-python/raqeem.pyi +53 -0
- raqeem-0.2.1/raqeem-python/src/lib.rs +162 -0
- raqeem-0.2.1/raqeem-python/tests/test_raqeem.py +80 -0
raqeem-0.2.1/PKG-INFO
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: raqeem
|
|
3
|
+
Version: 0.2.1
|
|
4
|
+
Classifier: Development Status :: 4 - Beta
|
|
5
|
+
Classifier: Intended Audience :: Developers
|
|
6
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
7
|
+
Classifier: Natural Language :: Arabic
|
|
8
|
+
Classifier: Programming Language :: Rust
|
|
9
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
10
|
+
Classifier: Topic :: Multimedia :: Sound/Audio :: Speech
|
|
11
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
12
|
+
Summary: رقيم — the easy way to use Cohere's open Arabic speech-recognition model from Python.
|
|
13
|
+
Keywords: arabic,asr,speech-to-text,transcription,cohere
|
|
14
|
+
Home-Page: https://github.com/SufficientDaikon/raqeem
|
|
15
|
+
Author-email: Ahmed Taha <tahaa755@gmail.com>
|
|
16
|
+
License: Apache-2.0
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
19
|
+
Project-URL: Changelog, https://github.com/SufficientDaikon/raqeem/blob/main/CHANGELOG.md
|
|
20
|
+
Project-URL: Homepage, https://github.com/SufficientDaikon/raqeem
|
|
21
|
+
Project-URL: Repository, https://github.com/SufficientDaikon/raqeem
|
|
22
|
+
|
|
23
|
+
# رقيم · raqeem
|
|
24
|
+
|
|
25
|
+
**The easy way to use Cohere's open Arabic speech-recognition model from Python.**
|
|
26
|
+
|
|
27
|
+
`raqeem` wraps
|
|
28
|
+
[`CohereLabs/cohere-transcribe-arabic-07-2026`](https://huggingface.co/CohereLabs/cohere-transcribe-arabic-07-2026)
|
|
29
|
+
— the most accurate open-source Arabic ASR model (dialects + Arabic/English
|
|
30
|
+
code-switching), Apache-2.0.
|
|
31
|
+
|
|
32
|
+
It is a **compiled Rust extension**, not a pure-Python wrapper: no `torch`, no model
|
|
33
|
+
weights, no subprocess. Inference is delegated to an endpoint you choose — Cohere's
|
|
34
|
+
hosted API, or your own vLLM.
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
pip install raqeem
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
import raqeem
|
|
42
|
+
|
|
43
|
+
# Cohere hosted API (reads $COHERE_API_KEY)
|
|
44
|
+
t = raqeem.transcribe("voice_note.ogg", lang="ar")
|
|
45
|
+
print(t.text) # verbatim, for humans
|
|
46
|
+
print(t.text_normalized) # Arabic-folded, for parsing
|
|
47
|
+
print(t.to_dict())
|
|
48
|
+
|
|
49
|
+
# your own vLLM — the Cohere key is never sent to a self-hosted endpoint
|
|
50
|
+
t = raqeem.transcribe(
|
|
51
|
+
"clip.wav",
|
|
52
|
+
provider="openai",
|
|
53
|
+
endpoint="http://localhost:8000/v1/audio/transcriptions",
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
# the Arabic normalizer on its own
|
|
57
|
+
raqeem.normalize_ar("الطماطم بـ ١٢٫٥ جنيه") # 'الطماطم ب 12.5 جنيه'
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
`text_normalized` folds alef/hamza, taa-marbuta, strips tatweel and diacritics, and
|
|
61
|
+
converts Arabic-Indic and Persian digits to ASCII — note `١٢٫٥` becomes `12.5`, **one**
|
|
62
|
+
number rather than two, which matters if anything downstream parses prices or quantities.
|
|
63
|
+
|
|
64
|
+
Failures raise `raqeem.TranscriptionError`; a bad provider or a missing key/endpoint
|
|
65
|
+
raises `ValueError`.
|
|
66
|
+
|
|
67
|
+
Full docs, the CLI, and the roadmap (subtitles, diarization, more bindings):
|
|
68
|
+
**https://github.com/SufficientDaikon/raqeem**
|
|
69
|
+
|
|
70
|
+
All model accuracy credit belongs to Cohere Labs — this package is the ergonomics.
|
|
71
|
+
Apache-2.0.
|
|
72
|
+
|
raqeem-0.2.1/README.md
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# رقيم · raqeem
|
|
2
|
+
|
|
3
|
+
**The easy way to use Cohere's open Arabic speech-recognition model from Python.**
|
|
4
|
+
|
|
5
|
+
`raqeem` wraps
|
|
6
|
+
[`CohereLabs/cohere-transcribe-arabic-07-2026`](https://huggingface.co/CohereLabs/cohere-transcribe-arabic-07-2026)
|
|
7
|
+
— the most accurate open-source Arabic ASR model (dialects + Arabic/English
|
|
8
|
+
code-switching), Apache-2.0.
|
|
9
|
+
|
|
10
|
+
It is a **compiled Rust extension**, not a pure-Python wrapper: no `torch`, no model
|
|
11
|
+
weights, no subprocess. Inference is delegated to an endpoint you choose — Cohere's
|
|
12
|
+
hosted API, or your own vLLM.
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
pip install raqeem
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
```python
|
|
19
|
+
import raqeem
|
|
20
|
+
|
|
21
|
+
# Cohere hosted API (reads $COHERE_API_KEY)
|
|
22
|
+
t = raqeem.transcribe("voice_note.ogg", lang="ar")
|
|
23
|
+
print(t.text) # verbatim, for humans
|
|
24
|
+
print(t.text_normalized) # Arabic-folded, for parsing
|
|
25
|
+
print(t.to_dict())
|
|
26
|
+
|
|
27
|
+
# your own vLLM — the Cohere key is never sent to a self-hosted endpoint
|
|
28
|
+
t = raqeem.transcribe(
|
|
29
|
+
"clip.wav",
|
|
30
|
+
provider="openai",
|
|
31
|
+
endpoint="http://localhost:8000/v1/audio/transcriptions",
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
# the Arabic normalizer on its own
|
|
35
|
+
raqeem.normalize_ar("الطماطم بـ ١٢٫٥ جنيه") # 'الطماطم ب 12.5 جنيه'
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
`text_normalized` folds alef/hamza, taa-marbuta, strips tatweel and diacritics, and
|
|
39
|
+
converts Arabic-Indic and Persian digits to ASCII — note `١٢٫٥` becomes `12.5`, **one**
|
|
40
|
+
number rather than two, which matters if anything downstream parses prices or quantities.
|
|
41
|
+
|
|
42
|
+
Failures raise `raqeem.TranscriptionError`; a bad provider or a missing key/endpoint
|
|
43
|
+
raises `ValueError`.
|
|
44
|
+
|
|
45
|
+
Full docs, the CLI, and the roadmap (subtitles, diarization, more bindings):
|
|
46
|
+
**https://github.com/SufficientDaikon/raqeem**
|
|
47
|
+
|
|
48
|
+
All model accuracy credit belongs to Cohere Labs — this package is the ergonomics.
|
|
49
|
+
Apache-2.0.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["maturin>=1.7,<2.0"]
|
|
3
|
+
build-backend = "maturin"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "raqeem"
|
|
7
|
+
description = "رقيم — the easy way to use Cohere's open Arabic speech-recognition model from Python."
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
requires-python = ">=3.9"
|
|
10
|
+
license = { text = "Apache-2.0" }
|
|
11
|
+
authors = [{ name = "Ahmed Taha", email = "tahaa755@gmail.com" }]
|
|
12
|
+
keywords = ["arabic", "asr", "speech-to-text", "transcription", "cohere"]
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Development Status :: 4 - Beta",
|
|
15
|
+
"Intended Audience :: Developers",
|
|
16
|
+
"License :: OSI Approved :: Apache Software License",
|
|
17
|
+
"Natural Language :: Arabic",
|
|
18
|
+
"Programming Language :: Rust",
|
|
19
|
+
"Programming Language :: Python :: Implementation :: CPython",
|
|
20
|
+
"Topic :: Multimedia :: Sound/Audio :: Speech",
|
|
21
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
22
|
+
]
|
|
23
|
+
dynamic = ["version"]
|
|
24
|
+
|
|
25
|
+
[project.urls]
|
|
26
|
+
Homepage = "https://github.com/SufficientDaikon/raqeem"
|
|
27
|
+
Repository = "https://github.com/SufficientDaikon/raqeem"
|
|
28
|
+
Changelog = "https://github.com/SufficientDaikon/raqeem/blob/main/CHANGELOG.md"
|
|
29
|
+
|
|
30
|
+
[tool.maturin]
|
|
31
|
+
features = ["pyo3/extension-module"]
|
|
32
|
+
module-name = "raqeem"
|
|
33
|
+
include = [{ path = "raqeem.pyi", format = "wheel" }]
|
|
34
|
+
manifest-path = "raqeem-python/Cargo.toml"
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
[package]
|
|
2
|
+
name = "raqeem-core"
|
|
3
|
+
version = "0.2.1"
|
|
4
|
+
edition = "2021"
|
|
5
|
+
license = "Apache-2.0"
|
|
6
|
+
repository = "https://github.com/SufficientDaikon/raqeem"
|
|
7
|
+
authors = ["Ahmed Taha <tahaa755@gmail.com>"]
|
|
8
|
+
homepage = "https://github.com/SufficientDaikon/raqeem"
|
|
9
|
+
keywords = ["arabic", "asr", "speech-to-text", "transcription", "cohere"]
|
|
10
|
+
categories = ["command-line-utilities", "api-bindings", "multimedia::audio"]
|
|
11
|
+
description = "Core library for raqeem — an easy client for Cohere's open Arabic ASR model."
|
|
12
|
+
|
|
13
|
+
[dependencies]
|
|
14
|
+
serde = { version = "^1", features = ["derive"] }
|
|
15
|
+
serde_json = "^1"
|
|
16
|
+
reqwest = { version = "^0.12", default-features = false, features = ["blocking", "multipart", "rustls-tls"] }
|
|
17
|
+
thiserror = "^2"
|
|
18
|
+
|
|
19
|
+
[dev-dependencies]
|
|
20
|
+
mockito = "1"
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
//! Arabic text normalization — a Rust port of scout's verified `normalize_ar`
|
|
2
|
+
//! (from the `arabic-first` convention). We fold audio transcripts to a
|
|
3
|
+
//! matching-stable form so downstream parsers (price extraction, commodity
|
|
4
|
+
//! matching) never fuzzy-match raw, unnormalized Arabic.
|
|
5
|
+
//!
|
|
6
|
+
//! Faithful to the Python reference — identical output on all Arabic and ASCII
|
|
7
|
+
//! input: strip diacritics / tatweel / Quranic marks, unify alef and hamza
|
|
8
|
+
//! carriers, taa-marbuta → haa, Arabic-Indic & Persian digits → ASCII, and the
|
|
9
|
+
//! Arabic decimal separator U+066B → `.` (so «١٢٫٥» folds to `12.5`, one number,
|
|
10
|
+
//! not two). Context-sensitive Unicode case tailoring aside — Python lower-cases a
|
|
11
|
+
//! word-final Greek `Σ` to `ς`, Rust to `σ` — which is out of domain for Arabic
|
|
12
|
+
//! transcripts, and scout re-normalizes via the Python function downstream anyway.
|
|
13
|
+
|
|
14
|
+
/// Fold Arabic text to a matching-stable form. Idempotent; ASCII passes through
|
|
15
|
+
/// lower-cased, so it is safe to run unconditionally on any transcript.
|
|
16
|
+
pub fn normalize_ar(text: &str) -> String {
|
|
17
|
+
let mut out = String::with_capacity(text.len());
|
|
18
|
+
for ch in text.chars() {
|
|
19
|
+
match ch {
|
|
20
|
+
// Strip: Quranic annotations, harakat/diacritics, tatweel, superscript alef.
|
|
21
|
+
'\u{0610}'..='\u{061A}'
|
|
22
|
+
| '\u{064B}'..='\u{065F}'
|
|
23
|
+
| '\u{0640}'
|
|
24
|
+
| '\u{0670}'
|
|
25
|
+
| '\u{06D6}'..='\u{06ED}' => {}
|
|
26
|
+
// Alef variants → bare alef.
|
|
27
|
+
'آ' | 'أ' | 'إ' | 'ٱ' => out.push('ا'),
|
|
28
|
+
// Hamza-carrier and taa-marbuta folds.
|
|
29
|
+
'ى' => out.push('ي'), // alef maqsura → yaa
|
|
30
|
+
'ة' => out.push('ه'), // taa marbuta → haa
|
|
31
|
+
'ؤ' => out.push('و'),
|
|
32
|
+
'ئ' => out.push('ي'),
|
|
33
|
+
// Arabic-Indic (U+0660–0669) and Persian (U+06F0–06F9) digits → ASCII.
|
|
34
|
+
'\u{0660}'..='\u{0669}' => out.push((b'0' + (ch as u32 - 0x0660) as u8) as char),
|
|
35
|
+
'\u{06F0}'..='\u{06F9}' => out.push((b'0' + (ch as u32 - 0x06F0) as u8) as char),
|
|
36
|
+
// Arabic decimal separator → ASCII point.
|
|
37
|
+
'\u{066B}' => out.push('.'),
|
|
38
|
+
// Everything else: lower-case ASCII, pass Arabic/other through unchanged.
|
|
39
|
+
other => out.extend(other.to_lowercase()),
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
// Collapse internal whitespace runs and trim.
|
|
43
|
+
out.split_whitespace().collect::<Vec<_>>().join(" ")
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
#[cfg(test)]
|
|
47
|
+
mod tests {
|
|
48
|
+
use super::normalize_ar;
|
|
49
|
+
|
|
50
|
+
#[test]
|
|
51
|
+
fn folds_alef_and_hamza_carriers() {
|
|
52
|
+
assert_eq!(normalize_ar("أ"), "ا");
|
|
53
|
+
assert_eq!(normalize_ar("إ"), "ا");
|
|
54
|
+
assert_eq!(normalize_ar("آ"), "ا");
|
|
55
|
+
assert_eq!(normalize_ar("ٱ"), "ا");
|
|
56
|
+
assert_eq!(normalize_ar("مصطفى"), "مصطفي"); // alef maqsura → yaa
|
|
57
|
+
assert_eq!(normalize_ar("مسؤول"), "مسوول"); // ؤ → و
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
#[test]
|
|
61
|
+
fn taa_marbuta_becomes_haa() {
|
|
62
|
+
let r = normalize_ar("طماطة");
|
|
63
|
+
assert!(!r.contains('ة'));
|
|
64
|
+
assert!(r.ends_with('ه'));
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
#[test]
|
|
68
|
+
fn strips_tatweel_and_diacritics() {
|
|
69
|
+
assert_eq!(normalize_ar("سـلام"), "سلام"); // tatweel U+0640
|
|
70
|
+
assert_eq!(normalize_ar("سَلاَم"), "سلام"); // harakat
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
#[test]
|
|
74
|
+
fn digits_and_decimal_separator() {
|
|
75
|
+
assert_eq!(normalize_ar("١٢٣"), "123"); // arabic-indic
|
|
76
|
+
assert_eq!(normalize_ar("۱۲۳"), "123"); // persian
|
|
77
|
+
assert_eq!(normalize_ar("١٢٫٥"), "12.5"); // U+066B decimal → one number
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
#[test]
|
|
81
|
+
fn ascii_lowercased_and_whitespace_collapsed() {
|
|
82
|
+
assert_eq!(normalize_ar("Tomato"), "tomato");
|
|
83
|
+
assert_eq!(normalize_ar(" a b "), "a b");
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
#[test]
|
|
87
|
+
fn idempotent() {
|
|
88
|
+
let samples = ["أحمد", "طماطة ١٢٫٥ جنيه", "Hello World", "سـلامٌ عليكم"];
|
|
89
|
+
for s in samples {
|
|
90
|
+
let once = normalize_ar(s);
|
|
91
|
+
assert_eq!(normalize_ar(&once), once, "not idempotent on {s:?}");
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
//! The one adapter. Cohere-hosted and self-hosted vLLM differ only in URL,
|
|
2
|
+
//! auth header, and model id — so a single `Endpoint` struct + two constructors
|
|
3
|
+
//! covers both, and any future OpenAI-compatible backend.
|
|
4
|
+
|
|
5
|
+
use crate::provider::Provider;
|
|
6
|
+
|
|
7
|
+
/// Cohere's hosted transcription endpoint.
|
|
8
|
+
pub const COHERE_URL: &str = "https://api.cohere.com/v2/audio/transcriptions";
|
|
9
|
+
/// Default model id for the Arabic transcriber. Cohere requires a **dated** model
|
|
10
|
+
/// id — undated aliases (`cohere-transcribe-arabic`) return HTTP 404 "model not
|
|
11
|
+
/// found". Bump this when Cohere ships a newer dated Arabic transcription model.
|
|
12
|
+
pub const DEFAULT_COHERE_MODEL: &str = "cohere-transcribe-arabic-07-2026";
|
|
13
|
+
|
|
14
|
+
/// Where and how to reach a transcription server.
|
|
15
|
+
#[derive(Debug, Clone)]
|
|
16
|
+
pub struct Endpoint {
|
|
17
|
+
/// Full URL to POST the multipart form to.
|
|
18
|
+
pub url: String,
|
|
19
|
+
/// Bearer token, if the backend requires auth.
|
|
20
|
+
pub api_key: Option<String>,
|
|
21
|
+
/// Model id sent as the `model` form field.
|
|
22
|
+
pub model: String,
|
|
23
|
+
/// Provenance tag recorded on the transcript.
|
|
24
|
+
pub provider: Provider,
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
impl Endpoint {
|
|
28
|
+
/// Cohere's hosted API. `model` defaults to [`DEFAULT_COHERE_MODEL`].
|
|
29
|
+
pub fn cohere(api_key: impl Into<String>, model: Option<String>) -> Self {
|
|
30
|
+
Endpoint {
|
|
31
|
+
url: COHERE_URL.to_string(),
|
|
32
|
+
api_key: Some(api_key.into()),
|
|
33
|
+
model: model.unwrap_or_else(|| DEFAULT_COHERE_MODEL.to_string()),
|
|
34
|
+
provider: Provider::Cohere,
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/// A self-hosted OpenAI-compatible endpoint — pass the full route URL
|
|
39
|
+
/// (e.g. `http://localhost:8000/v1/audio/transcriptions`).
|
|
40
|
+
pub fn openai_compatible(
|
|
41
|
+
url: impl Into<String>,
|
|
42
|
+
model: impl Into<String>,
|
|
43
|
+
api_key: Option<String>,
|
|
44
|
+
) -> Self {
|
|
45
|
+
Endpoint {
|
|
46
|
+
url: url.into(),
|
|
47
|
+
api_key,
|
|
48
|
+
model: model.into(),
|
|
49
|
+
provider: Provider::OpenAiCompatible,
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
use std::path::PathBuf;
|
|
2
|
+
|
|
3
|
+
/// Everything that can go wrong on the way from an audio file to a transcript.
|
|
4
|
+
#[derive(Debug, thiserror::Error)]
|
|
5
|
+
pub enum Error {
|
|
6
|
+
#[error("could not read audio file {path}: {source}")]
|
|
7
|
+
ReadFile {
|
|
8
|
+
path: PathBuf,
|
|
9
|
+
source: std::io::Error,
|
|
10
|
+
},
|
|
11
|
+
|
|
12
|
+
#[error("request to {url} failed: {source}")]
|
|
13
|
+
Http { url: String, source: reqwest::Error },
|
|
14
|
+
|
|
15
|
+
#[error("endpoint returned HTTP {status}: {body}")]
|
|
16
|
+
Api { status: u16, body: String },
|
|
17
|
+
|
|
18
|
+
#[error("could not parse endpoint response (expected a JSON object with a string \"text\" field), got: {body}")]
|
|
19
|
+
BadResponse { body: String },
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
pub type Result<T> = std::result::Result<T, Error>;
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
//! `raqeem-core` — رقيم. Turn an audio file into Arabic text by delegating
|
|
2
|
+
//! inference to a transcription endpoint (Cohere's hosted API, or your own
|
|
3
|
+
//! vLLM). This crate never loads model weights: it decodes nothing, runs no
|
|
4
|
+
//! model — it POSTs the audio and folds the result. That is what keeps it
|
|
5
|
+
//! light and callable from any language over the CLI.
|
|
6
|
+
//!
|
|
7
|
+
//! ```no_run
|
|
8
|
+
//! use raqeem_core::{Endpoint, Transcriber};
|
|
9
|
+
//!
|
|
10
|
+
//! let endpoint = Endpoint::cohere(std::env::var("COHERE_API_KEY").unwrap(), None);
|
|
11
|
+
//! let transcript = Transcriber::new(endpoint)
|
|
12
|
+
//! .language("ar")
|
|
13
|
+
//! .transcribe(std::path::Path::new("voice_note.ogg"))?;
|
|
14
|
+
//! println!("{}", transcript.text);
|
|
15
|
+
//! # Ok::<(), raqeem_core::Error>(())
|
|
16
|
+
//! ```
|
|
17
|
+
|
|
18
|
+
mod arabic;
|
|
19
|
+
mod endpoint;
|
|
20
|
+
mod error;
|
|
21
|
+
mod output;
|
|
22
|
+
mod provider;
|
|
23
|
+
|
|
24
|
+
pub use arabic::normalize_ar;
|
|
25
|
+
pub use endpoint::{Endpoint, COHERE_URL, DEFAULT_COHERE_MODEL};
|
|
26
|
+
pub use error::{Error, Result};
|
|
27
|
+
pub use output::OutputFormat;
|
|
28
|
+
pub use provider::Provider;
|
|
29
|
+
|
|
30
|
+
use std::path::Path;
|
|
31
|
+
use std::time::Duration;
|
|
32
|
+
|
|
33
|
+
/// Default transcription language (ISO-639-1). Arabic-first, by design.
|
|
34
|
+
pub const DEFAULT_LANGUAGE: &str = "ar";
|
|
35
|
+
|
|
36
|
+
/// Default total-request timeout, in seconds. Generous on purpose: it must cover
|
|
37
|
+
/// upload + model inference + download, and CPU inference of a voice note can take
|
|
38
|
+
/// many seconds. (reqwest's blocking client otherwise silently defaults to 30s,
|
|
39
|
+
/// which truncates a slow transcription mid-inference.)
|
|
40
|
+
pub const DEFAULT_TIMEOUT_SECS: u64 = 300;
|
|
41
|
+
|
|
42
|
+
/// A completed transcription. `text` is the model's verbatim output; the folded
|
|
43
|
+
/// `text_normalized` (see [`normalize_ar`]) is what downstream parsers consume.
|
|
44
|
+
#[derive(Debug, Clone, serde::Serialize)]
|
|
45
|
+
pub struct Transcript {
|
|
46
|
+
pub text: String,
|
|
47
|
+
pub text_normalized: String,
|
|
48
|
+
pub provider: String,
|
|
49
|
+
pub model: String,
|
|
50
|
+
pub language: String,
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/// Transcribes audio by POSTing it to a configured [`Endpoint`].
|
|
54
|
+
pub struct Transcriber {
|
|
55
|
+
endpoint: Endpoint,
|
|
56
|
+
language: String,
|
|
57
|
+
client: reqwest::blocking::Client,
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
impl Transcriber {
|
|
61
|
+
/// Build a transcriber for the given endpoint, defaulting to Arabic and a
|
|
62
|
+
/// [`DEFAULT_TIMEOUT_SECS`] total-request timeout.
|
|
63
|
+
pub fn new(endpoint: Endpoint) -> Self {
|
|
64
|
+
Self::with_timeout(endpoint, Duration::from_secs(DEFAULT_TIMEOUT_SECS))
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/// Like [`new`](Self::new) but with an explicit total-request timeout — raise it
|
|
68
|
+
/// for slow CPU inference or large files, lower it to fail faster. The timeout
|
|
69
|
+
/// spans the whole round-trip (connect + upload + inference + download).
|
|
70
|
+
pub fn with_timeout(endpoint: Endpoint, timeout: Duration) -> Self {
|
|
71
|
+
let client = reqwest::blocking::Client::builder()
|
|
72
|
+
.timeout(timeout)
|
|
73
|
+
.build()
|
|
74
|
+
.unwrap_or_else(|_| reqwest::blocking::Client::new());
|
|
75
|
+
Transcriber {
|
|
76
|
+
endpoint,
|
|
77
|
+
language: DEFAULT_LANGUAGE.to_string(),
|
|
78
|
+
client,
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/// Override the transcription language (ISO-639-1).
|
|
83
|
+
pub fn language(mut self, lang: impl Into<String>) -> Self {
|
|
84
|
+
self.language = lang.into();
|
|
85
|
+
self
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/// Read `audio`, send it to the endpoint, and return the transcript.
|
|
89
|
+
pub fn transcribe(&self, audio: &Path) -> Result<Transcript> {
|
|
90
|
+
let bytes = std::fs::read(audio).map_err(|source| Error::ReadFile {
|
|
91
|
+
path: audio.to_path_buf(),
|
|
92
|
+
source,
|
|
93
|
+
})?;
|
|
94
|
+
let filename = audio
|
|
95
|
+
.file_name()
|
|
96
|
+
.and_then(|n| n.to_str())
|
|
97
|
+
.unwrap_or("audio")
|
|
98
|
+
.to_string();
|
|
99
|
+
|
|
100
|
+
// Same multipart shape for every backend: model + language + file.
|
|
101
|
+
// Order matters: Cohere rejects the request unless the text fields
|
|
102
|
+
// (model, language) appear BEFORE the file part in the body.
|
|
103
|
+
let file_part = reqwest::blocking::multipart::Part::bytes(bytes).file_name(filename);
|
|
104
|
+
let form = reqwest::blocking::multipart::Form::new()
|
|
105
|
+
.text("model", self.endpoint.model.clone())
|
|
106
|
+
.text("language", self.language.clone())
|
|
107
|
+
.part("file", file_part);
|
|
108
|
+
|
|
109
|
+
let mut req = self.client.post(&self.endpoint.url).multipart(form);
|
|
110
|
+
if let Some(key) = &self.endpoint.api_key {
|
|
111
|
+
req = req.bearer_auth(key);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
let resp = req.send().map_err(|source| Error::Http {
|
|
115
|
+
url: self.endpoint.url.clone(),
|
|
116
|
+
source,
|
|
117
|
+
})?;
|
|
118
|
+
let status = resp.status();
|
|
119
|
+
let body = resp.text().map_err(|source| Error::Http {
|
|
120
|
+
url: self.endpoint.url.clone(),
|
|
121
|
+
source,
|
|
122
|
+
})?;
|
|
123
|
+
if !status.is_success() {
|
|
124
|
+
return Err(Error::Api {
|
|
125
|
+
status: status.as_u16(),
|
|
126
|
+
body,
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
let text = extract_text(&body).ok_or_else(|| Error::BadResponse { body: body.clone() })?;
|
|
131
|
+
Ok(Transcript {
|
|
132
|
+
text_normalized: normalize_ar(&text),
|
|
133
|
+
text,
|
|
134
|
+
provider: self.endpoint.provider.as_str().to_string(),
|
|
135
|
+
model: self.endpoint.model.clone(),
|
|
136
|
+
language: self.language.clone(),
|
|
137
|
+
})
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/// Pull the transcript out of an endpoint response.
|
|
142
|
+
///
|
|
143
|
+
// ponytail: assumes the OpenAI-standard `{"text": "..."}` shape, which vLLM and
|
|
144
|
+
// (per Cohere's docs) the hosted API both return. If a backend nests it
|
|
145
|
+
// differently, add a branch here rather than a whole response type — the raw
|
|
146
|
+
// body is surfaced in `Error::BadResponse` so a mismatch is obvious on first run.
|
|
147
|
+
fn extract_text(body: &str) -> Option<String> {
|
|
148
|
+
let v: serde_json::Value = serde_json::from_str(body).ok()?;
|
|
149
|
+
v.get("text")?.as_str().map(str::to_string)
|
|
150
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
//! Render a [`Transcript`] for a caller. `Text` is the human-readable verbatim
|
|
2
|
+
//! transcription; `Json` carries both the verbatim and normalized forms plus
|
|
3
|
+
//! provenance (what scout consumes).
|
|
4
|
+
|
|
5
|
+
use crate::Transcript;
|
|
6
|
+
|
|
7
|
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
8
|
+
pub enum OutputFormat {
|
|
9
|
+
Text,
|
|
10
|
+
Json,
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
impl OutputFormat {
|
|
14
|
+
pub fn render(self, t: &Transcript) -> String {
|
|
15
|
+
match self {
|
|
16
|
+
OutputFormat::Text => t.text.clone(),
|
|
17
|
+
OutputFormat::Json => {
|
|
18
|
+
// Transcript derives Serialize; serialization of these plain
|
|
19
|
+
// fields cannot fail, so the fallback is unreachable in practice.
|
|
20
|
+
serde_json::to_string_pretty(t).unwrap_or_else(|_| "{}".to_string())
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
//! Backend presets. Every supported transcription server speaks the same
|
|
2
|
+
//! OpenAI-compatible multipart `/audio/transcriptions` shape, so a "provider"
|
|
3
|
+
//! is just a label + the defaults ([`crate::Endpoint`] carries the actual
|
|
4
|
+
//! URL / auth / model). Adding a backend = adding a variant here and a
|
|
5
|
+
//! constructor on `Endpoint` — see `.claude/skills/add-endpoint-adapter`.
|
|
6
|
+
|
|
7
|
+
/// A known transcription backend.
|
|
8
|
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
9
|
+
pub enum Provider {
|
|
10
|
+
/// Cohere's hosted API — `https://api.cohere.com/v2/audio/transcriptions`.
|
|
11
|
+
Cohere,
|
|
12
|
+
/// Any self-hosted OpenAI-compatible server (e.g. vLLM at
|
|
13
|
+
/// `/v1/audio/transcriptions`), reached via an explicit URL.
|
|
14
|
+
OpenAiCompatible,
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
impl Provider {
|
|
18
|
+
/// Stable machine-readable tag, recorded on every [`crate::Transcript`].
|
|
19
|
+
pub fn as_str(self) -> &'static str {
|
|
20
|
+
match self {
|
|
21
|
+
Provider::Cohere => "cohere",
|
|
22
|
+
Provider::OpenAiCompatible => "openai-compatible",
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
//! End-to-end test of the transcribe path against a mocked HTTP endpoint — no
|
|
2
|
+
//! network, no API key, no model. Asserts the multipart body carries the fields
|
|
3
|
+
//! every backend expects and that an Arabic response parses + normalizes.
|
|
4
|
+
|
|
5
|
+
use raqeem_core::{Endpoint, Transcriber};
|
|
6
|
+
|
|
7
|
+
#[test]
|
|
8
|
+
fn posts_multipart_with_auth_and_parses_arabic_text() {
|
|
9
|
+
let mut server = mockito::Server::new();
|
|
10
|
+
|
|
11
|
+
let mock = server
|
|
12
|
+
.mock("POST", "/v2/audio/transcriptions")
|
|
13
|
+
.match_header("authorization", "Bearer testkey")
|
|
14
|
+
.match_body(mockito::Matcher::AllOf(vec![
|
|
15
|
+
mockito::Matcher::Regex("cohere-transcribe-arabic".to_string()),
|
|
16
|
+
// Order matters: Cohere requires model + language BEFORE the file part.
|
|
17
|
+
mockito::Matcher::Regex(
|
|
18
|
+
r#"(?s)name="model".*name="language".*name="file""#.to_string(),
|
|
19
|
+
),
|
|
20
|
+
]))
|
|
21
|
+
.with_status(200)
|
|
22
|
+
.with_header("content-type", "application/json")
|
|
23
|
+
// Arabic-Indic digits + U+066B on purpose, to prove normalization runs.
|
|
24
|
+
.with_body(r#"{"text": "الطماطم بـ ١٢٫٥ جنيه"}"#)
|
|
25
|
+
.create();
|
|
26
|
+
|
|
27
|
+
let url = format!("{}/v2/audio/transcriptions", server.url());
|
|
28
|
+
let endpoint =
|
|
29
|
+
Endpoint::openai_compatible(url, "cohere-transcribe-arabic", Some("testkey".into()));
|
|
30
|
+
|
|
31
|
+
let audio = std::env::temp_dir().join("RAQEEM_test_clip.wav");
|
|
32
|
+
std::fs::write(&audio, b"RIFF....WAVEfake").unwrap();
|
|
33
|
+
|
|
34
|
+
let t = Transcriber::new(endpoint)
|
|
35
|
+
.language("ar")
|
|
36
|
+
.transcribe(&audio)
|
|
37
|
+
.expect("transcribe should succeed against the mock");
|
|
38
|
+
|
|
39
|
+
assert_eq!(t.text, "الطماطم بـ ١٢٫٥ جنيه"); // verbatim, untouched
|
|
40
|
+
assert!(t.text_normalized.contains("12.5")); // digits + decimal folded
|
|
41
|
+
assert!(!t.text_normalized.contains('ـ')); // tatweel stripped
|
|
42
|
+
assert_eq!(t.language, "ar");
|
|
43
|
+
assert_eq!(t.provider, "openai-compatible");
|
|
44
|
+
|
|
45
|
+
mock.assert();
|
|
46
|
+
let _ = std::fs::remove_file(&audio);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
#[test]
|
|
50
|
+
fn surfaces_api_errors_with_status_and_body() {
|
|
51
|
+
let mut server = mockito::Server::new();
|
|
52
|
+
let _mock = server
|
|
53
|
+
.mock("POST", "/v1/audio/transcriptions")
|
|
54
|
+
.with_status(401)
|
|
55
|
+
.with_body("unauthorized")
|
|
56
|
+
.create();
|
|
57
|
+
|
|
58
|
+
let url = format!("{}/v1/audio/transcriptions", server.url());
|
|
59
|
+
let endpoint = Endpoint::openai_compatible(url, "m", None);
|
|
60
|
+
|
|
61
|
+
let audio = std::env::temp_dir().join("RAQEEM_test_clip2.wav");
|
|
62
|
+
std::fs::write(&audio, b"x").unwrap();
|
|
63
|
+
|
|
64
|
+
let err = Transcriber::new(endpoint).transcribe(&audio).unwrap_err();
|
|
65
|
+
let msg = err.to_string();
|
|
66
|
+
assert!(msg.contains("401"), "expected status in error, got: {msg}");
|
|
67
|
+
let _ = std::fs::remove_file(&audio);
|
|
68
|
+
}
|