gotoken 0.1.0__tar.gz

gotoken-0.1.0/Cargo.lock

@@ -0,0 +1,341 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "aho-corasick"
+version = "1.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
+
+[[package]]
+name = "either"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
+
+[[package]]
+name = "gotoken"
+version = "0.1.0"
+dependencies = [
+ "phf",
+ "pyo3",
+ "rayon",
+ "regex",
+]
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "indoc"
+version = "2.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
+dependencies = [
+ "rustversion",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.186"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
+
+[[package]]
+name = "memchr"
+version = "2.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
+
+[[package]]
+name = "memoffset"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "phf"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
+dependencies = [
+ "phf_macros",
+ "phf_shared",
+]
+
+[[package]]
+name = "phf_generator"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
+dependencies = [
+ "phf_shared",
+ "rand",
+]
+
+[[package]]
+name = "phf_macros"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216"
+dependencies = [
+ "phf_generator",
+ "phf_shared",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
+dependencies = [
+ "siphasher",
+]
+
+[[package]]
+name = "portable-atomic"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.106"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.22.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884"
+dependencies = [
+ "cfg-if",
+ "indoc",
+ "libc",
+ "memoffset",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+ "unindent",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.22.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38"
+dependencies = [
+ "once_cell",
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.22.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.22.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.22.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "pyo3-build-config",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rand"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a"
+dependencies = [
+ "rand_core",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
+
+[[package]]
+name = "rayon"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "regex"
+version = "1.12.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
+
+[[package]]
+name = "rustversion"
+version = "1.0.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
+
+[[package]]
+name = "siphasher"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649"
+
+[[package]]
+name = "syn"
+version = "2.0.117"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "target-lexicon"
+version = "0.12.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+
+[[package]]
+name = "unindent"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"

gotoken-0.1.0/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name        = "gotoken"
+version     = "0.1.0"
+edition     = "2021"
+description = "Syntax-aware tokenizer for Bash and formal languages"
+license     = "MIT"
+
+[lib]
+name        = "gotoken"
+crate-types = ["rlib", "cdylib"]
+
+[features]
+default = ["rayon"]
+rayon   = ["dep:rayon"]
+python  = ["dep:pyo3"]
+
+[dependencies]
+phf   = { version = "0.11", features = ["macros"] }
+regex = "1.10"
+rayon = { version = "1.10", optional = true }
+# 0.22.x supporta Python 3.13
+pyo3  = { version = "0.22", features = ["extension-module"], optional = true }
+
+[dev-dependencies]
+
+[profile.release]
+opt-level     = 3
+lto           = "fat"
+codegen-units = 1
+panic         = "abort"
+strip         = "symbols"

gotoken-0.1.0/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: gotoken
+Version: 0.1.0
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Summary: Syntax-aware Bash tokenizer — Rust core, Python bindings
+License: MIT
+Requires-Python: >=3.9

gotoken-0.1.0/patch_lexar.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+"""
+Fix 2 bug in src/lexer.rs:
+  1. try_dquote ora gestisce anche UnclosedQuote (stessa firma di try_squote)
+  2. Pattern operator: || e |& devono precedere | nell'alternazione
+  3. Il loop principale smista " a try_dquote e ' a try_squote
+
+Esegui da ~/gotoken/: python3 patch_lexer2.py
+"""
+
+path = "src/lexer.rs"
+with open(path, "r") as f:
+    src = f.read()
+
+# ── Fix 1: regex operator — || e |& prima di | ─────────────────────────────
+OLD_REGEX = (
+    r'            operator: Regex::new(\n'
+    r'                r"^(?:2>&1|&>>|&>|>>=|<<=|<<<|<<-|<<|>>|>\||\|&|&&|\|\|;;|;;|;|2>>|2>|<>|>>|>|<|&|\|)"\n'
+    r'            )?,'
+)
+NEW_REGEX = (
+    r'            operator: Regex::new(\n'
+    r'                r"^(?:2>&1|&>>|&>|>>=|<<=|<<<|<<-|<<|>>|>\||&&|\|\||;;|\|&|;|2>>|2>|<>|>|<|&|\|)"\n'
+    r'            )?,'
+)
+
+# Usiamo replace su stringa letterale
+OLD_REGEX_LIT = '''            operator: Regex::new(
+                r"^(?:2>&1|&>>|&>|>>=|<<=|<<<|<<-|<<|>>|>\\|\\|&|&&|\\|\\|;;|;;|;|2>>|2>|<>|>>|>|<|&|\\|)"
+            )?,'''
+
+# Leggiamo la riga esatta dal file
+import re
+op_match = re.search(r'operator: Regex::new\(\s*r"([^"]+)"', src)
+if op_match:
+    old_pattern = op_match.group(1)
+    print(f"Pattern operatore trovato: {old_pattern}")
+    # Nuova alternazione: || e |& prima del singolo |
+    new_pattern = r"^(?:2>&1|&>>|&>|>>=|<<=|<<<|<<-|<<|>>|>\||&&|\|\||;;|\|&|;|2>>|2>|<>|>|<|&|\|)"
+    src = src.replace(
+        f'r"{old_pattern}"',
+        f'r"{new_pattern}"',
+        1
+    )
+    print("✓ Fix 1 applicato: priorità || e |& corretta nel pattern operator")
+else:
+    print("WARN: pattern operator non trovato, Fix 1 saltato")
+
+# ── Fix 2: try_dquote ora ritorna Option<Result<Token>> ────────────────────
+
+OLD_DQUOTE = '''    /// Double-quoted string.  Returns `None` if no opening `"` is present.
+    /// If `"` is present but unclosed the regex simply won\'t match, so we
+    /// detect it here and return an error via the `ZeroLengthFallback`
+    /// guard path — actually we surface it as `UnclosedQuote`.
+    fn try_dquote(&self, tail: &str, offset: usize) -> Option<Token> {
+        if !tail.starts_with(\'"\') {
+            return None;
+        }
+        // If the regex matches we have a closed string.
+        if let Some(m) = self.patterns.dquote.find(tail) {
+            let text = &tail[m.start()..m.end()];
+            return Some(Token::new(text, TokenKind::StringLiteral, offset));
+        }
+        // `"` found but no closing quote — we return a sentinel token whose
+        // `kind` is set to a special value; the caller checks this and
+        // converts it to an error via `transpose()`.
+        // We encode the error as a `ByteFallback` with len 0 as a signal —
+        // the caller sees `ZeroLengthFallback` ... actually cleaner to use
+        // a dedicated approach: return the error inline.
+        // We exploit the Option<Result<Token>> path: NOT this function.
+        // This function stays Option<Token>; unclosed " is caught below.
+        None
+    }'''
+
+NEW_DQUOTE = '''    /// Double-quoted string. Returns `Option<Result<Token>>` so that
+    /// `UnclosedQuote` can bubble up when the closing `"` is missing.
+    fn try_dquote(
+        &self,
+        tail:   &str,
+        offset: usize,
+    ) -> Option<std::result::Result<Token, TokenizerError>> {
+        if !tail.starts_with(\'"\') {
+            return None;
+        }
+        if let Some(m) = self.patterns.dquote.find(tail) {
+            let text = &tail[m.start()..m.end()];
+            return Some(Ok(Token::new(text, TokenKind::StringLiteral, offset)));
+        }
+        // Opening `"` found but no closing quote.
+        Some(Err(TokenizerError::UnclosedQuote {
+            quote_char: \'"\',
+            opened_at:  offset,
+        }))
+    }'''
+
+if OLD_DQUOTE in src:
+    src = src.replace(OLD_DQUOTE, NEW_DQUOTE, 1)
+    print("✓ Fix 2 applicato: try_dquote ora ritorna Option<Result<Token>>")
+else:
+    print("WARN: corpo try_dquote non trovato esattamente — applico fix alternativo")
+    # Approccio alternativo: cerca e rimpiazza con regex
+    src = re.sub(
+        r'fn try_dquote\(&self, tail: &str, offset: usize\) -> Option<Token> \{[^}]+\}',
+        '''fn try_dquote(
+        &self,
+        tail:   &str,
+        offset: usize,
+    ) -> Option<std::result::Result<Token, TokenizerError>> {
+        if !tail.starts_with(\'"\') {
+            return None;
+        }
+        if let Some(m) = self.patterns.dquote.find(tail) {
+            let text = &tail[m.start()..m.end()];
+            return Some(Ok(Token::new(text, TokenKind::StringLiteral, offset)));
+        }
+        Some(Err(TokenizerError::UnclosedQuote {
+            quote_char: \'"\',
+            opened_at:  offset,
+        }))
+    }''',
+        src,
+        count=1,
+        flags=re.DOTALL
+    )
+    print("✓ Fix 2 applicato via regex fallback")
+
+# ── Fix 3: loop principale — smista " a try_dquote, ' a try_squote ─────────
+
+OLD_LOOP = '''            } else if tail.starts_with(\'"\') || tail.starts_with(\'\\'\') {
+                match self.try_squote(tail, cursor, input) {
+                    Some(Ok(t))  => t,
+                    Some(Err(e)) => return Err(e),
+                    None         => self.byte_fallback(tail, cursor),
+                }'''
+
+NEW_LOOP = '''            } else if tail.starts_with(\'"\') {
+                match self.try_dquote(tail, cursor) {
+                    Some(Ok(t))  => t,
+                    Some(Err(e)) => return Err(e),
+                    None         => self.byte_fallback(tail, cursor),
+                }
+            } else if tail.starts_with(\'\\'\') {
+                match self.try_squote(tail, cursor, input) {
+                    Some(Ok(t))  => t,
+                    Some(Err(e)) => return Err(e),
+                    None         => self.byte_fallback(tail, cursor),
+                }'''
+
+if OLD_LOOP in src:
+    src = src.replace(OLD_LOOP, NEW_LOOP, 1)
+    print("✓ Fix 3 applicato: loop smista \" a try_dquote e \\' a try_squote")
+else:
+    print("WARN: pattern loop non trovato esattamente")
+    # Regex fallback
+    src = re.sub(
+        r"} else if tail\.starts_with\('\"'\) \|\| tail\.starts_with\('\\\\'\) \{.*?match self\.try_squote\(tail, cursor, input\) \{.*?None\s*=> self\.byte_fallback\(tail, cursor\),\s*\}",
+        """} else if tail.starts_with('"') {
+                match self.try_dquote(tail, cursor) {
+                    Some(Ok(t))  => t,
+                    Some(Err(e)) => return Err(e),
+                    None         => self.byte_fallback(tail, cursor),
+                }
+            } else if tail.starts_with('\\'') {
+                match self.try_squote(tail, cursor, input) {
+                    Some(Ok(t))  => t,
+                    Some(Err(e)) => return Err(e),
+                    None         => self.byte_fallback(tail, cursor),
+                }""",
+        src,
+        count=1,
+        flags=re.DOTALL
+    )
+    print("✓ Fix 3 applicato via regex fallback")
+
+# ── Fix 4: rimuovi il check " da try_squote (ora lo gestisce try_dquote) ───
+
+OLD_SQUOTE_DQUOTE_CHECK = '''        if tail.starts_with(\'"\') {
+            // Double-quote unclosed detection (try_dquote returned None above)
+            if self.patterns.dquote.find(tail).is_none() {
+                return Some(Err(TokenizerError::UnclosedQuote {
+                    quote_char: \'"\',
+                    opened_at:  offset,
+                }));
+            }
+        }
+        if !tail.starts_with(\'\\'\') {'''
+
+NEW_SQUOTE_START = '''        if !tail.starts_with(\'\\'\') {'''
+
+if OLD_SQUOTE_DQUOTE_CHECK in src:
+    src = src.replace(OLD_SQUOTE_DQUOTE_CHECK, NEW_SQUOTE_START, 1)
+    print("✓ Fix 4 applicato: rimosso check \" duplicato da try_squote")
+else:
+    print("WARN: check \" in try_squote non trovato — potrebbe essere già rimosso")
+
+with open(path, "w") as f:
+    f.write(src)
+
+print("\nTutti i fix applicati. Esegui: cargo test 2>&1")

gotoken-0.1.0/patch_lib_python.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+"""
+Aggiunge i binding Python a src/lib.rs.
+Esegui da ~/gotoken/: python3 patch_lib_python.py
+"""
+
+path = "src/lib.rs"
+with open(path, "r") as f:
+    src = f.read()
+
+BLOCK = '''
+// ── Python bindings (compilati solo con --features python) ───────────────────
+#[cfg(feature = "python")]
+pub mod python;
+
+#[cfg(feature = "python")]
+use pyo3::prelude::*;
+
+/// Entry-point del modulo Python `gotoken`.
+/// Chiamato automaticamente da maturin quando Python importa la libreria.
+#[cfg(feature = "python")]
+#[pymodule]
+fn gotoken(m: &Bound<\'_, PyModule>) -> PyResult<()> {
+    python::gotoken(m)
+}
+'''
+
+MARKER = "pub mod vocab;"
+
+if "pub mod python;" in src:
+    print("Binding Python già presenti in lib.rs — nessuna modifica necessaria.")
+else:
+    if MARKER not in src:
+        print(f"ERRORE: marker '{MARKER}' non trovato in lib.rs")
+        raise SystemExit(1)
+    src = src.replace(MARKER, MARKER + BLOCK, 1)
+    with open(path, "w") as f:
+        f.write(src)
+    print("✓ Binding Python aggiunti a lib.rs")
+
+print("Ora esegui:")
+print("  pip install maturin")
+print("  maturin develop --features python,rayon")
+print("  python3 test_gotoken.py")

gotoken-0.1.0/pyproject.toml

@@ -0,0 +1,23 @@
+[build-system]
+requires      = ["maturin>=1.5,<2.0"]
+build-backend = "maturin"
+
+[project]
+name            = "gotoken"
+version         = "0.1.0"
+description     = "Syntax-aware Bash tokenizer — Rust core, Python bindings"
+requires-python = ">=3.9"
+license         = { text = "MIT" }
+classifiers     = [
+    "Programming Language :: Rust",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+
+[tool.maturin]
+# Attiva la feature "python" di Cargo che include pyo3
+features       = ["python", "rayon"]
+# Funzione di entry-point del modulo PyO3 (deve matchare #[pymodule] fn gotoken)
+module-name    = "gotoken"
+# Cartella sorgente Rust
+manifest-path  = "Cargo.toml"