infinity-sdk 0.7.0.dev5__tar.gz → 0.7.0.dev6__tar.gz

{infinity_sdk-0.7.0.dev5/python/infinity_sdk/infinity_sdk.egg-info → infinity_sdk-0.7.0.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: infinity-sdk
-Version: 0.7.0.dev5
+Version: 0.7.0.dev6
 Summary: infinity
 License-Expression: Apache-2.0
 Requires-Python: <3.14,>=3.11
@@ -97,7 +97,7 @@ Infinity supports two working modes, embedded mode and client-server mode. The f
 
 2. Install the `infinity-sdk` package:
    ```bash
-   pip install infinity-sdk==0.7.0.dev5
+   pip install infinity-sdk==0.7.0.dev6
    ```
 
 3. Use Infinity to conduct a dense vector search:

{infinity_sdk-0.7.0.dev5 → infinity_sdk-0.7.0.dev6}/README.md

@@ -96,7 +96,7 @@ If you are on Windows 10+, you must enable WSL or WSL2 to deploy Infinity using
 ### Install Infinity client
 
 ```
-pip install infinity-sdk==0.7.0.dev5
+pip install infinity-sdk==0.7.0.dev6
 ```
 
 ### Run a vector search

{infinity_sdk-0.7.0.dev5 → infinity_sdk-0.7.0.dev6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "infinity-sdk"
-version = "0.7.0.dev5"
+version = "0.7.0.dev6"
 description = "infinity"
 readme = "python/infinity_sdk/README.md"
 license = "Apache-2.0"

{infinity_sdk-0.7.0.dev5 → infinity_sdk-0.7.0.dev6}/python/infinity_sdk/README.md

@@ -63,7 +63,7 @@ Infinity supports two working modes, embedded mode and client-server mode. The f
 
 2. Install the `infinity-sdk` package:
    ```bash
-   pip install infinity-sdk==0.7.0.dev5
+   pip install infinity-sdk==0.7.0.dev6
    ```
 
 3. Use Infinity to conduct a dense vector search:

{infinity_sdk-0.7.0.dev5 → infinity_sdk-0.7.0.dev6}/python/infinity_sdk/infinity/rag_tokenizer.py

@@ -43,6 +43,29 @@ from nltk import word_tokenize
 from nltk.stem import SnowballStemmer, WordNetLemmatizer
 
 
+# Map language names (lowercase) to NLTK SnowballStemmer language names.
+# Used by set_language() to configure language-specific stemming.
+_SNOWBALL_LANGUAGE_MAP = {
+    "english": "english",
+    "dutch": "dutch",
+    "german": "german",
+    "french": "french",
+    "spanish": "spanish",
+    "italian": "italian",
+    "portuguese": "portuguese",
+    "portuguese br": "portuguese",
+    "russian": "russian",
+    "arabic": "arabic",
+    "danish": "danish",
+    "finnish": "finnish",
+    "hungarian": "hungarian",
+    "norwegian": "norwegian",
+    "romanian": "romanian",
+    "swedish": "swedish",
+    "turkish": "turkish",
+}
+
+
 class RagTokenizer:
     def key_(self, line):
         return str(line.lower().encode("utf-8"))[2:-1]
@@ -98,6 +121,7 @@ class RagTokenizer:
 
         self.stemmer = SnowballStemmer("english")
         self.lemmatizer = WordNetLemmatizer()
+        self._use_lemmatizer = True  # WordNet only supports English
 
         self.SPLIT_CHAR = r"([ ,\.<>/?;:'\[\]\\`!@#$%^&*\(\)\{\}\|_+=《》，。？、；‘’：“”【】~！￥%……（）——-]+|[a-zA-Z0-9,\.-]+)"
 
@@ -131,6 +155,38 @@ class RagTokenizer:
     def add_user_dict(self, fnm):
         self._load_dict(fnm)
 
+    def set_language(self, language: str):
+        """Configure stemmer/lemmatizer for the given language.
+
+        Args:
+            language: Language name (e.g. "English", "Dutch", "Chinese").
+                      Case-insensitive.
+        """
+        lang_key = language.strip().lower()
+        snowball_lang = _SNOWBALL_LANGUAGE_MAP.get(lang_key)
+
+        if snowball_lang is not None:
+            self.stemmer = SnowballStemmer(snowball_lang)
+            if snowball_lang == "english":
+                self.lemmatizer = WordNetLemmatizer()
+                self._use_lemmatizer = True
+            else:
+                # WordNet only supports English; disable lemmatizer for
+                # other languages and rely on Snowball stemming alone.
+                self._use_lemmatizer = False
+            logging.debug(
+                "Tokenizer language set to '%s' (Snowball: %s, lemmatizer: %s)",
+                language, snowball_lang, self._use_lemmatizer,
+            )
+        else:
+            # Unsupported language (Chinese, Japanese, Korean, etc.) –
+            # keep defaults.  CJK text uses dictionary segmentation,
+            # not stemming.
+            logging.debug(
+                "Language '%s' has no Snowball stemmer; keeping defaults",
+                language,
+            )
+
     def _strQ2B(self, ustring):
         """Convert full-width characters to half-width characters"""
         rstring = ""
@@ -326,7 +382,20 @@ class RagTokenizer:
         return self.score_(res[::-1])
 
     def english_normalize_(self, tks):
-        return [self.stemmer.stem(self.lemmatizer.lemmatize(t)) if re.match(r"[a-zA-Z_-]+$", t) else t for t in tks]
+        return [self._normalize_token(t) for t in tks]
+
+    def _normalize_token(self, t: str) -> str:
+        """Stem (and optionally lemmatize) a single alphabetic token.
+
+        When the lemmatizer is enabled (English), applies lemmatization
+        before stemming.  For other Snowball-supported languages, only
+        stemming is applied.  Non-alphabetic tokens are returned as-is.
+        """
+        if re.match(r"[a-zA-Z_-]+$", t):
+            if self._use_lemmatizer:
+                return self.stemmer.stem(self.lemmatizer.lemmatize(t))
+            return self.stemmer.stem(t)
+        return t
 
     def _split_by_lang(self, line):
         txt_lang_pairs = []
@@ -360,7 +429,7 @@ class RagTokenizer:
         res = []
         for L, lang in arr:
             if not lang:
-                res.extend([self.stemmer.stem(self.lemmatizer.lemmatize(t)) for t in word_tokenize(L)])
+                res.extend([self._normalize_token(t) for t in word_tokenize(L)])
                 continue
             if len(L) < 2 or re.match(r"[a-z\.-]+$", L) or re.match(r"[0-9\.-]+$", L):
                 res.append(L)
@@ -501,11 +570,16 @@ if __name__ == '__main__':
     parser.add_argument('--fine-grained', action='store_true',
                         help='Use fine-grained tokenization')
     parser.add_argument('--user-dict', help='User dictionary file')
+    parser.add_argument('-l', '--language', help='Language for stemming (e.g., english, dutch)')
 
     args = parser.parse_args()
 
     tokenizer = RagTokenizer(debug=True, user_dict=args.user_dict)
 
+    # Set language if specified
+    if args.language:
+        tokenizer.set_language(args.language)
+
     # Process input
     if args.file:
         # File mode

{infinity_sdk-0.7.0.dev5 → infinity_sdk-0.7.0.dev6}/python/infinity_sdk/infinity/remote_thrift/client.py

@@ -123,8 +123,8 @@ class ThriftInfinityClient:
         # version: 0.6.8 and 0.6.9 and 0.6.10, client_version: 33
         # version: 0.6.13, client_version: 34
         # version: 0.6.15, client_version: 35
-        # version: 0.7.0, 0.7.0.dev1, 0.7.0.dev2, 0.7.0.dev3 and 0.7.0.dev4, 0.7.0.dev5, client_version: 36
-        res = self.client.Connect(ConnectRequest(client_version=36))  # 0.7.0.dev5
+        # version: 0.7.0, 0.7.0.dev1, 0.7.0.dev2, 0.7.0.dev3, 0.7.0.dev4, 0.7.0.dev5 and 0.7.0.dev6, client_version: 36
+        res = self.client.Connect(ConnectRequest(client_version=36))  # 0.7.0.dev6
         if res.error_code != 0:
             raise InfinityException(res.error_code, res.error_msg)
         self.session_id = res.session_id

{infinity_sdk-0.7.0.dev5 → infinity_sdk-0.7.0.dev6/python/infinity_sdk/infinity_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: infinity-sdk
-Version: 0.7.0.dev5
+Version: 0.7.0.dev6
 Summary: infinity
 License-Expression: Apache-2.0
 Requires-Python: <3.14,>=3.11
@@ -97,7 +97,7 @@ Infinity supports two working modes, embedded mode and client-server mode. The f
 
 2. Install the `infinity-sdk` package:
    ```bash
-   pip install infinity-sdk==0.7.0.dev5
+   pip install infinity-sdk==0.7.0.dev6
    ```
 
 3. Use Infinity to conduct a dense vector search: