maxframe 2.4.0rc1__cp312-cp312-win32.whl

maxframe/learn/contrib/llm/multi_modal.py

@@ -0,0 +1,135 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict
+
+from ....dataframe.core import DATAFRAME_TYPE, SERIES_TYPE
+from .core import LLM
+
+
+class MultiModalLLM(LLM):
+    def generate(
+        self,
+        data,
+        prompt_template: Dict[str, Any],
+        params: Dict[str, Any] = None,
+    ):
+        raise NotImplementedError
+
+
+def generate(
+    data,
+    model: MultiModalLLM,
+    prompt_template: Dict[str, Any],
+    params: Dict[str, Any] = None,
+):
+    """
+    Generate text with multi model llm based on given data and prompt template.
+
+    Parameters
+    ----------
+    data : DataFrame or Series
+        Input data used for generation. Can be maxframe DataFrame, Series that contain text to be processed.
+    model : MultiModalLLM
+        Language model instance support **MultiModal** inputs used for text generation.
+    prompt_template : List[Dict[str, List[Dict[str, str]]]]
+        List of message with column names as placeholders. Each message contains a role and content. Content is a list of dict, each dict contains a text or image, the value can reference column data from input.
+
+        Here is an example of prompt template.
+
+        .. code-block:: python
+
+            [
+                {
+                    "role": "<role>",  # e.g. "user" or "assistant"
+                    "content": [
+                        {
+                            # At least one of these fields is required
+                            "image": "<image_data_url>",  # optional
+                            "text": "<prompt_text_template>"  # optional
+                        },
+                        ...
+                    ]
+                }
+            ]
+
+        Where:
+
+        - ``text`` can be a Python format string using column names from input data as parameters (e.g. ``"{column_name}"``)
+        - ``image`` should be a DataURL string following `RFC2397 <https://en.wikipedia.org/wiki/Data_URI_scheme>`_ standard with format.
+
+        .. code-block:: none
+
+            data:<mime_type>[;base64],<column_name>
+
+
+    params : Dict[str, Any], optional
+        Additional parameters for generation configuration, by default None.
+        Can include settings like temperature, max_tokens, etc.
+
+    Returns
+    -------
+    DataFrame
+        Generated text raw response and success status. If the success is False, the generated text will return the
+        error message.
+
+    Notes
+    -----
+    - The ``api_key_resource`` parameter should reference a text file resource in MaxCompute that contains only your DashScope API key.
+
+    - Using DashScope services requires enabling public network access for your MaxCompute project. This can be configured through the MaxCompute console by `enabling the Internet access feature <https://help.aliyun.com/zh/maxcompute/user-guide/network-connection-process>`_ for your project. Without this configuration, the API calls to DashScope will fail due to network connectivity issues.
+
+    Examples
+    --------
+    You can initialize a DashScope multi-modal model (such as qwen-vl-max) by providing a model name and an ``api_key_resource``.
+    The ``api_key_resource`` is a MaxCompute resource name that points to a text file containing a `DashScope <https://dashscope.aliyun.com/>`_ API key.
+
+    >>> from maxframe.learn.contrib.llm.models.dashscope import DashScopeMultiModalLLM
+    >>> import maxframe.dataframe as md
+    >>>
+    >>> model = DashScopeMultiModalLLM(
+    ...     name="qwen-vl-max",
+    ...     api_key_resource="<api-key-resource-name>"
+    ... )
+
+    We use Data Url Schema to provide multi modal input in prompt template, here is an example to fill in the image from table.
+
+    Assuming you have a MaxCompute table with two columns: ``image_id`` (as the index) and ``encoded_image_data_base64`` (containing Base64 encoded image data),
+    you can construct a prompt message template as follows:
+
+    >>> df = md.read_odps_table("image_content", index_col="image_id")
+
+    >>> prompt_template = [
+    ...     {
+    ...         "role": "user",
+    ...         "content": [
+    ...             {
+    ...                 "image": "data:image/png;base64,encoded_image_data_base64",
+    ...             },
+    ...             {
+    ...                 "text": "Analyze this image in detail",
+    ...             },
+    ...         ],
+    ...     },
+    ... ]
+    >>> result = model.generate(df, prompt_template)
+    >>> result.execute()
+    """
+    if not isinstance(data, DATAFRAME_TYPE) and not isinstance(data, SERIES_TYPE):
+        raise ValueError("data must be a maxframe dataframe or series object")
+    if not isinstance(model, MultiModalLLM):
+        raise ValueError("model must be a MultiModalLLM object")
+    params = params if params is not None else dict()
+    model.validate_params(params)
+    return model.generate(data, prompt_template, params)

maxframe/learn/contrib/llm/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

maxframe/learn/contrib/llm/tests/test_core.py

@@ -0,0 +1,34 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from .. import core as llm_core
+
+
+def test_text_gen_operator_output_dtypes():
+    op = llm_core.LLMTextGenOperator()
+    dtypes = op.get_output_dtypes()
+    assert dtypes["response"] == np.dtype("O")
+    assert dtypes["success"] == np.dtype("bool")
+
+
+def test_text_embedding_operator_output_dtypes_and_defaults():
+    op = llm_core.LLMTextEmbeddingOp()
+    dtypes = op.get_output_dtypes()
+    assert dtypes["response"] == np.dtype("O")
+    assert dtypes["success"] == np.dtype("bool")
+    assert op.simple_output is False
+    assert op.dimensions is None
+    assert op.encoding_format is None

maxframe/learn/contrib/llm/tests/test_openai.py

@@ -0,0 +1,187 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+import maxframe.dataframe as md
+
+from ..models.openai import (
+    OpenAICompatibleLLM,
+    OpenAICompatibleTextGenOp,
+    OpenAICompatibleTextLLM,
+)
+
+
+def test_openai_compatible_llm_field_assignment():
+    """Test OpenAICompatibleLLM field assignment."""
+    llm = OpenAICompatibleLLM()
+    llm.base_url = "https://api.openai.com/v1"
+    llm.api_key = "test-key"
+    llm.batch_size = 10
+    llm.batch_timeout = 300
+
+    assert llm.base_url == "https://api.openai.com/v1"
+    assert llm.api_key == "test-key"
+    assert llm.batch_size == 10
+    assert llm.batch_timeout == 300
+
+
+def test_openai_compatible_text_llm_initialization():
+    """Test OpenAICompatibleTextLLM initialization."""
+    model = OpenAICompatibleTextLLM(
+        name="gpt-3.5-turbo", base_url="https://api.openai.com/v1", api_key="test-key"
+    )
+
+    assert model.name == "gpt-3.5-turbo"
+    assert model.base_url == "https://api.openai.com/v1"
+    assert model.api_key == "test-key"
+    # Test inherited default values
+    assert model.batch_size is None
+    assert model.batch_timeout is None
+
+
+def test_openai_compatible_text_llm_generate_method():
+    """Test OpenAICompatibleTextLLM generate method."""
+    model = OpenAICompatibleTextLLM(
+        name="gpt-3.5-turbo", base_url="https://api.openai.com/v1", api_key="test-key"
+    )
+
+    # Create test data
+    df = md.DataFrame({"query": ["Hello world"]})
+    prompt_template = [{"role": "user", "content": "{query}"}]
+    params = {"temperature": 0.7}
+
+    # Test that generate method returns a DataFrame (result of operator execution)
+    result = model.generate(
+        data=df,
+        prompt_template=prompt_template,
+        simple_output=True,
+        params=params,
+        extra_param="test",
+    )
+
+    # Verify result is a DataFrame
+    assert hasattr(result, "index_value")
+    assert hasattr(result, "dtypes")
+
+    # Verify the operator that was created has the correct parameters
+    # We can access the operator through the result's op attribute
+    op = result.op
+    assert isinstance(op, OpenAICompatibleTextGenOp)
+    assert op.model == model
+    assert op.prompt_template == prompt_template
+    assert op.simple_output is True
+    assert op.params == params
+
+
+def test_openai_compatible_text_llm_generate_with_defaults():
+    """Test OpenAICompatibleTextLLM generate method with default parameters."""
+    model = OpenAICompatibleTextLLM(
+        name="gpt-3.5-turbo", base_url="https://api.openai.com/v1", api_key="test-key"
+    )
+
+    df = md.DataFrame({"query": ["Hello world"]})
+    prompt_template = [{"role": "user", "content": "{query}"}]
+
+    # Test that generate method returns a DataFrame (result of operator execution)
+    result = model.generate(data=df, prompt_template=prompt_template)
+
+    # Verify result is a DataFrame
+    assert hasattr(result, "index_value")
+    assert hasattr(result, "dtypes")
+
+    # Verify the operator that was created has the correct parameters
+    # We can access the operator through the result's op attribute
+    op = result.op
+    assert isinstance(op, OpenAICompatibleTextGenOp)
+    assert op.model == model
+    assert op.prompt_template == prompt_template
+    assert op.simple_output is False
+    assert op.params is None
+
+
+def test_openai_compatible_text_generation_operator_default_values():
+    """Test OpenAICompatibleTextGenOperator default field values."""
+    op = OpenAICompatibleTextGenOp()
+    assert op.simple_output is False
+
+
+def test_openai_compatible_text_generation_operator_field_assignment():
+    """Test OpenAICompatibleTextGenOperator field assignment."""
+    op = OpenAICompatibleTextGenOp()
+    op.simple_output = True
+    assert op.simple_output is True
+
+
+def test_openai_compatible_text_generation_operator_output_dtypes():
+    """Test OpenAICompatibleTextGenOperator inherits correct output dtypes."""
+    op = OpenAICompatibleTextGenOp()
+    dtypes = op.get_output_dtypes()
+    assert dtypes["response"] == np.dtype("O")
+    assert dtypes["success"] == np.dtype("bool")
+
+
+def test_openai_compatible_text_generation_operator_with_model():
+    """Test OpenAICompatibleTextGenOperator with model parameter."""
+    model = OpenAICompatibleTextLLM(
+        name="gpt-3.5-turbo", base_url="https://api.openai.com/v1", api_key="test-key"
+    )
+
+    prompt_template = [{"role": "user", "content": "Hello"}]
+    params = {"temperature": 0.5}
+
+    op = OpenAICompatibleTextGenOp(
+        model=model, prompt_template=prompt_template, simple_output=True, params=params
+    )
+
+    assert op.model == model
+    assert op.prompt_template == prompt_template
+    assert op.simple_output is True
+    assert op.params == params
+
+
+def test_openai_compatible_text_llm_inheritance():
+    """Test that OpenAICompatibleTextLLM properly inherits from both parent classes."""
+    model = OpenAICompatibleTextLLM(
+        name="gpt-3.5-turbo", base_url="https://api.openai.com/v1", api_key="test-key"
+    )
+
+    # Test TextGenLLM inheritance - should have validate_params method
+    assert hasattr(model, "validate_params")
+    assert callable(getattr(model, "validate_params"))
+
+    # Test OpenAICompatibleLLM inheritance - should have OpenAI-specific fields
+    assert hasattr(model, "base_url")
+    assert hasattr(model, "api_key")
+    assert hasattr(model, "batch_size")
+    assert hasattr(model, "batch_timeout")
+
+    # Test that validate_params doesn't raise error with empty params
+    model.validate_params({})
+
+
+def test_openai_compatible_text_llm_validate_params():
+    """Test OpenAICompatibleTextLLM validate_params method."""
+    model = OpenAICompatibleTextLLM(
+        name="gpt-3.5-turbo", base_url="https://api.openai.com/v1", api_key="test-key"
+    )
+
+    # Should not raise exception with valid params
+    model.validate_params({"temperature": 0.7, "max_tokens": 100})
+
+    # Should not raise exception with empty params
+    model.validate_params({})
+
+    # Should not raise exception with None params
+    model.validate_params(None)

maxframe/learn/contrib/llm/tests/test_text_gen.py

@@ -0,0 +1,155 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+import numpy as np
+import pytest
+
+import maxframe.dataframe as md
+
+from .. import text as llm_text
+
+
+def test_generate_invalid_data_type_raises():
+    model = mock.create_autospec(llm_text.TextGenLLM, instance=True)
+    with pytest.raises(ValueError):
+        llm_text.generate(
+            123, model, prompt_template=[{"role": "user", "content": "x"}]
+        )
+
+
+def test_generate_invalid_model_type_raises():
+    df = md.DataFrame({"query": ["x"]})
+    with pytest.raises(TypeError):
+        llm_text.generate(
+            df, object(), prompt_template=[{"role": "user", "content": "x"}]
+        )
+
+
+def test_generate_calls_validate_params_with_default_and_forwards():
+    df = md.DataFrame({"query": ["hello"]})
+    model = mock.create_autospec(llm_text.TextGenLLM, instance=True)
+    sentinel = object()
+    model.generate.return_value = sentinel
+
+    ret = llm_text.generate(
+        df, model, prompt_template=[{"role": "user", "content": "x"}]
+    )
+
+    assert ret is sentinel
+    model.validate_params.assert_called_once()
+    assert model.validate_params.call_args[0][0] == {}
+    assert model.generate.call_args.kwargs["prompt_template"] == [
+        {"role": "user", "content": "x"}
+    ]
+    assert model.generate.call_args.kwargs["params"] == {}
+
+
+def test_summary_type_and_dtype_validation_and_forward():
+    model = mock.create_autospec(llm_text.TextGenLLM, instance=True)
+
+    with pytest.raises(ValueError):
+        llm_text.summary("not_series", model)
+
+    s_wrong = md.Series(np.array([1], dtype=np.int_))
+    with pytest.raises(ValueError):
+        llm_text.summary(s_wrong, model)
+
+    s_ok = md.Series(np.array(["a"], dtype=np.str_))
+    model.summarize.return_value = "OK_SUM"
+    with mock.patch.object(llm_text.np, "str_", s_ok.dtype, create=True):
+        ret = llm_text.summary(s_ok, model)
+    assert ret == "OK_SUM"
+    model.summarize.assert_called_once()
+
+
+def test_translate_type_and_dtype_validation_and_forward():
+    model = mock.create_autospec(llm_text.TextGenLLM, instance=True)
+
+    with pytest.raises(ValueError):
+        llm_text.translate(
+            "not_series", model, source_language="en", target_language="zh"
+        )
+
+    s_wrong = md.Series(np.array([1], dtype=np.int_))
+    with pytest.raises(ValueError):
+        llm_text.translate(s_wrong, model, source_language="en", target_language="zh")
+
+    s_ok = md.Series(np.array(["hello"], dtype=np.str_))
+    model.translate.return_value = "OK_TRANS"
+    with mock.patch.object(llm_text.np, "str_", s_ok.dtype, create=True):
+        ret = llm_text.translate(
+            s_ok, model, source_language="en", target_language="zh"
+        )
+    assert ret == "OK_TRANS"
+    model.translate.assert_called_once()
+
+
+def test_classify_validation_and_forward():
+    model = mock.create_autospec(llm_text.TextGenLLM, instance=True)
+
+    with pytest.raises(ValueError):
+        llm_text.classify("not_series", model, labels=["A", "B"])
+
+    s_wrong = md.Series(np.array([1], dtype=np.int_))
+    with pytest.raises(ValueError):
+        llm_text.classify(s_wrong, model, labels=["A", "B"])
+
+    s_ok = md.Series(np.array(["text"], dtype=np.str_))
+    with pytest.raises(TypeError):
+        with mock.patch.object(llm_text.np, "str_", s_ok.dtype, create=True):
+            llm_text.classify(s_ok, model, labels="not_list")
+
+    with pytest.raises(ValueError):
+        with mock.patch.object(llm_text.np, "str_", s_ok.dtype, create=True):
+            llm_text.classify(s_ok, model, labels=[])
+
+    model.classify.return_value = "OK_CLS"
+    with mock.patch.object(llm_text.np, "str_", s_ok.dtype, create=True):
+        ret = llm_text.classify(
+            s_ok,
+            model,
+            labels=["A", "B"],
+            description="desc",
+            examples=[{"text": "t", "label": "A", "reason": "r"}],
+        )
+    assert ret == "OK_CLS"
+    model.classify.assert_called_once()
+
+
+def test_extract_validation_and_forward():
+    model = mock.create_autospec(llm_text.TextGenLLM, instance=True)
+
+    with pytest.raises(ValueError):
+        llm_text.extract("not_series", model, schema={"a": "b"})
+
+    s_wrong = md.Series(np.array([1], dtype=np.int_))
+    with pytest.raises(ValueError):
+        llm_text.extract(s_wrong, model, schema={"a": "b"})
+
+    s_ok = md.Series(np.array(["text"], dtype=np.str_))
+    with pytest.raises(ValueError):
+        llm_text.extract(s_ok, model, schema=None)
+
+    with pytest.raises(ValueError):
+        llm_text.extract(s_ok, model, schema={"a": "b"}, examples="not_list")
+
+    with pytest.raises(ValueError):
+        llm_text.extract(s_ok, model, schema={"a": "b"}, examples=[{"not": "tuple"}])
+
+    model.extract.return_value = "OK_EXT"
+    with mock.patch.object(llm_text.np, "str_", s_ok.dtype, create=True):
+        ret = llm_text.extract(s_ok, model, schema={"a": "b"}, examples=[("in", "out")])
+    assert ret == "OK_EXT"
+    model.extract.assert_called_once()