jcclang 0.1.1__tar.gz

jcclang-0.1.1/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: jcclang
+Version: 0.1.1
+Summary: A platform-adaptive task framework for cloud-edge scenarios
+Author-email: jeshua <ren1366929814@gmail.com>
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown

jcclang-0.1.1/jcclang/adapter/__init__.py

@@ -0,0 +1,17 @@
+from .modelarts import ModelArtsAdapter
+from .octopus import OctopusAdapter
+from .openi import OpenIAdapter
+from ..core.const import Platform
+from ..core.logger import jcwLogger
+
+
+def get_adapter(platform: str):
+    platform = platform.lower()
+    if platform == Platform.MODELARTS:
+        return ModelArtsAdapter()
+    elif platform == Platform.OPENI:
+        return OpenIAdapter()
+    elif platform == Platform.OCTOPUS:
+        return OctopusAdapter()
+    jcwLogger.error(f"Unsupported platform: {platform}")
+    return None

jcclang-0.1.1/jcclang/adapter/base_adapter.py

@@ -0,0 +1,10 @@
+
+class BaseAdapter:
+
+    def before_task(self, inputs, context: dict): ...
+
+    def after_task(self, outputs, context: dict): ...
+
+    def input_prepare(self, data_type: str, file_path: str): ...
+
+    def output_prepare(self, data_type: str, file_path: str): ...

jcclang-0.1.1/jcclang/adapter/modelarts.py

@@ -0,0 +1,56 @@
+import argparse
+from pathlib import Path
+
+from jcclang.adapter.base_adapter import BaseAdapter
+from jcclang.core.const import DataType
+from jcclang.core.logger import jcwLogger
+
+
+class ModelArtsAdapter(BaseAdapter):
+    def __init__(self):
+        self.parser = argparse.ArgumentParser(description='Model Training with input parameter')
+        self.output = ""
+
+    def before_task(self, inputs, context: dict):
+        jcwLogger.info("modelarts before task")
+
+    def after_task(self, outputs, context: dict):
+        jcwLogger.info("modelarts after task, output", self.output)
+
+    def input_prepare(self, data_type: str, file_path: str):
+        if data_type == DataType.DATASET:
+            if not any(a.dest == 'dataset_input' for a in self.parser._actions):
+                self.parser.add_argument('--dataset_input', default='./data', type=str,
+                                         help='dataset_input (default: %(default)s)')
+        elif data_type == DataType.MODEL:
+            if not any(a.dest == 'model_input' for a in self.parser._actions):
+                self.parser.add_argument('--model_input', default='./models', type=str,
+                                         help='model_input (default: %(default)s)')
+        elif data_type == DataType.CODE:
+            if not any(a.dest == 'code_input' for a in self.parser._actions):
+                self.parser.add_argument('--code_input', default='./src', type=str,
+                                         help='code_input (default: %(default)s)')
+        else:
+            jcwLogger.error(f"Unknown data type for input: {data_type}")
+            return ""
+
+        args, _ = self.parser.parse_known_args()
+
+        base_path = ""
+        if data_type == DataType.DATASET:
+            base_path = args.dataset_input
+        elif data_type == DataType.MODEL:
+            base_path = args.model_input
+        elif data_type == DataType.CODE:
+            base_path = args.code_input
+
+        path = Path(base_path) / file_path
+        return path
+
+    def output_prepare(self, data_type: str, file_path: str):
+        if not any(a.dest == 'output' for a in self.parser._actions):
+            self.parser.add_argument('--output', default='/output', type=str,
+                                     help='output (default: %(default)s)')
+        args, _ = self.parser.parse_known_args()
+        path = Path(args.output) / file_path
+        return path.as_posix()

jcclang-0.1.1/jcclang/adapter/octopus.py

@@ -0,0 +1,49 @@
+import os
+
+from jcclang.adapter.base_adapter import BaseAdapter
+from jcclang.core.const import DataType
+from jcclang.core.logger import jcwLogger
+
+
+class OctopusAdapter(BaseAdapter):
+    def __init__(self):
+        self.output = ""
+
+    def before_task(self, inputs, context: dict):
+        jcwLogger.info("execute before task")
+
+    def after_task(self, outputs, context: dict):
+        jcwLogger.info("execute after task")
+
+    def input_prepare(self, data_type: str, file_path: str):
+        if data_type == DataType.DATASET:
+            data_path = os.environ.get("dataset_input")
+            if not data_path:
+                jcwLogger.error("dataset_input is not set")
+                return ""
+            return os.path.join(data_path, file_path)
+
+        if data_type == DataType.MODEL:
+            data_path = os.environ.get("model_input")
+            if not data_path:
+                jcwLogger.error("model_input is not set")
+                return ""
+            return os.path.join(data_path, file_path)
+
+        if data_type == DataType.CODE:
+            data_path = os.environ.get("code_input")
+            if not data_path:
+                jcwLogger.error("code_input is not set")
+                return ""
+            return os.path.join(data_path, file_path)
+        jcwLogger.error(f"Unknown data type for input: {data_type}")
+        return ""
+
+    def output_prepare(self, data_type: str, file_path: str):
+        data_path = os.environ.get("output",  "./output")
+
+        if not os.path.exists(data_path):
+            jcwLogger.info(f"create output directory: {data_path}")
+            os.makedirs(data_path)
+
+        return os.path.join(data_path, file_path)

jcclang-0.1.1/jcclang/adapter/openi.py

@@ -0,0 +1,50 @@
+import argparse
+import os
+
+from jcclang.adapter.base_adapter import BaseAdapter
+from jcclang.core.const import DataType
+from jcclang.core.logger import jcwLogger
+
+
+class OpenIAdapter(BaseAdapter):
+    def __init__(self):
+        from c2net.context import prepare, upload_output
+        self._prepare = prepare()
+        self._upload_output = upload_output
+        self.output = ""
+        self.parser = argparse.ArgumentParser(description='Model Training with input parameter')
+
+    def before_task(self, inputs, context: dict):
+        jcwLogger.info("execute before task")
+        pass
+
+    def after_task(self, outputs, context: dict):
+        jcwLogger.info("execute after task")
+        self._upload_output()
+
+    def input_prepare(self, data_type: str, file_path: str):
+        if data_type == DataType.DATASET:
+            self.parser.add_argument('--dataset_input', default='./data', type=str,
+                                     help='dataset_input (default: %(default)s)')
+            args, _ = self.parser.parse_known_args()
+            name_no_ext, _ = os.path.splitext(args.dataset_input)
+            return os.path.join(self._prepare.dataset_path, name_no_ext)
+
+        if data_type == DataType.MODEL:
+            self.parser.add_argument('--model_input', default='./data', type=str,
+                                     help='dataset_input (default: %(default)s)')
+            args, _ = self.parser.parse_known_args()
+            name_no_ext, _ = os.path.splitext(args.model_input)
+            return os.path.join(self._prepare.pretrain_model_path, name_no_ext)
+        if data_type == DataType.CODE:
+            self.parser.add_argument('--code_input', default='./data', type=str,
+                                     help='dataset_input (default: %(default)s)')
+            args, _ = self.parser.parse_known_args()
+            name_no_ext, _ = os.path.splitext(args.code_input)
+            return os.path.join(self._prepare.code_path, name_no_ext)
+        jcwLogger.error(f"Unknown data type for input: {data_type}")
+        return ""
+
+    def output_prepare(self, data_type: str, file_path: str):
+        self.output = os.path.join(self._prepare.output_path, file_path)
+        return self.output

jcclang-0.1.1/jcclang/api/__init__.py

@@ -0,0 +1,4 @@
+from .data_prepare import *
+from .decorators import *
+
+__all__ = ["input_prepare", "output_prepare", "lifecycle", "register_task"]

jcclang-0.1.1/jcclang/api/context.py

@@ -0,0 +1,24 @@
+import trio
+from api.jobhub import JobHub
+
+
+class Context:
+    def __init__(
+        self, nursery: trio.Nursery, job_set_id: str, job_id: str, job_hub: JobHub
+    ):
+        self._nursery = nursery
+        self._job_set_id = job_set_id
+        self._job_id = job_id
+        self._job_hub = job_hub
+
+    def nursery(self):
+        return self._nursery
+
+    def job_set_id(self) -> str:
+        return self._job_set_id
+
+    def job_id(self) -> str:
+        return self._job_id
+
+    def job_hub(self) -> JobHub:
+        return self._job_hub

jcclang-0.1.1/jcclang/api/data_loader/llama_loader.py

@@ -0,0 +1,99 @@
+import os
+import torch
+import torch.nn as nn
+from safetensors.torch import safe_open
+
+# ============================
+# 1️⃣ 模型配置类
+# ============================
+
+class ModelConfig:
+    """
+    简单配置类，描述模型结构
+    """
+    def __init__(self, vocab_size, hidden_size, num_layers, num_heads, max_seq_len):
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.num_layers = num_layers
+        self.num_heads = num_heads
+        self.max_seq_len = max_seq_len
+
+# ============================
+# 2️⃣ 自定义模型类
+# ============================
+
+class GPTLayer(nn.Module):
+    def __init__(self, hidden_size, num_heads):
+        super().__init__()
+        self.attn = nn.MultiheadAttention(embed_dim=hidden_size, num_heads=num_heads, batch_first=True)
+        self.ffn = nn.Sequential(
+            nn.Linear(hidden_size, 4 * hidden_size),
+            nn.GELU(),
+            nn.Linear(4 * hidden_size, hidden_size)
+        )
+        self.ln1 = nn.LayerNorm(hidden_size)
+        self.ln2 = nn.LayerNorm(hidden_size)
+
+    def forward(self, x):
+        attn_out, _ = self.attn(x, x, x)
+        x = self.ln1(x + attn_out)
+        x = self.ln2(x + self.ffn(x))
+        return x
+
+class GPTModel(nn.Module):
+    def __init__(self, config: ModelConfig):
+        super().__init__()
+        self.config = config
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
+        self.layers = nn.ModuleList([GPTLayer(config.hidden_size, config.num_heads)
+                                     for _ in range(config.num_layers)])
+        self.ln_f = nn.LayerNorm(config.hidden_size)
+        self.head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+    def forward(self, input_ids):
+        x = self.embed_tokens(input_ids)
+        for layer in self.layers:
+            x = layer(x)
+        x = self.ln_f(x)
+        logits = self.head(x)
+        return logits
+
+# ============================
+# 3️⃣ 流式 Safetensors 权重加载器
+# ============================
+
+class StreamingSafeTensorsLoader:
+    """
+    支持按需加载 safetensors 权重到 CPU/GPU
+    """
+    def __init__(self, filename: str, device="cpu"):
+        self.filename = filename
+        self.device = device
+        self._file = safe_open(filename, framework="pt")
+        self.keys = self._file.keys()
+
+    def load_tensor(self, name: str, device=None):
+        """按需加载单个 tensor"""
+        dev = device or self.device
+        tensor = self._file.get_tensor(name).to(dev)
+        return tensor
+
+    def load_state_dict(self, model: nn.Module, device=None):
+        """
+        按 tensor 名称加载权重到模型
+        支持流式加载：可选择只加载部分层
+        """
+        dev = device or self.device
+        state_dict = {}
+        for name, param in model.named_parameters():
+            if name in self.keys:
+                tensor = self._file.get_tensor(name).to(dev)
+                state_dict[name] = tensor
+            else:
+                print(f"[Warning] weight {name} not found in safetensors")
+        model.load_state_dict(state_dict, strict=False)
+
+# ============================
+# 4️⃣ 使用示例
+# ============================
+

jcclang-0.1.1/jcclang/api/data_loader/pandas_loader.py

@@ -0,0 +1,52 @@
+import io
+
+import pandas as pd
+
+from jcclang.core.model import VirtualFileParams, Source
+from jcclang.virtualfile.virtual_file import VirtualFile
+
+
+class Pandas:
+    """虚拟化的 pandas I/O 层"""
+
+    def __init__(self, source: Source, virtual_file_params: VirtualFileParams = None):
+        if virtual_file_params is None:
+            virtual_file_params = VirtualFileParams()
+        self.virtual_file_params = virtual_file_params
+        self.source = source
+
+    def _open_virtual(self):
+        """返回一个基于 VirtualFile 的文件对象"""
+        vf = VirtualFile(source=self.source, params=self.virtual_file_params)
+        return vf
+
+    def _read_bytes(self) -> bytes:
+        """统一从虚拟文件系统读取数据"""
+        vf = self._open_virtual()
+        data = vf.read()
+        vf.close()
+        return data
+
+    # ==============================
+    # Pandas-like API
+    # ==============================
+
+    def read_csv(self, **kwargs) -> pd.DataFrame:
+        raw = self._read_bytes()
+        buf = io.BytesIO(raw)
+        return pd.read_csv(buf, **kwargs)
+
+    def read_json(self, **kwargs) -> pd.DataFrame:
+        raw = self._read_bytes()
+        buf = io.BytesIO(raw)
+        return pd.read_json(buf, **kwargs)
+
+    def read_parquet(self, **kwargs) -> pd.DataFrame:
+        raw = self._read_bytes()
+        buf = io.BytesIO(raw)
+        return pd.read_parquet(buf, **kwargs)
+
+    def read_excel(self, **kwargs) -> pd.DataFrame:
+        raw = self._read_bytes()
+        buf = io.BytesIO(raw)
+        return pd.read_excel(buf, **kwargs)

jcclang-0.1.1/jcclang/api/data_loader/tensorflow_loader.py

@@ -0,0 +1,53 @@
+import tensorflow as tf
+from jcclang.core.model import Sources, VirtualFileParams
+from jcclang.virtualfile.virtual_file import VirtualFile
+
+
+class Dataset:
+    def __init__(self, sources: Sources, decoder=None,
+                 virtual_file_params: VirtualFileParams = None):
+        """
+        :param sources: Sources 对象
+        :param decode_fn: 数据解码函数，输入 raw bytes，输出 numpy / tensor
+        :param virtual_file_params: 虚拟文件参数
+        """
+        if virtual_file_params is None:
+            virtual_file_params = VirtualFileParams()
+        self.virtual_file_params = virtual_file_params
+        self.sources = sources
+        self.decode_fn = decoder or (lambda x: x)  # 默认不解码
+
+    def generator(self):
+        """
+        数据生成器，每次 yield (sample, label)
+        """
+        for info in self.sources.items:
+            # 读取数据
+            vf = VirtualFile(info, params=self.virtual_file_params)
+            raw = vf.read()
+            vf.close()
+
+            # 解码
+            sample = self.decode_fn(raw)
+
+            # 输出 sample 和 label
+            yield sample, info.label
+
+    def to_tf_dataset(self, output_types=tf.float32, output_shapes=None, batch_size=32, shuffle=True):
+        """
+        转为 tf.data.Dataset
+        :param output_types: 输出类型，可以是 tf.float32, tf.int32 等
+        :param output_shapes: 输出形状，如 (28,28) 或 (None,)
+        :param batch_size: 批大小
+        :param shuffle: 是否打乱
+        """
+        ds = tf.data.Dataset.from_generator(
+            self.generator,
+            output_types=(output_types, tf.int32),
+            output_shapes=(output_shapes, ())
+        )
+        if shuffle:
+            ds = ds.shuffle(buffer_size=len(self.sources.items))
+        ds = ds.batch(batch_size)
+        ds = ds.prefetch(tf.data.AUTOTUNE)
+        return ds

jcclang-0.1.1/jcclang/api/data_loader/torch_loader.py

@@ -0,0 +1,78 @@
+from io import BytesIO
+
+import torch
+from torch.utils.data import Dataset as Ds
+
+from jcclang.core.model import VirtualFileParams, Sources
+from jcclang.virtualfile.virtual_file import VirtualFile
+
+
+class Dataset(Ds):
+    def __init__(self, sources: Sources, transform=None, decoder=None,
+                 virtual_file_params: VirtualFileParams = None):
+        if virtual_file_params is None:
+            virtual_file_params = VirtualFileParams()
+        self.virtual_file_params = virtual_file_params
+        self.sources = sources
+        self.transform = transform
+        self.decoder = decoder or (lambda x: x)
+
+    def __len__(self):
+        return len(self.sources.items)
+
+    def __getitem__(self, idx):
+        info = self.sources.items[idx]
+
+        # 读取数据
+        vf = VirtualFile(info, params=self.virtual_file_params)
+        raw = vf.read()
+        vf.close()
+
+        # 解码
+        sample = self.decoder(raw)
+
+        # transform
+        if self.transform:
+            sample = self.transform(sample)
+
+        return sample, info.label
+
+
+class JCWeaverModel:
+    """
+    通过 JCWeaver VirtualFile 加载模型权重，支持 PyTorch
+    """
+
+    def __init__(self, sources: Sources, virtual_file_params: VirtualFileParams = None):
+        if virtual_file_params is None:
+            virtual_file_params = VirtualFileParams()
+        self.virtual_file_params = virtual_file_params
+        self.sources = sources
+
+    def load_state_dict(self, model_class, map_location="cpu"):
+        """
+        从 JCWeaver 读取模型数据，并加载到指定 PyTorch model_class
+        """
+        vf = VirtualFile(info, params=self.virtual_file_params)
+        raw = vf.read()
+        vf.close()
+
+        buf = BytesIO(raw)
+        state_dict = torch.load(buf, map_location=map_location)
+        if isinstance(model_class, type):
+            model = model_class()
+        else:
+            model = model_class
+        model.load_state_dict(state_dict)
+        return model
+
+    def load_torch_model(self, map_location="cpu"):
+        """
+        直接读取 PyTorch 完整模型对象
+        """
+        vf = VirtualFile(info, params=self.virtual_file_params)
+        raw = vf.read()
+        vf.close()
+        buf = BytesIO(raw)
+        model = torch.load(buf, map_location=map_location)
+        return model

jcclang-0.1.1/jcclang/api/data_loader/transformers_hook/vfile_io2.py

@@ -0,0 +1,57 @@
+import io
+
+from jcclang.core.model import Sources, VirtualFileParams
+from jcclang.virtualfile.virtual_file import VirtualFile
+
+
+class VirtualFileIO(io.RawIOBase):
+    """
+    用 VirtualFile 封装成类文件对象，支持 seek/tell/read。
+    """
+
+    def __init__(self, vf: VirtualFile):
+        self.vf = vf
+        self.closed_flag = False
+
+    def read(self, size=-1):
+        return self.vf.read(size)
+
+    def seek(self, offset, whence=0):
+        self.vf.seek(offset, whence)
+        return self.vf.tell()
+
+    def tell(self):
+        return self.vf.tell()
+
+    def close(self):
+        if not self.closed_flag:
+            self.vf.close()
+            self.closed_flag = True
+
+    @property
+    def closed(self):
+        return self.closed_flag
+
+
+def virtual_cached_file(pretrained_model_name_or_path, filename, **kwargs):
+    """
+    替代 transformers.utils.cached_file。
+    从 VirtualFile 系统获取指定文件内容，返回 BytesIO 对象。
+    """
+    sources: Sources = kwargs.pop("sources")
+    vparams: VirtualFileParams = kwargs.pop("vparams")
+
+    # 找到对应 Source
+    src_map = {s.path: s for s in sources.items}
+    if filename not in src_map:
+        # 尝试只匹配文件名（去掉路径）
+        file_name_only_map = {s.path.split("/")[-1]: s for s in sources.items}
+        src = file_name_only_map.get(filename)
+        if src is None:
+            raise FileNotFoundError(f"{filename} not found in virtual sources")
+    else:
+        src = src_map[filename]
+
+    vf = VirtualFile(src, params=vparams)
+    # 直接返回类文件对象，支持 seek/read
+    return VirtualFileIO(vf)