tketool.datasets 1.3.5__tar.gz

tketool_datasets-1.3.5/PKG-INFO

@@ -0,0 +1,42 @@
+Metadata-Version: 2.4
+Name: tketool.datasets
+Version: 1.3.5
+Summary: Dataset contracts and local, memory, SSH, MinIO, and document adapters for tketool
+Author-email: Ke <jiangke1207@icloud.com>
+License-Expression: MIT
+Project-URL: Homepage, https://pypi.org/project/tketool.datasets/
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: tketool.core[console]==1.3.5
+Requires-Dist: paramiko<6,>=4
+Requires-Dist: minio<8,>=7.2.20
+Requires-Dist: numpy<3,>=1.26
+Requires-Dist: prettytable<4,>=3.12
+Requires-Dist: pypdf<7,>=6
+Requires-Dist: docx2txt<1,>=0.9
+Requires-Dist: python-docx<2,>=1.2
+Provides-Extra: test
+Requires-Dist: pytest<9,>=8; extra == "test"
+
+# tketool.datasets
+
+Dataset and sample storage APIs for local files, memory, SSH/SFTP, MinIO,
+PDF, Word, and random sampling workflows.
+
+```bash
+pip install tketool.datasets
+```
+
+```python
+from tketool.datasets import Dataset, LocalDatasetSource
+
+source = LocalDatasetSource("./data")
+samples = Dataset(source, "train")
+```
+
+The retired `tketool.sample` and `tketool.mlsample` paths are not shipped.

tketool_datasets-1.3.5/README.md

@@ -0,0 +1,17 @@
+# tketool.datasets
+
+Dataset and sample storage APIs for local files, memory, SSH/SFTP, MinIO,
+PDF, Word, and random sampling workflows.
+
+```bash
+pip install tketool.datasets
+```
+
+```python
+from tketool.datasets import Dataset, LocalDatasetSource
+
+source = LocalDatasetSource("./data")
+samples = Dataset(source, "train")
+```
+
+The retired `tketool.sample` and `tketool.mlsample` paths are not shipped.

tketool_datasets-1.3.5/pyproject.toml

@@ -0,0 +1,43 @@
+[build-system]
+requires = ["setuptools>=77", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "tketool.datasets"
+version = "1.3.5"
+description = "Dataset contracts and local, memory, SSH, MinIO, and document adapters for tketool"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+authors = [{ name = "Ke", email = "jiangke1207@icloud.com" }]
+dependencies = [
+  "tketool.core[console]==1.3.5",
+  "paramiko>=4,<6",
+  "minio>=7.2.20,<8",
+  "numpy>=1.26,<3",
+  "prettytable>=3.12,<4",
+  "pypdf>=6,<7",
+  "docx2txt>=0.9,<1",
+  "python-docx>=1.2,<2",
+]
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Operating System :: OS Independent",
+]
+
+[project.optional-dependencies]
+test = ["pytest>=8,<9"]
+
+[project.scripts]
+tketool-datasets = "tketool.datasets.cli:main"
+
+[project.urls]
+Homepage = "https://pypi.org/project/tketool.datasets/"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["tketool*"]
+namespaces = true

tketool_datasets-1.3.5/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

tketool_datasets-1.3.5/src/tketool/datasets/__init__.py

@@ -0,0 +1,19 @@
+"""Dataset contracts and local, memory, SSH, and MinIO adapters."""
+
+from .dataset import Dataset
+from .local import LocalDatasetSource
+from .memory import MemoryDatasetSource
+from .minio import MinioDatasetSource
+from .random import create_dataset
+from .source import DatasetSource
+from .ssh import SshDatasetSource
+
+__all__ = [
+    "Dataset",
+    "DatasetSource",
+    "LocalDatasetSource",
+    "MemoryDatasetSource",
+    "MinioDatasetSource",
+    "SshDatasetSource",
+    "create_dataset",
+]

tketool_datasets-1.3.5/src/tketool/datasets/cli.py

@@ -0,0 +1,42 @@
+# pass_generate pass_doc
+import argparse
+import inspect
+from tketool.core.commands import add_command_parser
+from tketool.datasets.operations import (
+    capture_str,
+    delete_set,
+    download,
+    find_s,
+    output_csv,
+    set_data_info,
+    set_info,
+    set_list,
+    upload,
+)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="A simple command line tool")
+    commands = {}
+
+    # Only create subparsers once
+    subparsers = parser.add_subparsers(dest="command", help='Commands')
+
+    add_command_parser(subparsers, set_list, commands)  # Add your functions here
+    add_command_parser(subparsers, set_info, commands)
+    add_command_parser(subparsers, delete_set, commands)
+    add_command_parser(subparsers, set_data_info, commands)
+    add_command_parser(subparsers, capture_str, commands)
+    add_command_parser(subparsers, upload, commands)
+    add_command_parser(subparsers, find_s, commands)
+    add_command_parser(subparsers, download, commands)
+    add_command_parser(subparsers, output_csv, commands)
+
+    args = parser.parse_args()
+
+    if args.command in commands:
+        command_params = inspect.signature(commands[args.command]).parameters
+        params = {name: getattr(args, name, None) for name in command_params}
+        commands[args.command](**params)
+    else:
+        parser.print_help()

tketool_datasets-1.3.5/src/tketool/datasets/converts.py

@@ -0,0 +1,82 @@
+# pass_generate
+from tketool.datasets.source import DatasetSource
+from tketool.datasets.dataset import Dataset
+import json
+from tketool.core.files import write_file_lines
+import docx2txt
+import pypdf
+from docx import Document
+from zipfile import BadZipFile
+
+
+
+def convert_jsonl(datasource: DatasetSource, set_name: str, prompt_completion_fun,
+                  target_path: str):
+    """
+    这个函数用于将数据源中的数据转换为jsonl文件。函数首先会从数据源中读取指定的“set_name”的样本集，然后使用“prompt_completion_fun”函数将每个样本的问题和答案转换为字典形式，最后将所有的行写入到“target_path”指定的文件中。
+    
+    参数:
+        datasource(DatasetSource): 数据源，必须是DatasetSource类型或其子类的实例。
+        set_name(str): 从数据源中获取样本集的名称。
+        prompt_completion_fun(function): 用于将样本转换为字典形式的函数。该函数必须接受一个参数（样本），并返回一个元组，元组的第一个元素是问题，第二个元素是答案。
+        target_path(str): 输出jsonl文件的路径。
+    
+    返回:
+        None
+    
+    示例：
+    
+        def prompt_completion(item):
+            return item.question, item.answer
+    
+        datasource = MyDataSource()
+        convert_jsonl(datasource, "train", prompt_completion, "train.jsonl")
+    
+    注意：这个函数不会检查“target_path”是否已经存在，如果存在，它会直接覆盖旧文件。
+    
+    错误与异常：
+        如果数据源中不存在指定的“set_name”，函数会抛出异常。
+        如果“prompt_completion_fun”函数不能正确处理样本，也会抛出异常。
+    """
+
+    row_lines = []
+    for item in Dataset(datasource, set_name):
+        prompt, completion = prompt_completion_fun(item)
+        row_data = {
+            'prompt': prompt,
+            'completion': completion
+        }
+        row_str = json.dumps(row_data)
+        row_lines.append(row_str)
+
+    write_file_lines(target_path, row_lines)
+
+
+def doc_to_txt(file_path, txt_path):
+    try:
+        text = docx2txt.process(file_path)
+    except BadZipFile:
+        print(f'Failed to process {file_path}: not a .doc file or file is corrupted.')
+        text = ''
+    with open(txt_path, 'w') as output:
+        output.write(text)
+
+
+def docx_to_txt(file_path, txt_path):
+    try:
+        doc = Document(file_path)
+        text = '\n'.join([paragraph.text for paragraph in doc.paragraphs])
+    except BadZipFile:
+        print(f'Failed to process {file_path}: not a .docx file or file is corrupted.')
+        text = ''
+    with open(txt_path, 'w') as output:
+        output.write(text)
+
+
+def pdf_to_txt(file_path, txt_path):
+    with open(file_path, 'rb') as pdf_file_obj:
+        pdf_reader = pypdf.PdfReader(pdf_file_obj)
+        num_pages = len(pdf_reader.pages)
+        text = '\n'.join([page.extract_text() for page in pdf_reader.pages])
+        with open(txt_path, 'w') as output:
+            output.write(text)