ml-management 0.6.3rc1__tar.gz → 0.6.3rc3__tar.gz

ml_management-0.6.3rc3/ML_management/__init__.py

@@ -0,0 +1,3 @@
+from ML_management.version import get_version
+
+__version__ = get_version()

ml_management-0.6.3rc3/ML_management/test_sdk/local_sdk_test.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+set -e
+
+export PYTHONPATH=$(pwd)/mlmanagement/client/  # this equals mlmanagement/client
+
+REGUSER=$1
+REGPASS=$2
+
+if [ -f $(pwd)/introspection/output/introspection.json ]; then
+  echo 'Using existing introspection.json'
+fi
+
+# if introspection.json doesn't exist create it
+if ! [ -f $(pwd)/introspection/output/introspection.json ]; then
+  echo 'Generate introspection.'
+  python3 -m venv introspection_venv
+  source introspection_venv/bin/activate
+  pip install --upgrade pip
+  pip install --upgrade pip setuptools wheel
+  pip install --no-cache-dir -r mlmanagement/server/requirements.txt --index-url https://$REGUSER:$REGPASS@nexus.intra.ispras.ru/repository/pypi-public/simple/
+  pip install mlmanagement/client/
+  sh introspect.sh
+  deactivate
+  rm -r introspection_venv
+fi
+
+NAME=test_sdk
+
+# start docker container with nodeJS and python3 to mock_server
+sudo docker run -d -v ${PWD}:/mlmanagement -p 4000:4000 --name $NAME nikolaik/python-nodejs:python3.10-nodejs18-slim sh -c "cd mlmanagement ; npm install --prefix mlmanagement/client/ML_management/test_sdk/ --no-save ; node mlmanagement/client/ML_management/test_sdk mock_server.js"
+
+# run test_sdk.py
+{
+  python3 -m venv ${NAME}_env
+  source ${NAME}_env/bin/activate
+  pip install -r mlmanagement/client/ML_management/test_sdk/requirements.txt
+  python3 -m unittest -v $(pwd)/mlmanagement/client/ML_management/test_sdk/test_sdk.py
+} || {
+    echo 'Test failed'
+}
+
+# stop and rm docker container
+sudo docker stop $NAME
+
+sudo docker rm $NAME
+
+# rm created env
+rm -r ${NAME}_env

ml_management-0.6.3rc3/ML_management/test_sdk/mock_server.js

@@ -0,0 +1,37 @@
+// Mocking server by introspection.json
+const { buildClientSchema } = require('graphql');
+
+// create path to introspection.json
+const path = require('path');
+const introspectionPath = path.join(process.cwd(), '/introspection/output/introspection.json');
+const introspectionResult = require(introspectionPath);
+
+const { ApolloServer } = require('apollo-server');
+
+console.log(introspectionResult);
+
+const schema = buildClientSchema(introspectionResult);  
+
+var json = {}
+var json_str = "{}"
+
+// mock some types
+const mocks = {
+    LONG: () => 1000000,
+    JSON: () => json,
+    MethodSchema: () => ({
+        schemaName: "predict_function",
+    }),
+    RoleMethodSchema: () => ({
+        role: "single"
+    })
+  };
+
+const server = new ApolloServer({
+  schema,
+  mocks: mocks,
+});
+
+server.listen().then(({ url }) => {
+  console.log(`🚀 Server ready at ${url}`)
+});

ml_management-0.6.3rc3/ML_management/test_sdk/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "server",
+  "version": "1.0.0",
+  "description": "",
+  "main": "mock_server.js",
+  "dependencies": {
+    "apollo-server": "^3.13.0",
+    "graphql": "^16.8.1"
+  },
+  "devDependencies": {},
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC"
+}

ml_management-0.6.3rc3/ML_management/test_sdk/requirements.txt

@@ -0,0 +1,11 @@
+pandas==2.0.3
+pydantic>=1.10.4, < 2.0
+jsf==0.7.1
+smart-open==6.4.0
+sgqlc==16.1
+httpx<1
+scipy
+PyYAML==6.0.1
+matplotlib==3.7.2
+websocket-client==1.6.1
+numpy==1.26.4

ml_management-0.6.3rc3/ML_management/tests/requirements.txt

@@ -0,0 +1,11 @@
+boto3==1.21.21
+jsonschema==2.6.0
+numpy<=1.23.5
+protobuf<4.0.0
+pydantic==1.10.4
+httpx<1
+sgqlc==16.1
+tqdm==4.65.0
+PyYAML==6.0.1
+pandas==2.0.3
+websocket-client==1.6.1

ml_management-0.6.3rc3/ML_management/tests/run_all_tests.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -e
+
+export PYTHONPATH=$(pwd)  # this equals mlmanagement/client
+
+python3 -m unittest discover -v -s $(pwd)/ML_management/tests/ -p 'test*.py'

ml_management-0.6.3rc3/ML_management/version.py

@@ -0,0 +1,6 @@
+def get_version():
+    try:
+        with open("./VERSION", encoding="utf-8") as f:
+            return f.read()
+    except Exception:
+        return "unknown"

ml_management-0.6.3rc3/PKG-INFO

@@ -0,0 +1,26 @@
+Metadata-Version: 2.1
+Name: ml-management
+Version: 0.6.3rc3
+Summary: Python implementation of model pattern, dataset
+Author-Email: ISPRAS MODIS <modis@ispras.ru>
+Maintainer: Maxim Ryndin
+Requires-Python: >=3.9
+Requires-Dist: sgqlc>=16
+Requires-Dist: boto3<2,>1
+Requires-Dist: aioboto3<14,>=13
+Requires-Dist: jsonschema<5,>=4.18
+Requires-Dist: tqdm<5,>=4.66.6
+Requires-Dist: pydantic<2,>=1
+Requires-Dist: jsf==0.7.1
+Requires-Dist: pylint<4,>=3
+Requires-Dist: httpx<1
+Requires-Dist: websocket-client<2,>1
+Requires-Dist: pandas<3,>2
+Requires-Dist: PyYAML<7,>6
+Requires-Dist: numpy<2,>1.26
+Requires-Dist: matplotlib<4,>=3.7
+Description-Content-Type: text/markdown
+
+# mlmanagement
+
+implementation of model pattern, dataset

ml_management-0.6.3rc3/pyproject.toml

@@ -0,0 +1,39 @@
+[project]
+name = "ml-management"
+dynamic = []
+description = "Python implementation of model pattern, dataset"
+authors = [
+    { name = "ISPRAS MODIS", email = "modis@ispras.ru" },
+]
+dependencies = [
+    "sgqlc>=16",
+    "boto3>1,<2",
+    "aioboto3>=13,<14",
+    "jsonschema>=4.18,<5",
+    "tqdm>=4.66.6,<5",
+    "pydantic>=1,<2",
+    "jsf==0.7.1",
+    "pylint>=3,<4",
+    "httpx<1",
+    "websocket-client>1,<2",
+    "pandas>2,<3",
+    "PyYAML>6,<7",
+    "numpy>1.26,<2",
+    "matplotlib>=3.7,<4",
+]
+requires-python = ">=3.9"
+readme = "README.md"
+maintainers = [
+    { name = "Maxim Ryndin" },
+]
+version = "0.6.3rc3"
+
+[build-system]
+requires = [
+    "pdm-backend==2.4.3",
+]
+build-backend = "pdm.backend"
+
+[tool.pdm.version]
+source = "call"
+getter = "ML_management.version:get_version"

ml_management-0.6.3rc1/MANIFEST.in

@@ -1 +0,0 @@
-include README.md

ml_management-0.6.3rc1/ML_management/__init__.py

@@ -1,3 +0,0 @@
-import ML_management.version
-
-__version__ = ML_management.version.__version__

ml_management-0.6.3rc1/ML_management/version.py

@@ -1,6 +0,0 @@
-from importlib.metadata import version
-
-try:
-    __version__ = version("ml-management")
-except Exception:
-    __version__ = "unknown"

ml_management-0.6.3rc1/PKG-INFO

@@ -1,27 +0,0 @@
-Metadata-Version: 2.1
-Name: ml-management
-Version: 0.6.3rc1
-Summary: Python implementation of model pattern, dataset
-Author: ISPRAS MODIS
-Author-email: modis@ispras.ru
-Maintainer: Maxim Ryndin
-Requires-Python: >=3.8
-Description-Content-Type: text/markdown
-Requires-Dist: sgqlc==16.1
-Requires-Dist: boto3==1.34.70
-Requires-Dist: aioboto3==13.1.1
-Requires-Dist: jsonschema==4.18.4
-Requires-Dist: tqdm==4.66.6
-Requires-Dist: pydantic<2,>=1
-Requires-Dist: jsf==0.7.1
-Requires-Dist: pylint<4,>=3
-Requires-Dist: httpx<1
-Requires-Dist: websocket-client==1.6.1
-Requires-Dist: pandas==2.0.3
-Requires-Dist: PyYAML==6.0.1
-Requires-Dist: numpy==1.26.4
-Requires-Dist: matplotlib==3.7.2
-
-# mlmanagement
-
-implementation of model pattern, dataset

ml_management-0.6.3rc1/VERSION

@@ -1 +0,0 @@
-0.6.3rc1

ml_management-0.6.3rc1/ml_management.egg-info/PKG-INFO

@@ -1,27 +0,0 @@
-Metadata-Version: 2.1
-Name: ml-management
-Version: 0.6.3rc1
-Summary: Python implementation of model pattern, dataset
-Author: ISPRAS MODIS
-Author-email: modis@ispras.ru
-Maintainer: Maxim Ryndin
-Requires-Python: >=3.8
-Description-Content-Type: text/markdown
-Requires-Dist: sgqlc==16.1
-Requires-Dist: boto3==1.34.70
-Requires-Dist: aioboto3==13.1.1
-Requires-Dist: jsonschema==4.18.4
-Requires-Dist: tqdm==4.66.6
-Requires-Dist: pydantic<2,>=1
-Requires-Dist: jsf==0.7.1
-Requires-Dist: pylint<4,>=3
-Requires-Dist: httpx<1
-Requires-Dist: websocket-client==1.6.1
-Requires-Dist: pandas==2.0.3
-Requires-Dist: PyYAML==6.0.1
-Requires-Dist: numpy==1.26.4
-Requires-Dist: matplotlib==3.7.2
-
-# mlmanagement
-
-implementation of model pattern, dataset

ml_management-0.6.3rc1/ml_management.egg-info/SOURCES.txt

@@ -1,102 +0,0 @@
-MANIFEST.in
-README.md
-VERSION
-setup.py
-ML_management/__init__.py
-ML_management/base_exceptions.py
-ML_management/session.py
-ML_management/singleton_pattern.py
-ML_management/variables.py
-ML_management/version.py
-ML_management/collectors/__init__.py
-ML_management/collectors/collector_pattern.py
-ML_management/collectors/collector_pattern_to_methods_map.py
-ML_management/collectors/collectors.py
-ML_management/collectors/dummy/__init__.py
-ML_management/collectors/dummy/dummy_collector.py
-ML_management/collectors/s3/__init__.py
-ML_management/collectors/s3/s3collector.py
-ML_management/collectors/topic_markers/__init__.py
-ML_management/collectors/topic_markers/api_schema.py
-ML_management/collectors/topic_markers/topic_markers_collector.py
-ML_management/dataset_loader/__init__.py
-ML_management/dataset_loader/base_splits_dataset_loader.py
-ML_management/dataset_loader/dataset_loader_pattern.py
-ML_management/dataset_loader/dataset_loader_pattern_to_methods_map.py
-ML_management/dataset_loader/poisoned_images_dataset_loader.py
-ML_management/dataset_loader/templates/__init__.py
-ML_management/dataset_loader/templates/upload.py
-ML_management/dataset_loader/templates/dummy_dataset_loader/__init__.py
-ML_management/dataset_loader/templates/dummy_dataset_loader/conda.yaml
-ML_management/dataset_loader/templates/dummy_dataset_loader/dummy_dataset.py
-ML_management/executor/__init__.py
-ML_management/executor/base_executor.py
-ML_management/executor/executor_pattern.py
-ML_management/executor/executor_pattern_to_methods_map.py
-ML_management/executor/no_model_executor_pattern.py
-ML_management/executor/patterns.py
-ML_management/executor/upload_model_mode.py
-ML_management/executor/templates/__init__.py
-ML_management/executor/templates/upload.py
-ML_management/executor/templates/eval/__init__.py
-ML_management/executor/templates/eval/conda.yaml
-ML_management/executor/templates/eval/eval_executor.py
-ML_management/executor/templates/finetune/__init__.py
-ML_management/executor/templates/finetune/conda.yaml
-ML_management/executor/templates/finetune/finetune_executor.py
-ML_management/executor/templates/train/__init__.py
-ML_management/executor/templates/train/conda.yaml
-ML_management/executor/templates/train/train_executor.py
-ML_management/graphql/__init__.py
-ML_management/graphql/schema.py
-ML_management/graphql/send_graphql_request.py
-ML_management/mlmanagement/__init__.py
-ML_management/mlmanagement/backend_api.py
-ML_management/mlmanagement/jsonschema_exceptions.py
-ML_management/mlmanagement/jsonschema_inference.py
-ML_management/mlmanagement/load_api.py
-ML_management/mlmanagement/log_api.py
-ML_management/mlmanagement/model_type.py
-ML_management/mlmanagement/module_finder.py
-ML_management/mlmanagement/server_mlmanager_exceptions.py
-ML_management/mlmanagement/utils.py
-ML_management/mlmanagement/visibility_options.py
-ML_management/model/__init__.py
-ML_management/model/model_type_to_methods_map.py
-ML_management/model/patterns/__init__.py
-ML_management/model/patterns/evaluatable_model.py
-ML_management/model/patterns/gradient_model.py
-ML_management/model/patterns/model_pattern.py
-ML_management/model/patterns/model_with_losses.py
-ML_management/model/patterns/model_with_metrics.py
-ML_management/model/patterns/preprocessor.py
-ML_management/model/patterns/retrainable_model.py
-ML_management/model/patterns/target_layer.py
-ML_management/model/patterns/torch_model.py
-ML_management/model/patterns/trainable_model.py
-ML_management/model/patterns/transformer.py
-ML_management/registry/__init__.py
-ML_management/registry/exceptions.py
-ML_management/s3/__init__.py
-ML_management/s3/manager.py
-ML_management/s3/utils.py
-ML_management/sdk/__init__.py
-ML_management/sdk/dataset_loader.py
-ML_management/sdk/executor.py
-ML_management/sdk/experiment.py
-ML_management/sdk/job.py
-ML_management/sdk/model.py
-ML_management/sdk/parameters.py
-ML_management/sdk/sdk.py
-ML_management/test_sdk/__init__.py
-ML_management/test_sdk/test_sdk.py
-ML_management/tests/__init__.py
-ML_management/tests/test_jsonschema_inference.py
-ML_management/uploader_data/__init__.py
-ML_management/uploader_data/s3_uploader.py
-ML_management/uploader_data/utils.py
-ml_management.egg-info/PKG-INFO
-ml_management.egg-info/SOURCES.txt
-ml_management.egg-info/dependency_links.txt
-ml_management.egg-info/requires.txt
-ml_management.egg-info/top_level.txt

ml_management-0.6.3rc1/ml_management.egg-info/dependency_links.txt

@@ -1 +0,0 @@
-

ml_management-0.6.3rc1/ml_management.egg-info/requires.txt

@@ -1,14 +0,0 @@
-sgqlc==16.1
-boto3==1.34.70
-aioboto3==13.1.1
-jsonschema==4.18.4
-tqdm==4.66.6
-pydantic<2,>=1
-jsf==0.7.1
-pylint<4,>=3
-httpx<1
-websocket-client==1.6.1
-pandas==2.0.3
-PyYAML==6.0.1
-numpy==1.26.4
-matplotlib==3.7.2

ml_management-0.6.3rc1/ml_management.egg-info/top_level.txt

@@ -1 +0,0 @@
-ML_management

ml_management-0.6.3rc1/setup.cfg

@@ -1,4 +0,0 @@
-[egg_info]
-tag_build = 
-tag_date = 0
-

ml_management-0.6.3rc1/setup.py

@@ -1,42 +0,0 @@
-"""Setup for ml-management package."""
-from setuptools import find_packages, setup
-
-with open("README.md", encoding="utf-8") as f:
-    long_description = f.read()
-
-with open("VERSION", encoding="utf-8") as f:
-    version = f.read()
-
-setup(
-    name="ml-management",
-    version=version,
-    description="Python implementation of model pattern, dataset",
-    long_description=long_description,
-    long_description_content_type="text/markdown",
-    author="ISPRAS MODIS",
-    author_email="modis@ispras.ru",
-    maintainer="Maxim Ryndin",
-    packages=find_packages(include=["ML_management", "ML_management.*"]),
-    include_package_data=True,
-    install_requires=[
-        "sgqlc==16.1",
-        "boto3==1.34.70",
-        "aioboto3==13.1.1",
-        "jsonschema==4.18.4",
-        "tqdm==4.66.6",
-        "pydantic>=1,<2",
-        "jsf==0.7.1",  # update when TDM will support pydantic>=2.0.0
-        "pylint>=3,<4",
-        "httpx<1",
-        "websocket-client==1.6.1",
-        "pandas==2.0.3",
-        "PyYAML==6.0.1",
-        "numpy==1.26.4",
-        "matplotlib==3.7.2",
-    ],
-    package_data={
-        "": ["*.yaml"],
-    },
-    data_files=[("", ["VERSION"])],
-    python_requires=">=3.8",
-)