pybiolib 1.2.1056__py3-none-any.whl → 1.2.1642__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pybiolib might be problematic. Click here for more details.
- biolib/__init__.py +33 -10
- biolib/_data_record/data_record.py +24 -11
- biolib/_index/__init__.py +0 -0
- biolib/_index/index.py +51 -0
- biolib/_index/types.py +7 -0
- biolib/_internal/data_record/data_record.py +1 -1
- biolib/_internal/data_record/push_data.py +1 -1
- biolib/_internal/data_record/remote_storage_endpoint.py +3 -3
- biolib/_internal/file_utils.py +7 -4
- biolib/_internal/index/__init__.py +1 -0
- biolib/_internal/index/index.py +18 -0
- biolib/_internal/lfs/cache.py +4 -2
- biolib/_internal/push_application.py +89 -23
- biolib/_internal/runtime.py +2 -0
- biolib/_internal/templates/gui_template/App.tsx +38 -2
- biolib/_internal/templates/gui_template/Dockerfile +2 -0
- biolib/_internal/templates/gui_template/biolib-sdk.ts +37 -0
- biolib/_internal/templates/gui_template/dev-data/output.json +7 -0
- biolib/_internal/templates/gui_template/package.json +1 -0
- biolib/_internal/templates/gui_template/vite-plugin-dev-data.ts +49 -0
- biolib/_internal/templates/gui_template/vite.config.mts +2 -1
- biolib/_internal/templates/init_template/.github/workflows/biolib.yml +6 -1
- biolib/_internal/templates/init_template/Dockerfile +2 -0
- biolib/_internal/utils/__init__.py +25 -0
- biolib/_internal/utils/job_url.py +33 -0
- biolib/_runtime/runtime.py +9 -0
- biolib/_session/session.py +7 -5
- biolib/_shared/__init__.py +0 -0
- biolib/_shared/types/__init__.py +69 -0
- biolib/_shared/types/resource.py +17 -0
- biolib/_shared/types/resource_deploy_key.py +11 -0
- biolib/{_internal → _shared}/types/resource_permission.py +1 -1
- biolib/_shared/utils/__init__.py +7 -0
- biolib/_shared/utils/resource_uri.py +75 -0
- biolib/api/client.py +1 -1
- biolib/app/app.py +56 -23
- biolib/biolib_api_client/app_types.py +1 -6
- biolib/biolib_api_client/biolib_app_api.py +17 -0
- biolib/biolib_binary_format/module_input.py +8 -0
- biolib/biolib_binary_format/remote_endpoints.py +3 -3
- biolib/biolib_binary_format/remote_stream_seeker.py +39 -25
- biolib/cli/__init__.py +2 -1
- biolib/cli/data_record.py +17 -0
- biolib/cli/index.py +32 -0
- biolib/cli/lfs.py +1 -1
- biolib/cli/start.py +14 -1
- biolib/compute_node/job_worker/executors/docker_executor.py +31 -9
- biolib/compute_node/job_worker/executors/docker_types.py +1 -1
- biolib/compute_node/job_worker/executors/types.py +6 -5
- biolib/compute_node/job_worker/job_worker.py +149 -93
- biolib/compute_node/job_worker/large_file_system.py +2 -6
- biolib/compute_node/job_worker/network_alloc.py +99 -0
- biolib/compute_node/job_worker/network_buffer.py +240 -0
- biolib/compute_node/job_worker/utilization_reporter_thread.py +2 -2
- biolib/compute_node/remote_host_proxy.py +125 -67
- biolib/compute_node/utils.py +2 -0
- biolib/compute_node/webserver/compute_node_results_proxy.py +188 -0
- biolib/compute_node/webserver/proxy_utils.py +28 -0
- biolib/compute_node/webserver/webserver.py +64 -19
- biolib/experiments/experiment.py +98 -16
- biolib/jobs/job.py +119 -29
- biolib/jobs/job_result.py +70 -33
- biolib/jobs/types.py +1 -0
- biolib/sdk/__init__.py +17 -2
- biolib/typing_utils.py +1 -1
- biolib/utils/cache_state.py +2 -2
- biolib/utils/seq_util.py +1 -1
- {pybiolib-1.2.1056.dist-info → pybiolib-1.2.1642.dist-info}/METADATA +4 -2
- {pybiolib-1.2.1056.dist-info → pybiolib-1.2.1642.dist-info}/RECORD +84 -66
- {pybiolib-1.2.1056.dist-info → pybiolib-1.2.1642.dist-info}/WHEEL +1 -1
- biolib/_internal/types/__init__.py +0 -6
- biolib/utils/app_uri.py +0 -57
- /biolib/{_internal → _shared}/types/account.py +0 -0
- /biolib/{_internal → _shared}/types/account_member.py +0 -0
- /biolib/{_internal → _shared}/types/app.py +0 -0
- /biolib/{_internal → _shared}/types/data_record.py +0 -0
- /biolib/{_internal → _shared}/types/experiment.py +0 -0
- /biolib/{_internal → _shared}/types/file_node.py +0 -0
- /biolib/{_internal → _shared}/types/push.py +0 -0
- /biolib/{_internal/types/resource.py → _shared/types/resource_types.py} +0 -0
- /biolib/{_internal → _shared}/types/resource_version.py +0 -0
- /biolib/{_internal → _shared}/types/result.py +0 -0
- /biolib/{_internal → _shared}/types/typing.py +0 -0
- /biolib/{_internal → _shared}/types/user.py +0 -0
- {pybiolib-1.2.1056.dist-info → pybiolib-1.2.1642.dist-info}/entry_points.txt +0 -0
- {pybiolib-1.2.1056.dist-info → pybiolib-1.2.1642.dist-info/licenses}/LICENSE +0 -0
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
interface IBioLibGlobals {
|
|
2
|
+
getOutputFileData: (path: string) => Promise<Uint8Array>;
|
|
3
|
+
}
|
|
4
|
+
|
|
5
|
+
declare global {
|
|
6
|
+
const biolib: IBioLibGlobals;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
// DO NOT MODIFY: Development data files are injected at build time from gui/dev-data/ folder
|
|
10
|
+
const DEV_DATA_FILES: Record<string, string> = {};
|
|
11
|
+
|
|
12
|
+
const devSdkBioLib: IBioLibGlobals = {
|
|
13
|
+
getOutputFileData: async (path: string): Promise<Uint8Array> => {
|
|
14
|
+
console.log(`[SDK] getOutputFileData called with path: ${path}`);
|
|
15
|
+
|
|
16
|
+
const normalizedPath = path.startsWith('/') ? path.slice(1) : path;
|
|
17
|
+
|
|
18
|
+
if (typeof DEV_DATA_FILES !== 'undefined' && normalizedPath in DEV_DATA_FILES) {
|
|
19
|
+
const base64Data = DEV_DATA_FILES[normalizedPath];
|
|
20
|
+
const binaryString = atob(base64Data);
|
|
21
|
+
const bytes = new Uint8Array(binaryString.length);
|
|
22
|
+
for (let i = 0; i < binaryString.length; i++) {
|
|
23
|
+
bytes[i] = binaryString.charCodeAt(i);
|
|
24
|
+
}
|
|
25
|
+
return bytes;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
throw new Error(`File not found: ${path}. Add this file to the dev-data/ folder for local development.`);
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
const biolib: IBioLibGlobals =
|
|
33
|
+
process.env.NODE_ENV === "development"
|
|
34
|
+
? devSdkBioLib
|
|
35
|
+
: (window as any).biolib;
|
|
36
|
+
|
|
37
|
+
export default biolib;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { Plugin } from 'vite';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
|
|
5
|
+
export function devDataPlugin(): Plugin {
|
|
6
|
+
let isDev = false;
|
|
7
|
+
|
|
8
|
+
return {
|
|
9
|
+
name: 'dev-data-plugin',
|
|
10
|
+
configResolved(config) {
|
|
11
|
+
isDev = config.mode === 'development';
|
|
12
|
+
},
|
|
13
|
+
transform(code: string, id: string) {
|
|
14
|
+
if (id.endsWith('biolib-sdk.ts')) {
|
|
15
|
+
let injectedCode: string;
|
|
16
|
+
|
|
17
|
+
if (isDev) {
|
|
18
|
+
const devDataDir = path.join(__dirname, 'dev-data');
|
|
19
|
+
const devDataMap: Record<string, string> = {};
|
|
20
|
+
|
|
21
|
+
if (fs.existsSync(devDataDir)) {
|
|
22
|
+
const files = fs.readdirSync(devDataDir);
|
|
23
|
+
for (const file of files) {
|
|
24
|
+
const filePath = path.join(devDataDir, file);
|
|
25
|
+
if (fs.statSync(filePath).isFile()) {
|
|
26
|
+
const content = fs.readFileSync(filePath);
|
|
27
|
+
const base64Content = content.toString('base64');
|
|
28
|
+
devDataMap[file] = base64Content;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const devDataJson = JSON.stringify(devDataMap);
|
|
34
|
+
injectedCode = code.replace(
|
|
35
|
+
"const DEV_DATA_FILES = {};",
|
|
36
|
+
`const DEV_DATA_FILES = ${devDataJson};`
|
|
37
|
+
);
|
|
38
|
+
} else {
|
|
39
|
+
injectedCode = code;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
return {
|
|
43
|
+
code: injectedCode,
|
|
44
|
+
map: null
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
}
|
|
@@ -2,7 +2,8 @@ import { defineConfig } from "vite";
|
|
|
2
2
|
import react from "@vitejs/plugin-react";
|
|
3
3
|
import tailwindcss from "@tailwindcss/vite";
|
|
4
4
|
import { viteSingleFile } from "vite-plugin-singlefile";
|
|
5
|
+
import { devDataPlugin } from "./gui/vite-plugin-dev-data";
|
|
5
6
|
|
|
6
7
|
export default defineConfig({
|
|
7
|
-
plugins: [react(), tailwindcss(), viteSingleFile()],
|
|
8
|
+
plugins: [react(), tailwindcss(), devDataPlugin(), viteSingleFile()],
|
|
8
9
|
});
|
|
@@ -11,6 +11,11 @@ jobs:
|
|
|
11
11
|
- name: Build
|
|
12
12
|
run: docker build -t BIOLIB_REPLACE_DOCKER_TAG:latest .
|
|
13
13
|
- name: Push
|
|
14
|
-
run:
|
|
14
|
+
run: |
|
|
15
|
+
if [ "$GITHUB_REF_NAME" == "main" ]; then
|
|
16
|
+
biolib push BIOLIB_REPLACE_APP_URI
|
|
17
|
+
else
|
|
18
|
+
biolib push --dev BIOLIB_REPLACE_APP_URI:latest-dev
|
|
19
|
+
fi
|
|
15
20
|
env:
|
|
16
21
|
BIOLIB_TOKEN: ${{ secrets.BIOLIB_TOKEN }}
|
|
@@ -1,5 +1,30 @@
|
|
|
1
1
|
import time
|
|
2
2
|
import uuid
|
|
3
|
+
from fnmatch import fnmatch
|
|
4
|
+
|
|
5
|
+
from biolib.biolib_binary_format.utils import LazyLoadedFile
|
|
6
|
+
from biolib.typing_utils import Callable, List, Union, cast
|
|
7
|
+
|
|
8
|
+
PathFilter = Union[str, Callable[[str], bool]]
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def filter_lazy_loaded_files(files: List[LazyLoadedFile], path_filter: PathFilter) -> List[LazyLoadedFile]:
|
|
12
|
+
if not (isinstance(path_filter, str) or callable(path_filter)):
|
|
13
|
+
raise Exception('Expected path_filter to be a string or a function')
|
|
14
|
+
|
|
15
|
+
if callable(path_filter):
|
|
16
|
+
return list(filter(lambda x: path_filter(x.path), files)) # type: ignore
|
|
17
|
+
|
|
18
|
+
glob_filter = cast(str, path_filter)
|
|
19
|
+
|
|
20
|
+
# since all file paths start with /, make sure filter does too
|
|
21
|
+
if not glob_filter.startswith('/'):
|
|
22
|
+
glob_filter = '/' + glob_filter
|
|
23
|
+
|
|
24
|
+
def _filter_function(file: LazyLoadedFile) -> bool:
|
|
25
|
+
return fnmatch(file.path, glob_filter)
|
|
26
|
+
|
|
27
|
+
return list(filter(_filter_function, files))
|
|
3
28
|
|
|
4
29
|
|
|
5
30
|
def open_browser_window_from_notebook(url_to_open: str) -> None:
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from urllib.parse import urlparse
|
|
3
|
+
|
|
4
|
+
import biolib.utils
|
|
5
|
+
from biolib.typing_utils import Optional, Tuple
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def parse_result_id_or_url(result_id_or_url: str, default_token: Optional[str] = None) -> Tuple[str, Optional[str]]:
|
|
9
|
+
result_id_or_url = result_id_or_url.strip()
|
|
10
|
+
|
|
11
|
+
if '/' not in result_id_or_url:
|
|
12
|
+
return (result_id_or_url, default_token)
|
|
13
|
+
|
|
14
|
+
if not result_id_or_url.startswith('http://') and not result_id_or_url.startswith('https://'):
|
|
15
|
+
result_id_or_url = 'https://' + result_id_or_url
|
|
16
|
+
|
|
17
|
+
parsed_url = urlparse(result_id_or_url)
|
|
18
|
+
|
|
19
|
+
if biolib.utils.BIOLIB_BASE_URL:
|
|
20
|
+
expected_base = urlparse(biolib.utils.BIOLIB_BASE_URL)
|
|
21
|
+
if parsed_url.scheme != expected_base.scheme or parsed_url.netloc != expected_base.netloc:
|
|
22
|
+
raise ValueError(f'URL must start with {biolib.utils.BIOLIB_BASE_URL}, got: {result_id_or_url}')
|
|
23
|
+
|
|
24
|
+
pattern = r'/results?/(?P<uuid>[a-f0-9-]+)/?(?:\?token=(?P<token>[^&]+))?'
|
|
25
|
+
match = re.search(pattern, result_id_or_url, re.IGNORECASE)
|
|
26
|
+
|
|
27
|
+
if not match:
|
|
28
|
+
raise ValueError(f'URL must be in format <base_url>/results/<UUID>/?token=<token>, got: {result_id_or_url}')
|
|
29
|
+
|
|
30
|
+
uuid = match.group('uuid')
|
|
31
|
+
token = match.group('token') or default_token
|
|
32
|
+
|
|
33
|
+
return (uuid, token)
|
biolib/_runtime/runtime.py
CHANGED
|
@@ -37,10 +37,19 @@ class Runtime:
|
|
|
37
37
|
return None
|
|
38
38
|
return job_requested_machine
|
|
39
39
|
|
|
40
|
+
@staticmethod
|
|
41
|
+
def get_job_requested_spot_machine() -> bool:
|
|
42
|
+
job_data = Runtime._get_job_data()
|
|
43
|
+
return job_data.get('job_requested_machine_spot', False)
|
|
44
|
+
|
|
40
45
|
@staticmethod
|
|
41
46
|
def get_app_uri() -> str:
|
|
42
47
|
return Runtime._get_job_data()['app_uri']
|
|
43
48
|
|
|
49
|
+
@staticmethod
|
|
50
|
+
def get_max_workers() -> int:
|
|
51
|
+
return Runtime._get_job_data()['job_reserved_machines']
|
|
52
|
+
|
|
44
53
|
@staticmethod
|
|
45
54
|
def get_secret(secret_name: str) -> bytes:
|
|
46
55
|
assert re.match(
|
biolib/_session/session.py
CHANGED
|
@@ -1,21 +1,23 @@
|
|
|
1
1
|
from biolib import utils
|
|
2
|
-
from biolib.
|
|
2
|
+
from biolib.typing_utils import Optional
|
|
3
3
|
from biolib.api.client import ApiClient, ApiClientInitDict
|
|
4
4
|
from biolib.app import BioLibApp
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
class Session:
|
|
8
|
-
def __init__(self, _init_dict: ApiClientInitDict) -> None:
|
|
8
|
+
def __init__(self, _init_dict: ApiClientInitDict, _experiment: Optional[str] = None) -> None:
|
|
9
9
|
self._api = ApiClient(_init_dict=_init_dict)
|
|
10
|
+
self._experiment = _experiment
|
|
10
11
|
|
|
11
12
|
@staticmethod
|
|
12
|
-
def get_session(refresh_token: str, base_url: Optional[str] = None, client_type: Optional[str] = None) -> 'Session':
|
|
13
|
+
def get_session(refresh_token: str, base_url: Optional[str] = None, client_type: Optional[str] = None, experiment: Optional[str] = None) -> 'Session':
|
|
13
14
|
return Session(
|
|
14
15
|
_init_dict=ApiClientInitDict(
|
|
15
16
|
refresh_token=refresh_token,
|
|
16
17
|
base_url=base_url or utils.load_base_url_from_env(),
|
|
17
18
|
client_type=client_type,
|
|
18
|
-
)
|
|
19
|
+
),
|
|
20
|
+
_experiment=experiment,
|
|
19
21
|
)
|
|
20
22
|
|
|
21
23
|
def load(self, uri: str, suppress_version_warning: bool = False) -> BioLibApp:
|
|
@@ -39,4 +41,4 @@ class Session:
|
|
|
39
41
|
>>> app = biolib.load('https://biolib.com/biolib/myapp/')
|
|
40
42
|
>>> result = app.cli('--help')
|
|
41
43
|
"""
|
|
42
|
-
return BioLibApp(uri=uri, _api_client=self._api, suppress_version_warning=suppress_version_warning)
|
|
44
|
+
return BioLibApp(uri=uri, _api_client=self._api, suppress_version_warning=suppress_version_warning, _experiment=self._experiment)
|
|
File without changes
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
from .account import AccountDict, AccountDetailedDict
|
|
2
|
+
from .account_member import AccountMemberDict
|
|
3
|
+
from .app import AppDetailedDict, AppSlimDict
|
|
4
|
+
from .data_record import (
|
|
5
|
+
DataRecordDetailedDict,
|
|
6
|
+
DataRecordSlimDict,
|
|
7
|
+
DataRecordTypeDict,
|
|
8
|
+
DataRecordValidationRuleDict,
|
|
9
|
+
SqliteV1Column,
|
|
10
|
+
SqliteV1DatabaseSchema,
|
|
11
|
+
SqliteV1ForeignKey,
|
|
12
|
+
SqliteV1Table,
|
|
13
|
+
)
|
|
14
|
+
from .experiment import (
|
|
15
|
+
DeprecatedExperimentDict,
|
|
16
|
+
ExperimentDetailedDict,
|
|
17
|
+
ExperimentDict,
|
|
18
|
+
ResultCounts,
|
|
19
|
+
)
|
|
20
|
+
from .file_node import FileNodeDict, FileZipMetadataDict, ZipFileNodeDict
|
|
21
|
+
from .push import PushResponseDict
|
|
22
|
+
from .resource import ResourceUriDict, SemanticVersionDict
|
|
23
|
+
from .resource_deploy_key import ResourceDeployKeyDict, ResourceDeployKeyWithSecretDict
|
|
24
|
+
from .resource_permission import ResourcePermissionDetailedDict, ResourcePermissionDict
|
|
25
|
+
from .resource_types import ResourceDetailedDict, ResourceDict
|
|
26
|
+
from .resource_version import ResourceVersionDetailedDict, ResourceVersionDict
|
|
27
|
+
from .result import ResultDetailedDict, ResultDict
|
|
28
|
+
from .typing import Optional
|
|
29
|
+
from .user import EnterpriseSettingsDict, UserDetailedDict, UserDict
|
|
30
|
+
|
|
31
|
+
__all__ = [
|
|
32
|
+
'AccountDetailedDict',
|
|
33
|
+
'AccountDict',
|
|
34
|
+
'AccountMemberDict',
|
|
35
|
+
'AppDetailedDict',
|
|
36
|
+
'AppSlimDict',
|
|
37
|
+
'DataRecordDetailedDict',
|
|
38
|
+
'DataRecordSlimDict',
|
|
39
|
+
'DataRecordTypeDict',
|
|
40
|
+
'DataRecordValidationRuleDict',
|
|
41
|
+
'DeprecatedExperimentDict',
|
|
42
|
+
'EnterpriseSettingsDict',
|
|
43
|
+
'ExperimentDetailedDict',
|
|
44
|
+
'ExperimentDict',
|
|
45
|
+
'FileNodeDict',
|
|
46
|
+
'FileZipMetadataDict',
|
|
47
|
+
'Optional',
|
|
48
|
+
'PushResponseDict',
|
|
49
|
+
'ResourceDeployKeyDict',
|
|
50
|
+
'ResourceDeployKeyWithSecretDict',
|
|
51
|
+
'ResourceDetailedDict',
|
|
52
|
+
'ResourceDict',
|
|
53
|
+
'ResourcePermissionDetailedDict',
|
|
54
|
+
'ResourcePermissionDict',
|
|
55
|
+
'ResourceUriDict',
|
|
56
|
+
'ResourceVersionDetailedDict',
|
|
57
|
+
'ResourceVersionDict',
|
|
58
|
+
'ResultCounts',
|
|
59
|
+
'ResultDetailedDict',
|
|
60
|
+
'ResultDict',
|
|
61
|
+
'SemanticVersionDict',
|
|
62
|
+
'SqliteV1Column',
|
|
63
|
+
'SqliteV1DatabaseSchema',
|
|
64
|
+
'SqliteV1ForeignKey',
|
|
65
|
+
'SqliteV1Table',
|
|
66
|
+
'UserDetailedDict',
|
|
67
|
+
'UserDict',
|
|
68
|
+
'ZipFileNodeDict',
|
|
69
|
+
]
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from .typing import Optional, TypedDict
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class SemanticVersionDict(TypedDict):
|
|
5
|
+
major: int
|
|
6
|
+
minor: int
|
|
7
|
+
patch: int
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class ResourceUriDict(TypedDict):
|
|
11
|
+
account_handle_normalized: str
|
|
12
|
+
account_handle: str
|
|
13
|
+
resource_name_normalized: Optional[str]
|
|
14
|
+
resource_name: Optional[str]
|
|
15
|
+
resource_prefix: Optional[str]
|
|
16
|
+
version: Optional[SemanticVersionDict]
|
|
17
|
+
tag: Optional[str]
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import re
|
|
2
|
+
|
|
3
|
+
from biolib._shared.types import Optional
|
|
4
|
+
from biolib._shared.types.resource import ResourceUriDict, SemanticVersionDict
|
|
5
|
+
from biolib.biolib_errors import BioLibError
|
|
6
|
+
|
|
7
|
+
URI_REGEX = re.compile(
|
|
8
|
+
r'^(@(?P<resource_prefix>[\w._-]+)/)?'
|
|
9
|
+
r'(?P<account_handle>[\w-]+)'
|
|
10
|
+
r'(/(?P<resource_name>[\w-]+))?'
|
|
11
|
+
r'(?::(?P<suffix>[^:]+))?$'
|
|
12
|
+
)
|
|
13
|
+
SEMVER_REGEX = re.compile(r'^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)$')
|
|
14
|
+
TAG_REGEX = re.compile(r'^[a-z0-9-]{1,128}$')
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def normalize_resource_name(string: str) -> str:
|
|
18
|
+
return string.replace('-', '_').lower()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def parse_semantic_version(semantic_version: str) -> SemanticVersionDict:
|
|
22
|
+
if match := SEMVER_REGEX.fullmatch(semantic_version):
|
|
23
|
+
return SemanticVersionDict(
|
|
24
|
+
major=int(match.group('major')),
|
|
25
|
+
minor=int(match.group('minor')),
|
|
26
|
+
patch=int(match.group('patch')),
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
raise ValueError('The version must be a valid semantic version in the format of major.minor.patch (1.2.3).')
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def parse_resource_uri(uri: str, use_account_as_name_default: bool = True) -> ResourceUriDict:
|
|
33
|
+
matches = URI_REGEX.match(uri)
|
|
34
|
+
if matches is None:
|
|
35
|
+
raise BioLibError(f"Could not parse resource uri '{uri}', uri did not match regex")
|
|
36
|
+
|
|
37
|
+
version: Optional[SemanticVersionDict] = None
|
|
38
|
+
tag: Optional[str] = None
|
|
39
|
+
|
|
40
|
+
suffix = matches.group('suffix')
|
|
41
|
+
if suffix and suffix != '*':
|
|
42
|
+
try:
|
|
43
|
+
version = parse_semantic_version(suffix)
|
|
44
|
+
except ValueError:
|
|
45
|
+
if TAG_REGEX.fullmatch(suffix):
|
|
46
|
+
tag = suffix
|
|
47
|
+
else:
|
|
48
|
+
raise BioLibError(
|
|
49
|
+
f'Invalid version or tag "{suffix}". '
|
|
50
|
+
'Versions must be semantic versions like "1.2.3". '
|
|
51
|
+
'Tags must be lowercase alphanumeric or dashes and at most 128 characters.'
|
|
52
|
+
) from None
|
|
53
|
+
|
|
54
|
+
resource_prefix_raw: Optional[str] = matches.group('resource_prefix')
|
|
55
|
+
resource_prefix = resource_prefix_raw.lower() if resource_prefix_raw is not None else None
|
|
56
|
+
account_handle: str = matches.group('account_handle')
|
|
57
|
+
account_handle_normalized: str = normalize_resource_name(account_handle)
|
|
58
|
+
resource_name: Optional[str] = matches.group('resource_name')
|
|
59
|
+
|
|
60
|
+
if resource_name:
|
|
61
|
+
resource_name_normalized = normalize_resource_name(resource_name)
|
|
62
|
+
elif use_account_as_name_default:
|
|
63
|
+
resource_name_normalized = account_handle_normalized
|
|
64
|
+
else:
|
|
65
|
+
resource_name_normalized = None
|
|
66
|
+
|
|
67
|
+
return ResourceUriDict(
|
|
68
|
+
resource_prefix=resource_prefix,
|
|
69
|
+
account_handle=account_handle,
|
|
70
|
+
account_handle_normalized=account_handle_normalized,
|
|
71
|
+
resource_name_normalized=resource_name_normalized,
|
|
72
|
+
resource_name=resource_name if resource_name is not None or not use_account_as_name_default else account_handle,
|
|
73
|
+
version=version,
|
|
74
|
+
tag=tag,
|
|
75
|
+
)
|
biolib/api/client.py
CHANGED
|
@@ -8,7 +8,7 @@ from urllib.parse import urlencode, urljoin
|
|
|
8
8
|
import importlib_metadata
|
|
9
9
|
|
|
10
10
|
from biolib._internal.http_client import HttpClient, HttpResponse
|
|
11
|
-
from biolib.
|
|
11
|
+
from biolib._shared.types.typing import Any, Dict, Optional, TypedDict, Union, cast
|
|
12
12
|
from biolib.biolib_api_client import BiolibApiClient as DeprecatedApiClient
|
|
13
13
|
from biolib.biolib_errors import BioLibError
|
|
14
14
|
from biolib.biolib_logging import logger
|
biolib/app/app.py
CHANGED
|
@@ -1,11 +1,16 @@
|
|
|
1
|
+
import copy
|
|
1
2
|
import io
|
|
2
3
|
import json
|
|
3
4
|
import os
|
|
5
|
+
import posixpath
|
|
4
6
|
import random
|
|
5
7
|
import string
|
|
6
8
|
from pathlib import Path
|
|
7
9
|
|
|
8
10
|
from biolib import utils
|
|
11
|
+
from biolib._internal.file_utils import path_to_renamed_path
|
|
12
|
+
from biolib._runtime.runtime import Runtime
|
|
13
|
+
from biolib._shared.utils import parse_resource_uri
|
|
9
14
|
from biolib.api.client import ApiClient
|
|
10
15
|
from biolib.biolib_api_client import JobState
|
|
11
16
|
from biolib.biolib_api_client.app_types import App, AppVersion
|
|
@@ -18,14 +23,24 @@ from biolib.compute_node.job_worker.job_worker import JobWorker
|
|
|
18
23
|
from biolib.experiments.experiment import Experiment
|
|
19
24
|
from biolib.jobs.job import Result
|
|
20
25
|
from biolib.typing_utils import Dict, Optional
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class JsonStringIO(io.StringIO):
|
|
29
|
+
pass
|
|
24
30
|
|
|
25
31
|
|
|
26
32
|
class BioLibApp:
|
|
27
|
-
def __init__(
|
|
33
|
+
def __init__(
|
|
34
|
+
self,
|
|
35
|
+
uri: str,
|
|
36
|
+
_api_client: Optional[ApiClient] = None,
|
|
37
|
+
suppress_version_warning: bool = False,
|
|
38
|
+
_experiment: Optional[str] = None,
|
|
39
|
+
):
|
|
28
40
|
self._api_client: Optional[ApiClient] = _api_client
|
|
41
|
+
self._experiment = _experiment
|
|
42
|
+
self._input_uri = uri
|
|
43
|
+
self._parsed_input_uri = parse_resource_uri(uri)
|
|
29
44
|
|
|
30
45
|
app_response = BiolibAppApi.get_by_uri(uri=uri, api_client=self._api_client)
|
|
31
46
|
self._app: App = app_response['app']
|
|
@@ -33,16 +48,19 @@ class BioLibApp:
|
|
|
33
48
|
self._app_version: AppVersion = app_response['app_version']
|
|
34
49
|
|
|
35
50
|
if not suppress_version_warning:
|
|
36
|
-
|
|
37
|
-
if parsed_uri['version'] is None:
|
|
51
|
+
if self._parsed_input_uri['version'] is None:
|
|
38
52
|
if Runtime.check_is_environment_biolib_app():
|
|
39
53
|
logger.warning(
|
|
40
54
|
f"No version specified in URI '{uri}'. This will use the default version, "
|
|
41
|
-
f
|
|
55
|
+
f'which may change behaviour over time. Consider locking down the exact version, '
|
|
42
56
|
f"e.g. '{uri}:1.2.3'"
|
|
43
57
|
)
|
|
44
58
|
|
|
45
|
-
|
|
59
|
+
if self._parsed_input_uri['tag']:
|
|
60
|
+
semantic_version = f"{self._app_version['major']}.{self._app_version['minor']}.{self._app_version['patch']}"
|
|
61
|
+
logger.info(f'Loaded {self._input_uri} (resolved to {semantic_version})')
|
|
62
|
+
else:
|
|
63
|
+
logger.info(f'Loaded {self._app_uri}')
|
|
46
64
|
|
|
47
65
|
def __str__(self) -> str:
|
|
48
66
|
return self._app_uri
|
|
@@ -71,7 +89,7 @@ class BioLibApp:
|
|
|
71
89
|
result_prefix: Optional[str] = None,
|
|
72
90
|
timeout: Optional[int] = None,
|
|
73
91
|
notify: bool = False,
|
|
74
|
-
|
|
92
|
+
max_workers: Optional[int] = None,
|
|
75
93
|
experiment: Optional[str] = None,
|
|
76
94
|
temporary_client_secrets: Optional[Dict[str, str]] = None,
|
|
77
95
|
check: bool = False,
|
|
@@ -84,7 +102,12 @@ class BioLibApp:
|
|
|
84
102
|
raise ValueError('The argument "check" cannot be True when blocking is False')
|
|
85
103
|
|
|
86
104
|
if not experiment_id:
|
|
87
|
-
|
|
105
|
+
experiment_to_use = experiment if experiment is not None else self._experiment
|
|
106
|
+
experiment_instance: Optional[Experiment]
|
|
107
|
+
if experiment_to_use:
|
|
108
|
+
experiment_instance = Experiment(experiment_to_use, _api_client=self._api_client)
|
|
109
|
+
else:
|
|
110
|
+
experiment_instance = Experiment.get_experiment_in_context()
|
|
88
111
|
experiment_id = experiment_instance.uuid if experiment_instance else None
|
|
89
112
|
|
|
90
113
|
module_input_serialized = self._get_serialized_module_input(args, stdin, files)
|
|
@@ -111,11 +134,12 @@ class BioLibApp:
|
|
|
111
134
|
override_command=override_command,
|
|
112
135
|
result_prefix=result_prefix,
|
|
113
136
|
timeout=timeout,
|
|
114
|
-
requested_machine_count=
|
|
137
|
+
requested_machine_count=max_workers,
|
|
115
138
|
temporary_client_secrets=temporary_client_secrets,
|
|
116
139
|
api_client=self._api_client,
|
|
117
140
|
)
|
|
118
|
-
|
|
141
|
+
if utils.IS_RUNNING_IN_NOTEBOOK:
|
|
142
|
+
logger.info(f'View the result in your browser at: {utils.BIOLIB_BASE_URL}/results/{job.id}/')
|
|
119
143
|
if blocking:
|
|
120
144
|
# TODO: Deprecate utils.STREAM_STDOUT and always stream logs by simply calling job.stream_logs()
|
|
121
145
|
if utils.IS_RUNNING_IN_NOTEBOOK:
|
|
@@ -152,6 +176,8 @@ Example: "app.cli('--help')"
|
|
|
152
176
|
def _get_serialized_module_input(args=None, stdin=None, files=None) -> bytes:
|
|
153
177
|
if args is None:
|
|
154
178
|
args = []
|
|
179
|
+
else:
|
|
180
|
+
args = copy.copy(args)
|
|
155
181
|
|
|
156
182
|
if stdin is None:
|
|
157
183
|
stdin = b''
|
|
@@ -173,18 +199,21 @@ Example: "app.cli('--help')"
|
|
|
173
199
|
for file_path in files:
|
|
174
200
|
path = Path(file_path)
|
|
175
201
|
if path.is_dir():
|
|
202
|
+
renamed_dir = path_to_renamed_path(file_path)
|
|
176
203
|
for filename in path.rglob('*'):
|
|
177
204
|
if filename.is_dir():
|
|
178
205
|
continue
|
|
179
206
|
with open(filename, 'rb') as f:
|
|
180
|
-
|
|
181
|
-
files_dict[
|
|
207
|
+
relative_to_dir = filename.resolve().relative_to(path.resolve())
|
|
208
|
+
files_dict[posixpath.join(renamed_dir, relative_to_dir.as_posix())] = f.read()
|
|
182
209
|
else:
|
|
183
210
|
with open(path, 'rb') as f:
|
|
184
211
|
files_dict[path_to_renamed_path(str(path))] = f.read()
|
|
185
212
|
elif isinstance(files, dict):
|
|
186
213
|
files_dict = {}
|
|
187
214
|
for key, value in files.items():
|
|
215
|
+
if '//' in key:
|
|
216
|
+
raise BioLibError(f"File path '{key}' contains double slashes which are not allowed")
|
|
188
217
|
if not key.startswith('/'):
|
|
189
218
|
key = '/' + key
|
|
190
219
|
files_dict[key] = value
|
|
@@ -199,11 +228,13 @@ Example: "app.cli('--help')"
|
|
|
199
228
|
files_dict[path_to_renamed_path(arg)] = f.read()
|
|
200
229
|
elif os.path.isdir(arg):
|
|
201
230
|
path = Path(arg)
|
|
231
|
+
renamed_dir = path_to_renamed_path(arg)
|
|
202
232
|
for filename in path.rglob('*'):
|
|
203
233
|
if filename.is_dir():
|
|
204
234
|
continue
|
|
205
235
|
with open(filename, 'rb') as f:
|
|
206
|
-
|
|
236
|
+
relative_to_dir = filename.resolve().relative_to(path.resolve())
|
|
237
|
+
files_dict[posixpath.join(renamed_dir, relative_to_dir.as_posix())] = f.read()
|
|
207
238
|
args[idx] = path_to_renamed_path(arg, prefix_with_slash=False)
|
|
208
239
|
|
|
209
240
|
# support --myarg=file.txt
|
|
@@ -214,20 +245,22 @@ Example: "app.cli('--help')"
|
|
|
214
245
|
files_dict[path_to_renamed_path(file_path)] = f.read()
|
|
215
246
|
elif os.path.isdir(file_path):
|
|
216
247
|
path = Path(file_path)
|
|
248
|
+
renamed_dir = path_to_renamed_path(file_path)
|
|
217
249
|
for filename in path.rglob('*'):
|
|
218
250
|
if filename.is_dir():
|
|
219
251
|
continue
|
|
220
252
|
with open(filename, 'rb') as f:
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
path_to_renamed_path(file_path, prefix_with_slash=False)
|
|
225
|
-
)
|
|
253
|
+
relative_to_dir = filename.resolve().relative_to(path.resolve())
|
|
254
|
+
files_dict[posixpath.join(renamed_dir, relative_to_dir.as_posix())] = f.read()
|
|
255
|
+
args[idx] = arg.split('=')[0] + '=' + path_to_renamed_path(file_path, prefix_with_slash=False)
|
|
226
256
|
else:
|
|
227
257
|
pass # a normal string arg was given
|
|
228
258
|
else:
|
|
229
259
|
tmp_filename = f'input_{"".join(random.choices(string.ascii_letters + string.digits, k=7))}'
|
|
230
|
-
if isinstance(arg,
|
|
260
|
+
if isinstance(arg, JsonStringIO):
|
|
261
|
+
file_data = arg.getvalue().encode()
|
|
262
|
+
tmp_filename += '.json'
|
|
263
|
+
elif isinstance(arg, io.StringIO):
|
|
231
264
|
file_data = arg.getvalue().encode()
|
|
232
265
|
elif isinstance(arg, io.BytesIO):
|
|
233
266
|
file_data = arg.getvalue()
|
|
@@ -246,7 +279,7 @@ Example: "app.cli('--help')"
|
|
|
246
279
|
def _run_locally(self, module_input_serialized: bytes) -> Result:
|
|
247
280
|
job_dict = BiolibJobApi.create(
|
|
248
281
|
app_version_id=self._app_version['public_id'],
|
|
249
|
-
app_resource_name_prefix=
|
|
282
|
+
app_resource_name_prefix=parse_resource_uri(self._app_uri)['resource_prefix'],
|
|
250
283
|
)
|
|
251
284
|
job = Result(job_dict)
|
|
252
285
|
|
|
@@ -271,7 +304,7 @@ Example: "app.cli('--help')"
|
|
|
271
304
|
continue
|
|
272
305
|
|
|
273
306
|
if isinstance(value, dict):
|
|
274
|
-
value =
|
|
307
|
+
value = JsonStringIO(json.dumps(value))
|
|
275
308
|
elif isinstance(value, (int, float)): # Cast numeric values to strings
|
|
276
309
|
value = str(value)
|
|
277
310
|
|