pybiolib 1.2.1056__py3-none-any.whl → 1.2.1642__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of pybiolib might be problematic. Click here for more details.

Files changed (86) hide show
  1. biolib/__init__.py +33 -10
  2. biolib/_data_record/data_record.py +24 -11
  3. biolib/_index/__init__.py +0 -0
  4. biolib/_index/index.py +51 -0
  5. biolib/_index/types.py +7 -0
  6. biolib/_internal/data_record/data_record.py +1 -1
  7. biolib/_internal/data_record/push_data.py +1 -1
  8. biolib/_internal/data_record/remote_storage_endpoint.py +3 -3
  9. biolib/_internal/file_utils.py +7 -4
  10. biolib/_internal/index/__init__.py +1 -0
  11. biolib/_internal/index/index.py +18 -0
  12. biolib/_internal/lfs/cache.py +4 -2
  13. biolib/_internal/push_application.py +89 -23
  14. biolib/_internal/runtime.py +2 -0
  15. biolib/_internal/templates/gui_template/App.tsx +38 -2
  16. biolib/_internal/templates/gui_template/Dockerfile +2 -0
  17. biolib/_internal/templates/gui_template/biolib-sdk.ts +37 -0
  18. biolib/_internal/templates/gui_template/dev-data/output.json +7 -0
  19. biolib/_internal/templates/gui_template/package.json +1 -0
  20. biolib/_internal/templates/gui_template/vite-plugin-dev-data.ts +49 -0
  21. biolib/_internal/templates/gui_template/vite.config.mts +2 -1
  22. biolib/_internal/templates/init_template/.github/workflows/biolib.yml +6 -1
  23. biolib/_internal/templates/init_template/Dockerfile +2 -0
  24. biolib/_internal/utils/__init__.py +25 -0
  25. biolib/_internal/utils/job_url.py +33 -0
  26. biolib/_runtime/runtime.py +9 -0
  27. biolib/_session/session.py +7 -5
  28. biolib/_shared/__init__.py +0 -0
  29. biolib/_shared/types/__init__.py +69 -0
  30. biolib/_shared/types/resource.py +17 -0
  31. biolib/_shared/types/resource_deploy_key.py +11 -0
  32. biolib/{_internal → _shared}/types/resource_permission.py +1 -1
  33. biolib/_shared/utils/__init__.py +7 -0
  34. biolib/_shared/utils/resource_uri.py +75 -0
  35. biolib/api/client.py +1 -1
  36. biolib/app/app.py +56 -23
  37. biolib/biolib_api_client/app_types.py +1 -6
  38. biolib/biolib_api_client/biolib_app_api.py +17 -0
  39. biolib/biolib_binary_format/module_input.py +8 -0
  40. biolib/biolib_binary_format/remote_endpoints.py +3 -3
  41. biolib/biolib_binary_format/remote_stream_seeker.py +39 -25
  42. biolib/cli/__init__.py +2 -1
  43. biolib/cli/data_record.py +17 -0
  44. biolib/cli/index.py +32 -0
  45. biolib/cli/lfs.py +1 -1
  46. biolib/cli/start.py +14 -1
  47. biolib/compute_node/job_worker/executors/docker_executor.py +31 -9
  48. biolib/compute_node/job_worker/executors/docker_types.py +1 -1
  49. biolib/compute_node/job_worker/executors/types.py +6 -5
  50. biolib/compute_node/job_worker/job_worker.py +149 -93
  51. biolib/compute_node/job_worker/large_file_system.py +2 -6
  52. biolib/compute_node/job_worker/network_alloc.py +99 -0
  53. biolib/compute_node/job_worker/network_buffer.py +240 -0
  54. biolib/compute_node/job_worker/utilization_reporter_thread.py +2 -2
  55. biolib/compute_node/remote_host_proxy.py +125 -67
  56. biolib/compute_node/utils.py +2 -0
  57. biolib/compute_node/webserver/compute_node_results_proxy.py +188 -0
  58. biolib/compute_node/webserver/proxy_utils.py +28 -0
  59. biolib/compute_node/webserver/webserver.py +64 -19
  60. biolib/experiments/experiment.py +98 -16
  61. biolib/jobs/job.py +119 -29
  62. biolib/jobs/job_result.py +70 -33
  63. biolib/jobs/types.py +1 -0
  64. biolib/sdk/__init__.py +17 -2
  65. biolib/typing_utils.py +1 -1
  66. biolib/utils/cache_state.py +2 -2
  67. biolib/utils/seq_util.py +1 -1
  68. {pybiolib-1.2.1056.dist-info → pybiolib-1.2.1642.dist-info}/METADATA +4 -2
  69. {pybiolib-1.2.1056.dist-info → pybiolib-1.2.1642.dist-info}/RECORD +84 -66
  70. {pybiolib-1.2.1056.dist-info → pybiolib-1.2.1642.dist-info}/WHEEL +1 -1
  71. biolib/_internal/types/__init__.py +0 -6
  72. biolib/utils/app_uri.py +0 -57
  73. /biolib/{_internal → _shared}/types/account.py +0 -0
  74. /biolib/{_internal → _shared}/types/account_member.py +0 -0
  75. /biolib/{_internal → _shared}/types/app.py +0 -0
  76. /biolib/{_internal → _shared}/types/data_record.py +0 -0
  77. /biolib/{_internal → _shared}/types/experiment.py +0 -0
  78. /biolib/{_internal → _shared}/types/file_node.py +0 -0
  79. /biolib/{_internal → _shared}/types/push.py +0 -0
  80. /biolib/{_internal/types/resource.py → _shared/types/resource_types.py} +0 -0
  81. /biolib/{_internal → _shared}/types/resource_version.py +0 -0
  82. /biolib/{_internal → _shared}/types/result.py +0 -0
  83. /biolib/{_internal → _shared}/types/typing.py +0 -0
  84. /biolib/{_internal → _shared}/types/user.py +0 -0
  85. {pybiolib-1.2.1056.dist-info → pybiolib-1.2.1642.dist-info}/entry_points.txt +0 -0
  86. {pybiolib-1.2.1056.dist-info → pybiolib-1.2.1642.dist-info/licenses}/LICENSE +0 -0
@@ -0,0 +1,37 @@
1
+ interface IBioLibGlobals {
2
+ getOutputFileData: (path: string) => Promise<Uint8Array>;
3
+ }
4
+
5
+ declare global {
6
+ const biolib: IBioLibGlobals;
7
+ }
8
+
9
+ // DO NOT MODIFY: Development data files are injected at build time from gui/dev-data/ folder
10
+ const DEV_DATA_FILES: Record<string, string> = {};
11
+
12
+ const devSdkBioLib: IBioLibGlobals = {
13
+ getOutputFileData: async (path: string): Promise<Uint8Array> => {
14
+ console.log(`[SDK] getOutputFileData called with path: ${path}`);
15
+
16
+ const normalizedPath = path.startsWith('/') ? path.slice(1) : path;
17
+
18
+ if (typeof DEV_DATA_FILES !== 'undefined' && normalizedPath in DEV_DATA_FILES) {
19
+ const base64Data = DEV_DATA_FILES[normalizedPath];
20
+ const binaryString = atob(base64Data);
21
+ const bytes = new Uint8Array(binaryString.length);
22
+ for (let i = 0; i < binaryString.length; i++) {
23
+ bytes[i] = binaryString.charCodeAt(i);
24
+ }
25
+ return bytes;
26
+ }
27
+
28
+ throw new Error(`File not found: ${path}. Add this file to the dev-data/ folder for local development.`);
29
+ },
30
+ };
31
+
32
+ const biolib: IBioLibGlobals =
33
+ process.env.NODE_ENV === "development"
34
+ ? devSdkBioLib
35
+ : (window as any).biolib;
36
+
37
+ export default biolib;
@@ -0,0 +1,7 @@
1
+ {
2
+ "message": "Example JSON data for development",
3
+ "results": [
4
+ { "id": 1, "value": "Sample result 1" },
5
+ { "id": 2, "value": "Sample result 2" }
6
+ ]
7
+ }
@@ -14,6 +14,7 @@
14
14
  },
15
15
  "devDependencies": {
16
16
  "@tailwindcss/vite": "4.0.14",
17
+ "@types/node": "20.17.10",
17
18
  "@types/react": "18.3.3",
18
19
  "@types/react-dom": "18.3.0",
19
20
  "@vitejs/plugin-react": "4.2.1",
@@ -0,0 +1,49 @@
1
+ import type { Plugin } from 'vite';
2
+ import fs from 'fs';
3
+ import path from 'path';
4
+
5
+ export function devDataPlugin(): Plugin {
6
+ let isDev = false;
7
+
8
+ return {
9
+ name: 'dev-data-plugin',
10
+ configResolved(config) {
11
+ isDev = config.mode === 'development';
12
+ },
13
+ transform(code: string, id: string) {
14
+ if (id.endsWith('biolib-sdk.ts')) {
15
+ let injectedCode: string;
16
+
17
+ if (isDev) {
18
+ const devDataDir = path.join(__dirname, 'dev-data');
19
+ const devDataMap: Record<string, string> = {};
20
+
21
+ if (fs.existsSync(devDataDir)) {
22
+ const files = fs.readdirSync(devDataDir);
23
+ for (const file of files) {
24
+ const filePath = path.join(devDataDir, file);
25
+ if (fs.statSync(filePath).isFile()) {
26
+ const content = fs.readFileSync(filePath);
27
+ const base64Content = content.toString('base64');
28
+ devDataMap[file] = base64Content;
29
+ }
30
+ }
31
+ }
32
+
33
+ const devDataJson = JSON.stringify(devDataMap);
34
+ injectedCode = code.replace(
35
+ "const DEV_DATA_FILES = {};",
36
+ `const DEV_DATA_FILES = ${devDataJson};`
37
+ );
38
+ } else {
39
+ injectedCode = code;
40
+ }
41
+
42
+ return {
43
+ code: injectedCode,
44
+ map: null
45
+ };
46
+ }
47
+ }
48
+ };
49
+ }
@@ -2,7 +2,8 @@ import { defineConfig } from "vite";
2
2
  import react from "@vitejs/plugin-react";
3
3
  import tailwindcss from "@tailwindcss/vite";
4
4
  import { viteSingleFile } from "vite-plugin-singlefile";
5
+ import { devDataPlugin } from "./gui/vite-plugin-dev-data";
5
6
 
6
7
  export default defineConfig({
7
- plugins: [react(), tailwindcss(), viteSingleFile()],
8
+ plugins: [react(), tailwindcss(), devDataPlugin(), viteSingleFile()],
8
9
  });
@@ -11,6 +11,11 @@ jobs:
11
11
  - name: Build
12
12
  run: docker build -t BIOLIB_REPLACE_DOCKER_TAG:latest .
13
13
  - name: Push
14
- run: biolib push $([ "$GITHUB_REF_NAME" != "main" ] && echo -n "--dev") BIOLIB_REPLACE_APP_URI
14
+ run: |
15
+ if [ "$GITHUB_REF_NAME" == "main" ]; then
16
+ biolib push BIOLIB_REPLACE_APP_URI
17
+ else
18
+ biolib push --dev BIOLIB_REPLACE_APP_URI:latest-dev
19
+ fi
15
20
  env:
16
21
  BIOLIB_TOKEN: ${{ secrets.BIOLIB_TOKEN }}
@@ -1,3 +1,5 @@
1
+ # syntax=docker/dockerfile:1
2
+
1
3
  FROM python:3.13.3-slim
2
4
  WORKDIR /home/biolib/
3
5
 
@@ -1,5 +1,30 @@
1
1
  import time
2
2
  import uuid
3
+ from fnmatch import fnmatch
4
+
5
+ from biolib.biolib_binary_format.utils import LazyLoadedFile
6
+ from biolib.typing_utils import Callable, List, Union, cast
7
+
8
+ PathFilter = Union[str, Callable[[str], bool]]
9
+
10
+
11
+ def filter_lazy_loaded_files(files: List[LazyLoadedFile], path_filter: PathFilter) -> List[LazyLoadedFile]:
12
+ if not (isinstance(path_filter, str) or callable(path_filter)):
13
+ raise Exception('Expected path_filter to be a string or a function')
14
+
15
+ if callable(path_filter):
16
+ return list(filter(lambda x: path_filter(x.path), files)) # type: ignore
17
+
18
+ glob_filter = cast(str, path_filter)
19
+
20
+ # since all file paths start with /, make sure filter does too
21
+ if not glob_filter.startswith('/'):
22
+ glob_filter = '/' + glob_filter
23
+
24
+ def _filter_function(file: LazyLoadedFile) -> bool:
25
+ return fnmatch(file.path, glob_filter)
26
+
27
+ return list(filter(_filter_function, files))
3
28
 
4
29
 
5
30
  def open_browser_window_from_notebook(url_to_open: str) -> None:
@@ -0,0 +1,33 @@
1
+ import re
2
+ from urllib.parse import urlparse
3
+
4
+ import biolib.utils
5
+ from biolib.typing_utils import Optional, Tuple
6
+
7
+
8
+ def parse_result_id_or_url(result_id_or_url: str, default_token: Optional[str] = None) -> Tuple[str, Optional[str]]:
9
+ result_id_or_url = result_id_or_url.strip()
10
+
11
+ if '/' not in result_id_or_url:
12
+ return (result_id_or_url, default_token)
13
+
14
+ if not result_id_or_url.startswith('http://') and not result_id_or_url.startswith('https://'):
15
+ result_id_or_url = 'https://' + result_id_or_url
16
+
17
+ parsed_url = urlparse(result_id_or_url)
18
+
19
+ if biolib.utils.BIOLIB_BASE_URL:
20
+ expected_base = urlparse(biolib.utils.BIOLIB_BASE_URL)
21
+ if parsed_url.scheme != expected_base.scheme or parsed_url.netloc != expected_base.netloc:
22
+ raise ValueError(f'URL must start with {biolib.utils.BIOLIB_BASE_URL}, got: {result_id_or_url}')
23
+
24
+ pattern = r'/results?/(?P<uuid>[a-f0-9-]+)/?(?:\?token=(?P<token>[^&]+))?'
25
+ match = re.search(pattern, result_id_or_url, re.IGNORECASE)
26
+
27
+ if not match:
28
+ raise ValueError(f'URL must be in format <base_url>/results/<UUID>/?token=<token>, got: {result_id_or_url}')
29
+
30
+ uuid = match.group('uuid')
31
+ token = match.group('token') or default_token
32
+
33
+ return (uuid, token)
@@ -37,10 +37,19 @@ class Runtime:
37
37
  return None
38
38
  return job_requested_machine
39
39
 
40
+ @staticmethod
41
+ def get_job_requested_spot_machine() -> bool:
42
+ job_data = Runtime._get_job_data()
43
+ return job_data.get('job_requested_machine_spot', False)
44
+
40
45
  @staticmethod
41
46
  def get_app_uri() -> str:
42
47
  return Runtime._get_job_data()['app_uri']
43
48
 
49
+ @staticmethod
50
+ def get_max_workers() -> int:
51
+ return Runtime._get_job_data()['job_reserved_machines']
52
+
44
53
  @staticmethod
45
54
  def get_secret(secret_name: str) -> bytes:
46
55
  assert re.match(
@@ -1,21 +1,23 @@
1
1
  from biolib import utils
2
- from biolib._internal.types import Optional
2
+ from biolib.typing_utils import Optional
3
3
  from biolib.api.client import ApiClient, ApiClientInitDict
4
4
  from biolib.app import BioLibApp
5
5
 
6
6
 
7
7
  class Session:
8
- def __init__(self, _init_dict: ApiClientInitDict) -> None:
8
+ def __init__(self, _init_dict: ApiClientInitDict, _experiment: Optional[str] = None) -> None:
9
9
  self._api = ApiClient(_init_dict=_init_dict)
10
+ self._experiment = _experiment
10
11
 
11
12
  @staticmethod
12
- def get_session(refresh_token: str, base_url: Optional[str] = None, client_type: Optional[str] = None) -> 'Session':
13
+ def get_session(refresh_token: str, base_url: Optional[str] = None, client_type: Optional[str] = None, experiment: Optional[str] = None) -> 'Session':
13
14
  return Session(
14
15
  _init_dict=ApiClientInitDict(
15
16
  refresh_token=refresh_token,
16
17
  base_url=base_url or utils.load_base_url_from_env(),
17
18
  client_type=client_type,
18
- )
19
+ ),
20
+ _experiment=experiment,
19
21
  )
20
22
 
21
23
  def load(self, uri: str, suppress_version_warning: bool = False) -> BioLibApp:
@@ -39,4 +41,4 @@ class Session:
39
41
  >>> app = biolib.load('https://biolib.com/biolib/myapp/')
40
42
  >>> result = app.cli('--help')
41
43
  """
42
- return BioLibApp(uri=uri, _api_client=self._api, suppress_version_warning=suppress_version_warning)
44
+ return BioLibApp(uri=uri, _api_client=self._api, suppress_version_warning=suppress_version_warning, _experiment=self._experiment)
File without changes
@@ -0,0 +1,69 @@
1
+ from .account import AccountDict, AccountDetailedDict
2
+ from .account_member import AccountMemberDict
3
+ from .app import AppDetailedDict, AppSlimDict
4
+ from .data_record import (
5
+ DataRecordDetailedDict,
6
+ DataRecordSlimDict,
7
+ DataRecordTypeDict,
8
+ DataRecordValidationRuleDict,
9
+ SqliteV1Column,
10
+ SqliteV1DatabaseSchema,
11
+ SqliteV1ForeignKey,
12
+ SqliteV1Table,
13
+ )
14
+ from .experiment import (
15
+ DeprecatedExperimentDict,
16
+ ExperimentDetailedDict,
17
+ ExperimentDict,
18
+ ResultCounts,
19
+ )
20
+ from .file_node import FileNodeDict, FileZipMetadataDict, ZipFileNodeDict
21
+ from .push import PushResponseDict
22
+ from .resource import ResourceUriDict, SemanticVersionDict
23
+ from .resource_deploy_key import ResourceDeployKeyDict, ResourceDeployKeyWithSecretDict
24
+ from .resource_permission import ResourcePermissionDetailedDict, ResourcePermissionDict
25
+ from .resource_types import ResourceDetailedDict, ResourceDict
26
+ from .resource_version import ResourceVersionDetailedDict, ResourceVersionDict
27
+ from .result import ResultDetailedDict, ResultDict
28
+ from .typing import Optional
29
+ from .user import EnterpriseSettingsDict, UserDetailedDict, UserDict
30
+
31
+ __all__ = [
32
+ 'AccountDetailedDict',
33
+ 'AccountDict',
34
+ 'AccountMemberDict',
35
+ 'AppDetailedDict',
36
+ 'AppSlimDict',
37
+ 'DataRecordDetailedDict',
38
+ 'DataRecordSlimDict',
39
+ 'DataRecordTypeDict',
40
+ 'DataRecordValidationRuleDict',
41
+ 'DeprecatedExperimentDict',
42
+ 'EnterpriseSettingsDict',
43
+ 'ExperimentDetailedDict',
44
+ 'ExperimentDict',
45
+ 'FileNodeDict',
46
+ 'FileZipMetadataDict',
47
+ 'Optional',
48
+ 'PushResponseDict',
49
+ 'ResourceDeployKeyDict',
50
+ 'ResourceDeployKeyWithSecretDict',
51
+ 'ResourceDetailedDict',
52
+ 'ResourceDict',
53
+ 'ResourcePermissionDetailedDict',
54
+ 'ResourcePermissionDict',
55
+ 'ResourceUriDict',
56
+ 'ResourceVersionDetailedDict',
57
+ 'ResourceVersionDict',
58
+ 'ResultCounts',
59
+ 'ResultDetailedDict',
60
+ 'ResultDict',
61
+ 'SemanticVersionDict',
62
+ 'SqliteV1Column',
63
+ 'SqliteV1DatabaseSchema',
64
+ 'SqliteV1ForeignKey',
65
+ 'SqliteV1Table',
66
+ 'UserDetailedDict',
67
+ 'UserDict',
68
+ 'ZipFileNodeDict',
69
+ ]
@@ -0,0 +1,17 @@
1
+ from .typing import Optional, TypedDict
2
+
3
+
4
+ class SemanticVersionDict(TypedDict):
5
+ major: int
6
+ minor: int
7
+ patch: int
8
+
9
+
10
+ class ResourceUriDict(TypedDict):
11
+ account_handle_normalized: str
12
+ account_handle: str
13
+ resource_name_normalized: Optional[str]
14
+ resource_name: Optional[str]
15
+ resource_prefix: Optional[str]
16
+ version: Optional[SemanticVersionDict]
17
+ tag: Optional[str]
@@ -0,0 +1,11 @@
1
+ from .typing import TypedDict
2
+
3
+
4
+ class ResourceDeployKeyDict(TypedDict):
5
+ uuid: str
6
+ created_at: str
7
+ name: str
8
+
9
+
10
+ class ResourceDeployKeyWithSecretDict(ResourceDeployKeyDict):
11
+ secret_key: str
@@ -1,4 +1,4 @@
1
- from .resource import ResourceDict
1
+ from .resource_types import ResourceDict
2
2
  from .typing import TypedDict
3
3
 
4
4
 
@@ -0,0 +1,7 @@
1
+ from biolib._shared.utils.resource_uri import normalize_resource_name, parse_resource_uri, parse_semantic_version
2
+
3
+ __all__ = [
4
+ 'normalize_resource_name',
5
+ 'parse_resource_uri',
6
+ 'parse_semantic_version',
7
+ ]
@@ -0,0 +1,75 @@
1
+ import re
2
+
3
+ from biolib._shared.types import Optional
4
+ from biolib._shared.types.resource import ResourceUriDict, SemanticVersionDict
5
+ from biolib.biolib_errors import BioLibError
6
+
7
+ URI_REGEX = re.compile(
8
+ r'^(@(?P<resource_prefix>[\w._-]+)/)?'
9
+ r'(?P<account_handle>[\w-]+)'
10
+ r'(/(?P<resource_name>[\w-]+))?'
11
+ r'(?::(?P<suffix>[^:]+))?$'
12
+ )
13
+ SEMVER_REGEX = re.compile(r'^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)$')
14
+ TAG_REGEX = re.compile(r'^[a-z0-9-]{1,128}$')
15
+
16
+
17
+ def normalize_resource_name(string: str) -> str:
18
+ return string.replace('-', '_').lower()
19
+
20
+
21
+ def parse_semantic_version(semantic_version: str) -> SemanticVersionDict:
22
+ if match := SEMVER_REGEX.fullmatch(semantic_version):
23
+ return SemanticVersionDict(
24
+ major=int(match.group('major')),
25
+ minor=int(match.group('minor')),
26
+ patch=int(match.group('patch')),
27
+ )
28
+
29
+ raise ValueError('The version must be a valid semantic version in the format of major.minor.patch (1.2.3).')
30
+
31
+
32
+ def parse_resource_uri(uri: str, use_account_as_name_default: bool = True) -> ResourceUriDict:
33
+ matches = URI_REGEX.match(uri)
34
+ if matches is None:
35
+ raise BioLibError(f"Could not parse resource uri '{uri}', uri did not match regex")
36
+
37
+ version: Optional[SemanticVersionDict] = None
38
+ tag: Optional[str] = None
39
+
40
+ suffix = matches.group('suffix')
41
+ if suffix and suffix != '*':
42
+ try:
43
+ version = parse_semantic_version(suffix)
44
+ except ValueError:
45
+ if TAG_REGEX.fullmatch(suffix):
46
+ tag = suffix
47
+ else:
48
+ raise BioLibError(
49
+ f'Invalid version or tag "{suffix}". '
50
+ 'Versions must be semantic versions like "1.2.3". '
51
+ 'Tags must be lowercase alphanumeric or dashes and at most 128 characters.'
52
+ ) from None
53
+
54
+ resource_prefix_raw: Optional[str] = matches.group('resource_prefix')
55
+ resource_prefix = resource_prefix_raw.lower() if resource_prefix_raw is not None else None
56
+ account_handle: str = matches.group('account_handle')
57
+ account_handle_normalized: str = normalize_resource_name(account_handle)
58
+ resource_name: Optional[str] = matches.group('resource_name')
59
+
60
+ if resource_name:
61
+ resource_name_normalized = normalize_resource_name(resource_name)
62
+ elif use_account_as_name_default:
63
+ resource_name_normalized = account_handle_normalized
64
+ else:
65
+ resource_name_normalized = None
66
+
67
+ return ResourceUriDict(
68
+ resource_prefix=resource_prefix,
69
+ account_handle=account_handle,
70
+ account_handle_normalized=account_handle_normalized,
71
+ resource_name_normalized=resource_name_normalized,
72
+ resource_name=resource_name if resource_name is not None or not use_account_as_name_default else account_handle,
73
+ version=version,
74
+ tag=tag,
75
+ )
biolib/api/client.py CHANGED
@@ -8,7 +8,7 @@ from urllib.parse import urlencode, urljoin
8
8
  import importlib_metadata
9
9
 
10
10
  from biolib._internal.http_client import HttpClient, HttpResponse
11
- from biolib._internal.types.typing import Any, Dict, Optional, TypedDict, Union, cast
11
+ from biolib._shared.types.typing import Any, Dict, Optional, TypedDict, Union, cast
12
12
  from biolib.biolib_api_client import BiolibApiClient as DeprecatedApiClient
13
13
  from biolib.biolib_errors import BioLibError
14
14
  from biolib.biolib_logging import logger
biolib/app/app.py CHANGED
@@ -1,11 +1,16 @@
1
+ import copy
1
2
  import io
2
3
  import json
3
4
  import os
5
+ import posixpath
4
6
  import random
5
7
  import string
6
8
  from pathlib import Path
7
9
 
8
10
  from biolib import utils
11
+ from biolib._internal.file_utils import path_to_renamed_path
12
+ from biolib._runtime.runtime import Runtime
13
+ from biolib._shared.utils import parse_resource_uri
9
14
  from biolib.api.client import ApiClient
10
15
  from biolib.biolib_api_client import JobState
11
16
  from biolib.biolib_api_client.app_types import App, AppVersion
@@ -18,14 +23,24 @@ from biolib.compute_node.job_worker.job_worker import JobWorker
18
23
  from biolib.experiments.experiment import Experiment
19
24
  from biolib.jobs.job import Result
20
25
  from biolib.typing_utils import Dict, Optional
21
- from biolib.utils.app_uri import parse_app_uri
22
- from biolib._runtime.runtime import Runtime
23
- from biolib._internal.file_utils import path_to_renamed_path
26
+
27
+
28
+ class JsonStringIO(io.StringIO):
29
+ pass
24
30
 
25
31
 
26
32
  class BioLibApp:
27
- def __init__(self, uri: str, _api_client: Optional[ApiClient] = None, suppress_version_warning: bool = False):
33
+ def __init__(
34
+ self,
35
+ uri: str,
36
+ _api_client: Optional[ApiClient] = None,
37
+ suppress_version_warning: bool = False,
38
+ _experiment: Optional[str] = None,
39
+ ):
28
40
  self._api_client: Optional[ApiClient] = _api_client
41
+ self._experiment = _experiment
42
+ self._input_uri = uri
43
+ self._parsed_input_uri = parse_resource_uri(uri)
29
44
 
30
45
  app_response = BiolibAppApi.get_by_uri(uri=uri, api_client=self._api_client)
31
46
  self._app: App = app_response['app']
@@ -33,16 +48,19 @@ class BioLibApp:
33
48
  self._app_version: AppVersion = app_response['app_version']
34
49
 
35
50
  if not suppress_version_warning:
36
- parsed_uri = parse_app_uri(uri)
37
- if parsed_uri['version'] is None:
51
+ if self._parsed_input_uri['version'] is None:
38
52
  if Runtime.check_is_environment_biolib_app():
39
53
  logger.warning(
40
54
  f"No version specified in URI '{uri}'. This will use the default version, "
41
- f"which may change behaviour over time. Consider locking down the exact version, "
55
+ f'which may change behaviour over time. Consider locking down the exact version, '
42
56
  f"e.g. '{uri}:1.2.3'"
43
57
  )
44
58
 
45
- logger.info(f'Loaded project {self._app_uri}')
59
+ if self._parsed_input_uri['tag']:
60
+ semantic_version = f"{self._app_version['major']}.{self._app_version['minor']}.{self._app_version['patch']}"
61
+ logger.info(f'Loaded {self._input_uri} (resolved to {semantic_version})')
62
+ else:
63
+ logger.info(f'Loaded {self._app_uri}')
46
64
 
47
65
  def __str__(self) -> str:
48
66
  return self._app_uri
@@ -71,7 +89,7 @@ class BioLibApp:
71
89
  result_prefix: Optional[str] = None,
72
90
  timeout: Optional[int] = None,
73
91
  notify: bool = False,
74
- machine_count: Optional[int] = None,
92
+ max_workers: Optional[int] = None,
75
93
  experiment: Optional[str] = None,
76
94
  temporary_client_secrets: Optional[Dict[str, str]] = None,
77
95
  check: bool = False,
@@ -84,7 +102,12 @@ class BioLibApp:
84
102
  raise ValueError('The argument "check" cannot be True when blocking is False')
85
103
 
86
104
  if not experiment_id:
87
- experiment_instance = Experiment(experiment) if experiment else Experiment.get_experiment_in_context()
105
+ experiment_to_use = experiment if experiment is not None else self._experiment
106
+ experiment_instance: Optional[Experiment]
107
+ if experiment_to_use:
108
+ experiment_instance = Experiment(experiment_to_use, _api_client=self._api_client)
109
+ else:
110
+ experiment_instance = Experiment.get_experiment_in_context()
88
111
  experiment_id = experiment_instance.uuid if experiment_instance else None
89
112
 
90
113
  module_input_serialized = self._get_serialized_module_input(args, stdin, files)
@@ -111,11 +134,12 @@ class BioLibApp:
111
134
  override_command=override_command,
112
135
  result_prefix=result_prefix,
113
136
  timeout=timeout,
114
- requested_machine_count=machine_count,
137
+ requested_machine_count=max_workers,
115
138
  temporary_client_secrets=temporary_client_secrets,
116
139
  api_client=self._api_client,
117
140
  )
118
- logger.info(f'View the result in your browser at: {utils.BIOLIB_BASE_URL}/results/{job.id}/')
141
+ if utils.IS_RUNNING_IN_NOTEBOOK:
142
+ logger.info(f'View the result in your browser at: {utils.BIOLIB_BASE_URL}/results/{job.id}/')
119
143
  if blocking:
120
144
  # TODO: Deprecate utils.STREAM_STDOUT and always stream logs by simply calling job.stream_logs()
121
145
  if utils.IS_RUNNING_IN_NOTEBOOK:
@@ -152,6 +176,8 @@ Example: "app.cli('--help')"
152
176
  def _get_serialized_module_input(args=None, stdin=None, files=None) -> bytes:
153
177
  if args is None:
154
178
  args = []
179
+ else:
180
+ args = copy.copy(args)
155
181
 
156
182
  if stdin is None:
157
183
  stdin = b''
@@ -173,18 +199,21 @@ Example: "app.cli('--help')"
173
199
  for file_path in files:
174
200
  path = Path(file_path)
175
201
  if path.is_dir():
202
+ renamed_dir = path_to_renamed_path(file_path)
176
203
  for filename in path.rglob('*'):
177
204
  if filename.is_dir():
178
205
  continue
179
206
  with open(filename, 'rb') as f:
180
- relative_path = filename.relative_to(Path.cwd())
181
- files_dict[path_to_renamed_path(str(relative_path))] = f.read()
207
+ relative_to_dir = filename.resolve().relative_to(path.resolve())
208
+ files_dict[posixpath.join(renamed_dir, relative_to_dir.as_posix())] = f.read()
182
209
  else:
183
210
  with open(path, 'rb') as f:
184
211
  files_dict[path_to_renamed_path(str(path))] = f.read()
185
212
  elif isinstance(files, dict):
186
213
  files_dict = {}
187
214
  for key, value in files.items():
215
+ if '//' in key:
216
+ raise BioLibError(f"File path '{key}' contains double slashes which are not allowed")
188
217
  if not key.startswith('/'):
189
218
  key = '/' + key
190
219
  files_dict[key] = value
@@ -199,11 +228,13 @@ Example: "app.cli('--help')"
199
228
  files_dict[path_to_renamed_path(arg)] = f.read()
200
229
  elif os.path.isdir(arg):
201
230
  path = Path(arg)
231
+ renamed_dir = path_to_renamed_path(arg)
202
232
  for filename in path.rglob('*'):
203
233
  if filename.is_dir():
204
234
  continue
205
235
  with open(filename, 'rb') as f:
206
- files_dict[path_to_renamed_path(str(filename))] = f.read()
236
+ relative_to_dir = filename.resolve().relative_to(path.resolve())
237
+ files_dict[posixpath.join(renamed_dir, relative_to_dir.as_posix())] = f.read()
207
238
  args[idx] = path_to_renamed_path(arg, prefix_with_slash=False)
208
239
 
209
240
  # support --myarg=file.txt
@@ -214,20 +245,22 @@ Example: "app.cli('--help')"
214
245
  files_dict[path_to_renamed_path(file_path)] = f.read()
215
246
  elif os.path.isdir(file_path):
216
247
  path = Path(file_path)
248
+ renamed_dir = path_to_renamed_path(file_path)
217
249
  for filename in path.rglob('*'):
218
250
  if filename.is_dir():
219
251
  continue
220
252
  with open(filename, 'rb') as f:
221
- files_dict[path_to_renamed_path(str(filename))] = f.read()
222
- args[idx] = (
223
- arg.split('=')[0] + '=' +
224
- path_to_renamed_path(file_path, prefix_with_slash=False)
225
- )
253
+ relative_to_dir = filename.resolve().relative_to(path.resolve())
254
+ files_dict[posixpath.join(renamed_dir, relative_to_dir.as_posix())] = f.read()
255
+ args[idx] = arg.split('=')[0] + '=' + path_to_renamed_path(file_path, prefix_with_slash=False)
226
256
  else:
227
257
  pass # a normal string arg was given
228
258
  else:
229
259
  tmp_filename = f'input_{"".join(random.choices(string.ascii_letters + string.digits, k=7))}'
230
- if isinstance(arg, io.StringIO):
260
+ if isinstance(arg, JsonStringIO):
261
+ file_data = arg.getvalue().encode()
262
+ tmp_filename += '.json'
263
+ elif isinstance(arg, io.StringIO):
231
264
  file_data = arg.getvalue().encode()
232
265
  elif isinstance(arg, io.BytesIO):
233
266
  file_data = arg.getvalue()
@@ -246,7 +279,7 @@ Example: "app.cli('--help')"
246
279
  def _run_locally(self, module_input_serialized: bytes) -> Result:
247
280
  job_dict = BiolibJobApi.create(
248
281
  app_version_id=self._app_version['public_id'],
249
- app_resource_name_prefix=parse_app_uri(self._app_uri)['resource_name_prefix'],
282
+ app_resource_name_prefix=parse_resource_uri(self._app_uri)['resource_prefix'],
250
283
  )
251
284
  job = Result(job_dict)
252
285
 
@@ -271,7 +304,7 @@ Example: "app.cli('--help')"
271
304
  continue
272
305
 
273
306
  if isinstance(value, dict):
274
- value = io.StringIO(json.dumps(value))
307
+ value = JsonStringIO(json.dumps(value))
275
308
  elif isinstance(value, (int, float)): # Cast numeric values to strings
276
309
  value = str(value)
277
310