bfabric 1.13.18__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bfabric-1.13.18/.gitignore +9 -0
- bfabric-1.13.18/PKG-INFO +50 -0
- bfabric-1.13.18/pyproject.toml +87 -0
- bfabric-1.13.18/src/bfabric/__init__.py +14 -0
- bfabric-1.13.18/src/bfabric/bfabric.py +389 -0
- bfabric-1.13.18/src/bfabric/bfabric2.py +7 -0
- bfabric-1.13.18/src/bfabric/bfabric_config.py +38 -0
- bfabric-1.13.18/src/bfabric/cli_formatting.py +34 -0
- bfabric-1.13.18/src/bfabric/config/__init__.py +5 -0
- bfabric-1.13.18/src/bfabric/config/bfabric_auth.py +17 -0
- bfabric-1.13.18/src/bfabric/config/bfabric_client_config.py +49 -0
- bfabric-1.13.18/src/bfabric/config/config_file.py +98 -0
- bfabric-1.13.18/src/bfabric/engine/__init__.py +0 -0
- bfabric-1.13.18/src/bfabric/engine/engine_suds.py +132 -0
- bfabric-1.13.18/src/bfabric/engine/engine_zeep.py +181 -0
- bfabric-1.13.18/src/bfabric/engine/response_format_suds.py +48 -0
- bfabric-1.13.18/src/bfabric/entities/__init__.py +29 -0
- bfabric-1.13.18/src/bfabric/entities/application.py +21 -0
- bfabric-1.13.18/src/bfabric/entities/core/__init__.py +0 -0
- bfabric-1.13.18/src/bfabric/entities/core/entity.py +153 -0
- bfabric-1.13.18/src/bfabric/entities/core/has_container_mixin.py +49 -0
- bfabric-1.13.18/src/bfabric/entities/core/has_many.py +107 -0
- bfabric-1.13.18/src/bfabric/entities/core/has_one.py +36 -0
- bfabric-1.13.18/src/bfabric/entities/core/relationship.py +22 -0
- bfabric-1.13.18/src/bfabric/entities/dataset.py +46 -0
- bfabric-1.13.18/src/bfabric/entities/executable.py +28 -0
- bfabric-1.13.18/src/bfabric/entities/externaljob.py +37 -0
- bfabric-1.13.18/src/bfabric/entities/multiplexid.py +15 -0
- bfabric-1.13.18/src/bfabric/entities/multiplexkit.py +29 -0
- bfabric-1.13.18/src/bfabric/entities/order.py +19 -0
- bfabric-1.13.18/src/bfabric/entities/parameter.py +18 -0
- bfabric-1.13.18/src/bfabric/entities/project.py +15 -0
- bfabric-1.13.18/src/bfabric/entities/resource.py +25 -0
- bfabric-1.13.18/src/bfabric/entities/sample.py +18 -0
- bfabric-1.13.18/src/bfabric/entities/storage.py +29 -0
- bfabric-1.13.18/src/bfabric/entities/workunit.py +77 -0
- bfabric-1.13.18/src/bfabric/errors.py +39 -0
- bfabric-1.13.18/src/bfabric/examples/compare_zeep_suds_pagination.py +107 -0
- bfabric-1.13.18/src/bfabric/examples/compare_zeep_suds_query.py +222 -0
- bfabric-1.13.18/src/bfabric/examples/exists_multi.py +34 -0
- bfabric-1.13.18/src/bfabric/examples/zeep_debug.py +74 -0
- bfabric-1.13.18/src/bfabric/experimental/README.md +4 -0
- bfabric-1.13.18/src/bfabric/experimental/__init__.py +3 -0
- bfabric-1.13.18/src/bfabric/experimental/entity_lookup_cache.py +115 -0
- bfabric-1.13.18/src/bfabric/experimental/multi_query.py +129 -0
- bfabric-1.13.18/src/bfabric/experimental/upload_dataset.py +89 -0
- bfabric-1.13.18/src/bfabric/experimental/workunit_definition.py +123 -0
- bfabric-1.13.18/src/bfabric/py.typed +0 -0
- bfabric-1.13.18/src/bfabric/results/__init__.py +0 -0
- bfabric-1.13.18/src/bfabric/results/response_format_dict.py +159 -0
- bfabric-1.13.18/src/bfabric/results/result_container.py +120 -0
- bfabric-1.13.18/src/bfabric/utils/__init__.py +0 -0
- bfabric-1.13.18/src/bfabric/utils/paginator.py +61 -0
- bfabric-1.13.18/src/bfabric/utils/polars_utils.py +19 -0
- bfabric-1.13.18/src/bfabric/wrapper_creator/__init__.py +0 -0
- bfabric-1.13.18/src/bfabric/wrapper_creator/bfabric_external_job.py +94 -0
- bfabric-1.13.18/src/bfabric/wrapper_creator/bfabric_submitter.py +277 -0
- bfabric-1.13.18/src/bfabric/wrapper_creator/bfabric_wrapper_creator.py +254 -0
- bfabric-1.13.18/src/bfabric/wrapper_creator/demo_config.yaml +40 -0
- bfabric-1.13.18/src/bfabric/wrapper_creator/gridengine.py +107 -0
- bfabric-1.13.18/src/bfabric/wrapper_creator/slurm.py +73 -0
bfabric-1.13.18/PKG-INFO
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: bfabric
|
|
3
|
+
Version: 1.13.18
|
|
4
|
+
Summary: Python client for the B-Fabric API
|
|
5
|
+
Project-URL: Homepage, https://github.com/fgcz/bfabricPy
|
|
6
|
+
Project-URL: Repository, https://github.com/fgcz/bfabricPy
|
|
7
|
+
Author: Aleksejs Fomins, Marco Schmidt, Maria d'Errico, Witold Eryk Wolski
|
|
8
|
+
Author-email: Christian Panse <cp@fgcz.ethz.ch>, Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
|
|
9
|
+
License: GPL-3.0
|
|
10
|
+
Requires-Python: >=3.9
|
|
11
|
+
Requires-Dist: cyclopts>=2.9.9
|
|
12
|
+
Requires-Dist: eval-type-backport; python_version < '3.10'
|
|
13
|
+
Requires-Dist: flask>=3.0.3
|
|
14
|
+
Requires-Dist: loguru>=0.7
|
|
15
|
+
Requires-Dist: polars-lts-cpu>=0.20.25; platform_machine == 'x86_64' and platform_system == 'Darwin'
|
|
16
|
+
Requires-Dist: polars>=0.20.25; platform_machine != 'x86_64' or platform_system != 'Darwin'
|
|
17
|
+
Requires-Dist: pydantic>=2.9.2
|
|
18
|
+
Requires-Dist: python-dateutil>=2.9.0
|
|
19
|
+
Requires-Dist: pyyaml>=6.0
|
|
20
|
+
Requires-Dist: rich>=13.7.1
|
|
21
|
+
Requires-Dist: suds>=1.1.2
|
|
22
|
+
Requires-Dist: zeep>=4.2.1
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: black; extra == 'dev'
|
|
25
|
+
Requires-Dist: ipython; extra == 'dev'
|
|
26
|
+
Requires-Dist: isort; extra == 'dev'
|
|
27
|
+
Requires-Dist: licensecheck; extra == 'dev'
|
|
28
|
+
Requires-Dist: logot[loguru,pytest]; extra == 'dev'
|
|
29
|
+
Requires-Dist: mkdocs; extra == 'dev'
|
|
30
|
+
Requires-Dist: mkdocs-material; extra == 'dev'
|
|
31
|
+
Requires-Dist: mkdocstrings[python]; extra == 'dev'
|
|
32
|
+
Requires-Dist: nox; extra == 'dev'
|
|
33
|
+
Requires-Dist: pytest; extra == 'dev'
|
|
34
|
+
Requires-Dist: pytest-mock; extra == 'dev'
|
|
35
|
+
Requires-Dist: ruff; extra == 'dev'
|
|
36
|
+
Requires-Dist: uv; extra == 'dev'
|
|
37
|
+
Provides-Extra: doc
|
|
38
|
+
Requires-Dist: mkdocs; extra == 'doc'
|
|
39
|
+
Requires-Dist: mkdocs-material; extra == 'doc'
|
|
40
|
+
Requires-Dist: mkdocstrings[python]; extra == 'doc'
|
|
41
|
+
Provides-Extra: test
|
|
42
|
+
Requires-Dist: logot[loguru,pytest]; extra == 'test'
|
|
43
|
+
Requires-Dist: pytest; extra == 'test'
|
|
44
|
+
Requires-Dist: pytest-mock; extra == 'test'
|
|
45
|
+
Provides-Extra: typing
|
|
46
|
+
Requires-Dist: lxml-stubs; extra == 'typing'
|
|
47
|
+
Requires-Dist: mypy; extra == 'typing'
|
|
48
|
+
Requires-Dist: pandas-stubs; extra == 'typing'
|
|
49
|
+
Requires-Dist: types-python-dateutil; extra == 'typing'
|
|
50
|
+
Requires-Dist: types-requests; extra == 'typing'
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "bfabric"
|
|
7
|
+
description = "Python client for the B-Fabric API"
|
|
8
|
+
version = "1.13.18"
|
|
9
|
+
license = { text = "GPL-3.0" }
|
|
10
|
+
authors = [
|
|
11
|
+
{ name = "Christian Panse", email = "cp@fgcz.ethz.ch" },
|
|
12
|
+
{ name = "Leonardo Schwarz", email = "leonardo.schwarz@fgcz.ethz.ch" },
|
|
13
|
+
{ name = "Aleksejs Fomins" },
|
|
14
|
+
{ name = "Marco Schmidt" },
|
|
15
|
+
{ name = "Maria d'Errico" },
|
|
16
|
+
{ name = "Witold Eryk Wolski" },
|
|
17
|
+
]
|
|
18
|
+
requires-python = ">=3.9"
|
|
19
|
+
dependencies = [
|
|
20
|
+
"suds >= 1.1.2",
|
|
21
|
+
"PyYAML >= 6.0",
|
|
22
|
+
"Flask >= 3.0.3",
|
|
23
|
+
"rich >= 13.7.1",
|
|
24
|
+
"zeep >= 4.2.1",
|
|
25
|
+
"polars-lts-cpu >= 0.20.25; platform_machine == 'x86_64' and platform_system == 'Darwin'",
|
|
26
|
+
"polars >= 0.20.25; platform_machine != 'x86_64' or platform_system != 'Darwin'",
|
|
27
|
+
"loguru>=0.7",
|
|
28
|
+
"pydantic>=2.9.2",
|
|
29
|
+
"eval_type_backport; python_version < '3.10'",
|
|
30
|
+
"python-dateutil >= 2.9.0",
|
|
31
|
+
"cyclopts >= 2.9.9",
|
|
32
|
+
#"platformdirs >= 4.3",
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
[project.optional-dependencies]
|
|
36
|
+
dev = [
|
|
37
|
+
"bfabric[doc,test]",
|
|
38
|
+
"black",
|
|
39
|
+
"isort",
|
|
40
|
+
"ruff",
|
|
41
|
+
"licensecheck",
|
|
42
|
+
"nox",
|
|
43
|
+
"uv",
|
|
44
|
+
"ipython",
|
|
45
|
+
]
|
|
46
|
+
doc = ["mkdocs", "mkdocs-material", "mkdocstrings[python]"]
|
|
47
|
+
test = ["pytest", "pytest-mock", "logot[pytest,loguru]"]
|
|
48
|
+
typing = ["mypy", "types-requests", "lxml-stubs", "pandas-stubs", "types-python-dateutil"]
|
|
49
|
+
|
|
50
|
+
[project.urls]
|
|
51
|
+
Homepage = "https://github.com/fgcz/bfabricPy"
|
|
52
|
+
Repository = "https://github.com/fgcz/bfabricPy"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
[tool.uv]
|
|
56
|
+
reinstall-package = ["bfabric", "bfabric_scripts", "app_runner"]
|
|
57
|
+
|
|
58
|
+
[tool.black]
|
|
59
|
+
line-length = 120
|
|
60
|
+
target-version = ["py39"]
|
|
61
|
+
|
|
62
|
+
[tool.ruff]
|
|
63
|
+
line-length = 120
|
|
64
|
+
indent-width = 4
|
|
65
|
+
target-version = "py39"
|
|
66
|
+
|
|
67
|
+
[tool.ruff.lint]
|
|
68
|
+
select = ["ANN", "BLE", "D103", "E", "EXE", "F", "N", "PLW", "PTH", "SIM", "TCH", "UP", "W191"]
|
|
69
|
+
ignore = ["ANN401"]
|
|
70
|
+
|
|
71
|
+
[tool.ruff.lint.per-file-ignores]
|
|
72
|
+
"**/bfabric_scripts/**" = ["ALL"]
|
|
73
|
+
"**/wrapper_creator/**" = ["ALL"]
|
|
74
|
+
"**/examples/**" = ["ALL"]
|
|
75
|
+
"**/tests/**" = ["ALL"]
|
|
76
|
+
"noxfile.py" = ["ALL"]
|
|
77
|
+
|
|
78
|
+
[tool.licensecheck]
|
|
79
|
+
using = "PEP631"
|
|
80
|
+
|
|
81
|
+
#[tool.pytest.ini_options]
|
|
82
|
+
#logot_capturer = "logot.loguru.LoguruCapturer"
|
|
83
|
+
|
|
84
|
+
#[tool.check-tests-structure]
|
|
85
|
+
#sources_path = "src/bfabric"
|
|
86
|
+
#tests_path = "tests/unit"
|
|
87
|
+
#allow_missing_tests = true
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import importlib.metadata
|
|
2
|
+
|
|
3
|
+
from bfabric.bfabric import Bfabric, BfabricAPIEngineType
|
|
4
|
+
from bfabric.config.bfabric_auth import BfabricAuth
|
|
5
|
+
from bfabric.config.bfabric_client_config import BfabricClientConfig
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"Bfabric",
|
|
9
|
+
"BfabricAPIEngineType",
|
|
10
|
+
"BfabricAuth",
|
|
11
|
+
"BfabricClientConfig",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
__version__ = importlib.metadata.version("bfabric")
|
|
@@ -0,0 +1,389 @@
|
|
|
1
|
+
"""B-Fabric Application Interface using WSDL
|
|
2
|
+
|
|
3
|
+
Copyright (C) 2014 - 2024 Functional Genomics Center Zurich ETHZ|UZH. All rights reserved.
|
|
4
|
+
|
|
5
|
+
Licensed under GPL version 3
|
|
6
|
+
|
|
7
|
+
Authors:
|
|
8
|
+
Marco Schmidt <marco.schmidt@fgcz.ethz.ch>
|
|
9
|
+
Christian Panse <cp@fgcz.ethz.ch>
|
|
10
|
+
Leonardo Schwarz
|
|
11
|
+
Aleksejs Fomins
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import base64
|
|
17
|
+
import importlib.metadata
|
|
18
|
+
import sys
|
|
19
|
+
from contextlib import contextmanager
|
|
20
|
+
from datetime import datetime
|
|
21
|
+
from enum import Enum
|
|
22
|
+
from functools import cached_property
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from pprint import pprint
|
|
25
|
+
from typing import Literal, Any, TYPE_CHECKING
|
|
26
|
+
|
|
27
|
+
from loguru import logger
|
|
28
|
+
from rich.console import Console
|
|
29
|
+
|
|
30
|
+
from bfabric.bfabric_config import read_config
|
|
31
|
+
from bfabric.cli_formatting import HostnameHighlighter, DEFAULT_THEME
|
|
32
|
+
from bfabric.config import BfabricAuth
|
|
33
|
+
from bfabric.config import BfabricClientConfig
|
|
34
|
+
from bfabric.engine.engine_suds import EngineSUDS
|
|
35
|
+
from bfabric.engine.engine_zeep import EngineZeep
|
|
36
|
+
from bfabric.results.result_container import ResultContainer
|
|
37
|
+
from bfabric.utils.paginator import compute_requested_pages, BFABRIC_QUERY_LIMIT
|
|
38
|
+
|
|
39
|
+
if TYPE_CHECKING:
|
|
40
|
+
from collections.abc import Generator
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class BfabricAPIEngineType(Enum):
|
|
44
|
+
"""Choice of engine to use."""
|
|
45
|
+
|
|
46
|
+
SUDS = 1
|
|
47
|
+
ZEEP = 2
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class Bfabric:
|
|
51
|
+
"""Bfabric client class, providing general functionality for interaction with the B-Fabric API.
|
|
52
|
+
Use `Bfabric.from_config` to create a new instance.
|
|
53
|
+
:param config: Configuration object
|
|
54
|
+
:param auth: Authentication object (if `None`, it has to be provided using the `with_auth` context manager)
|
|
55
|
+
:param engine: Engine type to use for the API. Default is `BfabricAPIEngineType.SUDS`.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
def __init__(
|
|
59
|
+
self,
|
|
60
|
+
config: BfabricClientConfig,
|
|
61
|
+
auth: BfabricAuth | None,
|
|
62
|
+
engine: BfabricAPIEngineType = BfabricAPIEngineType.SUDS,
|
|
63
|
+
) -> None:
|
|
64
|
+
self.query_counter = 0
|
|
65
|
+
self._config = config
|
|
66
|
+
self._auth = auth
|
|
67
|
+
self._engine_type = engine
|
|
68
|
+
self._log_version_message()
|
|
69
|
+
|
|
70
|
+
@cached_property
|
|
71
|
+
def _engine(self) -> EngineSUDS | EngineZeep:
|
|
72
|
+
if self._engine_type == BfabricAPIEngineType.SUDS:
|
|
73
|
+
return EngineSUDS(base_url=self._config.base_url)
|
|
74
|
+
elif self._engine_type == BfabricAPIEngineType.ZEEP:
|
|
75
|
+
return EngineZeep(base_url=self._config.base_url)
|
|
76
|
+
else:
|
|
77
|
+
raise ValueError(f"Unexpected engine type: {self._engine_type}")
|
|
78
|
+
|
|
79
|
+
@classmethod
|
|
80
|
+
def from_config(
|
|
81
|
+
cls,
|
|
82
|
+
config_env: str | None = None,
|
|
83
|
+
config_path: str | None = None,
|
|
84
|
+
auth: BfabricAuth | Literal["config"] | None = "config",
|
|
85
|
+
engine: BfabricAPIEngineType = BfabricAPIEngineType.SUDS,
|
|
86
|
+
) -> Bfabric:
|
|
87
|
+
"""Returns a new Bfabric instance, configured with the user configuration file.
|
|
88
|
+
If the `config_env` is specified then it will be used, if it is not specified the default environment will be
|
|
89
|
+
determined by checking the following in order (picking the first one that is found):
|
|
90
|
+
- The `BFABRICPY_CONFIG_ENV` environment variable
|
|
91
|
+
- The `default_config` field in the config file "GENERAL" section
|
|
92
|
+
:param config_env: Configuration environment to use. If not given, it is deduced as described above.
|
|
93
|
+
:param config_path: Path to the config file, in case it is different from default
|
|
94
|
+
:param auth: Authentication to use. If "config" is given, the authentication will be read from the config file.
|
|
95
|
+
If it is set to None, no authentication will be used.
|
|
96
|
+
:param engine: Engine to use for the API. Default is SUDS.
|
|
97
|
+
"""
|
|
98
|
+
config, auth_config = get_system_auth(
|
|
99
|
+
config_env=config_env, config_path=config_path
|
|
100
|
+
)
|
|
101
|
+
auth_used: BfabricAuth | None = auth_config if auth == "config" else auth
|
|
102
|
+
return cls(config, auth_used, engine=engine)
|
|
103
|
+
|
|
104
|
+
@property
|
|
105
|
+
def config(self) -> BfabricClientConfig:
|
|
106
|
+
"""Returns the config object."""
|
|
107
|
+
return self._config
|
|
108
|
+
|
|
109
|
+
@property
|
|
110
|
+
def auth(self) -> BfabricAuth:
|
|
111
|
+
"""Returns the auth object.
|
|
112
|
+
:raises ValueError: If authentication is not available
|
|
113
|
+
"""
|
|
114
|
+
if self._auth is None:
|
|
115
|
+
raise ValueError("Authentication not available")
|
|
116
|
+
return self._auth
|
|
117
|
+
|
|
118
|
+
@contextmanager
|
|
119
|
+
def with_auth(self, auth: BfabricAuth) -> Generator[None, None, None]:
|
|
120
|
+
"""Context manager that temporarily (within the scope of the context) sets the authentication for
|
|
121
|
+
the Bfabric object to the provided value. This is useful when authenticating multiple users, to avoid accidental
|
|
122
|
+
use of the wrong credentials.
|
|
123
|
+
"""
|
|
124
|
+
old_auth = self._auth
|
|
125
|
+
self._auth = auth
|
|
126
|
+
try:
|
|
127
|
+
yield
|
|
128
|
+
finally:
|
|
129
|
+
self._auth = old_auth
|
|
130
|
+
|
|
131
|
+
def read(
|
|
132
|
+
self,
|
|
133
|
+
endpoint: str,
|
|
134
|
+
obj: dict[str, Any],
|
|
135
|
+
max_results: int | None = 100,
|
|
136
|
+
offset: int = 0,
|
|
137
|
+
check: bool = True,
|
|
138
|
+
return_id_only: bool = False,
|
|
139
|
+
) -> ResultContainer:
|
|
140
|
+
"""Reads from the specified endpoint matching all specified attributes in `obj`.
|
|
141
|
+
By setting `max_results` it is possible to change the number of results that are returned.
|
|
142
|
+
:param endpoint: the endpoint to read from, e.g. "sample"
|
|
143
|
+
:param obj: a dictionary containing the query, for every field multiple possible values can be provided, the
|
|
144
|
+
final query requires the condition for each field to be met
|
|
145
|
+
:param max_results: cap on the number of results to query. The code will keep reading pages until all pages
|
|
146
|
+
are read or expected number of results has been reached. If None, load all available pages.
|
|
147
|
+
NOTE: max_results will be rounded upwards to the nearest multiple of BFABRIC_QUERY_LIMIT, because results
|
|
148
|
+
come in blocks, and there is little overhead to providing results over integer number of pages.
|
|
149
|
+
:param offset: the number of elements to skip before starting to return results (useful for pagination, default
|
|
150
|
+
is 0 which means no skipping)
|
|
151
|
+
:param check: whether to raise an error if the response is not successful
|
|
152
|
+
:param return_id_only: whether to return only the ids of the found objects
|
|
153
|
+
:return: List of responses, packaged in the results container
|
|
154
|
+
"""
|
|
155
|
+
# Get the first page.
|
|
156
|
+
logger.debug(f"Reading from endpoint {repr(endpoint)} with query {repr(obj)}")
|
|
157
|
+
results = self._engine.read(
|
|
158
|
+
endpoint=endpoint,
|
|
159
|
+
obj=obj,
|
|
160
|
+
auth=self.auth,
|
|
161
|
+
page=1,
|
|
162
|
+
return_id_only=return_id_only,
|
|
163
|
+
)
|
|
164
|
+
n_available_pages = results.total_pages_api
|
|
165
|
+
if not n_available_pages:
|
|
166
|
+
if check:
|
|
167
|
+
results.assert_success()
|
|
168
|
+
return results.get_first_n_results(max_results)
|
|
169
|
+
|
|
170
|
+
# Get results from other pages as well, if need be
|
|
171
|
+
requested_pages, initial_offset = compute_requested_pages(
|
|
172
|
+
n_page_total=n_available_pages,
|
|
173
|
+
n_item_per_page=BFABRIC_QUERY_LIMIT,
|
|
174
|
+
n_item_offset=offset,
|
|
175
|
+
n_item_return_max=max_results,
|
|
176
|
+
)
|
|
177
|
+
logger.debug(f"Requested pages: {requested_pages}")
|
|
178
|
+
|
|
179
|
+
# NOTE: Page numbering starts at 1
|
|
180
|
+
response_items: list[dict[str, Any]] = []
|
|
181
|
+
errors = results.errors
|
|
182
|
+
page_offset = initial_offset
|
|
183
|
+
for i_iter, i_page in enumerate(requested_pages):
|
|
184
|
+
if not (i_iter == 0 and i_page == 1):
|
|
185
|
+
logger.debug(f"Reading page {i_page} of {n_available_pages}")
|
|
186
|
+
results = self._engine.read(
|
|
187
|
+
endpoint=endpoint,
|
|
188
|
+
obj=obj,
|
|
189
|
+
auth=self.auth,
|
|
190
|
+
page=i_page,
|
|
191
|
+
return_id_only=return_id_only,
|
|
192
|
+
)
|
|
193
|
+
errors += results.errors
|
|
194
|
+
|
|
195
|
+
response_items += results[page_offset:]
|
|
196
|
+
page_offset = 0
|
|
197
|
+
|
|
198
|
+
result = ResultContainer(
|
|
199
|
+
response_items, total_pages_api=n_available_pages, errors=errors
|
|
200
|
+
)
|
|
201
|
+
if check:
|
|
202
|
+
result.assert_success()
|
|
203
|
+
return result.get_first_n_results(max_results)
|
|
204
|
+
|
|
205
|
+
def save(
|
|
206
|
+
self,
|
|
207
|
+
endpoint: str,
|
|
208
|
+
obj: dict[str, Any],
|
|
209
|
+
check: bool = True,
|
|
210
|
+
method: str = "save",
|
|
211
|
+
) -> ResultContainer:
|
|
212
|
+
"""Saves the provided object to the specified endpoint.
|
|
213
|
+
:param endpoint: the endpoint to save to, e.g. "sample"
|
|
214
|
+
:param obj: the object to save
|
|
215
|
+
:param check: whether to raise an error if the response is not successful
|
|
216
|
+
:param method: the method to use for saving, generally "save", but in some cases e.g. "checkandinsert" is more
|
|
217
|
+
appropriate to be used instead.
|
|
218
|
+
:return a ResultContainer describing the saved object if successful
|
|
219
|
+
"""
|
|
220
|
+
results = self._engine.save(
|
|
221
|
+
endpoint=endpoint, obj=obj, auth=self.auth, method=method
|
|
222
|
+
)
|
|
223
|
+
if check:
|
|
224
|
+
results.assert_success()
|
|
225
|
+
return results
|
|
226
|
+
|
|
227
|
+
def delete(
|
|
228
|
+
self, endpoint: str, id: int | list[int], check: bool = True
|
|
229
|
+
) -> ResultContainer:
|
|
230
|
+
"""Deletes the object with the specified ID from the specified endpoint.
|
|
231
|
+
:param endpoint: the endpoint to delete from, e.g. "sample"
|
|
232
|
+
:param id: the ID of the object to delete
|
|
233
|
+
:param check: whether to raise an error if the response is not successful
|
|
234
|
+
:return a ResultContainer describing the deleted object if successful
|
|
235
|
+
"""
|
|
236
|
+
results = self._engine.delete(endpoint=endpoint, id=id, auth=self.auth)
|
|
237
|
+
if check:
|
|
238
|
+
results.assert_success()
|
|
239
|
+
return results
|
|
240
|
+
|
|
241
|
+
def exists(
|
|
242
|
+
self,
|
|
243
|
+
endpoint: str,
|
|
244
|
+
key: str,
|
|
245
|
+
value: int | str,
|
|
246
|
+
query: dict[str, Any] | None = None,
|
|
247
|
+
check: bool = True,
|
|
248
|
+
) -> bool:
|
|
249
|
+
"""Returns whether an object with the specified key-value pair exists in the specified endpoint.
|
|
250
|
+
Further conditions can be specified in the query.
|
|
251
|
+
:param endpoint: the endpoint to check, e.g. "sample"
|
|
252
|
+
:param key: the key to check, e.g. "id"
|
|
253
|
+
:param value: the value to check, e.g. 123
|
|
254
|
+
:param query: additional query conditions (optional)
|
|
255
|
+
:param check: whether to raise an error if the response is not successful
|
|
256
|
+
"""
|
|
257
|
+
query = query or {}
|
|
258
|
+
results = self.read(
|
|
259
|
+
endpoint=endpoint,
|
|
260
|
+
obj={**query, key: value},
|
|
261
|
+
max_results=1,
|
|
262
|
+
check=check,
|
|
263
|
+
return_id_only=True,
|
|
264
|
+
)
|
|
265
|
+
return len(results) > 0
|
|
266
|
+
|
|
267
|
+
def upload_resource(
|
|
268
|
+
self, resource_name: str, content: bytes, workunit_id: int, check: bool = True
|
|
269
|
+
) -> ResultContainer:
|
|
270
|
+
"""Uploads a resource to B-Fabric, only intended for relatively small files that will be tracked by B-Fabric
|
|
271
|
+
and not one of the dedicated experimental data stores.
|
|
272
|
+
:param resource_name: the name of the resource to create (the same name can only exist once per workunit)
|
|
273
|
+
:param content: the content of the resource as bytes
|
|
274
|
+
:param workunit_id: the workunit ID to which the resource belongs
|
|
275
|
+
:param check: whether to check for errors in the response
|
|
276
|
+
"""
|
|
277
|
+
content_encoded = base64.b64encode(content).decode()
|
|
278
|
+
return self.save(
|
|
279
|
+
endpoint="resource",
|
|
280
|
+
obj={
|
|
281
|
+
"base64": content_encoded,
|
|
282
|
+
"name": resource_name,
|
|
283
|
+
"description": "base64 encoded file",
|
|
284
|
+
"workunitid": workunit_id,
|
|
285
|
+
},
|
|
286
|
+
check=check,
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
def _get_version_message(self) -> tuple[str, str]:
|
|
290
|
+
"""Returns the version message as a string."""
|
|
291
|
+
package_version = importlib.metadata.version("bfabric")
|
|
292
|
+
year = datetime.now().year
|
|
293
|
+
engine_name = self._engine.__class__.__name__
|
|
294
|
+
base_url = self.config.base_url
|
|
295
|
+
user_name = f"U={self._auth.login if self._auth else None}"
|
|
296
|
+
python_version = f"PY={sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
|
|
297
|
+
return (
|
|
298
|
+
f"bfabricPy v{package_version} ({engine_name}, {base_url}, {user_name}, {python_version})",
|
|
299
|
+
f"Copyright (C) 2014-{year} Functional Genomics Center Zurich",
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
def _log_version_message(self) -> None:
|
|
303
|
+
"""Logs the version message describing bfabricpy version, engine and base url."""
|
|
304
|
+
console = Console(highlighter=HostnameHighlighter(), theme=DEFAULT_THEME)
|
|
305
|
+
for line in self._get_version_message():
|
|
306
|
+
with console.capture() as capture:
|
|
307
|
+
console.print(line, style="bright_yellow", end="")
|
|
308
|
+
logger.info(capture.get())
|
|
309
|
+
|
|
310
|
+
def __repr__(self) -> str:
|
|
311
|
+
return f"Bfabric(config={repr(self.config)}, auth={repr(self.auth)}, engine={self._engine})"
|
|
312
|
+
|
|
313
|
+
__str__ = __repr__
|
|
314
|
+
|
|
315
|
+
def __getstate__(self) -> dict[str, Any]:
|
|
316
|
+
return {
|
|
317
|
+
"config": self._config,
|
|
318
|
+
"auth": self._auth,
|
|
319
|
+
"engine_type": self._engine_type,
|
|
320
|
+
"query_counter": self.query_counter,
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
def __setstate__(self, state: dict[str, Any]) -> None:
|
|
324
|
+
self._config = state["config"]
|
|
325
|
+
self._auth = state["auth"]
|
|
326
|
+
self._engine_type = state["engine_type"]
|
|
327
|
+
self.query_counter = state["query_counter"]
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def get_system_auth(
|
|
331
|
+
login: str | None = None,
|
|
332
|
+
password: str | None = None,
|
|
333
|
+
base_url: str | None = None,
|
|
334
|
+
config_path: str | None = None,
|
|
335
|
+
config_env: str | None = None,
|
|
336
|
+
optional_auth: bool = True,
|
|
337
|
+
verbose: bool = False,
|
|
338
|
+
) -> tuple[BfabricClientConfig, BfabricAuth | None]:
|
|
339
|
+
"""
|
|
340
|
+
:param login: Login string for overriding config file
|
|
341
|
+
:param password: Password for overriding config file
|
|
342
|
+
:param base_url: Base server url for overriding config file
|
|
343
|
+
:param config_path: Path to the config file, in case it is different from default
|
|
344
|
+
:param config_env: Which config environment to use. Can also specify via environment variable or use
|
|
345
|
+
default in the config file (at your own risk)
|
|
346
|
+
:param optional_auth: Whether authentication is optional. If yes, missing authentication will be ignored,
|
|
347
|
+
otherwise an exception will be raised
|
|
348
|
+
:param verbose: Verbosity (TODO: resolve potential redundancy with logger)
|
|
349
|
+
"""
|
|
350
|
+
resolved_path = Path(config_path or "~/.bfabricpy.yml").expanduser()
|
|
351
|
+
|
|
352
|
+
# Use the provided config data from arguments instead of the file
|
|
353
|
+
if not resolved_path.is_file():
|
|
354
|
+
if config_path:
|
|
355
|
+
# NOTE: If user explicitly specifies a path to a wrong config file, this has to be an exception
|
|
356
|
+
raise OSError(
|
|
357
|
+
f"Explicitly specified config file does not exist: {resolved_path}"
|
|
358
|
+
)
|
|
359
|
+
# TODO: Convert to log
|
|
360
|
+
print(
|
|
361
|
+
f"Warning: could not find the config file in the default location: {resolved_path}"
|
|
362
|
+
)
|
|
363
|
+
config = BfabricClientConfig(base_url=base_url)
|
|
364
|
+
auth = (
|
|
365
|
+
None
|
|
366
|
+
if login is None or password is None
|
|
367
|
+
else BfabricAuth(login=login, password=password)
|
|
368
|
+
)
|
|
369
|
+
|
|
370
|
+
# Load config from file, override some of the fields with the provided ones
|
|
371
|
+
else:
|
|
372
|
+
config, auth = read_config(resolved_path, config_env=config_env)
|
|
373
|
+
config = config.copy_with(base_url=base_url)
|
|
374
|
+
if (login is not None) and (password is not None):
|
|
375
|
+
auth = BfabricAuth(login=login, password=password)
|
|
376
|
+
elif (login is None) and (password is None):
|
|
377
|
+
pass
|
|
378
|
+
else:
|
|
379
|
+
raise OSError("Must provide both username and password, or neither.")
|
|
380
|
+
|
|
381
|
+
if not config.base_url:
|
|
382
|
+
raise ValueError("base_url missing")
|
|
383
|
+
if not optional_auth and (not auth or not auth.login or not auth.password):
|
|
384
|
+
raise ValueError("Authentication not initialized but required")
|
|
385
|
+
|
|
386
|
+
if verbose:
|
|
387
|
+
pprint(config)
|
|
388
|
+
|
|
389
|
+
return config, auth
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import yaml
|
|
6
|
+
from loguru import logger
|
|
7
|
+
|
|
8
|
+
from bfabric.config import BfabricAuth
|
|
9
|
+
from bfabric.config import BfabricClientConfig
|
|
10
|
+
from bfabric.config import ConfigFile
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def read_config(
|
|
14
|
+
config_path: str | Path,
|
|
15
|
+
config_env: str | None = None,
|
|
16
|
+
) -> tuple[BfabricClientConfig, BfabricAuth | None]:
|
|
17
|
+
"""
|
|
18
|
+
Reads bfabricpy.yml file, parses it, extracting authentication and configuration data
|
|
19
|
+
:param config_path: Path to the configuration file. It is assumed the file exists
|
|
20
|
+
:param config_env: Configuration environment to use. If not given, it is deduced.
|
|
21
|
+
:return: Configuration and Authentication class instances
|
|
22
|
+
|
|
23
|
+
NOTE: BFabricPy expects a .bfabricpy.yml of the format, as seen in bfabricPy/tests/unit/example_config.yml
|
|
24
|
+
* The general field always has to be present
|
|
25
|
+
* There may be any number of environments, with arbitrary names. Here, they are called PRODUCTION and TEST
|
|
26
|
+
* Must specify correct login, password and base_url for each environment.
|
|
27
|
+
* application and job_notification_emails fields are optional
|
|
28
|
+
* The default environment will be selected as follows:
|
|
29
|
+
- First, parser will check if the optional argument `config_env` is provided directly to the parser function
|
|
30
|
+
- If not, secondly, the parser will check if the environment variable `BFABRICPY_CONFIG_ENV` is declared
|
|
31
|
+
- If not, finally, the parser will select the default_config specified in [GENERAL] of the .bfabricpy.yml file
|
|
32
|
+
"""
|
|
33
|
+
logger.debug(f"Reading configuration from: {config_path}")
|
|
34
|
+
config_file = ConfigFile.model_validate(
|
|
35
|
+
yaml.safe_load(Path(config_path).read_text())
|
|
36
|
+
)
|
|
37
|
+
env_config = config_file.get_selected_config(explicit_config_env=config_env)
|
|
38
|
+
return env_config.config, env_config.auth
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
from loguru import logger
|
|
5
|
+
from rich.highlighter import RegexHighlighter
|
|
6
|
+
from rich.theme import Theme
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class HostnameHighlighter(RegexHighlighter):
|
|
10
|
+
"""Highlights hostnames in URLs."""
|
|
11
|
+
|
|
12
|
+
base_style = "bfabric."
|
|
13
|
+
highlights = [r"https://(?P<hostname>[^.]+)"]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
DEFAULT_THEME = Theme({"bfabric.hostname": "bold red"})
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def setup_script_logging(debug: bool = False) -> None:
|
|
20
|
+
"""Sets up the logging for the command line scripts."""
|
|
21
|
+
setup_flag_key = "BFABRICPY_SCRIPT_LOGGING_SETUP"
|
|
22
|
+
if os.environ.get(setup_flag_key, "0") == "1":
|
|
23
|
+
return
|
|
24
|
+
logger.remove()
|
|
25
|
+
packages = ["bfabric", "bfabric_scripts", "app_runner", "__main__"]
|
|
26
|
+
if not (debug or os.environ.get("BFABRICPY_DEBUG")):
|
|
27
|
+
for package in packages:
|
|
28
|
+
logger.add(
|
|
29
|
+
sys.stderr, filter=package, level="INFO", format="{level} {message}"
|
|
30
|
+
)
|
|
31
|
+
else:
|
|
32
|
+
for package in packages:
|
|
33
|
+
logger.add(sys.stderr, filter=package, level="DEBUG")
|
|
34
|
+
os.environ[setup_flag_key] = "1"
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Annotated
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, Field
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class BfabricAuth(BaseModel):
|
|
9
|
+
"""Holds the authentication data for the B-Fabric client."""
|
|
10
|
+
|
|
11
|
+
login: Annotated[str, Field(min_length=3)]
|
|
12
|
+
password: Annotated[str, Field(min_length=32, max_length=32)]
|
|
13
|
+
|
|
14
|
+
def __repr__(self) -> str:
|
|
15
|
+
return f"BfabricAuth(login={repr(self.login)}, password=...)"
|
|
16
|
+
|
|
17
|
+
__str__ = __repr__
|