q8s 0.8.0.dev2__tar.gz → 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {q8s-0.8.0.dev2 → q8s-0.9.0}/PKG-INFO +7 -1
- {q8s-0.8.0.dev2 → q8s-0.9.0}/pyproject.toml +7 -1
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s/cli.py +16 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s/execution.py +0 -32
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s/plugins/cpu_job.py +7 -3
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s/plugins/cuda_job.py +7 -3
- q8s-0.9.0/q8s/plugins/job.py +53 -0
- q8s-0.9.0/q8s/plugins/utils/git_info.py +111 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s/project.py +46 -6
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s/utils.py +5 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s/workload.py +15 -15
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s.egg-info/PKG-INFO +7 -1
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s.egg-info/SOURCES.txt +5 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s.egg-info/requires.txt +1 -0
- q8s-0.9.0/tests/__init__.py +0 -0
- q8s-0.9.0/tests/test_git_info.py +102 -0
- q8s-0.9.0/tests/test_job_plugin.py +64 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/tests/test_job_template_spec.py +4 -1
- {q8s-0.8.0.dev2 → q8s-0.9.0}/LICENSE +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/README.md +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s/__init__.py +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s/__main__.py +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s/constants.py +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s/deps/__init__.py +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s/deps/code_analyzer.py +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s/deps/parser.py +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s/enums.py +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s/install.py +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s/kernel.py +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s/matplotlib/backend.py +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s/multifiles.py +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s/plugins/__init__.py +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s/plugins/job_template_spec.py +0 -0
- {q8s-0.8.0.dev2/tests → q8s-0.9.0/q8s/plugins/utils}/__init__.py +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s.egg-info/dependency_links.txt +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s.egg-info/entry_points.txt +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/q8s.egg-info/top_level.txt +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/setup.cfg +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/tests/fixtures/cache/app.py +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/tests/fixtures/simple/qiskit.py +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/tests/test_backend.py +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/tests/test_deps.py +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/tests/test_kernel.py +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/tests/test_multifiles.py +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/tests/test_project.py +0 -0
- {q8s-0.8.0.dev2 → q8s-0.9.0}/tests/test_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: q8s
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.0
|
|
4
4
|
Summary: Kernel extension for executing quantum programs in simulators on q8s clusters
|
|
5
5
|
Author-email: Vlad Stirbu <vstirbu@gmail.com>
|
|
6
6
|
License: Apache License
|
|
@@ -211,6 +211,11 @@ Keywords: jupyter,quantum,kernel
|
|
|
211
211
|
Classifier: Development Status :: 4 - Beta
|
|
212
212
|
Classifier: Intended Audience :: Developers
|
|
213
213
|
Classifier: Programming Language :: Python :: 3
|
|
214
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
215
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
216
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
217
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
218
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
214
219
|
Classifier: License :: OSI Approved :: MIT License
|
|
215
220
|
Classifier: Operating System :: OS Independent
|
|
216
221
|
Requires-Python: >=3.10
|
|
@@ -224,6 +229,7 @@ Requires-Dist: pluggy==1.5.0
|
|
|
224
229
|
Requires-Dist: python-dotenv==1.0.1
|
|
225
230
|
Requires-Dist: pyyaml==6.0.2
|
|
226
231
|
Requires-Dist: typer>=0.15.4
|
|
232
|
+
Requires-Dist: GitPython>=3.1.43
|
|
227
233
|
Provides-Extra: benchmark
|
|
228
234
|
Requires-Dist: pandas; extra == "benchmark"
|
|
229
235
|
Requires-Dist: testbook; extra == "benchmark"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "q8s"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.9.0"
|
|
4
4
|
description = "Kernel extension for executing quantum programs in simulators on q8s clusters"
|
|
5
5
|
authors = [{ name = "Vlad Stirbu", email = "vstirbu@gmail.com" }]
|
|
6
6
|
readme = "README.md"
|
|
@@ -10,6 +10,11 @@ classifiers = [
|
|
|
10
10
|
"Development Status :: 4 - Beta",
|
|
11
11
|
"Intended Audience :: Developers",
|
|
12
12
|
"Programming Language :: Python :: 3",
|
|
13
|
+
"Programming Language :: Python :: 3.10",
|
|
14
|
+
"Programming Language :: Python :: 3.11",
|
|
15
|
+
"Programming Language :: Python :: 3.12",
|
|
16
|
+
"Programming Language :: Python :: 3.13",
|
|
17
|
+
"Programming Language :: Python :: 3.14",
|
|
13
18
|
"License :: OSI Approved :: MIT License",
|
|
14
19
|
"Operating System :: OS Independent",
|
|
15
20
|
]
|
|
@@ -22,6 +27,7 @@ dependencies = [
|
|
|
22
27
|
"python-dotenv==1.0.1",
|
|
23
28
|
"pyyaml==6.0.2",
|
|
24
29
|
"typer>=0.15.4",
|
|
30
|
+
"GitPython>=3.1.43",
|
|
25
31
|
]
|
|
26
32
|
requires-python = ">= 3.10"
|
|
27
33
|
|
|
@@ -16,6 +16,22 @@ from q8s.workload import Workload
|
|
|
16
16
|
app = typer.Typer()
|
|
17
17
|
|
|
18
18
|
|
|
19
|
+
@app.command()
|
|
20
|
+
def init(
|
|
21
|
+
images: Annotated[
|
|
22
|
+
bool, typer.Option(help="Initialize images cache if build in a CI pipeline")
|
|
23
|
+
] = False,
|
|
24
|
+
):
|
|
25
|
+
project = Project()
|
|
26
|
+
project.init_cache()
|
|
27
|
+
|
|
28
|
+
if images:
|
|
29
|
+
project.images_from_ci()
|
|
30
|
+
project.update_images_cache()
|
|
31
|
+
|
|
32
|
+
print(f"Project {project.name} initialized")
|
|
33
|
+
|
|
34
|
+
|
|
19
35
|
@app.command()
|
|
20
36
|
def build(
|
|
21
37
|
init: Annotated[bool, typer.Option(help="Initialize project")] = False,
|
|
@@ -146,38 +146,6 @@ class K8sContext:
|
|
|
146
146
|
self.__progress.advance(prepare_task, 1)
|
|
147
147
|
return job
|
|
148
148
|
|
|
149
|
-
def __create_config_map_object(self, job: client.V1Job, code: str):
|
|
150
|
-
"""
|
|
151
|
-
Create a ConfigMap object with the given code.
|
|
152
|
-
"""
|
|
153
|
-
|
|
154
|
-
data = {}
|
|
155
|
-
data["main.py"] = code
|
|
156
|
-
|
|
157
|
-
# Configureate ConfigMap from a local file
|
|
158
|
-
configmap = client.V1ConfigMap(
|
|
159
|
-
api_version="v1",
|
|
160
|
-
kind="ConfigMap",
|
|
161
|
-
data=data,
|
|
162
|
-
metadata=client.V1ObjectMeta(
|
|
163
|
-
name=self.name,
|
|
164
|
-
owner_references=[
|
|
165
|
-
client.V1OwnerReference(
|
|
166
|
-
api_version="batch/v1",
|
|
167
|
-
kind="Job",
|
|
168
|
-
name=job.metadata.name,
|
|
169
|
-
uid=job.metadata.uid,
|
|
170
|
-
# block_owner_deletion=True,
|
|
171
|
-
# controller=True,
|
|
172
|
-
)
|
|
173
|
-
],
|
|
174
|
-
),
|
|
175
|
-
)
|
|
176
|
-
|
|
177
|
-
self.core_api_instance.create_namespaced_config_map(
|
|
178
|
-
namespace=self.namespace, body=configmap
|
|
179
|
-
)
|
|
180
|
-
|
|
181
149
|
def __create_config_map_object_from_workload(
|
|
182
150
|
self, job: client.V1Job, workload: Workload
|
|
183
151
|
):
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
from typing import Dict
|
|
2
|
+
from q8s.plugins.job import JobPlugin
|
|
2
3
|
from q8s.plugins.job_template_spec import hookimpl
|
|
3
4
|
from q8s.enums import Target
|
|
4
5
|
from kubernetes import client
|
|
@@ -6,7 +7,7 @@ from q8s.constants import WORKSPACE
|
|
|
6
7
|
from q8s.workload import Workload
|
|
7
8
|
|
|
8
9
|
|
|
9
|
-
class CPUJobTemplatePlugin:
|
|
10
|
+
class CPUJobTemplatePlugin(JobPlugin):
|
|
10
11
|
|
|
11
12
|
@hookimpl
|
|
12
13
|
def makejob(
|
|
@@ -25,13 +26,16 @@ class CPUJobTemplatePlugin:
|
|
|
25
26
|
|
|
26
27
|
volume_name = f"app-volume-{name}"
|
|
27
28
|
|
|
29
|
+
env_var = list(env)
|
|
28
30
|
if workload.is_src_project:
|
|
29
|
-
|
|
31
|
+
env_var.append(client.V1EnvVar(name="PYTHONPATH", value=f"{WORKSPACE}/src"))
|
|
32
|
+
|
|
33
|
+
self.patch_environment_with_git_info(env_var)
|
|
30
34
|
|
|
31
35
|
container = client.V1Container(
|
|
32
36
|
name="quantum-routine",
|
|
33
37
|
image=container_image,
|
|
34
|
-
env=
|
|
38
|
+
env=env_var,
|
|
35
39
|
command=["python"],
|
|
36
40
|
args=(
|
|
37
41
|
["-m", workload.entry_module] + workload.args
|
|
@@ -3,13 +3,14 @@ from typing import Dict
|
|
|
3
3
|
from kubernetes import client
|
|
4
4
|
from q8s.constants import WORKSPACE
|
|
5
5
|
from q8s.enums import Target
|
|
6
|
+
from q8s.plugins.job import JobPlugin
|
|
6
7
|
from q8s.plugins.job_template_spec import hookimpl
|
|
7
8
|
from q8s.workload import Workload
|
|
8
9
|
|
|
9
10
|
MEMORY = os.environ.get("MEMORY", "32Gi")
|
|
10
11
|
|
|
11
12
|
|
|
12
|
-
class CUDAJobTemplatePlugin:
|
|
13
|
+
class CUDAJobTemplatePlugin(JobPlugin):
|
|
13
14
|
"""
|
|
14
15
|
This plugin is used to create a job template for a GPU job.
|
|
15
16
|
"""
|
|
@@ -31,13 +32,16 @@ class CUDAJobTemplatePlugin:
|
|
|
31
32
|
|
|
32
33
|
volume_name = f"app-volume-{name}"
|
|
33
34
|
|
|
35
|
+
env_var = list(env)
|
|
34
36
|
if workload.is_src_project:
|
|
35
|
-
|
|
37
|
+
env_var.append(client.V1EnvVar(name="PYTHONPATH", value=f"{WORKSPACE}/src"))
|
|
38
|
+
|
|
39
|
+
self.patch_environment_with_git_info(env_var)
|
|
36
40
|
|
|
37
41
|
container = client.V1Container(
|
|
38
42
|
name="quantum-routine",
|
|
39
43
|
image=container_image,
|
|
40
|
-
env=
|
|
44
|
+
env=env_var,
|
|
41
45
|
command=["python"],
|
|
42
46
|
args=(
|
|
43
47
|
["-m", workload.entry_module] + workload.args
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from kubernetes import client
|
|
3
|
+
|
|
4
|
+
from q8s.plugins.utils.git_info import get_git_info
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class JobPlugin:
|
|
8
|
+
"""
|
|
9
|
+
Base class for job template plugins.
|
|
10
|
+
This class provides functionality to patch job environment variables with Git information,
|
|
11
|
+
such as commit hash, branch name, and remote repository URL. It is intended to be extended
|
|
12
|
+
by specific job plugins that require Git metadata in their execution environment.
|
|
13
|
+
The following environment variables are added:
|
|
14
|
+
- Q8S_GIT_COMMIT, MLFLOW_GIT_COMMIT: The current Git commit hash.
|
|
15
|
+
- Q8S_GIT_BRANCH, MLFLOW_GIT_BRANCH: The current Git branch name.
|
|
16
|
+
- Q8S_GIT_REPO_URL, MLFLOW_GIT_REPO_URL: The remote repository URL.
|
|
17
|
+
- GIT_PYTHON_REFRESH: Set to "quiet" to suppress GitPython refresh warnings.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
def patch_environment_with_git_info(
|
|
21
|
+
self, env: list[client.V1EnvVar]
|
|
22
|
+
) -> list[client.V1EnvVar]:
|
|
23
|
+
"""
|
|
24
|
+
Patch environment variables for the job with Git information
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
env (list[client.V1EnvVar]): Original environment variables
|
|
28
|
+
Returns:
|
|
29
|
+
list[client.V1EnvVar]: Patched environment variables
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
# Get Git information from the q8sctl current working directory
|
|
33
|
+
git_info = get_git_info(os.getcwd())
|
|
34
|
+
|
|
35
|
+
if git_info.commit:
|
|
36
|
+
env.append(client.V1EnvVar(name="Q8S_GIT_COMMIT", value=git_info.commit))
|
|
37
|
+
env.append(client.V1EnvVar(name="MLFLOW_GIT_COMMIT", value=git_info.commit))
|
|
38
|
+
|
|
39
|
+
if git_info.branch:
|
|
40
|
+
env.append(client.V1EnvVar(name="Q8S_GIT_BRANCH", value=git_info.branch))
|
|
41
|
+
env.append(client.V1EnvVar(name="MLFLOW_GIT_BRANCH", value=git_info.branch))
|
|
42
|
+
|
|
43
|
+
if git_info.remote_url:
|
|
44
|
+
env.append(
|
|
45
|
+
client.V1EnvVar(name="Q8S_GIT_REPO_URL", value=git_info.remote_url)
|
|
46
|
+
)
|
|
47
|
+
env.append(
|
|
48
|
+
client.V1EnvVar(name="MLFLOW_GIT_REPO_URL", value=git_info.remote_url)
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
env.append(client.V1EnvVar(name="GIT_PYTHON_REFRESH", value="quiet"))
|
|
52
|
+
|
|
53
|
+
return env
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from typing import Optional, Dict, Any
|
|
3
|
+
import os
|
|
4
|
+
|
|
5
|
+
from git import Repo, InvalidGitRepositoryError, NoSuchPathError
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass
|
|
9
|
+
class GitExtraInfo:
|
|
10
|
+
"""Additional Git information dataclass"""
|
|
11
|
+
|
|
12
|
+
branch_source: Optional[str]
|
|
13
|
+
git_dir: Optional[str]
|
|
14
|
+
working_tree_dir: Optional[str]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class GitExtraReason:
|
|
19
|
+
"""Reason for missing Git information"""
|
|
20
|
+
|
|
21
|
+
reason: str
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class GitInfo:
|
|
26
|
+
"""Git information dataclass"""
|
|
27
|
+
|
|
28
|
+
commit: Optional[str]
|
|
29
|
+
branch: Optional[str]
|
|
30
|
+
remote_url: Optional[str]
|
|
31
|
+
extra: GitExtraInfo | GitExtraReason
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _detect_branch_from_env() -> Optional[str]:
|
|
35
|
+
"""
|
|
36
|
+
Try to infer branch name from common CI environment variables.
|
|
37
|
+
Extend this for your CI setup as needed.
|
|
38
|
+
"""
|
|
39
|
+
# GitHub Actions
|
|
40
|
+
if ref_name := os.getenv("GITHUB_REF_NAME"):
|
|
41
|
+
return ref_name
|
|
42
|
+
|
|
43
|
+
# GitLab CI
|
|
44
|
+
if ref_name := os.getenv("CI_COMMIT_REF_NAME"):
|
|
45
|
+
return ref_name
|
|
46
|
+
|
|
47
|
+
# Generic
|
|
48
|
+
return os.getenv("BRANCH_NAME")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def get_git_info(path: str = ".") -> GitInfo:
|
|
52
|
+
"""
|
|
53
|
+
Collect commit, branch and remote URL info using GitPython.
|
|
54
|
+
|
|
55
|
+
Returns GitInfo with None fields if not in a git repo.
|
|
56
|
+
"""
|
|
57
|
+
try:
|
|
58
|
+
repo = Repo(path, search_parent_directories=True)
|
|
59
|
+
except (InvalidGitRepositoryError, NoSuchPathError):
|
|
60
|
+
return GitInfo(
|
|
61
|
+
commit=None,
|
|
62
|
+
branch=None,
|
|
63
|
+
remote_url=None,
|
|
64
|
+
extra=GitExtraReason(reason="not_a_git_repo"),
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
# Commit SHA
|
|
68
|
+
try:
|
|
69
|
+
commit = repo.head.commit.hexsha
|
|
70
|
+
except ValueError:
|
|
71
|
+
commit = None
|
|
72
|
+
|
|
73
|
+
# Branch (handle detached HEAD, CI environments, etc.)
|
|
74
|
+
branch = None
|
|
75
|
+
if repo.head.is_detached:
|
|
76
|
+
branch = _detect_branch_from_env()
|
|
77
|
+
branch_source = "env" if branch else "detached_head"
|
|
78
|
+
else:
|
|
79
|
+
branch = repo.active_branch.name
|
|
80
|
+
branch_source = "repo"
|
|
81
|
+
|
|
82
|
+
# Remote URL
|
|
83
|
+
try:
|
|
84
|
+
origin = repo.remotes.origin
|
|
85
|
+
remote_url = origin.url
|
|
86
|
+
if remote_url.startswith("git@"):
|
|
87
|
+
# git@github.com:org/repo.git -> https://github.com/org/repo.git
|
|
88
|
+
host, _, url_path = remote_url.partition(":")
|
|
89
|
+
remote_url = f"https://{host.removeprefix('git@')}/{url_path}"
|
|
90
|
+
elif remote_url.startswith("ssh://"):
|
|
91
|
+
# ssh://git@github.com/org/repo.git -> https://github.com/org/repo.git
|
|
92
|
+
stripped = remote_url.removeprefix("ssh://")
|
|
93
|
+
if stripped.startswith("git@"):
|
|
94
|
+
stripped = stripped.removeprefix("git@")
|
|
95
|
+
host, _, url_path = stripped.partition("/")
|
|
96
|
+
remote_url = f"https://{host}/{url_path}"
|
|
97
|
+
except (AttributeError, IndexError, ValueError):
|
|
98
|
+
remote_url = None
|
|
99
|
+
|
|
100
|
+
extra = GitExtraInfo(
|
|
101
|
+
branch_source=branch_source,
|
|
102
|
+
git_dir=repo.git_dir,
|
|
103
|
+
working_tree_dir=repo.working_tree_dir,
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
return GitInfo(
|
|
107
|
+
commit=commit,
|
|
108
|
+
branch=branch,
|
|
109
|
+
remote_url=remote_url,
|
|
110
|
+
extra=extra,
|
|
111
|
+
)
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import configparser
|
|
1
2
|
from dataclasses import dataclass
|
|
2
3
|
from datetime import datetime
|
|
3
4
|
from pathlib import Path
|
|
@@ -43,7 +44,7 @@ def _read_stream_lines(stream, progress, silent, is_error: bool = False):
|
|
|
43
44
|
"""
|
|
44
45
|
while True:
|
|
45
46
|
line = stream.readline()
|
|
46
|
-
if line ==
|
|
47
|
+
if line == "":
|
|
47
48
|
break
|
|
48
49
|
if not silent:
|
|
49
50
|
if is_error:
|
|
@@ -96,6 +97,7 @@ class Q8SDocker:
|
|
|
96
97
|
username: str
|
|
97
98
|
registry: Optional[str]
|
|
98
99
|
|
|
100
|
+
|
|
99
101
|
@dataclass
|
|
100
102
|
class Q8SProject:
|
|
101
103
|
name: str
|
|
@@ -112,9 +114,11 @@ class CacheNotBuiltException(Exception):
|
|
|
112
114
|
class ProjectNotFoundException(Exception):
|
|
113
115
|
pass
|
|
114
116
|
|
|
117
|
+
|
|
115
118
|
class ProjectInvalidConfigurationException(Exception):
|
|
116
119
|
pass
|
|
117
120
|
|
|
121
|
+
|
|
118
122
|
class Project:
|
|
119
123
|
name: str
|
|
120
124
|
__path: str
|
|
@@ -215,14 +219,15 @@ class Project:
|
|
|
215
219
|
stderr=STDOUT,
|
|
216
220
|
bufsize=1,
|
|
217
221
|
universal_newlines=True,
|
|
218
|
-
encoding="utf-8",
|
|
219
|
-
errors="replace",
|
|
222
|
+
encoding="utf-8", # force UTF-8 decoding
|
|
223
|
+
errors="replace", # avoid crashing on bad bytes
|
|
220
224
|
)
|
|
221
225
|
|
|
222
226
|
if sys.platform == "win32":
|
|
223
227
|
# Windows-safe: read line by line instead of using selectors
|
|
224
228
|
_handle_subprocess_output(build_process, progress, silent)
|
|
225
229
|
else:
|
|
230
|
+
|
|
226
231
|
def handle_output(stream, mask):
|
|
227
232
|
# Because the process' output is line buffered, there's only ever one
|
|
228
233
|
# line to read when this function is called
|
|
@@ -270,8 +275,8 @@ class Project:
|
|
|
270
275
|
stderr=STDOUT,
|
|
271
276
|
bufsize=1,
|
|
272
277
|
universal_newlines=True,
|
|
273
|
-
encoding="utf-8",
|
|
274
|
-
errors="replace",
|
|
278
|
+
encoding="utf-8", # force UTF-8 decoding
|
|
279
|
+
errors="replace", # avoid crashing on bad bytes
|
|
275
280
|
)
|
|
276
281
|
|
|
277
282
|
if sys.platform == "win32":
|
|
@@ -279,6 +284,7 @@ class Project:
|
|
|
279
284
|
_handle_subprocess_output(push_process, progress, silent)
|
|
280
285
|
|
|
281
286
|
else:
|
|
287
|
+
|
|
282
288
|
def handle_output(stream, mask):
|
|
283
289
|
# Because the process' output is line buffered, there's only ever one
|
|
284
290
|
# line to read when this function is called
|
|
@@ -306,11 +312,19 @@ class Project:
|
|
|
306
312
|
else:
|
|
307
313
|
progress.advance(task)
|
|
308
314
|
|
|
315
|
+
def images_from_ci(self):
|
|
316
|
+
"""
|
|
317
|
+
Images are build in a CI pipeline.
|
|
318
|
+
"""
|
|
319
|
+
for target in self.configuration.targets.keys():
|
|
320
|
+
self.__images[target] = self.__image_name(target)
|
|
321
|
+
|
|
309
322
|
def update_images_cache(self):
|
|
310
323
|
"""
|
|
311
324
|
Update the images cache
|
|
312
325
|
"""
|
|
313
326
|
with open(join(self.__path, ".q8s_cache", "images"), "w") as f:
|
|
327
|
+
print(f"# This file is autogenerated by q8sctl\n", file=f)
|
|
314
328
|
yaml.dump(self.__images, f)
|
|
315
329
|
|
|
316
330
|
def clear_cache(self):
|
|
@@ -325,7 +339,7 @@ class Project:
|
|
|
325
339
|
|
|
326
340
|
def __image_name(self, target: str):
|
|
327
341
|
registry = self.configuration.docker.registry
|
|
328
|
-
username = self.__docker_login()
|
|
342
|
+
username = self.__docker_login().lower()
|
|
329
343
|
|
|
330
344
|
# Build image name based on available information
|
|
331
345
|
if registry and username:
|
|
@@ -388,6 +402,15 @@ class Project:
|
|
|
388
402
|
file=f,
|
|
389
403
|
)
|
|
390
404
|
print(f"LABEL org.opencontainers.image.title={self.name}", file=f)
|
|
405
|
+
|
|
406
|
+
url = self.__get_project_url()
|
|
407
|
+
|
|
408
|
+
if url is not None:
|
|
409
|
+
print(
|
|
410
|
+
f"LABEL org.opencontainers.image.source={url.strip()}",
|
|
411
|
+
file=f,
|
|
412
|
+
)
|
|
413
|
+
|
|
391
414
|
print("", file=f)
|
|
392
415
|
|
|
393
416
|
print(f"WORKDIR {WORKSPACE}", file=f)
|
|
@@ -399,3 +422,20 @@ class Project:
|
|
|
399
422
|
raise Exception(f"Target {target} not found")
|
|
400
423
|
|
|
401
424
|
return getattr(self.configuration.targets, target)
|
|
425
|
+
|
|
426
|
+
def __get_project_url(self) -> Optional[str]:
|
|
427
|
+
"""
|
|
428
|
+
Get the project URL from setup.cfg if available
|
|
429
|
+
"""
|
|
430
|
+
cfg_path = Path(self.__path) / "setup.cfg"
|
|
431
|
+
if not cfg_path.exists():
|
|
432
|
+
return None
|
|
433
|
+
|
|
434
|
+
config = configparser.ConfigParser()
|
|
435
|
+
config.read(cfg_path)
|
|
436
|
+
|
|
437
|
+
url = config.get("metadata", "url", fallback=None)
|
|
438
|
+
if url is not None:
|
|
439
|
+
return url.strip()
|
|
440
|
+
|
|
441
|
+
return None
|
|
@@ -4,6 +4,11 @@ from q8s.project import CacheNotBuiltException, Project, ProjectNotFoundExceptio
|
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
def extract_non_none_value(arr):
|
|
7
|
+
"""
|
|
8
|
+
Extract the first non-None value from a list
|
|
9
|
+
|
|
10
|
+
Used to get the first available configuration value from multiple sources generated by job plugins
|
|
11
|
+
"""
|
|
7
12
|
non_none_values = [x for x in arr if x is not None]
|
|
8
13
|
return non_none_values[0] if non_none_values else None
|
|
9
14
|
|
|
@@ -7,34 +7,34 @@ from .multifiles import collect_imported_files
|
|
|
7
7
|
|
|
8
8
|
class Workload:
|
|
9
9
|
"""
|
|
10
|
-
Represents a
|
|
10
|
+
Represents a multifile workload consisting of an entry script and its imported files
|
|
11
11
|
"""
|
|
12
12
|
|
|
13
|
-
__pyproject_root: Path | None = None
|
|
14
|
-
__args: list[str] = []
|
|
15
|
-
|
|
16
13
|
def __init__(self, entry_script: Path | str = None, code: str = None):
|
|
14
|
+
self.__pyproject_root: Path | None = None
|
|
15
|
+
self.__args: list[str] = []
|
|
16
|
+
|
|
17
17
|
if entry_script is not None:
|
|
18
18
|
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
self.__entry_script = Path(os.path.abspath(entry_script))
|
|
20
|
+
|
|
21
|
+
if self.__detect_and_set_src_layout():
|
|
21
22
|
self.__base_path = self.__pyproject_root
|
|
22
23
|
|
|
23
24
|
self.__files = collect_imported_files(
|
|
24
25
|
self.__entry_script, is_project=True
|
|
25
26
|
)
|
|
26
|
-
|
|
27
27
|
else:
|
|
28
|
-
self.__entry_script = Path(os.path.abspath(entry_script))
|
|
29
28
|
self.__base_path = self.__entry_script.parent
|
|
30
29
|
|
|
31
30
|
self.__files = collect_imported_files(
|
|
32
31
|
self.__entry_script, is_project=False
|
|
33
32
|
)
|
|
34
33
|
|
|
35
|
-
self.__data = {
|
|
36
|
-
|
|
37
|
-
|
|
34
|
+
self.__data = {}
|
|
35
|
+
for f in self.__files:
|
|
36
|
+
with open(f, "r", encoding="utf-8") as file:
|
|
37
|
+
self.__data[self.__path_mapping(f)] = file.read()
|
|
38
38
|
|
|
39
39
|
elif code is not None:
|
|
40
40
|
self.__entry_script = Path(os.path.abspath("main.py"))
|
|
@@ -86,7 +86,7 @@ class Workload:
|
|
|
86
86
|
"""
|
|
87
87
|
Entry script module name
|
|
88
88
|
"""
|
|
89
|
-
return self.entry_script.replace(
|
|
89
|
+
return self.entry_script.replace(os.path.sep, ".")[:-3] # strip .py
|
|
90
90
|
|
|
91
91
|
@property
|
|
92
92
|
def data(self) -> dict[str, str]:
|
|
@@ -127,7 +127,7 @@ class Workload:
|
|
|
127
127
|
"""
|
|
128
128
|
return self.__relative_path(file).replace("/", "__")
|
|
129
129
|
|
|
130
|
-
def
|
|
130
|
+
def __find_pyproject_root(self) -> Path | None:
|
|
131
131
|
"""
|
|
132
132
|
Find the project root by looking for setup.cfg or pyproject.toml
|
|
133
133
|
"""
|
|
@@ -140,11 +140,11 @@ class Workload:
|
|
|
140
140
|
|
|
141
141
|
return None
|
|
142
142
|
|
|
143
|
-
def
|
|
143
|
+
def __detect_and_set_src_layout(self) -> bool:
|
|
144
144
|
"""
|
|
145
145
|
Determine if the project uses a src/ layout
|
|
146
146
|
"""
|
|
147
|
-
root = self.
|
|
147
|
+
root = self.__find_pyproject_root()
|
|
148
148
|
|
|
149
149
|
if root is None:
|
|
150
150
|
return False
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: q8s
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.0
|
|
4
4
|
Summary: Kernel extension for executing quantum programs in simulators on q8s clusters
|
|
5
5
|
Author-email: Vlad Stirbu <vstirbu@gmail.com>
|
|
6
6
|
License: Apache License
|
|
@@ -211,6 +211,11 @@ Keywords: jupyter,quantum,kernel
|
|
|
211
211
|
Classifier: Development Status :: 4 - Beta
|
|
212
212
|
Classifier: Intended Audience :: Developers
|
|
213
213
|
Classifier: Programming Language :: Python :: 3
|
|
214
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
215
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
216
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
217
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
218
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
214
219
|
Classifier: License :: OSI Approved :: MIT License
|
|
215
220
|
Classifier: Operating System :: OS Independent
|
|
216
221
|
Requires-Python: >=3.10
|
|
@@ -224,6 +229,7 @@ Requires-Dist: pluggy==1.5.0
|
|
|
224
229
|
Requires-Dist: python-dotenv==1.0.1
|
|
225
230
|
Requires-Dist: pyyaml==6.0.2
|
|
226
231
|
Requires-Dist: typer>=0.15.4
|
|
232
|
+
Requires-Dist: GitPython>=3.1.43
|
|
227
233
|
Provides-Extra: benchmark
|
|
228
234
|
Requires-Dist: pandas; extra == "benchmark"
|
|
229
235
|
Requires-Dist: testbook; extra == "benchmark"
|
|
@@ -26,10 +26,15 @@ q8s/matplotlib/backend.py
|
|
|
26
26
|
q8s/plugins/__init__.py
|
|
27
27
|
q8s/plugins/cpu_job.py
|
|
28
28
|
q8s/plugins/cuda_job.py
|
|
29
|
+
q8s/plugins/job.py
|
|
29
30
|
q8s/plugins/job_template_spec.py
|
|
31
|
+
q8s/plugins/utils/__init__.py
|
|
32
|
+
q8s/plugins/utils/git_info.py
|
|
30
33
|
tests/__init__.py
|
|
31
34
|
tests/test_backend.py
|
|
32
35
|
tests/test_deps.py
|
|
36
|
+
tests/test_git_info.py
|
|
37
|
+
tests/test_job_plugin.py
|
|
33
38
|
tests/test_job_template_spec.py
|
|
34
39
|
tests/test_kernel.py
|
|
35
40
|
tests/test_multifiles.py
|
|
File without changes
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import tempfile
|
|
3
|
+
import unittest
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from git import Actor, Repo
|
|
7
|
+
|
|
8
|
+
from q8s.plugins.utils.git_info import (
|
|
9
|
+
GitExtraInfo,
|
|
10
|
+
GitExtraReason,
|
|
11
|
+
_detect_branch_from_env,
|
|
12
|
+
get_git_info,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class TestDetectBranchFromEnv(unittest.TestCase):
|
|
17
|
+
|
|
18
|
+
def test_prefers_github_ref_name(self):
|
|
19
|
+
with unittest.mock.patch.dict(
|
|
20
|
+
os.environ,
|
|
21
|
+
{
|
|
22
|
+
"GITHUB_REF_NAME": "github-branch",
|
|
23
|
+
"CI_COMMIT_REF_NAME": "gitlab-branch",
|
|
24
|
+
"BRANCH_NAME": "generic-branch",
|
|
25
|
+
},
|
|
26
|
+
clear=True,
|
|
27
|
+
):
|
|
28
|
+
self.assertEqual(_detect_branch_from_env(), "github-branch")
|
|
29
|
+
|
|
30
|
+
def test_uses_gitlab_commit_ref_when_github_missing(self):
|
|
31
|
+
with unittest.mock.patch.dict(
|
|
32
|
+
os.environ,
|
|
33
|
+
{"CI_COMMIT_REF_NAME": "gitlab-branch", "BRANCH_NAME": "generic-branch"},
|
|
34
|
+
clear=True,
|
|
35
|
+
):
|
|
36
|
+
self.assertEqual(_detect_branch_from_env(), "gitlab-branch")
|
|
37
|
+
|
|
38
|
+
def test_falls_back_to_generic_branch_name(self):
|
|
39
|
+
with unittest.mock.patch.dict(
|
|
40
|
+
os.environ,
|
|
41
|
+
{"BRANCH_NAME": "generic-branch"},
|
|
42
|
+
clear=True,
|
|
43
|
+
):
|
|
44
|
+
self.assertEqual(_detect_branch_from_env(), "generic-branch")
|
|
45
|
+
|
|
46
|
+
def test_returns_none_when_no_env_vars_set(self):
|
|
47
|
+
with unittest.mock.patch.dict(os.environ, {}, clear=True):
|
|
48
|
+
self.assertIsNone(_detect_branch_from_env())
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class TestGetGitInfo(unittest.TestCase):
|
|
52
|
+
|
|
53
|
+
def _create_repo_with_commit(self):
|
|
54
|
+
tempdir = tempfile.TemporaryDirectory()
|
|
55
|
+
self.addCleanup(tempdir.cleanup)
|
|
56
|
+
repo = Repo.init(tempdir.name)
|
|
57
|
+
file_path = Path(tempdir.name) / "tracked.txt"
|
|
58
|
+
file_path.write_text("data\n", encoding="utf-8")
|
|
59
|
+
repo.index.add([str(file_path)])
|
|
60
|
+
actor = Actor("Tester", "tester@example.com")
|
|
61
|
+
repo.index.commit("initial", author=actor, committer=actor)
|
|
62
|
+
return tempdir.name, repo
|
|
63
|
+
|
|
64
|
+
def test_returns_reason_when_path_not_repo(self):
|
|
65
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
66
|
+
info = get_git_info(tmpdir)
|
|
67
|
+
|
|
68
|
+
self.assertIsNone(info.commit)
|
|
69
|
+
self.assertIsNone(info.branch)
|
|
70
|
+
self.assertIsNone(info.remote_url)
|
|
71
|
+
self.assertEqual(type(info.extra), GitExtraReason)
|
|
72
|
+
self.assertEqual(info.extra.reason, "not_a_git_repo")
|
|
73
|
+
|
|
74
|
+
def test_collects_repo_branch_commit_and_remote(self):
|
|
75
|
+
path, repo = self._create_repo_with_commit()
|
|
76
|
+
repo.create_remote("origin", "git@github.com:org/project.git")
|
|
77
|
+
|
|
78
|
+
info = get_git_info(path)
|
|
79
|
+
|
|
80
|
+
self.assertEqual(info.commit, repo.head.commit.hexsha)
|
|
81
|
+
self.assertEqual(info.branch, repo.active_branch.name)
|
|
82
|
+
self.assertEqual(info.remote_url, "https://github.com/org/project.git")
|
|
83
|
+
self.assertEqual(type(info.extra), GitExtraInfo)
|
|
84
|
+
self.assertEqual(info.extra.branch_source, "repo")
|
|
85
|
+
self.assertEqual(info.extra.git_dir, repo.git_dir)
|
|
86
|
+
self.assertEqual(info.extra.working_tree_dir, repo.working_tree_dir)
|
|
87
|
+
|
|
88
|
+
def test_detached_head_uses_env_branch(self):
|
|
89
|
+
path, repo = self._create_repo_with_commit()
|
|
90
|
+
repo.git.checkout(repo.head.commit.hexsha) # detach HEAD
|
|
91
|
+
|
|
92
|
+
with unittest.mock.patch.dict(
|
|
93
|
+
os.environ, {"GITHUB_REF_NAME": "ci-branch"}, clear=True
|
|
94
|
+
):
|
|
95
|
+
info = get_git_info(path)
|
|
96
|
+
|
|
97
|
+
self.assertEqual(info.branch, "ci-branch")
|
|
98
|
+
self.assertEqual(info.extra.branch_source, "env")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
if __name__ == "__main__":
|
|
102
|
+
unittest.main()
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import unittest
|
|
2
|
+
from unittest.mock import patch
|
|
3
|
+
|
|
4
|
+
from kubernetes import client
|
|
5
|
+
|
|
6
|
+
from q8s.plugins.job import JobPlugin
|
|
7
|
+
from q8s.plugins.utils.git_info import GitInfo
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class TestJobPlugin(unittest.TestCase):
|
|
11
|
+
|
|
12
|
+
@patch("q8s.plugins.job.get_git_info")
|
|
13
|
+
@patch("q8s.plugins.job.os.getcwd")
|
|
14
|
+
def test_patch_environment_appends_git_metadata(
|
|
15
|
+
self,
|
|
16
|
+
mock_getcwd,
|
|
17
|
+
mock_get_git_info,
|
|
18
|
+
):
|
|
19
|
+
mock_getcwd.return_value = "/tmp/repo"
|
|
20
|
+
mock_get_git_info.return_value = GitInfo(
|
|
21
|
+
commit="abc123",
|
|
22
|
+
branch="main",
|
|
23
|
+
remote_url="https://example.com/repo.git",
|
|
24
|
+
extra={},
|
|
25
|
+
)
|
|
26
|
+
env = [client.V1EnvVar(name="EXISTING", value="1")]
|
|
27
|
+
|
|
28
|
+
plugin = JobPlugin()
|
|
29
|
+
result = plugin.patch_environment_with_git_info(env)
|
|
30
|
+
|
|
31
|
+
self.assertIs(result, env)
|
|
32
|
+
mock_get_git_info.assert_called_once_with("/tmp/repo")
|
|
33
|
+
env_pairs = {(var.name, var.value) for var in env}
|
|
34
|
+
self.assertIn(("Q8S_GIT_COMMIT", "abc123"), env_pairs)
|
|
35
|
+
self.assertIn(("MLFLOW_GIT_COMMIT", "abc123"), env_pairs)
|
|
36
|
+
self.assertIn(("Q8S_GIT_BRANCH", "main"), env_pairs)
|
|
37
|
+
self.assertIn(("MLFLOW_GIT_BRANCH", "main"), env_pairs)
|
|
38
|
+
self.assertIn(("Q8S_GIT_REPO_URL", "https://example.com/repo.git"), env_pairs)
|
|
39
|
+
self.assertIn(
|
|
40
|
+
("MLFLOW_GIT_REPO_URL", "https://example.com/repo.git"), env_pairs
|
|
41
|
+
)
|
|
42
|
+
self.assertIn(("GIT_PYTHON_REFRESH", "quiet"), env_pairs)
|
|
43
|
+
|
|
44
|
+
@patch("q8s.plugins.job.get_git_info")
|
|
45
|
+
def test_patch_environment_handles_missing_git_metadata(self, mock_get_git_info):
|
|
46
|
+
mock_get_git_info.return_value = GitInfo(
|
|
47
|
+
commit=None,
|
|
48
|
+
branch=None,
|
|
49
|
+
remote_url=None,
|
|
50
|
+
extra={"reason": "Not a git repository"},
|
|
51
|
+
)
|
|
52
|
+
env: list[client.V1EnvVar] = []
|
|
53
|
+
|
|
54
|
+
plugin = JobPlugin()
|
|
55
|
+
result = plugin.patch_environment_with_git_info(env)
|
|
56
|
+
|
|
57
|
+
self.assertIs(result, env)
|
|
58
|
+
self.assertEqual(
|
|
59
|
+
[(var.name, var.value) for var in env], [("GIT_PYTHON_REFRESH", "quiet")]
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
if __name__ == "__main__":
|
|
64
|
+
unittest.main()
|
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import unittest
|
|
2
2
|
from unittest.mock import patch, MagicMock
|
|
3
3
|
from kubernetes import client
|
|
4
|
-
|
|
4
|
+
|
|
5
|
+
# from q8s.plugins.job_template_spec import CPUJobTemplatePlugin, CUDAJobTemplatePlugin
|
|
5
6
|
from q8s.enums import Target
|
|
7
|
+
from q8s.plugins.cpu_job import CPUJobTemplatePlugin
|
|
8
|
+
from q8s.plugins.cuda_job import CUDAJobTemplatePlugin
|
|
6
9
|
|
|
7
10
|
|
|
8
11
|
class TestCPUandGPUJobTemplatePlugin(unittest.TestCase):
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|