elastic-kernel 0.0.3__py3-none-any.whl → 0.0.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- elastic_kernel/kernel.py +1 -7
- {elastic_kernel-0.0.3.dist-info → elastic_kernel-0.0.5.dist-info}/METADATA +4 -4
- {elastic_kernel-0.0.3.dist-info → elastic_kernel-0.0.5.dist-info}/RECORD +15 -15
- elastic_notebook/__init__.py +1 -0
- elastic_notebook/core/common/profile_migration_speed.py +1 -8
- elastic_notebook/core/io/migrate.py +0 -7
- elastic_notebook/core/io/recover.py +3 -6
- elastic_notebook/core/mutation/fingerprint.py +0 -1
- elastic_notebook/core/notebook/checkpoint.py +1 -21
- elastic_notebook/core/notebook/restore_notebook.py +0 -4
- elastic_notebook/elastic_notebook.py +2 -21
- {elastic_kernel-0.0.3.dist-info → elastic_kernel-0.0.5.dist-info}/WHEEL +0 -0
- {elastic_kernel-0.0.3.dist-info → elastic_kernel-0.0.5.dist-info}/entry_points.txt +0 -0
- {elastic_kernel-0.0.3.dist-info → elastic_kernel-0.0.5.dist-info}/licenses/LICENSE +0 -0
- {elastic_kernel-0.0.3.dist-info → elastic_kernel-0.0.5.dist-info}/top_level.txt +0 -0
elastic_kernel/kernel.py
CHANGED
@@ -6,7 +6,7 @@ from logging.handlers import RotatingFileHandler
|
|
6
6
|
|
7
7
|
from ipykernel.ipkernel import IPythonKernel
|
8
8
|
|
9
|
-
from elastic_notebook
|
9
|
+
from elastic_notebook import ElasticNotebook
|
10
10
|
|
11
11
|
|
12
12
|
class JSTFormatter(logging.Formatter):
|
@@ -146,12 +146,6 @@ class ElasticKernel(IPythonKernel):
|
|
146
146
|
"%Y-%m-%d %H:%M:%S.%f",
|
147
147
|
)
|
148
148
|
|
149
|
-
# コンソールハンドラー
|
150
|
-
console_handler = logging.StreamHandler()
|
151
|
-
console_handler.setLevel(log_level)
|
152
|
-
console_handler.setFormatter(formatter)
|
153
|
-
self.logger.addHandler(console_handler)
|
154
|
-
|
155
149
|
# ローテーティングファイルハンドラー
|
156
150
|
rotating_file_handler = RotatingFileHandler(
|
157
151
|
self.log_file_path,
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: elastic-kernel
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.5
|
4
4
|
Summary: Elastic kernel for Jupyter with automatic checkpoint and dependency tracking
|
5
5
|
Author-email: ryutarom <m2002r1028@icloud.com>
|
6
6
|
License: Apache License
|
@@ -257,12 +257,12 @@ Dynamic: license-file
|
|
257
257
|
|
258
258
|
1. ライブラリをインストールする
|
259
259
|
```
|
260
|
-
pip install elastic-
|
260
|
+
uv pip install elastic-kernel
|
261
261
|
```
|
262
262
|
|
263
263
|
2. カーネルをインストールする
|
264
264
|
```
|
265
|
-
|
265
|
+
elastic-kernel install
|
266
266
|
```
|
267
267
|
|
268
268
|
|
@@ -271,7 +271,7 @@ jupyter kernelspec install --user elastic_kernel
|
|
271
271
|
### 自動でアップロードする方法
|
272
272
|
|
273
273
|
```
|
274
|
-
pip install bump-my-version # 初回のみ実行する
|
274
|
+
uv pip install bump-my-version # 初回のみ実行する
|
275
275
|
bump-my-version bump {hogehoge} # コマンドは以下のいずれかから選択する
|
276
276
|
git push --follow-tags # コミットとタグの両方をプッシュする
|
277
277
|
```
|
@@ -1,10 +1,10 @@
|
|
1
1
|
elastic_kernel/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
2
2
|
elastic_kernel/command.py,sha256=w5aj8BHmWPRi5ZCjVXRLvkFMSmUU7dh6WSlMnPAZRnY,1092
|
3
3
|
elastic_kernel/kernel.json,sha256=U-CuoCvvBnwdAL_OQyW0Ob5m7s3pWYjtC1ZCZ7SrHRk,147
|
4
|
-
elastic_kernel/kernel.py,sha256=
|
5
|
-
elastic_kernel-0.0.
|
6
|
-
elastic_notebook/__init__.py,sha256=
|
7
|
-
elastic_notebook/elastic_notebook.py,sha256=
|
4
|
+
elastic_kernel/kernel.py,sha256=zESQKG8akcw2QKFuqJmlyIsVJIIdryD6OKeQLVNdy44,9793
|
5
|
+
elastic_kernel-0.0.5.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
|
6
|
+
elastic_notebook/__init__.py,sha256=KTDqFlUl6XAKlWIulvvr15T_OB1UqJP2etAUN0-iIqM,54
|
7
|
+
elastic_notebook/elastic_notebook.py,sha256=KoGGok2Ikh-U6zv9hQ0DnSPZm8PTYPaHExzs5BpHr5Y,12141
|
8
8
|
elastic_notebook/algorithm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
9
9
|
elastic_notebook/algorithm/baseline.py,sha256=H37jSR00xSqOUfHtXkA3DlBHPw62e1ocxOPiT6tqTzI,723
|
10
10
|
elastic_notebook/algorithm/optimizer_exact.py,sha256=sH1yvJv7CaJ9sobE7VkyaZRT3KRib4yeBZ_zHp1qWPs,4873
|
@@ -13,7 +13,7 @@ elastic_notebook/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
|
|
13
13
|
elastic_notebook/core/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
14
14
|
elastic_notebook/core/common/checkpoint_file.py,sha256=M6vdE_AroFy8WS5X3NQkdEfl3wyVE_07nt1-E3dXHrc,4053
|
15
15
|
elastic_notebook/core/common/profile_graph_size.py,sha256=ys_vfYr7hTA1SXyFCvTFlo-i_qvbjacVBH46p1Yxw8g,1439
|
16
|
-
elastic_notebook/core/common/profile_migration_speed.py,sha256=
|
16
|
+
elastic_notebook/core/common/profile_migration_speed.py,sha256=o8fGH1X01fcjLCsew-uYGn7NBQwMJtE_wMdoO75y5t8,2106
|
17
17
|
elastic_notebook/core/common/profile_variable_size.py,sha256=2GOJWu1pfMiNr_URoEqSlNqikkbKDYzkq73K2otKsFs,2467
|
18
18
|
elastic_notebook/core/graph/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
19
19
|
elastic_notebook/core/graph/cell_execution.py,sha256=RCmfkCfyFbYXmQJu1ACSdll7l7XQnUL_0I6szr0pXF0,1290
|
@@ -22,21 +22,21 @@ elastic_notebook/core/graph/variable_snapshot.py,sha256=HPUz7Tli858rm2m3JBsbBFLY
|
|
22
22
|
elastic_notebook/core/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
23
23
|
elastic_notebook/core/io/adapter.py,sha256=exo6NRzPROtw0Dcq3Ra42bLTkJ-3etW_RWVvD7yW_4w,391
|
24
24
|
elastic_notebook/core/io/filesystem_adapter.py,sha256=8Lo5Sh3q90NxKYFoWEmmGJYoS_d5Hezd3KM5UlcHGKo,758
|
25
|
-
elastic_notebook/core/io/migrate.py,sha256=
|
25
|
+
elastic_notebook/core/io/migrate.py,sha256=lV-H1VKQ2dgRiSl2CBUpkAEnYP7P7kl_6OjLVXs45gE,2969
|
26
26
|
elastic_notebook/core/io/pickle.py,sha256=amOyNoyGdGqgt1J9KRbUfJhzcn-3TqZHwftN8EbcvlI,1815
|
27
|
-
elastic_notebook/core/io/recover.py,sha256=
|
27
|
+
elastic_notebook/core/io/recover.py,sha256=jgCWXZ8ykSnXJ5hsA-NfnTgo2VHOvcFVHNLiPzh7AFQ,1673
|
28
28
|
elastic_notebook/core/mutation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
29
|
-
elastic_notebook/core/mutation/fingerprint.py,sha256=
|
29
|
+
elastic_notebook/core/mutation/fingerprint.py,sha256=BAWwJ3qR4qVMqiMwRAG1UWiAp4JBoTn-CZIBI29Cgvs,5808
|
30
30
|
elastic_notebook/core/mutation/id_graph.py,sha256=nyNRLAuKIfJbFzePrX13GxH6nnfCZaQfBRUJSN0k8IY,4613
|
31
31
|
elastic_notebook/core/mutation/object_hash.py,sha256=P0ow_mbHHuAXK6U9JvJAjASe_ONr1zWa2qlR-pM5u8o,5103
|
32
32
|
elastic_notebook/core/notebook/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
33
|
-
elastic_notebook/core/notebook/checkpoint.py,sha256=
|
33
|
+
elastic_notebook/core/notebook/checkpoint.py,sha256=IphcjWizROIsdDmY_I277CYuyfsxxxDcOKZ2ez9GQYI,6996
|
34
34
|
elastic_notebook/core/notebook/find_input_vars.py,sha256=RjVs45BdPI7uOIXWnOOI84qEqV_1YnCt8Q1r7dGy0mI,4165
|
35
35
|
elastic_notebook/core/notebook/find_output_vars.py,sha256=nAR4R73MPKdN3zgsgOOykD8swp7Lj7J42BU43V_A-Uc,655
|
36
|
-
elastic_notebook/core/notebook/restore_notebook.py,sha256=
|
36
|
+
elastic_notebook/core/notebook/restore_notebook.py,sha256=ALckAX9O2QzlrsMfPEJfd_Yc58qTM-CVu-yN0T4W_Bk,3067
|
37
37
|
elastic_notebook/core/notebook/update_graph.py,sha256=O0QgCFIKxvz9CIfb8nmTxH760LOF4scu0zaOTLNJUXI,1578
|
38
|
-
elastic_kernel-0.0.
|
39
|
-
elastic_kernel-0.0.
|
40
|
-
elastic_kernel-0.0.
|
41
|
-
elastic_kernel-0.0.
|
42
|
-
elastic_kernel-0.0.
|
38
|
+
elastic_kernel-0.0.5.dist-info/METADATA,sha256=QYGNJJ7WfYevJsaOqmVqH3tG_NjEqVjoooMC3BtX4ik,15978
|
39
|
+
elastic_kernel-0.0.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
40
|
+
elastic_kernel-0.0.5.dist-info/entry_points.txt,sha256=FKLcUx5zdRvFVgrvd_Efl3mg5Q9DEe4AEsRhAFSJmxU,63
|
41
|
+
elastic_kernel-0.0.5.dist-info/top_level.txt,sha256=8HRztRxwCCwc-DNqfW6oUOmGcWopm7LI8cYmoRMFZXE,32
|
42
|
+
elastic_kernel-0.0.5.dist-info/RECORD,,
|
elastic_notebook/__init__.py
CHANGED
@@ -0,0 +1 @@
|
|
1
|
+
from .elastic_notebook import ElasticNotebook # noqa
|
@@ -58,12 +58,5 @@ def profile_migration_speed(dirname: str, alpha=1) -> float:
|
|
58
58
|
os.system("rm -rf {}".format(testing_dir))
|
59
59
|
|
60
60
|
migration_speed_bps = total_bytes / (total_read_time + total_write_time * alpha)
|
61
|
-
|
62
|
-
"migration speed (bps) (total_bytes / (total_read_time + total_write_time * alpha)):",
|
63
|
-
migration_speed_bps,
|
64
|
-
)
|
65
|
-
print("|- total bytes:", total_bytes)
|
66
|
-
print("|- total read time:", total_read_time)
|
67
|
-
print("|- total write time:", total_write_time)
|
68
|
-
print("|- alpha:", alpha)
|
61
|
+
|
69
62
|
return migration_speed_bps
|
@@ -78,7 +78,6 @@ def migrate(
|
|
78
78
|
)
|
79
79
|
|
80
80
|
if filename:
|
81
|
-
print("Checkpoint saved to:", filename)
|
82
81
|
write_path = filename
|
83
82
|
else:
|
84
83
|
write_path = FILENAME
|
@@ -90,9 +89,3 @@ def migrate(
|
|
90
89
|
for vs in vs_list:
|
91
90
|
obj_list.append(shell.user_ns[vs.name])
|
92
91
|
dill.dump(obj_list, output_file)
|
93
|
-
# Write the JSON file to the specified location. Uses the default location if a file path isn't specified.
|
94
|
-
# if filename:
|
95
|
-
# print("Checkpoint saved to:", filename)
|
96
|
-
# adapter.write_all(Path(filename), metadata)
|
97
|
-
# else:
|
98
|
-
# adapter.write_all(Path(FILENAME), metadata)
|
@@ -39,13 +39,10 @@ def resume(filename):
|
|
39
39
|
|
40
40
|
return graph, fingerprint_dict, udfs, recomputation_ces, overlapping_vss
|
41
41
|
|
42
|
-
except _pickle.UnpicklingError
|
43
|
-
print(f"Warning: Checkpoint file is corrupted: {e}")
|
42
|
+
except _pickle.UnpicklingError:
|
44
43
|
return DependencyGraph(), {}, set(), {}, []
|
45
|
-
except EOFError
|
46
|
-
print(f"Warning: Checkpoint file is incomplete: {e}")
|
44
|
+
except EOFError:
|
47
45
|
return DependencyGraph(), {}, set(), {}, []
|
48
46
|
|
49
|
-
except Exception
|
50
|
-
print(f"Error loading checkpoint: {e}")
|
47
|
+
except Exception:
|
51
48
|
return DependencyGraph(), {}, set(), {}, []
|
@@ -42,20 +42,15 @@ def checkpoint(
|
|
42
42
|
|
43
43
|
# Retrieve active VSs from the graph. Active VSs are correspond to the latest instances/versions of each variable.
|
44
44
|
active_vss = set()
|
45
|
-
print("---------------------------")
|
46
|
-
print("all variables:")
|
47
|
-
# print(graph.variable_snapshots)
|
48
45
|
for vs_list in graph.variable_snapshots.values():
|
49
46
|
if not vs_list[-1].deleted:
|
50
|
-
print(f"name: {vs_list[-1].name}")
|
51
47
|
active_vss.add(vs_list[-1])
|
52
48
|
|
53
49
|
# Profile the size of each variable defined in the current session.
|
54
50
|
for active_vs in active_vss:
|
55
51
|
# 変数がfingerprint_dictに存在するかチェック
|
56
52
|
if active_vs.name not in fingerprint_dict:
|
57
|
-
|
58
|
-
continue
|
53
|
+
pass
|
59
54
|
|
60
55
|
attr_str = getattr(shell.user_ns[active_vs.name], "__module__", None)
|
61
56
|
# Object is unserializable
|
@@ -132,26 +127,11 @@ def checkpoint(
|
|
132
127
|
write_log_location, notebook_name, optimizer_name
|
133
128
|
)
|
134
129
|
opt_end = time.time()
|
135
|
-
print("---------------------------")
|
136
|
-
print("variables to migrate:")
|
137
|
-
for vs in vss_to_migrate:
|
138
|
-
print(f"name: {vs.name}, size: {vs.size}")
|
139
130
|
|
140
131
|
difference_start = time.time()
|
141
132
|
vss_to_recompute = active_vss - vss_to_migrate
|
142
133
|
difference_end = time.time()
|
143
134
|
|
144
|
-
print("---------------------------")
|
145
|
-
print("variables to recompute:")
|
146
|
-
for vs in vss_to_recompute:
|
147
|
-
print(f"name: {vs.name}, size: {vs.size}")
|
148
|
-
print([vs.name for vs in vss_to_recompute])
|
149
|
-
|
150
|
-
print("---------------------------")
|
151
|
-
print("cells to recompute:")
|
152
|
-
for ce in ces_to_recompute:
|
153
|
-
print(f"cell num: {ce.cell_num}, cell runtime: {ce.cell_runtime}")
|
154
|
-
print(sorted([ce.cell_num + 1 for ce in ces_to_recompute]))
|
155
135
|
|
156
136
|
optimize_end = time.time()
|
157
137
|
if write_log_location:
|
@@ -34,9 +34,6 @@ def restore_notebook(
|
|
34
34
|
recompute_start = time.time()
|
35
35
|
for ce in graph.cell_executions:
|
36
36
|
if ce in ces_to_recompute:
|
37
|
-
# Rerun cell code; suppress stdout when rerunning.
|
38
|
-
print("Rerunning cell", ce.cell_num + 1)
|
39
|
-
|
40
37
|
# 出力を完全に抑制するための設定
|
41
38
|
ipython = get_ipython()
|
42
39
|
|
@@ -68,7 +65,6 @@ def restore_notebook(
|
|
68
65
|
|
69
66
|
# Define output variables in the CE.
|
70
67
|
for pair in variables[ce.cell_num]:
|
71
|
-
print("Declaring variable", pair[0].name)
|
72
68
|
shell.user_ns[pair[0].name] = pair[1]
|
73
69
|
|
74
70
|
recompute_end = time.time()
|
@@ -69,9 +69,6 @@ class ElasticNotebook:
|
|
69
69
|
# Dict for recording overhead of profiling operations.
|
70
70
|
self.profile_dict = {"idgraph": 0, "representation": 0}
|
71
71
|
|
72
|
-
# デバッグフラグ
|
73
|
-
self.debug = False
|
74
|
-
|
75
72
|
# マイグレーションと再計算の変数リスト
|
76
73
|
self._vss_to_migrate = []
|
77
74
|
self._vss_to_recompute = []
|
@@ -95,13 +92,7 @@ class ElasticNotebook:
|
|
95
92
|
"""文字列表現を定義"""
|
96
93
|
return f"マイグレーション対象: {self.vss_to_migrate}\n再計算対象: {self.vss_to_recompute}"
|
97
94
|
|
98
|
-
def set_debug(self, debug):
|
99
|
-
self.debug = debug
|
100
|
-
|
101
95
|
def record_event(self, cell):
|
102
|
-
if self.debug:
|
103
|
-
print("Recording event...")
|
104
|
-
|
105
96
|
pre_execution = set(self.shell.user_ns.keys())
|
106
97
|
|
107
98
|
# Create id trees for output variables
|
@@ -215,11 +206,9 @@ class ElasticNotebook:
|
|
215
206
|
if float(migration_speed) > 0:
|
216
207
|
self.migration_speed_bps = float(migration_speed)
|
217
208
|
self.manual_migration_speed = True
|
218
|
-
else:
|
219
|
-
pass
|
220
|
-
# print("Migration speed is not positive.")
|
221
209
|
except ValueError:
|
222
|
-
|
210
|
+
pass
|
211
|
+
|
223
212
|
self.selector.migration_speed_bps = self.migration_speed_bps
|
224
213
|
|
225
214
|
def set_optimizer(self, optimizer):
|
@@ -247,9 +236,6 @@ class ElasticNotebook:
|
|
247
236
|
|
248
237
|
def checkpoint(self, filename):
|
249
238
|
"""チェックポイントを作成"""
|
250
|
-
if self.debug:
|
251
|
-
print("Checkpointing...")
|
252
|
-
|
253
239
|
# Write overhead metrics to file (for experiments).
|
254
240
|
if self.write_log_location:
|
255
241
|
with open(
|
@@ -312,10 +298,6 @@ class ElasticNotebook:
|
|
312
298
|
)
|
313
299
|
|
314
300
|
def load_checkpoint(self, filename):
|
315
|
-
if self.debug:
|
316
|
-
print("Loading checkpoint...")
|
317
|
-
|
318
|
-
# start_time = time.time()
|
319
301
|
(
|
320
302
|
self.dependency_graph,
|
321
303
|
variables,
|
@@ -333,4 +315,3 @@ class ElasticNotebook:
|
|
333
315
|
self.notebook_name,
|
334
316
|
self.optimizer_name,
|
335
317
|
)
|
336
|
-
# print("Checkpoint load time:", time.time() - start_time)
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|