optimum-rbln 0.1.15__py3-none-any.whl → 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. optimum/rbln/__init__.py +26 -33
  2. optimum/rbln/__version__.py +2 -2
  3. optimum/rbln/diffusers/__init__.py +4 -0
  4. optimum/rbln/{modeling_diffusers.py → diffusers/modeling_diffusers.py} +66 -24
  5. optimum/rbln/diffusers/models/__init__.py +2 -0
  6. optimum/rbln/diffusers/models/autoencoders/autoencoder_kl.py +38 -12
  7. optimum/rbln/diffusers/models/autoencoders/vae.py +0 -1
  8. optimum/rbln/diffusers/models/controlnet.py +1 -1
  9. optimum/rbln/diffusers/models/transformers/transformer_sd3.py +1 -1
  10. optimum/rbln/diffusers/models/unets/unet_2d_condition.py +5 -7
  11. optimum/rbln/diffusers/pipelines/__init__.py +1 -0
  12. optimum/rbln/diffusers/pipelines/controlnet/multicontrolnet.py +8 -7
  13. optimum/rbln/diffusers/pipelines/controlnet/pipeline_controlnet.py +17 -2
  14. optimum/rbln/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py +17 -2
  15. optimum/rbln/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py +17 -2
  16. optimum/rbln/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py +17 -2
  17. optimum/rbln/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +1 -2
  18. optimum/rbln/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +1 -2
  19. optimum/rbln/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +1 -2
  20. optimum/rbln/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3.py +1 -2
  21. optimum/rbln/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_img2img.py +1 -2
  22. optimum/rbln/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_inpaint.py +1 -2
  23. optimum/rbln/diffusers/pipelines/stable_diffusion_xl/__init__.py +23 -0
  24. optimum/rbln/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py +1 -2
  25. optimum/rbln/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py +1 -2
  26. optimum/rbln/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py +1 -2
  27. optimum/rbln/modeling.py +13 -347
  28. optimum/rbln/modeling_base.py +24 -4
  29. optimum/rbln/modeling_config.py +31 -7
  30. optimum/rbln/ops/__init__.py +26 -0
  31. optimum/rbln/ops/attn.py +221 -0
  32. optimum/rbln/ops/flash_attn.py +70 -0
  33. optimum/rbln/ops/kv_cache_update.py +69 -0
  34. optimum/rbln/transformers/__init__.py +20 -0
  35. optimum/rbln/{modeling_alias.py → transformers/modeling_alias.py} +5 -1
  36. optimum/rbln/transformers/modeling_generic.py +385 -0
  37. optimum/rbln/transformers/models/auto/__init__.py +23 -0
  38. optimum/rbln/transformers/models/auto/modeling_auto.py +0 -1
  39. optimum/rbln/transformers/models/bart/__init__.py +0 -1
  40. optimum/rbln/transformers/models/bart/bart_architecture.py +107 -464
  41. optimum/rbln/transformers/models/bart/modeling_bart.py +8 -4
  42. optimum/rbln/transformers/models/clip/modeling_clip.py +1 -1
  43. optimum/rbln/transformers/models/decoderonly/__init__.py +0 -7
  44. optimum/rbln/transformers/models/decoderonly/decoderonly_architecture.py +329 -328
  45. optimum/rbln/transformers/models/decoderonly/modeling_decoderonly.py +92 -107
  46. optimum/rbln/transformers/models/exaone/exaone_architecture.py +2 -3
  47. optimum/rbln/transformers/models/gemma/gemma_architecture.py +1 -1
  48. optimum/rbln/transformers/models/gpt2/gpt2_architecture.py +10 -10
  49. optimum/rbln/transformers/models/gpt2/modeling_gpt2.py +1 -1
  50. optimum/rbln/transformers/models/llama/llama_architecture.py +0 -1
  51. optimum/rbln/transformers/models/llava_next/modeling_llava_next.py +1 -0
  52. optimum/rbln/transformers/models/midm/midm_architecture.py +11 -11
  53. optimum/rbln/transformers/models/midm/modeling_midm.py +0 -1
  54. optimum/rbln/transformers/models/mistral/mistral_architecture.py +0 -1
  55. optimum/rbln/transformers/models/phi/phi_architecture.py +2 -3
  56. optimum/rbln/transformers/models/qwen2/qwen2_architecture.py +0 -1
  57. optimum/rbln/transformers/models/seq2seq/modeling_seq2seq.py +57 -57
  58. optimum/rbln/transformers/models/seq2seq/seq2seq_architecture.py +498 -0
  59. optimum/rbln/transformers/models/t5/__init__.py +0 -1
  60. optimum/rbln/transformers/models/t5/modeling_t5.py +5 -2
  61. optimum/rbln/transformers/models/t5/t5_architecture.py +106 -448
  62. optimum/rbln/transformers/models/whisper/generation_whisper.py +42 -0
  63. optimum/rbln/transformers/models/whisper/modeling_whisper.py +77 -54
  64. optimum/rbln/transformers/models/whisper/whisper_architecture.py +219 -312
  65. optimum/rbln/transformers/utils/rbln_quantization.py +0 -1
  66. optimum/rbln/utils/decorator_utils.py +51 -15
  67. optimum/rbln/utils/import_utils.py +7 -0
  68. optimum/rbln/utils/logging.py +37 -0
  69. optimum/rbln/utils/model_utils.py +0 -1
  70. optimum/rbln/utils/runtime_utils.py +9 -3
  71. optimum/rbln/utils/save_utils.py +17 -0
  72. optimum/rbln/utils/submodule.py +23 -0
  73. {optimum_rbln-0.1.15.dist-info → optimum_rbln-0.2.0.dist-info}/METADATA +37 -26
  74. {optimum_rbln-0.1.15.dist-info → optimum_rbln-0.2.0.dist-info}/RECORD +76 -72
  75. optimum_rbln-0.2.0.dist-info/licenses/LICENSE +288 -0
  76. optimum/rbln/transformers/cache_utils.py +0 -107
  77. optimum/rbln/utils/timer_utils.py +0 -43
  78. optimum_rbln-0.1.15.dist-info/licenses/LICENSE +0 -201
  79. {optimum_rbln-0.1.15.dist-info → optimum_rbln-0.2.0.dist-info}/WHEEL +0 -0
@@ -0,0 +1,288 @@
1
+ Software User License Agreement
2
+
3
+ This User License Agreement (this "Agreement") is a binding agreement between
4
+ [Rebellions Inc.], a Korean company with its office located at [102-801, 239,
5
+ Jeongjail-ro, Bundang-gu, Seongnam-si, Gyeonggi-do, Republic of Korea], ("Licensor")
6
+ and you, your employer, your employees, or other entity for whose benefit you act as
7
+ applicable, as the licensee of the Software (“You” or "Licensee"). DO NOT download,
8
+ install, access, copy, or use the entirety or any portion of the Software until
9
+ You have read and agreed to the terms and conditions of this Agreement.
10
+
11
+ Licensor provides the Software solely on the terms and conditions set forth in
12
+ this Agreement and on the condition that Licensee accepts and complies with them.
13
+ By downloading, installing, accessing, copying, or using the Software You
14
+ (a) accept this Agreement and agree that licensee is legally bound by its terms;
15
+ and (b) represent and warrant that: (i) You are of legal age to enter into a
16
+ binding agreement; and (ii) if Licensee is a corporation, governmental organization,
17
+ or other legal entity, You have the right, power, and authority to enter into this
18
+ Agreement on behalf of Licensee and bind Licensee to its terms. If Licensee does
19
+ not agree to the terms of this Agreement, Licensor will not and does not license
20
+ the Software to Licensee and You must not download, install, access, copy, use or
21
+ otherwise utilize the Software or Documentation.
22
+
23
+ This Agreement expressly excludes any right, concerning any software that Licensee
24
+ did not acquire lawfully or that is not a legitimate, authorized copy of Licensor's
25
+ Software.
26
+
27
+ 1. Definitions. For purposes of this Agreement, the following terms have the
28
+ following meanings:
29
+
30
+ "Documentation" means user manuals, technical manuals, and any other materials
31
+ provided by Licensor, in printed, electronic, or other form, that describe the
32
+ installation, operation, use, or technical specifications of the Software.
33
+
34
+ “Hardware” means cloud service provider’s proprietary device(s) supplied by
35
+ Licensor and used for cloud services and which use right is purchased by
36
+ Licensee for the purpose of its direct use or to provide subsequent services
37
+ to end-customers.
38
+
39
+ "Intellectual Property Rights" means any and all registered and unregistered
40
+ rights granted, applied for, or otherwise now or hereafter in existence under
41
+ or related to any patent, copyright, trademark, trade secret, database
42
+ protection, or other intellectual property rights laws, and all similar or
43
+ equivalent rights or forms of protection, in any part of the world.
44
+
45
+ "Licensee" has the meaning set forth in the preamble.
46
+
47
+ "Licensor" has the meaning set forth in the preamble.
48
+
49
+ "PERSON" MEANS AN INDIVIDUAL, CORPORATION, PARTNERSHIP, JOINT VENTURE, LIMITED
50
+ LIABILITY COMPANY, GOVERNMENTAL AUTHORITY, UNINCORPORATED ORGANIZATION, TRUST,
51
+ ASSOCIATION, OR OTHER ENTITY.
52
+
53
+ "Software" means any one or more of the developer kits, libraries, runtimes
54
+ and drivers, together with any necessary updates and accompanying Documentation,
55
+ that are provided by Licensor in connection with the use and/or implementation
56
+ of one or more of Hardware as provided under this Agreement, excluding any
57
+ Third Party files, programs, or other materials.
58
+
59
+ "Term" has the meaning set forth in Section 11.
60
+
61
+ "Third Party" means any Person other than Licensee or Licensor.
62
+
63
+ 2. License Grant and Scope. Subject to and conditioned upon Licensee's strict
64
+ compliance with all terms and conditions set forth in this Agreement, Licensor
65
+ hereby grants to Licensee a non-exclusive, non-transferable, non-sublicensable,
66
+ limited license during the Term to use the Software and Documentation, solely
67
+ as set forth in this Section 2 and subject to all conditions and limitations set
68
+ forth in Section 3 or elsewhere in this Agreement. This license grants Licensee to:
69
+
70
+ (a) Download, access, copy, and install in accordance with the Documentation
71
+ one (1) copy of the Software on one (1) computer owned or leased, and controlled
72
+ by,Licensee for the sole purpose of utilizing the Hardware. Each such computer
73
+ shall be for a single user authorized by Licensee, unless agreed otherwise. All
74
+ copies of the Software made by the Licensee:
75
+
76
+ (i) will be the exclusive property of the Licensor;
77
+
78
+ (ii) will be subject to the terms and conditions of this Agreement; and
79
+
80
+ (iii) must include all trademark, copyright, patent, and other Intellectual
81
+ Property Rights notices contained in the original.
82
+
83
+ (b) Use and run the Software as properly installed in accordance with this
84
+ Agreement and the Documentation, solely as set forth in the Documentation and
85
+ solely for Licensee's internal business purposes. No source code of the Software
86
+ shall be made available or accessible to the Licensee except to the extent
87
+ explicitly permitted and necessary for Licensee’s use of the Software under
88
+ this Agreement.
89
+
90
+ 3. Use Restrictions. Licensee shall not, and shall require its authorized users not
91
+ to, directly or indirectly:
92
+
93
+ (a) use (including make any copies of) the Software or Documentation beyond the
94
+ scope of the license granted under Section 2;
95
+
96
+ (b) use the Software separately from the Hardware except explicitly permitted
97
+ by the Licensor or independently agreed between Licensor and Licensee;
98
+
99
+ (c) provide any other Person, including any subcontractor, independent contractor,
100
+ affiliate, or service provider of Licensee, with access to or use of the Software
101
+ or Documentation;
102
+
103
+ (d) modify, translate, adapt, or otherwise create derivative works or improvements,
104
+ whether or not patentable, of the Software or Documentation or any part thereof;
105
+
106
+ (e) combine the Software or any part thereof with, or incorporate the Software or
107
+ any part thereof in, any other programs;
108
+
109
+ (f) reverse engineer, disassemble, decompile, decode, or otherwise attempt to
110
+ derive or gain access to the source code of the Software or any part thereof;
111
+
112
+ (g) remove, delete, alter, or obscure any trademarks or any copyright, trademark,
113
+ patent, or other intellectual property or proprietary rights notices provided on
114
+ or with the Software or Documentation, including any copy thereof;
115
+
116
+ (h) except as expressly set forth in Section 2(a), copy the Software or
117
+ Documentation, in whole or in part;
118
+
119
+ (i) rent, lease, lend, sell, sublicense, assign, distribute, publish, transfer,
120
+ or otherwise make available the Software, or any features or functionality of the
121
+ Software, to any unauthorized Third Party by any way for any reason;
122
+
123
+ (j) use the Software or Documentation in, or in association with, the design,
124
+ construction, maintenance, or operation of any hazardous environments or systems;
125
+
126
+ (k) use the Software or Documentation in violation of any law, regulation, or rule; or
127
+
128
+ (l) use the Software or Documentation for purposes of competitive analysis of the
129
+ Software, the development of a competing software product or service, or any other
130
+ purpose that is to the Licensor's commercial disadvantage.
131
+
132
+ 4. Responsibility for Use of Software. Licensee is responsible and liable for all uses
133
+ of the Software and Documentation through access thereto provided by Licensee, directly
134
+ or indirectly. Specifically, and without limiting the generality of the foregoing,
135
+ Licensee is responsible and liable for all actions and failures to take required
136
+ actions with respect to the Software and Documentation by any Person to whom Licensee
137
+ may provide access to or use of the Software and/or Documentation, whether such access
138
+ or use is permitted by or in violation of this Agreement.
139
+
140
+ 5. Maintenance and Support Package. Licensee acknowledges that the scope of this Agreement
141
+ does not include any right for Licensee to receive maintenance, support, or mapping
142
+ services with regard to the Software. Licensor has full discretion to offer maintenance,
143
+ support, and/or mapping services for the Software on a separate subscription basis (the
144
+ “Support Subscription”).
145
+
146
+ 6. Intellectual Property Rights. Licensee acknowledges and agrees that the Software and
147
+ Documentation are provided under license, and not sold, to Licensee. Licensee does not
148
+ acquire any ownership interest in the Software or Documentation under this Agreement,
149
+ or any other rights thereto, other than to use the same in accordance with the license
150
+ granted and subject to all terms, conditions, and restrictions under this Agreement.
151
+ Licensor and its licensors and service providers reserve and shall retain their entire
152
+ right, title, and interest in and to the Software and all Intellectual Property Rights
153
+ arising out of or relating to the Software, except as expressly granted to the Licensee
154
+ in this Agreement. Licensee shall safeguard all Software (including all copies thereof)
155
+ from infringement, misappropriation, theft, misuse, or unauthorized access.
156
+
157
+ 7. Term and Termination.
158
+
159
+ (a) This Agreement and the license granted hereunder shall remain in effect until
160
+ terminated as set forth herein (the "Term").
161
+
162
+ (b) Licensor may terminate this Agreement, effective immediately, if its separate
163
+ agreement for use of the Hardware is terminated.
164
+
165
+ (c) Licensor may terminate this Agreement without cause and effective immediately
166
+ upon written notice to Licensee.
167
+
168
+ (d) Licensor may terminate this Agreement, effective upon notice to Licensee, if
169
+ Licensee fails to comply with any of the terms and conditions in this Agreement
170
+ during the Term.
171
+
172
+ (e) Upon expiration or termination of this Agreement, the license granted hereunder
173
+ shall also terminate, and Licensee shall cease using and destroy all copies of the
174
+ Software and Documentation.
175
+
176
+ (f) Licensor may suspend the Software licensed hereunder or terminate this Agreement,
177
+ if the Licensee commences, participates or threatens to commence or participate in
178
+ any legal proceeding against Licensor.
179
+
180
+ 8. Exclusion of Warranties. The Software is provided “as is” without any express or
181
+ implied warranty of any kind, including warranties of merchantability, non-infringement,
182
+ or fitness for a particular purpose. Licensor does not warrant or assume responsibility
183
+ for the accuracy or completeness of any information, text, graphics, links, or other
184
+ items within the Software.
185
+
186
+ 9. Limitation of Liability. TO THE FULLEST EXTENT PERMITTED UNDER APPLICABLE LAW:
187
+
188
+ (a) IN NO EVENT WILL LICENSOR OR ITS AFFILIATES, OR ANY OF ITS OR THEIR RESPECTIVE
189
+ LICENSORS OR SERVICE PROVIDERS, BE LIABLE TO LICENSEE OR ANY THIRD PARTY FOR ANY USE,
190
+ INTERRUPTION, DELAY, OR INABILITY TO USE THE SOFTWARE; LOST REVENUES OR PROFITS;
191
+ DELAYS, INTERRUPTION, OR LOSS OF SERVICES, BUSINESS, OR GOODWILL; LOSS OR CORRUPTION
192
+ OF DATA; LOSS RESULTING FROM SYSTEM OR SYSTEM SERVICE FAILURE, MALFUNCTION, OR
193
+ SHUTDOWN; FAILURE TO ACCURATELY TRANSFER, READ, OR TRANSMIT INFORMATION; FAILURE TO
194
+ UPDATE OR PROVIDE CORRECT INFORMATION; SYSTEM INCOMPATIBILITY OR PROVISION OF INCORRECT
195
+ COMPATIBILITY INFORMATION; OR BREACHES IN SYSTEM SECURITY; OR FOR ANY CONSEQUENTIAL,
196
+ INCIDENTAL, INDIRECT, EXEMPLARY, SPECIAL, OR PUNITIVE DAMAGES, WHETHER ARISING OUT
197
+ OF OR IN CONNECTION WITH THIS AGREEMENT, BREACH OF CONTRACT, TORT (INCLUDING NEGLIGENCE),
198
+ OR OTHERWISE, REGARDLESS OF WHETHER SUCH DAMAGES WERE FORESEEABLE AND WHETHER OR NOT THE
199
+ LICENSOR WAS ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
200
+
201
+ (b) THE LIMITATIONS SET FORTH IN SECTION 9(a) SHALL APPLY EVEN IF THE LICENSEE'S
202
+ REMEDIES UNDER THIS AGREEMENT FAIL OF THEIR ESSENTIAL PURPOSE.
203
+
204
+ 10. Open Source Statement. The Software may include open source software (“OSS”) licensed
205
+ pursuant to OSS license agreement(s) identified in the OSS comments in the applicable
206
+ source code file(s) or file header(s) provided with or otherwise associated with the
207
+ Software. Neither Licensee nor any OEM, ODM, customer, distributor or other end user
208
+ may subject any proprietary portion of the Software to any OSS license obligations
209
+ including, without limitation, combining or distributing the Software with OSS in a
210
+ manner that subjects Licensee, the Software or any portion thereof to any OSS license
211
+ obligation. Nothing in this Agreement limits any rights under, or grants rights that
212
+ supersede, the terms of any applicable OSS license.
213
+
214
+ 11. Export Regulation. The Software and Documentation may be subject to export control
215
+ laws of relevant jurisdictions. The Licensee shall not, directly or indirectly, export,
216
+ re-export, or release the Software or Documentation to, or make the Software or
217
+ Documentation accessible from, any jurisdiction or country to which export, re-export,
218
+ or release is prohibited by law, rule, or regulation. The Licensee shall comply with
219
+ all applicable laws, regulations, and rules, and complete all required undertakings
220
+ (including obtaining any necessary export license or other governmental approval),
221
+ prior to exporting, re-exporting, releasing, or otherwise making the Software or
222
+ Documentation available outside the Republic of Korea.
223
+
224
+ 12. Miscellaneous.
225
+
226
+ (a) This Agreement and any dispute arising out of or relating to it will be governed
227
+ by the laws of the Republic of Korea, without regard to conflict of laws principles.
228
+ The Parties exclude the application of the United Nations Convention on Contracts for
229
+ the International Sale of Goods (1980). The Seoul Central District Court will have
230
+ exclusive jurisdiction over any dispute arising out of or relating to this Agreement.
231
+ The Parties consent to personal jurisdiction and venue in the aforementioned court.
232
+ A Party that obtains a judgment against the other Party in the courts identified in
233
+ this section may enforce that judgment in any court that has jurisdiction over the
234
+ Parties.
235
+
236
+ (b) Licensor will not be responsible or liable to Licensee, or deemed in default or
237
+ breach hereunder by reason of any failure or delay in the performance of its obligations
238
+ hereunder where such failure or delay is due to strikes, labor disputes, civil
239
+ disturbances, riot, rebellion, invasion, epidemic, hostilities, war, terrorist attack,
240
+ embargo, natural disaster, acts of God, flood, fire, sabotage, fluctuations or
241
+ non-availability of electrical power, heat, light, air conditioning, or Licensee
242
+ equipment, loss and destruction of property, or any other circumstances or causes
243
+ beyond Licensor's reasonable control.
244
+
245
+ (c) This Agreement, together with all annexes, schedules, and exhibits attached hereto
246
+ and all other documents that are incorporated by reference herein, constitutes the sole
247
+ and entire agreement between Licensee and Licensor with respect to the subject matter
248
+ contained herein, and supersedes all prior and contemporaneous understandings,
249
+ agreements, representations, and warranties, both written and oral, with respect to
250
+ such subject matter.
251
+
252
+ (d) Licensee shall not assign or otherwise transfer any of its rights, or delegate or
253
+ otherwise transfer any of its obligations or performance, under this Agreement, in
254
+ each case whether voluntarily, involuntarily, by operation of law, or otherwise,
255
+ without Licensor's prior written consent, which consent Licensor may give or withhold
256
+ in its sole discretion. For purposes of the preceding sentence, and without limiting
257
+ its generality, any merger, consolidation, or reorganization involving Licensee
258
+ (regardless of whether Licensee is a surviving or disappearing entity) will be deemed
259
+ to be a transfer of rights, obligations, or performance under this Agreement for which
260
+ Licensor's prior written consent is required. No delegation or other transfer will
261
+ relieve Licensee of any of its obligations or performance under this Agreement.
262
+ Any purported assignment, delegation, or transfer in violation of this Section is void.
263
+ Licensor may freely assign or otherwise transfer all or any of its rights, or delegate
264
+ or otherwise transfer all or any of its obligations or performance, under this Agreement
265
+ without Licensee's consent. This Agreement is binding upon and inures to the benefit
266
+ of the parties hereto and their respective permitted successors and assigns.
267
+
268
+ (e) This Agreement is for the sole benefit of the parties hereto and their respective
269
+ successors and permitted assigns and nothing herein, express or implied, is intended
270
+ to or shall confer on any other Person any legal or equitable right, benefit, or remedy
271
+ of any nature whatsoever under or by reason of this Agreement.
272
+
273
+ (f) No waiver by any party of any of the provisions hereof shall be effective unless
274
+ explicitly set forth in writing and signed by the party so waiving. Except as otherwise
275
+ set forth in this Agreement, no failure to exercise, or delay in exercising, any right,
276
+ remedy, power, or privilege arising from this Agreement shall operate or be construed
277
+ as a waiver thereof; nor shall any single or partial exercise of any right, remedy,
278
+ power, or privilege hereunder preclude any other or further exercise thereof or the
279
+ exercise of any other right, remedy, power, or privilege.
280
+
281
+ (g) If any term or provision of this Agreement is invalid, illegal, or unenforceable in
282
+ any jurisdiction, such invalidity, illegality, or unenforceability shall not affect
283
+ any other term or provision of this Agreement or invalidate or render unenforceable
284
+ such term or provision in any other jurisdiction.
285
+
286
+ (h) The headings in this Agreement are for reference only and do not affect the
287
+ interpretation of this Agreement.
288
+
@@ -1,107 +0,0 @@
1
- from typing import Optional, Tuple
2
-
3
- import torch
4
- from transformers.cache_utils import DynamicCache
5
-
6
-
7
- class RebelDynamicCache(DynamicCache):
8
- """
9
- A cache that grows dynamically as more tokens are generated. This is the default for generative models.
10
-
11
- It stores the Key and Value states as a list of tensors, one for each layer. The expected shape for each tensor is
12
- `[batch_size, num_heads, seq_len, head_dim]`.
13
- """
14
-
15
- def __init__(self, position_ids) -> None:
16
- super().__init__()
17
- # batch, _ = position_ids.shape
18
- # current_steps = [position_ids[b][0] for b in range(batch)]
19
- self.current_steps = position_ids[:, 0]
20
-
21
- def assign(
22
- self,
23
- key_states: torch.Tensor,
24
- value_states: torch.Tensor,
25
- layer_idx: int,
26
- ) -> None:
27
- self.key_cache[layer_idx] = key_states.squeeze(2)
28
- self.value_cache[layer_idx] = value_states.squeeze(2)
29
-
30
- def update(
31
- self,
32
- key_states: torch.Tensor,
33
- value_states: torch.Tensor,
34
- layer_idx: int,
35
- batch_idx: int,
36
- read_first_step: Optional[bool] = False,
37
- ) -> Tuple[torch.Tensor, torch.Tensor]:
38
- """
39
- Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx` and the batch 'batch_inx'
40
- based on self.current_step,
41
- """
42
- current_step = self.current_steps[0 if read_first_step else batch_idx]
43
- kend = current_step + key_states.shape[-2]
44
- vend = current_step + value_states.shape[-2]
45
- update_key_states = (
46
- self.key_cache[layer_idx][batch_idx]
47
- .unsqueeze(0)
48
- .unsqueeze(2)
49
- .slice_scatter(key_states, dim=-2, start=current_step, end=kend)
50
- )
51
- update_value_states = (
52
- self.value_cache[layer_idx][batch_idx]
53
- .unsqueeze(0)
54
- .unsqueeze(2)
55
- .slice_scatter(value_states, dim=-2, start=current_step, end=vend)
56
- )
57
-
58
- return update_key_states, update_value_states
59
-
60
- @classmethod
61
- def from_input_format(cls, position_ids, num_hidden_layer, *past_key_values) -> "DynamicCache":
62
- """Converts a cache in the rbln cache format (list of past_kv) into an equivalent `DynamicCache`."""
63
- cache = cls(position_ids)
64
- for layer_idx in range(num_hidden_layer):
65
- key_states = past_key_values[layer_idx * 2]
66
- value_states = past_key_values[layer_idx * 2 + 1]
67
- cache.key_cache.append(key_states)
68
- cache.value_cache.append(value_states)
69
-
70
- return cache
71
-
72
-
73
- class RebelDynamicCache_4D(RebelDynamicCache):
74
- def assign(
75
- self,
76
- keys: torch.Tensor,
77
- values: torch.Tensor,
78
- layer_idx: int,
79
- ) -> None:
80
- self.key_cache[layer_idx] = keys
81
- self.value_cache[layer_idx] = values
82
-
83
- def update(
84
- self,
85
- keys: torch.Tensor,
86
- values: torch.Tensor,
87
- layer_idx: int,
88
- batch_idx: int,
89
- read_first_step: Optional[bool] = False,
90
- ) -> Tuple[torch.Tensor, torch.Tensor]:
91
- """
92
- Updates the cache with the new `keys` and `values` for the layer `layer_idx` and the batch 'batch_inx'
93
- based on self.current_step,
94
- """
95
- current_step = self.current_steps[0 if read_first_step else batch_idx]
96
- kend = current_step + keys.shape[-2]
97
- vend = current_step + values.shape[-2]
98
- update_keys = (
99
- self.key_cache[layer_idx][batch_idx].unsqueeze(0).slice_scatter(keys, dim=-2, start=current_step, end=kend)
100
- )
101
- update_values = (
102
- self.value_cache[layer_idx][batch_idx]
103
- .unsqueeze(0)
104
- .slice_scatter(values, dim=-2, start=current_step, end=vend)
105
- )
106
-
107
- return update_keys, update_values
@@ -1,43 +0,0 @@
1
- import os
2
- from datetime import datetime
3
-
4
- from halo import Halo
5
-
6
- from .logging import get_logger
7
-
8
-
9
- logger = get_logger()
10
-
11
-
12
- def rbln_timer(print_name):
13
- def decorator(function):
14
- def wrapper(*args, **kwargs):
15
- disable = os.getenv("OPTIMUM_RBLN_DISABLE_SPIN", "False").lower() in ("true", "1", "t")
16
- if disable:
17
- logger.info(f"{print_name} ...")
18
-
19
- spinner = Halo(text=f"{print_name} ...", spinner="dots", color="green", enabled=(not disable))
20
- spinner.start()
21
-
22
- # Start timer
23
- tick = datetime.now()
24
- try:
25
- result = function(*args, **kwargs)
26
- except Exception as e:
27
- spinner.fail(f"{print_name} failed.")
28
- raise e
29
-
30
- # Print elapsed time.
31
- if disable:
32
- logger.info(f"{print_name} done. Elasped time: {format_elapsed_time(tick)}")
33
-
34
- spinner.stop()
35
- spinner.succeed(text=f"{print_name} done. Elasped time: {format_elapsed_time(tick)}")
36
- return result
37
-
38
- return wrapper
39
-
40
- def format_elapsed_time(start_time: datetime) -> str:
41
- return str(datetime.now() - start_time)[:7]
42
-
43
- return decorator
@@ -1,201 +0,0 @@
1
- Apache License
2
- Version 2.0, January 2004
3
- http://www.apache.org/licenses/
4
-
5
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
-
7
- 1. Definitions.
8
-
9
- "License" shall mean the terms and conditions for use, reproduction,
10
- and distribution as defined by Sections 1 through 9 of this document.
11
-
12
- "Licensor" shall mean the copyright owner or entity authorized by
13
- the copyright owner that is granting the License.
14
-
15
- "Legal Entity" shall mean the union of the acting entity and all
16
- other entities that control, are controlled by, or are under common
17
- control with that entity. For the purposes of this definition,
18
- "control" means (i) the power, direct or indirect, to cause the
19
- direction or management of such entity, whether by contract or
20
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
- outstanding shares, or (iii) beneficial ownership of such entity.
22
-
23
- "You" (or "Your") shall mean an individual or Legal Entity
24
- exercising permissions granted by this License.
25
-
26
- "Source" form shall mean the preferred form for making modifications,
27
- including but not limited to software source code, documentation
28
- source, and configuration files.
29
-
30
- "Object" form shall mean any form resulting from mechanical
31
- transformation or translation of a Source form, including but
32
- not limited to compiled object code, generated documentation,
33
- and conversions to other media types.
34
-
35
- "Work" shall mean the work of authorship, whether in Source or
36
- Object form, made available under the License, as indicated by a
37
- copyright notice that is included in or attached to the work
38
- (an example is provided in the Appendix below).
39
-
40
- "Derivative Works" shall mean any work, whether in Source or Object
41
- form, that is based on (or derived from) the Work and for which the
42
- editorial revisions, annotations, elaborations, or other modifications
43
- represent, as a whole, an original work of authorship. For the purposes
44
- of this License, Derivative Works shall not include works that remain
45
- separable from, or merely link (or bind by name) to the interfaces of,
46
- the Work and Derivative Works thereof.
47
-
48
- "Contribution" shall mean any work of authorship, including
49
- the original version of the Work and any modifications or additions
50
- to that Work or Derivative Works thereof, that is intentionally
51
- submitted to Licensor for inclusion in the Work by the copyright owner
52
- or by an individual or Legal Entity authorized to submit on behalf of
53
- the copyright owner. For the purposes of this definition, "submitted"
54
- means any form of electronic, verbal, or written communication sent
55
- to the Licensor or its representatives, including but not limited to
56
- communication on electronic mailing lists, source code control systems,
57
- and issue tracking systems that are managed by, or on behalf of, the
58
- Licensor for the purpose of discussing and improving the Work, but
59
- excluding communication that is conspicuously marked or otherwise
60
- designated in writing by the copyright owner as "Not a Contribution."
61
-
62
- "Contributor" shall mean Licensor and any individual or Legal Entity
63
- on behalf of whom a Contribution has been received by Licensor and
64
- subsequently incorporated within the Work.
65
-
66
- 2. Grant of Copyright License. Subject to the terms and conditions of
67
- this License, each Contributor hereby grants to You a perpetual,
68
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
- copyright license to reproduce, prepare Derivative Works of,
70
- publicly display, publicly perform, sublicense, and distribute the
71
- Work and such Derivative Works in Source or Object form.
72
-
73
- 3. Grant of Patent License. Subject to the terms and conditions of
74
- this License, each Contributor hereby grants to You a perpetual,
75
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
- (except as stated in this section) patent license to make, have made,
77
- use, offer to sell, sell, import, and otherwise transfer the Work,
78
- where such license applies only to those patent claims licensable
79
- by such Contributor that are necessarily infringed by their
80
- Contribution(s) alone or by combination of their Contribution(s)
81
- with the Work to which such Contribution(s) was submitted. If You
82
- institute patent litigation against any entity (including a
83
- cross-claim or counterclaim in a lawsuit) alleging that the Work
84
- or a Contribution incorporated within the Work constitutes direct
85
- or contributory patent infringement, then any patent licenses
86
- granted to You under this License for that Work shall terminate
87
- as of the date such litigation is filed.
88
-
89
- 4. Redistribution. You may reproduce and distribute copies of the
90
- Work or Derivative Works thereof in any medium, with or without
91
- modifications, and in Source or Object form, provided that You
92
- meet the following conditions:
93
-
94
- (a) You must give any other recipients of the Work or
95
- Derivative Works a copy of this License; and
96
-
97
- (b) You must cause any modified files to carry prominent notices
98
- stating that You changed the files; and
99
-
100
- (c) You must retain, in the Source form of any Derivative Works
101
- that You distribute, all copyright, patent, trademark, and
102
- attribution notices from the Source form of the Work,
103
- excluding those notices that do not pertain to any part of
104
- the Derivative Works; and
105
-
106
- (d) If the Work includes a "NOTICE" text file as part of its
107
- distribution, then any Derivative Works that You distribute must
108
- include a readable copy of the attribution notices contained
109
- within such NOTICE file, excluding those notices that do not
110
- pertain to any part of the Derivative Works, in at least one
111
- of the following places: within a NOTICE text file distributed
112
- as part of the Derivative Works; within the Source form or
113
- documentation, if provided along with the Derivative Works; or,
114
- within a display generated by the Derivative Works, if and
115
- wherever such third-party notices normally appear. The contents
116
- of the NOTICE file are for informational purposes only and
117
- do not modify the License. You may add Your own attribution
118
- notices within Derivative Works that You distribute, alongside
119
- or as an addendum to the NOTICE text from the Work, provided
120
- that such additional attribution notices cannot be construed
121
- as modifying the License.
122
-
123
- You may add Your own copyright statement to Your modifications and
124
- may provide additional or different license terms and conditions
125
- for use, reproduction, or distribution of Your modifications, or
126
- for any such Derivative Works as a whole, provided Your use,
127
- reproduction, and distribution of the Work otherwise complies with
128
- the conditions stated in this License.
129
-
130
- 5. Submission of Contributions. Unless You explicitly state otherwise,
131
- any Contribution intentionally submitted for inclusion in the Work
132
- by You to the Licensor shall be under the terms and conditions of
133
- this License, without any additional terms or conditions.
134
- Notwithstanding the above, nothing herein shall supersede or modify
135
- the terms of any separate license agreement you may have executed
136
- with Licensor regarding such Contributions.
137
-
138
- 6. Trademarks. This License does not grant permission to use the trade
139
- names, trademarks, service marks, or product names of the Licensor,
140
- except as required for reasonable and customary use in describing the
141
- origin of the Work and reproducing the content of the NOTICE file.
142
-
143
- 7. Disclaimer of Warranty. Unless required by applicable law or
144
- agreed to in writing, Licensor provides the Work (and each
145
- Contributor provides its Contributions) on an "AS IS" BASIS,
146
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
- implied, including, without limitation, any warranties or conditions
148
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
- PARTICULAR PURPOSE. You are solely responsible for determining the
150
- appropriateness of using or redistributing the Work and assume any
151
- risks associated with Your exercise of permissions under this License.
152
-
153
- 8. Limitation of Liability. In no event and under no legal theory,
154
- whether in tort (including negligence), contract, or otherwise,
155
- unless required by applicable law (such as deliberate and grossly
156
- negligent acts) or agreed to in writing, shall any Contributor be
157
- liable to You for damages, including any direct, indirect, special,
158
- incidental, or consequential damages of any character arising as a
159
- result of this License or out of the use or inability to use the
160
- Work (including but not limited to damages for loss of goodwill,
161
- work stoppage, computer failure or malfunction, or any and all
162
- other commercial damages or losses), even if such Contributor
163
- has been advised of the possibility of such damages.
164
-
165
- 9. Accepting Warranty or Additional Liability. While redistributing
166
- the Work or Derivative Works thereof, You may choose to offer,
167
- and charge a fee for, acceptance of support, warranty, indemnity,
168
- or other liability obligations and/or rights consistent with this
169
- License. However, in accepting such obligations, You may act only
170
- on Your own behalf and on Your sole responsibility, not on behalf
171
- of any other Contributor, and only if You agree to indemnify,
172
- defend, and hold each Contributor harmless for any liability
173
- incurred by, or claims asserted against, such Contributor by reason
174
- of your accepting any such warranty or additional liability.
175
-
176
- END OF TERMS AND CONDITIONS
177
-
178
- APPENDIX: How to apply the Apache License to your work.
179
-
180
- To apply the Apache License to your work, attach the following
181
- boilerplate notice, with the fields enclosed by brackets "[]"
182
- replaced with your own identifying information. (Don't include
183
- the brackets!) The text should be enclosed in the appropriate
184
- comment syntax for the file format. We also recommend that a
185
- file or class name and description of purpose be included on the
186
- same "printed page" as the copyright notice for easier
187
- identification within third-party archives.
188
-
189
- Copyright [yyyy] [name of copyright owner]
190
-
191
- Licensed under the Apache License, Version 2.0 (the "License");
192
- you may not use this file except in compliance with the License.
193
- You may obtain a copy of the License at
194
-
195
- http://www.apache.org/licenses/LICENSE-2.0
196
-
197
- Unless required by applicable law or agreed to in writing, software
198
- distributed under the License is distributed on an "AS IS" BASIS,
199
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
- See the License for the specific language governing permissions and
201
- limitations under the License.