robo-lib 0.0.11__tar.gz → 1.0.1__tar.gz

robo_lib-1.0.1/.gitignore

@@ -0,0 +1 @@
+__pycache__/

{robo_lib-0.0.11 → robo_lib-1.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: robo_lib
-Version: 0.0.11
+Version: 1.0.1
 Summary: A package to create, configure, and train transformer models.
 Project-URL: Homepage, https://github.com/hamburgerfish/robo_pack
 Project-URL: Issues, https://github.com/hamburgerfish/robo_pack/issues
@@ -13,6 +13,7 @@ Requires-Python: >=3.8
 Requires-Dist: numpy
 Requires-Dist: tokenizers
 Requires-Dist: torch
+Requires-Dist: typing
 Description-Content-Type: text/markdown
 
 # robo-lib
@@ -83,10 +84,8 @@ proc.process_list(
     save_path="data/training",
     dec_data=french_train,
     dec_max_block_size=100,
-    dec_block_size_exceeded_policy="skip",
     enc_data=english_train,
-    enc_max_block_size=100,
-    enc_block_size_exceeded_policy="skip"
+    enc_max_block_size=100
 )
 
 # process and save validation data as data/validation*.pt
@@ -94,10 +93,8 @@ proc.process_list(
     save_path="data/validation",
     dec_data=french_val,
     dec_max_block_size=100,
-    dec_block_size_exceeded_policy="skip",
     enc_data=english_val,
-    enc_max_block_size=100,
-    enc_block_size_exceeded_policy="skip"
+    enc_max_block_size=100
 )
 ```
 - The `RoboConstructor` class is used to create and configure transformer models before trainin.
@@ -128,14 +125,8 @@ robo.train_robo(
     max_iters=20000,
     eval_interval=200,
     batch_size=128,
-    dec_training_path="data/training_decoder_data.pt",
-    dec_eval_path="data/validation_decoder_data.pt",
-    dec_training_masks_path="data/training_decoder_mask_data.pt",
-    dec_eval_masks_path="data/validation_decoder_mask_data.pt",
-    enc_training_path="data/training_encoder_data.pt",
-    enc_eval_path="data/validation_encoder_data.pt",
-    enc_training_masks_path="data/training_encoder_mask_data.pt",
-    enc_eval_masks_path="data/validation_encoder_mask_data.pt",
+    training_dir_path="data/training",
+    eval_dir_path="data/validation",
     dec_tokenizer=decoder_tok,
     save_path="models/eng_to_fr_robo.pkl"
 )
@@ -223,8 +214,8 @@ robo.train(
     max_iters=20000,
     eval_interval=200,
     batch_size=64,
-    dec_training_path="data/shakespeare_train_decoder_data.pt",
-    dec_eval_path="data/shakespeare_valid_decoder_data.pt",
+    training_dir_path="data/shakespeare_train",
+    eval_dir_path="data/shakespeare_valid",
     dec_tokenizer=tok,
     save_path="models/shakespeare_robo.pkl"
 )

{robo_lib-0.0.11 → robo_lib-1.0.1}/README.md

@@ -66,10 +66,8 @@ proc.process_list(
     save_path="data/training",
     dec_data=french_train,
     dec_max_block_size=100,
-    dec_block_size_exceeded_policy="skip",
     enc_data=english_train,
-    enc_max_block_size=100,
-    enc_block_size_exceeded_policy="skip"
+    enc_max_block_size=100
 )
 
 # process and save validation data as data/validation*.pt
@@ -77,10 +75,8 @@ proc.process_list(
     save_path="data/validation",
     dec_data=french_val,
     dec_max_block_size=100,
-    dec_block_size_exceeded_policy="skip",
     enc_data=english_val,
-    enc_max_block_size=100,
-    enc_block_size_exceeded_policy="skip"
+    enc_max_block_size=100
 )
 ```
 - The `RoboConstructor` class is used to create and configure transformer models before trainin.
@@ -111,14 +107,8 @@ robo.train_robo(
     max_iters=20000,
     eval_interval=200,
     batch_size=128,
-    dec_training_path="data/training_decoder_data.pt",
-    dec_eval_path="data/validation_decoder_data.pt",
-    dec_training_masks_path="data/training_decoder_mask_data.pt",
-    dec_eval_masks_path="data/validation_decoder_mask_data.pt",
-    enc_training_path="data/training_encoder_data.pt",
-    enc_eval_path="data/validation_encoder_data.pt",
-    enc_training_masks_path="data/training_encoder_mask_data.pt",
-    enc_eval_masks_path="data/validation_encoder_mask_data.pt",
+    training_dir_path="data/training",
+    eval_dir_path="data/validation",
     dec_tokenizer=decoder_tok,
     save_path="models/eng_to_fr_robo.pkl"
 )
@@ -206,8 +196,8 @@ robo.train(
     max_iters=20000,
     eval_interval=200,
     batch_size=64,
-    dec_training_path="data/shakespeare_train_decoder_data.pt",
-    dec_eval_path="data/shakespeare_valid_decoder_data.pt",
+    training_dir_path="data/shakespeare_train",
+    eval_dir_path="data/shakespeare_valid",
     dec_tokenizer=tok,
     save_path="models/shakespeare_robo.pkl"
 )

{robo_lib-0.0.11 → robo_lib-1.0.1}/pyproject.toml

@@ -4,14 +4,14 @@ build-backend = "hatchling.build"
 
 [project]
 name = "robo_lib"
-version = "0.0.11"
+version = "1.0.1"
 authors = [
   { name="Erik Papp", email="erik3papp@gmail.com" },
 ]
 description = "A package to create, configure, and train transformer models."
 readme = "README.md"
 requires-python = ">=3.8"
-dependencies = ["torch", "tokenizers", "numpy"]
+dependencies = ["torch", "tokenizers", "numpy", "typing"]
 classifiers = [
     "Programming Language :: Python :: 3",
     "License :: OSI Approved :: MIT License",

{robo_lib-0.0.11 → robo_lib-1.0.1}/robo_lib/__init__.py

@@ -1,8 +1,7 @@
 from .components import TokenizerConstructor as TokenizerConstructor
 from .components import create_mask as create_mask
-from .components import pad as pad
-from .components import process_row as process_row
-from .components import scan_max_block_size as scan_max_block_size
+from .components import pre_process_data as pre_process_data
+from .components import safe_stack as safe_stack
 from .components import DataProcessor as DataProcessor
 from .components import get_valid_samples as get_valid_samples
 from .components import get_batch as get_batch