sapiens-gpt 2.0.1__py3-none-any.whl

sapiens_gpt/__init__.py

@@ -0,0 +1,11 @@
+# This is a standard code of a GPT (Generative Pre-trained Transformer) model, developed by Sapiens Technology®️,
+# which faithfully follows the mathematical structure of the article “Attention Is All You Need” for the construction of the Transformer architecture
+# used in the pattern recognition of the model that is saved. Some optimizations that do not influence the Transformer architecture
+# were applied only to facilitate the adjustments of the parameters and variables of the training, saving, loading, fine-tuning and inference of the pre-trained model.
+# --------------------------> A SAPIENS TECHNOLOGY®️ PRODUCTION) <--------------------------
+from .sapiens_gpt import *
+# This is a standard code of a GPT (Generative Pre-trained Transformer) model, developed by Sapiens Technology®️,
+# which faithfully follows the mathematical structure of the article “Attention Is All You Need” for the construction of the Transformer architecture
+# used in the pattern recognition of the model that is saved. Some optimizations that do not influence the Transformer architecture
+# were applied only to facilitate the adjustments of the parameters and variables of the training, saving, loading, fine-tuning and inference of the pre-trained model.
+# --------------------------> A SAPIENS TECHNOLOGY®️ PRODUCTION) <--------------------------

sapiens_gpt/sapiens_gpt.py

@@ -0,0 +1,803 @@
+# This is a standard code of a GPT (Generative Pre-trained Transformer) model, developed by Sapiens Technology®️,
+# which faithfully follows the mathematical structure of the article “Attention Is All You Need” for the construction of the Transformer architecture
+# used in the pattern recognition of the model that is saved. Some optimizations that do not influence the Transformer architecture
+# were applied only to facilitate the adjustments of the parameters and variables of the training, saving, loading, fine-tuning and inference of the pre-trained model.
+# --------------------------> A SAPIENS TECHNOLOGY®️ PRODUCTION) <--------------------------
+class SapiensGPT:
+    def __init__(self, show_errors=True, display_error_point=False):
+        try:
+            self.__show_errors = bool(show_errors) if type(show_errors) in (bool, int, float) else True
+            self.__display_error_point = bool(display_error_point) if type(display_error_point) in (bool, int, float) else False
+            try:
+                from warnings import filterwarnings
+                from logging import getLogger, ERROR, disable, CRITICAL
+                from os import environ
+                from dotenv import load_dotenv
+                filterwarnings('ignore')
+                filterwarnings('ignore', category=UserWarning, module='torch.distributed')
+                getLogger('torch.distributed.elastic.multiprocessing.redirects').setLevel(ERROR)
+                environ['DISABLE_MODEL_SOURCE_CHECK'] = 'True'
+                load_dotenv()
+                disable(CRITICAL)
+            except: pass
+            from traceback import print_exc
+            self.__print_exc = print_exc
+            self.embedding_dim = 384
+            self.block_size = 500
+            self.batch_size = 32
+            self.number_heads = 6
+            self.number_layers = 6
+            self.dropout = 0.1
+            self.learning_rate = 3e-4
+            self.eval_interval = 500
+            self.epochs = 2000
+            self.string = ''
+            self.precision = 1.0
+            self.tokenizer = 'gpt'
+            self.context_window = 500
+            self.end_tag = '<|end|>'
+            self.validate = 0.0
+            self.max_tokens = 500
+            self.temperature = 0.5
+            self.top_k = 0
+            self.top_p = 1.0
+            self.delay = 0.01
+            self.device = None
+            from torch import cuda, device, backends
+            from torch.utils.data import Dataset, DataLoader
+            from torch.nn import Module, functional as Function, utils
+            from torch import nn as artificial_neural_network, triu, ones
+            from torch import tensor, no_grad, int64, multinomial, cat, topk, where, sort, cumsum, zeros_like, bool as torch_bool, save, load
+            from tiktoken import get_encoding
+            from json import load as json_load
+            from torch import optim
+            from tqdm import tqdm
+            from os import path as os_path, makedirs as os_makedirs
+            try:
+                import torch_xla.core.xla_model as xm
+                self.__xm = xm
+            except ImportError: self.__xm = None
+            if not self.device:
+                if cuda.is_available(): self.device = device('cuda')
+                elif self.__xm is not None: self.device = self.__xm.xla_device()
+                elif backends.mps.is_available(): self.device = device('mps')
+                else: self.device = device('cpu')
+            self.__Dataset = Dataset
+            self.__Module = Module
+            self.__neural_network = artificial_neural_network
+            self.__tensor = tensor
+            self.__triu = triu
+            self.__ones = ones
+            self.__no_grad = no_grad
+            self.__Function = Function
+            self.__int64 = int64
+            self.__multinomial = multinomial
+            self.__cat = cat
+            self.__topk = topk
+            self.__where = where
+            self.__sort = sort
+            self.__cumsum = cumsum
+            self.__zeros_like = zeros_like
+            self.__bool = torch_bool
+            self.__get_encoding = get_encoding
+            self.__json_load = json_load
+            self.__DataLoader = DataLoader
+            self.__optim = optim
+            self.__utils = utils
+            self.__tqdm = tqdm
+            self.__os_path = os_path
+            self.__os_makedirs = os_makedirs
+            self.__save = save
+            self.__load = load
+            self.__model = None
+            self.__encode = None
+            self.__decode = None
+            self.__end_tag = None
+            self.__string = ''
+            self.__vocab_size = 0
+            self.__char_to_idx = {}
+            self.__idx_to_char = {}
+            self.__tokenizer = 'gpt'
+            self.__optimizer = None
+            self.__train = False
+            self.parameters_number = 0
+            class TextDataset(self.__Dataset):
+                def __init__(self, data={}, block_size=0): self.data, self.block_size = data, block_size
+                def __len__(self): return len(self.data) - self.block_size
+                def __getitem__(self, index=0):
+                    input_sequence = self.data[index:index + self.block_size]
+                    target_sequence = self.data[index + 1:index + self.block_size + 1]
+                    return input_sequence, target_sequence
+            class Transformer(self.__Module):
+                def __init__(self, outer=None, vocab_size=0, embedding_dim=0, number_heads=0, number_layers=0, dropout=None, block_size=0):
+                    super().__init__()
+                    self.outer = outer
+                    self.positional_encoding = outer._SapiensGPT__neural_network.Parameter(outer._SapiensGPT__tensor([]).new_zeros(1, block_size, embedding_dim))
+                    self.dropout = outer._SapiensGPT__neural_network.Dropout(dropout)
+                    self.input_embedding = outer._SapiensGPT__neural_network.Embedding(vocab_size, embedding_dim)
+                    self.multi_head_attention = outer._SapiensGPT__neural_network.TransformerDecoder(outer._SapiensGPT__neural_network.TransformerDecoderLayer(d_model=embedding_dim, nhead=number_heads, dropout=dropout), num_layers=number_layers)
+                    self.output_function = outer._SapiensGPT__neural_network.Linear(embedding_dim, vocab_size)
+                    self.block_size = block_size
+                def forward(self, input_tensor=[]):
+                    outer = self.outer
+                    batch_size, sequence_length = input_tensor.size()
+                    positions = self.positional_encoding[:, :sequence_length, :].to(input_tensor.device)
+                    output_embedding = self.dropout(self.input_embedding(input_tensor) + positions)
+                    transposed = output_embedding.transpose(0, 1)
+                    masked_multi_head_attention = outer._SapiensGPT__triu(outer._SapiensGPT__ones(sequence_length, sequence_length, device=input_tensor.device) * float('-inf'), diagonal=1)
+                    add_and_norm = self.multi_head_attention(transposed, transposed, tgt_mask=masked_multi_head_attention)
+                    add_and_norm = add_and_norm.transpose(0, 1)
+                    return self.output_function(add_and_norm)
+            self.__TextDatasets = TextDataset
+            self.__Transformers = Transformer
+        except Exception as error:
+            try:
+                if self.__show_errors:
+                    error_message = 'ERROR in SapiensGPT.__init__: '+str(error)
+                    print(error_message)
+                    try: self.__print_exc() if self.__display_error_point else None
+                    except: pass
+            except: pass
+    def __compute_loss(self, loader=[]):
+        try:
+            self.__model.eval()
+            total_loss = 0
+            with self.__no_grad():
+                for input_batch, target_batch in loader:
+                    input_batch, target_batch = input_batch.to(self.device), target_batch.to(self.device)
+                    logits = self.__model(input_batch)
+                    loss = self.__Function.cross_entropy(logits.view(-1, logits.size(-1)), target_batch.view(-1))
+                    total_loss += loss.item()
+            return total_loss / len(loader)
+        except Exception as error:
+            try:
+                if self.__show_errors:
+                    error_message = 'ERROR in SapiensGPT.__compute_loss: '+str(error)
+                    print(error_message)
+                    try: self.__print_exc() if self.__display_error_point else None
+                    except: pass
+            except: pass
+            return 0
+    def __format_params(self, number_params=0):
+        try:
+            if number_params < 1_000: return f'{number_params}U'
+            elif number_params < 1_000_000: return f'{number_params // 1_000}K'
+            elif number_params < 1_000_000_000: return f'{number_params // 1_000_000}M'
+            elif number_params < 1_000_000_000_000: return f'{number_params // 1_000_000_000}B'
+            else: return f'{number_params // 1_000_000_000_000}T'
+        except Exception as error:
+            try:
+                if self.__show_errors:
+                    error_message = 'ERROR in SapiensGPT.__format_params: '+str(error)
+                    print(error_message)
+                    try: self.__print_exc() if self.__display_error_point else None
+                    except: pass
+            except: pass
+            return f'{number_params}U'
+    def __get_found_end_tag(self, decoded_token='', decoded_tokens='', limits=[]):
+        try:
+            if self.__end_tag is None: return False
+            decoded_token, decoded_tokens, limits = str(decoded_token).strip(), str(decoded_tokens).strip(), list(limits)
+            for limit in ['']+limits+[' ']:
+                if decoded_token.endswith(limit+self.__end_tag) or decoded_tokens.endswith(limit+self.__end_tag): return True
+                elif decoded_token.endswith(limit+self.__end_tag[0]) or decoded_tokens.endswith(limit+self.__end_tag[0]): return True
+            return False
+        except Exception as error:
+            try:
+                if self.__show_errors:
+                    error_message = 'ERROR in SapiensGPT.__get_found_end_tag: '+str(error)
+                    print(error_message)
+                    try: self.__print_exc() if self.__display_error_point else None
+                    except: pass
+            except: pass
+            return False
+    def __generate_tokens_x(self, prompt='', max_tokens=500, temperature=1.0):
+        try:
+            self.__model.eval()
+            encoded_prompt = self.__encode(prompt)
+            input_tensor = self.__tensor(encoded_prompt, dtype=self.__int64).unsqueeze(0).to(self.device)
+            limits = ('.', '\n', '!', '?', ';')
+            with self.__no_grad():
+                tokens_generated, decoded_tokens = 0, ''
+                while True:
+                    conditioned_input = input_tensor[:, -self.block_size:] if input_tensor.size(1) > self.block_size else input_tensor
+                    logits = self.__model(conditioned_input)
+                    logits = logits[:, -1, :] / temperature
+                    output_probabilities = self.__Function.softmax(logits, dim=-1)
+                    shifted_right = self.__multinomial(output_probabilities, num_samples=1)
+                    input_tensor = self.__cat((input_tensor, shifted_right), dim=1)
+                    token = shifted_right.item()
+                    decoded_token, found_end_tag = self.__decode([token]), False
+                    if tokens_generated == 0 and '\n' in decoded_token: continue
+                    tokens_generated += 1
+                    decoded_tokens += decoded_token
+                    found_end_tag = self.__get_found_end_tag(decoded_token=decoded_token, decoded_tokens=decoded_tokens, limits=limits)
+                    if found_end_tag and decoded_token.endswith(self.__end_tag[0]): decoded_token = decoded_token[:-1]
+                    yield decoded_token
+                    if found_end_tag or ((tokens_generated >= max_tokens) and (decoded_token[-1] in limits)) or (tokens_generated >= (max_tokens*2)): break
+        except Exception as error:
+            try:
+                if self.__show_errors:
+                    error_message = 'ERROR in SapiensGPT.__generate_tokens_x: '+str(error)
+                    print(error_message)
+                    try: self.__print_exc() if self.__display_error_point else None
+                    except: pass
+            except: pass
+            return ''
+    def __generate_tokens_y(self, prompt='', max_tokens=500, temperature=1.0, top_k=50, top_p=0.9):
+        try:
+            self.__model.eval()
+            encoded_prompt = self.__encode(prompt)
+            input_tensor = self.__tensor(encoded_prompt, dtype=self.__int64).unsqueeze(0).to(self.device)
+            limits = ('.', '\n', '!', '?', ';')
+            with self.__no_grad():
+                tokens_generated, decoded_tokens = 0, ''
+                while True:
+                    conditioned_input = (input_tensor[:, -self.block_size:] if input_tensor.size(1) > self.block_size else input_tensor)
+                    logits = self.__model(conditioned_input)
+                    logits = logits[:, -1, :] / temperature
+                    if top_k > 0:
+                        top_k = min(top_k, logits.size(-1))
+                        value, _ = self.__topk(logits, top_k)
+                        thresh = value[:, -1].unsqueeze(-1)
+                        logits = self.__where(logits < thresh, self.__tensor(float('-inf')).to(logits), logits)
+                    if top_p < 1.0:
+                        sorted_logits, sorted_index = self.__sort(logits, dim=-1, descending=True)
+                        sorted_probabilities = self.__Function.softmax(sorted_logits, dim=-1)
+                        cumulative_probabilities = self.__cumsum(sorted_probabilities, dim=-1)
+                        sorted_mask = cumulative_probabilities > top_p
+                        sorted_mask[:, 0] = False
+                        mask = self.__zeros_like(logits, dtype=self.__bool)
+                        mask.scatter_(-1, sorted_index, sorted_mask)
+                        logits = logits.masked_fill(mask, float('-inf'))
+                    output_probabilities = self.__Function.softmax(logits, dim=-1)
+                    shifted_right = self.__multinomial(output_probabilities, num_samples=1)
+                    input_tensor = self.__cat((input_tensor, shifted_right), dim=1)
+                    token = shifted_right.item()
+                    decoded_token, found_end_tag = self.__decode([token]), False
+                    if tokens_generated == 0 and '\n' in decoded_token: continue
+                    tokens_generated += 1
+                    decoded_tokens += decoded_token
+                    found_end_tag = self.__get_found_end_tag(decoded_token=decoded_token, decoded_tokens=decoded_tokens, limits=limits)
+                    if found_end_tag and decoded_token.endswith(self.__end_tag[0]): decoded_token = decoded_token[:-1]
+                    yield decoded_token
+                    if found_end_tag or ((tokens_generated >= max_tokens) and (decoded_token[-1] in limits)) or (tokens_generated >= (max_tokens*2)): break
+        except Exception as error:
+            try:
+                if self.__show_errors:
+                    error_message = 'ERROR in SapiensGPT.__generate_tokens_y: '+str(error)
+                    print(error_message)
+                    try: self.__print_exc() if self.__display_error_point else None
+                    except: pass
+            except: pass
+            return ''
+    def __generate_tokens(self, prompt='', max_tokens=500, temperature=1.0, top_k=0, top_p=1.0):
+        try:
+            prompt = '?' if len(str(prompt).strip()) < 1 else str(prompt).strip()
+            def get_last_n_tokens(text='', n=0):
+                if self.__tokenizer == 'sapi': return text[-n:]
+                else:
+                    encoding = self.__get_encoding('gpt2')
+                    tokens = encoding.encode(text)
+                    last_n_tokens = tokens[-n:]
+                    truncated_text = encoding.decode(last_n_tokens)
+                    return truncated_text
+            prompt = get_last_n_tokens(text=prompt, n=self.block_size)
+            if top_k > 0 or top_p < 1.0: return self.__generate_tokens_y(prompt=prompt, max_tokens=max_tokens, temperature=temperature, top_k=top_k, top_p=top_p)
+            else: return self.__generate_tokens_x(prompt=prompt, max_tokens=max_tokens, temperature=temperature)
+        except Exception as error:
+            try:
+                if self.__show_errors:
+                    error_message = 'ERROR in SapiensGPT.__generate_tokens: '+str(error)
+                    print(error_message)
+                    try: self.__print_exc() if self.__display_error_point else None
+                    except: pass
+            except: pass
+            return ''
+    def __addFit(self, prompt='', answer=''):
+        try:
+            prompt = str(prompt).strip()
+            answer = str(answer).strip()
+            if not self.__train:
+                if self.__end_tag is None: self.__end_tag = '<|end|>'
+                self.__string += prompt+'\n'+answer+self.__end_tag+'\n\n'
+            else:
+                if self.__model is None: raise ValueError('Model is not initialized. Call train or loadModel first.')
+                if self.__optimizer is None: self.__optimizer = self.__optim.AdamW(self.__model.parameters(), lr=self.learning_rate)
+                if self.__end_tag is None: formatted = prompt+'\n'+answer+'\n\n'
+                else: formatted = prompt+'\n'+answer+self.__end_tag+'\n\n'
+                encoded = self.__encode(formatted)
+                if len(encoded) > self.block_size: encoded = encoded[:self.block_size]
+                input_tensor = self.__tensor(encoded[:-1], dtype=self.__int64).unsqueeze(0).to(self.device)
+                target_tensor = self.__tensor(encoded[1:], dtype=self.__int64).unsqueeze(0).to(self.device)
+                self.__model.train()
+                logits = self.__model(input_tensor)
+                loss = self.__Function.cross_entropy(logits.view(-1, logits.size(-1)), target_tensor.view(-1))
+                self.__optimizer.zero_grad()
+                loss.backward()
+                self.__utils.clip_grad_norm_(self.__model.parameters(), 1.0)
+                if self.__xm is not None or str(self.device).lower().strip() == 'tpu': self.device = self.__xm.xla_device()
+                if self.__xm is not None: self.__xm.optimizer_step(self.__optimizer)
+                else: self.__optimizer.step()
+            return True
+        except Exception as error:
+            try:
+                if self.__show_errors:
+                    error_message = 'ERROR in SapiensGPT.__addFit: '+str(error)
+                    print(error_message)
+                    try: self.__print_exc() if self.__display_error_point else None
+                    except: pass
+            except: pass
+            return False
+    def train(self, dataset_path='', progress=True):
+        try:
+            training_metrics = {'val_loss': 0.0, 'loss': 0.0, 'generalization_rate': 0.0, 'precision': 0.0}
+            if self.__train: return training_metrics
+            dataset_path = str(dataset_path).strip()
+            string = str(self.string).strip()
+            precision = min((1.0, max((0.0, float(self.precision))))) if type(self.precision) in (bool, int, float) else 1.0
+            tokenizer = str(self.tokenizer).lower().strip()
+            self.block_size = max((1, int(self.context_window))) if type(self.context_window) in (bool, int, float) else 500
+            if self.end_tag is not None and self.__end_tag is None: self.__end_tag = str(self.end_tag)
+            validate = min((1.0, max((0.0, float(self.validate))))) if type(self.validate) in (bool, int, float) else 0.0
+            progress = bool(progress) if type(progress) in (bool, int, float) else True
+            self.embedding_dim = max((1, int(self.embedding_dim))) if type(self.embedding_dim) in (bool, int, float) else 384
+            self.block_size = max((1, int(self.block_size))) if type(self.block_size) in (bool, int, float) else 500
+            self.batch_size = max((1, int(self.batch_size))) if type(self.batch_size) in (bool, int, float) else 32
+            self.number_heads = max((1, int(self.number_heads))) if type(self.number_heads) in (bool, int, float) else 6
+            self.number_layers = max((1, int(self.number_layers))) if type(self.number_layers) in (bool, int, float) else 6
+            self.dropout = max((0, float(self.dropout))) if type(self.dropout) in (bool, int, float) else 0.1
+            self.learning_rate = max((0, float(self.learning_rate))) if type(self.learning_rate) in (bool, int, float) else 3e-4
+            self.eval_interval = max((1, int(self.eval_interval))) if type(self.eval_interval) in (bool, int, float) else 500
+            self.epochs = max((1, int(self.epochs))) if type(self.epochs) in (bool, int, float) else 2000
+            if tokenizer not in ('sapi', 'gpt'): tokenizer = 'gpt'
+            self.__string = str(self.__string+'\n\n'+string).strip()
+            loss_limit = min(1.0, max(0.0, 1.0 - precision))
+            is_txt, is_json, text_data = dataset_path.endswith('.txt'), dataset_path.endswith('.json'), ''
+            def prepare_json(json_data={}):
+                if type(json_data) == dict: pairs = json_data[list(json_data.keys())[0]]
+                else: pairs = json_data
+                if self.__end_tag is None: self.__end_tag = '<|end|>'
+                return '\n\n'.join([str(pair[list(pair.keys())[0]]+'\n'+pair[list(pair.keys())[1]]).replace(self.__end_tag, '').strip()+self.__end_tag for pair in pairs])
+            def is_web_address(url_path=''):
+                url_path = str(url_path).lower().strip()
+                return url_path.startswith('https://') or url_path.startswith('http://') or url_path.startswith('www.')
+            _is_web_address = is_web_address(url_path=dataset_path)
+            if _is_web_address:
+                is_json = True if '.json' in dataset_path.lower() else False
+                def read_remote_file(url_path=''):
+                    from urllib.request import urlopen
+                    with urlopen(url_path) as response: return str(response.read().decode('utf-8', errors='replace').replace('\r\n', '\n').replace('\r', '\n')).strip()
+                text_data = read_remote_file(url_path=dataset_path)
+                if is_json:
+                    def load_json(string_content=''):
+                        json_content = {}
+                        string_content = str(string_content)
+                        try:
+                            from json import loads
+                            json_content = loads(string_content)
+                        except:
+                            from ast import literal_eval
+                            json_content = literal_eval(string_content)
+                        return json_content
+                    json_data = load_json(string_content=text_data)
+                    text_data = prepare_json(json_data=json_data)
+            else:
+                if not is_txt and not is_json and len(self.__string) < 1: raise ValueError('Unsupported file format. Use .txt or .json.')
+                if is_txt:
+                    with open(dataset_path, 'r', encoding='utf-8') as file: text_data = str(file.read()).strip()
+                elif is_json:
+                    with open(dataset_path, 'r', encoding='utf-8') as file: json_data = self.__json_load(file)
+                    text_data = prepare_json(json_data=json_data)
+            if len(self.__string) > 0: text_data += '\n\n' + self.__string
+            text_data = text_data.strip()
+            if tokenizer == 'sapi':
+                chars = sorted(list(set(text_data)))
+                self.__vocab_size = len(chars)
+                self.__char_to_idx = {char: index for index, char in enumerate(chars)}
+                self.__idx_to_char = {index: char for index, char in enumerate(chars)}
+                self.__encode = lambda string: [self.__char_to_idx[char] for char in string]
+                self.__decode = lambda indices: ''.join([self.__idx_to_char[index] for index in indices])
+            else:
+                encode = self.__get_encoding('gpt2')
+                self.__vocab_size = encode.n_vocab
+                self.__encode = encode.encode
+                self.__decode = encode.decode
+            data = self.__tensor(self.__encode(text_data), dtype=self.__int64)
+            if validate > 0:
+                split_point = int((1-validate) * len(data))
+                train_data, validation_data = data[:split_point], data[split_point:]
+                minimum_length = min(len(train_data), len(validation_data))
+                if minimum_length >= 2:
+                    desired_block_size = int(context_window) if context_window else 500
+                    self.block_size = max(1, min(desired_block_size, minimum_length - 1))
+                else: self.block_size = 1
+            else:
+                train_data = data
+                data_length = len(train_data)
+                self.block_size = max(1, min(self.block_size, data_length - 1))
+            self.__tokenizer = tokenizer
+            train_dataset = self.__TextDatasets(train_data, self.block_size)
+            if validate > 0: validation_dataset = self.__TextDatasets(validation_data, self.block_size)
+            train_loader = self.__DataLoader(train_dataset, batch_size=self.batch_size, shuffle=True)
+            if validate > 0: validation_loader = self.__DataLoader(validation_dataset, batch_size=self.batch_size, shuffle=False)
+            self.__model = self.__Transformers(self, self.__vocab_size, self.embedding_dim, self.number_heads, self.number_layers, self.dropout, self.block_size).to(self.device)
+            self.__optimizer = self.__optim.AdamW(self.__model.parameters(), lr=self.learning_rate)
+            scheduler, feed_forward = self.__optim.lr_scheduler.ReduceLROnPlateau(self.__optimizer, mode='min', factor=0.5, patience=3), True
+            Nx, last_validation_loss, step, best_val_loss = 0, 1.0, 0, float('inf')
+            string_precision = f'{precision:.4f}'.ljust(5, '0')
+            formatted_string = '{desc}: {percentage:3.0f}%|{bar:10}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt:>9}]'
+            if self.__xm is not None or str(self.device).lower().strip() == 'tpu': self.device = self.__xm.xla_device()
+            while feed_forward:
+                self.__model.train()
+                loss_item, total_train_loss = 1.0, 1.0
+                epoch = str(Nx+1).rjust(10, '0')
+                for input_batch, target_batch in train_loader:
+                    input_batch, target_batch = input_batch.to(self.device), target_batch.to(self.device)
+                    logits = self.__model(input_batch)
+                    loss = self.__Function.cross_entropy(logits.view(-1, logits.size(-1)), target_batch.view(-1))
+                    self.__optimizer.zero_grad()
+                    loss.backward()
+                    self.__utils.clip_grad_norm_(self.__model.parameters(), 1.0)
+                    if self.__xm is not None: self.__xm.optimizer_step(self.__optimizer)
+                    else: self.__optimizer.step()
+                    loss_item = loss.item()
+                    total_train_loss += loss_item
+                    last_validation_loss = validation_loss = self.__compute_loss(validation_loader) if validate > 0 else 1.0
+                    training_metrics['generalization_rate'] = min((1.0, max((0.0, 1.0-validation_loss))))
+                    if step > 0 and step % self.eval_interval == 0:
+                        scheduler.step(validation_loss)
+                        if validation_loss < best_val_loss: best_val_loss = validation_loss
+                    step += 1
+                current_precision = min(1.0, max(0.0, 1.0 - loss_item))
+                average_train_loss = total_train_loss / max((1, len(train_loader)))
+                if current_precision >= precision or average_train_loss <= loss_limit or Nx >= self.epochs:
+                    training_metrics['loss'] = loss_item if current_precision >= precision else average_train_loss
+                    training_metrics['precision'] = current_precision
+                    if progress:
+                        description = f'Finalization of backpropagations... current precision is '+f'{current_precision:.4f}'.ljust(5, '0')+f'; aiming for precision >= {string_precision} in training'
+                        self.__tqdm(train_loader, desc=description, unit='it', unit_scale=True, unit_divisor=1000, smoothing=0.1, bar_format=formatted_string).update(len(train_loader))
+                        print()
+                    break
+                elif progress:
+                    description = f'Backpropagation epoch: {epoch} - current precision is '+f'{current_precision:.4f}'.ljust(5, '0')+f'; aiming for precision >= {string_precision} in training'
+                    train_loader = self.__tqdm(train_loader, desc=description, unit='it', unit_scale=True, unit_divisor=1000, smoothing=0.1, bar_format=formatted_string)
+                Nx += 1
+            training_metrics['val_loss'] = best_val_loss if best_val_loss < 1.0 else min((1.0, max((0.0, last_validation_loss))))
+            self.__train = True
+            return training_metrics
+        except Exception as error:
+            try:
+                if self.__show_errors:
+                    error_message = 'ERROR in SapiensGPT.train: '+str(error)
+                    print(error_message)
+                    try: self.__print_exc() if self.__display_error_point else None
+                    except: pass
+            except: pass
+            try: return training_metrics
+            except: return {'val_loss': 1.0, 'loss': 1.0, 'generalization_rate': 0.0, 'precision': 0.0}
+    def saveModel(self, model_path='', progress=True):
+        try:
+            model_path = str(model_path).strip()
+            progress = bool(progress) if type(progress) in (bool, int, float) else True
+            if self.__model is None: raise ValueError('Model is not initialized. Call train or loadModel first.')
+            self.parameters_number = sum(parameters.numel() for parameters in self.__model.parameters())
+            formatted_params = self.__format_params(self.parameters_number)
+            if len(model_path) > 0:
+                directory, file_name = self.__os_path.split(model_path)
+                if not file_name: file_name = 'model.gpt'
+                elif not file_name.endswith('.gpt'): file_name += '.gpt'
+            else: directory, file_name = str(model_path), 'model.gpt'
+            if directory and not self.__os_path.exists(directory): self.__os_makedirs(directory)
+            save_path = self.__os_path.join(directory, file_name)
+            save_dict = {
+                'tokenizer': str(self.__tokenizer).lower().strip(),
+                'embedding_dim': max((1, int(self.embedding_dim))) if type(self.embedding_dim) in (bool, int, float) else -1,
+                'vocab_size': max((0, int(self.__vocab_size))) if type(self.__vocab_size) in (bool, int, float) else 0,
+                'block_size': max((1, int(self.block_size))) if type(self.block_size) in (bool, int, float) else -1,
+                'end_tag': str(self.__end_tag) if self.__end_tag is not None else '',
+                'number_heads': max((1, int(self.number_heads))) if type(self.number_heads) in (bool, int, float) else -1,
+                'number_layers': max((1, int(self.number_layers))) if type(self.number_layers) in (bool, int, float) else -1,
+                'dropout': max((0, int(self.dropout))) if type(self.dropout) in (bool, int, float) else 0.1,
+                'parameters_number': max((0, int(self.parameters_number))) if type(self.parameters_number) in (bool, int, float) else 0,
+                'architecture_type': 'gpt_model',
+                'model_state_dict': self.__model.state_dict(),
+                'fine_tuning': [],
+                'precision': 1.0
+
+            }
+            if self.__tokenizer == 'sapi':
+                save_dict['char_to_idx'] = self.__char_to_idx if type(self.__char_to_idx) == dict else {}
+                save_dict['idx_to_char'] = self.__idx_to_char if type(self.__idx_to_char) == dict else {}
+            if progress:
+                for _ in self.__tqdm(range(10), desc=f'Saving model with {formatted_params} parameters', leave=False): self.__save(save_dict, save_path)
+            else: self.__save(save_dict, save_path)
+            return True
+        except Exception as error:
+            try:
+                if self.__show_errors:
+                    error_message = 'ERROR in SapiensGPT.saveModel: '+str(error)
+                    print(error_message)
+                    try: self.__print_exc() if self.__display_error_point else None
+                    except: pass
+            except: pass
+            return False
+    def loadModel(self, model_path='', progress=True):
+        try:
+            model_path = str(model_path).strip()
+            progress = bool(progress) if type(progress) in (bool, int, float) else True
+            if len(model_path) > 0:
+                directory, file_name = self.__os_path.split(model_path)
+                if not file_name: file_name = 'model.gpt'
+                elif not file_name.endswith('.gpt'): file_name += '.gpt'
+            else: directory, file_name = str(model_path), 'model.gpt'
+            model_file = self.__os_path.join(directory, file_name)
+            if progress:
+                for _ in self.__tqdm(range(10), desc='Loading model', leave=False):
+                    try: checkpoint = self.__load(model_file, map_location=self.device)
+                    except: checkpoint = self.__load(model_file)
+            else:
+                try: checkpoint = self.__load(model_file, map_location=self.device)
+                except: checkpoint = self.__load(model_file)
+            try: self.__tokenizer = str(checkpoint['tokenizer']).lower().strip()
+            except: self.__tokenizer = 'gpt'
+            try: self.embedding_dim = max((1, int(checkpoint['embedding_dim']))) if checkpoint['embedding_dim'] != -1 else None
+            except: self.embedding_dim = None
+            try: self.__vocab_size = max((0, int(checkpoint['vocab_size']))) if type(checkpoint['vocab_size']) in (bool, int, float) else 0
+            except: self.__vocab_size = 0
+            try: self.block_size = max((1, int(checkpoint['block_size']))) if type(checkpoint['block_size']) != -1 else None
+            except: self.block_size = None
+            try: self.__end_tag = str(checkpoint['end_tag'])
+            except: self.__end_tag = ''
+            try: self.number_heads = max((1, int(checkpoint['number_heads']))) if type(checkpoint['number_heads']) != -1 else None
+            except: self.number_heads = None
+            try: self.number_layers = max((1, int(checkpoint['number_layers']))) if type(checkpoint['number_layers']) != -1 else None
+            except: self.number_layers = None
+            try: self.dropout = max((0, float(checkpoint['dropout']))) if type(checkpoint['dropout']) in (bool, int, float) else 0.1
+            except: self.dropout = 0.1
+            try: self.parameters_number = max((0, int(checkpoint['parameters_number']))) if type(checkpoint['parameters_number']) in (bool, int, float) else 0
+            except: self.parameters_number = 0
+            if self.__tokenizer == 'sapi':
+                try: self.__char_to_idx = dict(checkpoint['char_to_idx'])
+                except: self.__char_to_idx = {}
+                try: self.__idx_to_char = dict(checkpoint['idx_to_char'])
+                except: self.__idx_to_char = {}
+                self.__encode = lambda string: [self.__char_to_idx[char] for char in string]
+                self.__decode = lambda indexes: ''.join([self.__idx_to_char[index] for index in indexes])
+            else:
+                encode = self.__get_encoding('gpt2')
+                self.__encode = encode.encode
+                self.__decode = encode.decode
+            if len(self.__end_tag) < 1: self.__end_tag = None
+            self.__model = self.__Transformers(outer=self, vocab_size=self.__vocab_size, embedding_dim=self.embedding_dim, number_heads=self.number_heads, number_layers=self.number_layers, dropout=self.dropout, block_size=self.block_size).to(self.device)
+            state_dict = checkpoint['model_state_dict']
+            self.__model.load_state_dict(state_dict)
+            self.__optimizer, self.__train = None, True
+            return True
+        except Exception as error:
+            try:
+                if self.__show_errors:
+                    error_message = 'ERROR in SapiensGPT.loadModel: '+str(error)
+                    print(error_message)
+                    try: self.__print_exc() if self.__display_error_point else None
+                    except: pass
+            except: pass
+            return False
+    def fineTuning(self, dataset_path='', progress=True):
+        try:
+            adjusted_model = False
+            dataset_path = str(dataset_path).strip()
+            progress = bool(progress) if type(progress) in (bool, int, float) else True
+            def _read_remote_file(remote_path=''):
+                remote_path = str(remote_path).strip()
+                from urllib.request import urlopen
+                try:
+                    from os import environ
+                    from certifi import where
+                    environ['SSL_CERT_FILE'] = where()
+                    from logging import getLogger, ERROR
+                    getLogger('urlopen').setLevel(ERROR)
+                except: pass
+                remote_stream = urlopen(remote_path)
+                content = remote_stream.read().decode('utf-8')
+                return str(content).strip()
+            is_json, string_content = dataset_path.lower().endswith('.json'), ''
+            if dataset_path.startswith(('https://', 'http://')): string_content = _read_remote_file(remote_path=dataset_path)
+            else:
+                with open(dataset_path, 'r', encoding='utf-8') as file_object: string_content = str(file_object.read()).strip()
+            if is_json:
+                fit_structure, data = [], []
+                if string_content:
+                    def _string_to_dictionary_or_json(string=''):
+                        dictionary_or_json = {}
+                        original_string = string
+                        string = str(string).strip()
+                        try:
+                            from json import loads
+                            try: dictionary_or_json = loads(string)
+                            except: dictionary_or_json = loads(original_string)
+                        except:
+                            from ast import literal_eval
+                            try: dictionary_or_json = literal_eval(string)
+                            except: dictionary_or_json = literal_eval(original_string)
+                        return dictionary_or_json
+                    fit_structure = _string_to_dictionary_or_json(string=string_content)
+                if fit_structure:
+                    if type(fit_structure) == dict:
+                        json_keys = list(fit_structure.keys())
+                        if 'data' in json_keys: data = list(fit_structure.get('data', []))
+                        else:
+                            data_key = str(json_keys[0]).strip()
+                            data = list(fit_structure.get(data_key, []))
+                    elif type(fit_structure) in (tuple, list): data = fit_structure
+                    if data:
+                        from tqdm import tqdm
+                        total_length = len(data)
+                        with tqdm(total=total_length, unit='item', disable=not progress) as progress_bar:
+                            for input_output in data:
+                                _input, _output, _file_path = '', '', ''
+                                if input_output and type(input_output) == dict:
+                                    json_keys = list(input_output.keys())
+                                    if 'input' in json_keys: _input = str(input_output.get('input', '')).strip()
+                                    elif 'Input' in json_keys: _input = str(input_output.get('Input', '')).strip()
+                                    elif 'INPUT' in json_keys: _input = str(input_output.get('INPUT', '')).strip()
+                                    elif 'question' in json_keys: _input = str(input_output.get('question', '')).strip()
+                                    elif 'Question' in json_keys: _input = str(input_output.get('Question', '')).strip()
+                                    elif 'QUESTION' in json_keys: _input = str(input_output.get('QUESTION', '')).strip()
+                                    elif 'prompt' in json_keys: _input = str(input_output.get('prompt', '')).strip()
+                                    elif 'Prompt' in json_keys: _input = str(input_output.get('Prompt', '')).strip()
+                                    elif 'PROMPT' in json_keys: _input = str(input_output.get('PROMPT', '')).strip()
+                                    if 'output' in json_keys: _output = str(input_output.get('output', '')).strip()
+                                    elif 'Output' in json_keys: _output = str(input_output.get('Output', '')).strip()
+                                    elif 'OUTPUT' in json_keys: _output = str(input_output.get('OUTPUT', '')).strip()
+                                    elif 'answer' in json_keys: _output = str(input_output.get('answer', '')).strip()
+                                    elif 'Answer' in json_keys: _output = str(input_output.get('Answer', '')).strip()
+                                    elif 'ANSWER' in json_keys: _output = str(input_output.get('ANSWER', '')).strip()
+                                    elif 'response' in json_keys: _output = str(input_output.get('response', '')).strip()
+                                    elif 'Response' in json_keys: _output = str(input_output.get('Response', '')).strip()
+                                    elif 'RESPONSE' in json_keys: _output = str(input_output.get('RESPONSE', '')).strip()
+                                    if 'file_path' in json_keys: _file_path = str(input_output.get('file_path', '')).strip()
+                                    elif 'File_path' in json_keys: _file_path = str(input_output.get('File_path', '')).strip()
+                                    elif 'FILE_PATH' in json_keys: _file_path = str(input_output.get('FILE_PATH', '')).strip()
+                                    if not _input: _input = str(input_output[json_keys[0]]).strip()
+                                    if not _output: _output = str(input_output[json_keys[1]]).strip()
+                                if _input and _output: self.__addFit(prompt=_input, answer=_output)
+                                if progress:
+                                    progress_bar.set_description('Adjusting')
+                                    progress_bar.update(1)
+                        adjusted_model = total_length > 0
+            elif self.__show_errors:
+                print('The file must be a JSON with a "data" key containing an array of objects with the keys "input" and "output".')
+                adjusted_model = False
+            return adjusted_model
+        except Exception as error:
+            try:
+                if self.__show_errors:
+                    error_message = 'ERROR in SapiensGPT.fineTuning: '+str(error)
+                    print(error_message)
+                    try: self.__print_exc() if self.__display_error_point else None
+                    except: pass
+            except: pass
+            return False
+    def inference(self, prompt='', stream=True):
+        try:
+            prompt = str(prompt).strip()
+            max_tokens = max((1, int(self.max_tokens))) if type(self.max_tokens) in (bool, int, float) else 500
+            temperature = max((0, float(self.temperature))) if type(self.temperature) in (bool, int, float) else 0.5
+            top_k = max((0, int(self.top_k))) if type(self.top_k) in (bool, int, float) else 0
+            top_p = min((1.0, max((0.0, float(self.top_p))))) if type(self.top_p) in (bool, int, float) else 1.0
+            stream = bool(stream) if type(stream) in (bool, int, float) else False
+            if self.__model is None: raise ValueError('Model is not initialized. Call train or loadModel first.')
+            if stream: return self.__generate_tokens(prompt=prompt, max_tokens=max_tokens, temperature=temperature, top_k=top_k, top_p=top_p)
+            tokens = list(self.__generate_tokens(prompt=prompt, max_tokens=max_tokens, temperature=temperature, top_k=top_k, top_p=top_p))
+            return ''.join(tokens)
+        except Exception as error:
+            try:
+                if self.__show_errors:
+                    error_message = 'ERROR in SapiensGPT.inference: '+str(error)
+                    print(error_message)
+                    try: self.__print_exc() if self.__display_error_point else None
+                    except: pass
+            except: pass
+            return ''
+    def completeMessages(self, messages=[], stream=False):
+        try:
+            complete_messages = {'answer': '', 'messages': [], 'next_token': ''}
+            messages = list(messages) if type(messages) in (tuple, list, dict) else []
+            stream = bool(stream) if type(stream) in (bool, int, float) else False
+            complete_messages['messages'] = messages
+            if messages:
+                prompt_template = ''
+                end_tag = self.end_tag if self.end_tag else ''
+                for message in messages:
+                    role = str(message.get('role', '')).strip()
+                    content = str(message.get('content', '')).strip()
+                    if role and content: prompt_template += f'{role}:\n{content}{end_tag}\n'
+                prompt_template = prompt_template.strip()
+                def _get_stream(prompt_template='', complete_messages={}):
+                    current_answer, message = '', {'role': 'assistant', 'content': ''}
+                    complete_messages['messages'].append(message)
+                    inference_function = self.inference(prompt=prompt_template, stream=True)
+                    for token in inference_function:
+                        current_answer += token
+                        complete_messages['answer'] = current_answer
+                        complete_messages['messages'][-1]['content'] = current_answer
+                        complete_messages['next_token'] = token
+                        yield complete_messages
+                def _get_string(prompt_template='', complete_messages={}):
+                    inference_function = self.inference(prompt=prompt_template, stream=False)
+                    complete_messages['answer'] = inference_function
+                    message = {'role': 'assistant', 'content': inference_function}
+                    complete_messages['messages'].append(message)
+                    token = inference_function.split(chr(32))[-1].rstrip()
+                    complete_messages['next_token'] = token
+                    return complete_messages
+                if stream: complete_messages = _get_stream(prompt_template=prompt_template, complete_messages=complete_messages)
+                else: complete_messages = _get_string(prompt_template=prompt_template, complete_messages=complete_messages)
+            return complete_messages
+        except Exception as error:
+            try:
+                if self.__show_errors:
+                    error_message = 'ERROR in SapiensGPT.completeMessages: '+str(error)
+                    print(error_message)
+                    try: self.__print_exc() if self.__display_error_point else None
+                    except: pass
+            except: pass
+            return {'answer': '', 'messages': [], 'next_token': ''}
+    def printInference(self, prompt='', stream=True):
+        try:
+            prompt = str(prompt).strip()
+            max_tokens = max((1, int(self.max_tokens))) if type(self.max_tokens) in (bool, int, float) else 500
+            temperature = max((0, float(self.temperature))) if type(self.temperature) in (bool, int, float) else 0.5
+            top_k = max((0, int(self.top_k))) if type(self.top_k) in (bool, int, float) else 0
+            top_p = min((1.0, max((0.0, float(self.top_p))))) if type(self.top_p) in (bool, int, float) else 1.0
+            stream = bool(stream) if type(stream) in (bool, int, float) else False
+            if self.__model is None: raise ValueError('Model is not initialized. Call train or loadModel first.')
+            if stream:
+                [print(token, end='', flush=True) for token in self.__generate_tokens(prompt=prompt, max_tokens=max_tokens, temperature=temperature, top_k=top_k, top_p=top_p)]
+                print()
+            else: print(self.predict(prompt=prompt, max_tokens=max_tokens, temperature=temperature, stream=stream))
+        except Exception as error:
+            try:
+                if self.__show_errors:
+                    error_message = 'ERROR in SapiensGPT.printInference: '+str(error)
+                    print(error_message)
+                    try: self.__print_exc() if self.__display_error_point else None
+                    except: pass
+            except: pass
+    def printCompleteMessages(self, messages=[], stream=True):
+        try:
+            inference = self.completeMessages(messages=messages, stream=stream)
+            if stream:
+                from time import sleep
+                for token in inference:
+                    print(token['next_token'], end='', flush=True)
+                    sleep(self.delay)
+                print()
+            else: print(inference['answer'])
+        except Exception as error:
+            try:
+                if self.__show_errors:
+                    error_message = 'ERROR in SapiensModel.printInference: '+str(error)
+                    print(error_message)
+                    try: self.__print_exc() if self.__display_error_point else None
+                    except: pass
+            except: pass
+    def close(self):
+        try:
+            self.__init__()
+            return True
+        except Exception as error:
+            try:
+                if self.__show_errors:
+                    error_message = 'ERROR in SapiensGPT.close: '+str(error)
+                    print(error_message)
+                    try: self.__print_exc() if self.__display_error_point else None
+                    except: pass
+            except: pass
+            return False
+# This is a standard code of a GPT (Generative Pre-trained Transformer) model, developed by Sapiens Technology®️,
+# which faithfully follows the mathematical structure of the article “Attention Is All You Need” for the construction of the Transformer architecture
+# used in the pattern recognition of the model that is saved. Some optimizations that do not influence the Transformer architecture
+# were applied only to facilitate the adjustments of the parameters and variables of the training, saving, loading, fine-tuning and inference of the pre-trained model.
+# --------------------------> A SAPIENS TECHNOLOGY®️ PRODUCTION) <--------------------------

sapiens_gpt-2.0.1.dist-info/LICENSE.txt

@@ -0,0 +1,3 @@
+This is proprietary code.
+
+Its copying, alteration and distribution outside official media is strictly prohibited.

sapiens_gpt-2.0.1.dist-info/METADATA

@@ -0,0 +1,17 @@
+Metadata-Version: 2.2
+Name: sapiens_gpt
+Version: 2.0.1
+Home-page: https://github.com/sapiens-technology/sapiens_gpt
+Author: SAPIENS TECHNOLOGY
+License: Proprietary Software
+License-File: LICENSE.txt
+Requires-Dist: torch
+Requires-Dist: numpy
+Requires-Dist: tiktoken
+Requires-Dist: tqdm
+Requires-Dist: certifi
+Requires-Dist: torch-xla==2.7.0; platform_system == "Linux" and platform_machine == "x86_64"
+Dynamic: author
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist

sapiens_gpt-2.0.1.dist-info/RECORD

@@ -0,0 +1,7 @@
+sapiens_gpt/__init__.py,sha256=_M8Z1PmKWzDj0sMmNRtpMZfP8YCN_gHS1dL1rlg_nIA,1361
+sapiens_gpt/sapiens_gpt.py,sha256=A1wTQ_5W9k7aF7dmxeEIUG8Qj-KYeydBNFBXHa30gwc,51718
+sapiens_gpt-2.0.1.dist-info/LICENSE.txt,sha256=WqB2vIA5tH5lqLTr53yT_oy1m0wYfuvCPQKxdDHWimg,115
+sapiens_gpt-2.0.1.dist-info/METADATA,sha256=15tefvU31tQ7Qj88ypt0d8Pot9Ee-XmBLG7gccckPvk,476
+sapiens_gpt-2.0.1.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+sapiens_gpt-2.0.1.dist-info/top_level.txt,sha256=vgTSwuajeQYPx5xRnkIQIjjTH5bb5JMPIbfHnhSutBM,12
+sapiens_gpt-2.0.1.dist-info/RECORD,,

sapiens_gpt-2.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (76.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

sapiens_gpt-2.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+sapiens_gpt