SinaTools 0.1.4__py2.py3-none-any.whl → 0.1.7__py2.py3-none-any.whl

nlptools/CLI/utils/latin_remove.py

@@ -1,51 +0,0 @@
-"""
-About:
-------
-The sina_remove_latin tool performs delete latin characters from the input text.
-
-Usage:
-------
-Below is the usage information that can be generated by running sina_remove_latin --help.
-
-.. code-block:: none
-
-    Usage:
-        sina_remove_latin --text=TEXT
-        sina_remove_latin --file "path/to/your/file.txt"
-
-Examples:
----------
-
-.. code-block:: none
-
-    sina_remove_punctuation --text "123test"
-    
-    sina_remove_punctuation --file "path/to/your/file.txt"
-
-Note:
------
-
-.. code-block:: none
-
-    - This tool is specific to Arabic text, as it focuses on Arabic linguistic elements.
-    - Ensure that the text input is appropriately encoded in UTF-8 or compatible formats.
-    - This tool for latin characters, if the input text is an Arabic characters or numbers the output will be the same input 
-
-"""
-
-import argparse
-from nlptools.utils.parser import remove_latin
-
-
-def main():
-    parser = argparse.ArgumentParser(description='remove latin characters from the text')
-
-    parser.add_argument('--text', type=str, required=True, help='The input text')
-    args = parser.parse_args()
-    result = remove_latin(args.text)
- 
-    print(result)
-    if __name__ == '__main__':
-        main()
-
-#sina_remove_latin --text "123test"

nlptools/CLI/utils/remove_Punc.py

@@ -1,53 +0,0 @@
-"""
-About:
-------
-The sina_remove_punctuation tool performs delete punctuation marks from the input text.
-
-Usage:
-------
-Below is the usage information that can be generated by running sina_remove_punctuation --help.
-
-.. code-block:: none
-
-    Usage:
-        sina_remove_punctuation --text=TEXT
-        sina_remove_punctuation --file "path/to/your/file.txt"
-
-Examples:
----------
-
-.. code-block:: none
-
-    sina_remove_punctuation --text "te%s@t...!!?"
-    
-    sina_remove_punctuation --file "path/to/your/file.txt"
-
-Note:
------
-
-.. code-block:: none
-
-    - This tool is specific to Arabic text, as it focuses on Arabic linguistic elements.
-    - Ensure that the text input is appropriately encoded in UTF-8 or compatible formats.
-"""
-
-import argparse
-from nlptools.utils.parser import remove_punctuation
-#from nlptools.utils.parser import read_file
-#from nlptools.utils.parser import write_file
-
-
-def main():
-    parser = argparse.ArgumentParser(description='remove punctuation marks from the text')
-
-    parser.add_argument('--text',required=True,help="input text")
-   # parser.add_argument('myFile', type=argparse.FileType('r'),help='Input file csv')
-    args = parser.parse_args()
-    result = remove_punctuation(args.text)
- 
-    print(result)
-    if __name__ == '__main__':
-        main()
-
-#sina_remove_punctuation --text "your text"
-

nlptools/VERSION

@@ -1 +0,0 @@
-0.1.4

nlptools/arabiner/bin/__init__.py

@@ -1,14 +0,0 @@
-from nlptools.DataDownload import downloader
-import os
-from nlptools.arabiner.utils.helpers import load_checkpoint
-import nlptools
-
-nlptools.tagger = None
-nlptools.tag_vocab = None
-nlptools.train_config = None
-
-filename = 'Wj27012000.tar'
-path =downloader.get_appdatadir()
-model_path = os.path.join(path, filename)
-print('1',model_path)
-nlptools.tagger, nlptools.tag_vocab, nlptools.train_config = load_checkpoint(model_path)

nlptools/arabiner/bin/eval.py

@@ -1,87 +0,0 @@
-import os
-import logging
-import argparse
-from collections import namedtuple
-from nlptools.arabiner.utils.helpers import load_checkpoint, make_output_dirs, logging_config
-from nlptools.arabiner.utils.data import get_dataloaders, parse_conll_files
-from nlptools.arabiner.utils.metrics import compute_single_label_metrics, compute_nested_metrics
-
-logger = logging.getLogger(__name__)
-
-
-def parse_args():
-    parser = argparse.ArgumentParser(
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-    )
-
-    parser.add_argument(
-        "--output_path",
-        type=str,
-        required=True,
-        help="Path to save results",
-    )
-
-    parser.add_argument(
-        "--model_path",
-        type=str,
-        required=True,
-        help="Model path",
-    )
-
-    parser.add_argument(
-        "--data_paths",
-        nargs="+",
-        type=str,
-        required=True,
-        help="Text or sequence to tag, this is in same format as training data with 'O' tag for all tokens",
-    )
-
-    parser.add_argument(
-        "--batch_size",
-        type=int,
-        default=32,
-        help="Batch size",
-    )
-
-    args = parser.parse_args()
-
-    return args
-
-
-def main(args):
-    # Create directory to save predictions
-    make_output_dirs(args.output_path, overwrite=True)
-    logging_config(log_file=os.path.join(args.output_path, "eval.log"))
-
-    # Load tagger
-    tagger, tag_vocab, train_config = load_checkpoint(args.model_path)
-
-    # Convert text to a tagger dataset and index the tokens in args.text
-    datasets, vocab = parse_conll_files(args.data_paths)
-
-    vocabs = namedtuple("Vocab", ["tags", "tokens"])
-    vocab = vocabs(tokens=vocab.tokens, tags=tag_vocab)
-
-    # From the datasets generate the dataloaders
-    dataloaders = get_dataloaders(
-        datasets, vocab,
-        train_config.data_config,
-        batch_size=args.batch_size,
-        shuffle=[False] * len(datasets)
-    )
-
-    # Evaluate the model on each dataloader
-    for dataloader, input_file in zip(dataloaders, args.data_paths):
-        filename = os.path.basename(input_file)
-        predictions_file = os.path.join(args.output_path, f"predictions_{filename}")
-        _, segments, _, _ = tagger.eval(dataloader)
-        tagger.segments_to_file(segments, predictions_file)
-
-        if "Nested" in train_config.trainer_config["fn"]:
-            compute_nested_metrics(segments, vocab.tags[1:])
-        else:
-            compute_single_label_metrics(segments)
-
-
-if __name__ == "__main__":
-    main(parse_args())

nlptools/arabiner/bin/process.py

@@ -1,140 +0,0 @@
-import os
-import argparse
-import csv
-import logging
-import numpy as np
-from nlptools.arabiner.utils.helpers import logging_config
-from nlptools.arabiner.utils.data import conll_to_segments
-
-logger = logging.getLogger(__name__)
-
-
-def to_conll_format(input_files, output_path, multi_label=False):
-    """
-    Parse data files and convert them into CoNLL format
-    :param input_files: List[str] - list of filenames
-    :param output_path: str - output path
-    :param multi_label: boolean - True to process data with mutli-class/multi-label
-    :return:
-    """
-    for input_file in input_files:
-        tokens = list()
-        prev_sent_id = None
-
-        with open(input_file, "r") as fh:
-            r = csv.reader(fh, delimiter="\t", quotechar=" ")
-            next(r)
-
-            for row in r:
-                sent_id, token, labels = row[1], row[3], row[4].split()
-                valid_labels = sum([1 for l in labels if "-" in l or l == "O"]) == len(labels)
-
-                if not valid_labels:
-                    logging.warning("Invalid labels found %s", str(row))
-                    continue
-                if not labels:
-                    logging.warning("Token %s has no label", str(row))
-                    continue
-                if not token:
-                    logging.warning("Token %s is missing", str(row))
-                    continue
-                if len(token.split()) > 1:
-                    logging.warning("Token %s has multiple tokens", str(row))
-                    continue
-
-                if prev_sent_id is not None and sent_id != prev_sent_id:
-                    tokens.append([])
-
-                if multi_label:
-                    tokens.append([token] + labels)
-                else:
-                    tokens.append([token, labels[0]])
-
-                prev_sent_id = sent_id
-
-        num_segments = sum([1 for token in tokens if not token])
-        logging.info("Found %d segments and %d tokens in %s", num_segments + 1, len(tokens) - num_segments, input_file)
-
-        filename = os.path.basename(input_file)
-        output_file = os.path.join(output_path, filename)
-
-        with open(output_file, "w") as fh:
-            fh.write("\n".join(" ".join(token) for token in tokens))
-            logging.info("Output file %s", output_file)
-
-
-def train_dev_test_split(input_files, output_path, train_ratio, dev_ratio):
-    segments = list()
-    filenames = ["train.txt", "val.txt", "test.txt"]
-
-    for input_file in input_files:
-        segments += conll_to_segments(input_file)
-
-    n = len(segments)
-    np.random.shuffle(segments)
-    datasets = np.split(segments, [int(train_ratio*n), int((train_ratio+dev_ratio)*n)])
-
-    # write data to files
-    for i in range(len(datasets)):
-        filename = os.path.join(output_path, filenames[i])
-
-        with open(filename, "w") as fh:
-            text = "\n\n".join(["\n".join([f"{token.text} {' '.join(token.gold_tag)}" for token in segment]) for segment in datasets[i]])
-            fh.write(text)
-            logging.info("Output file %s", filename)
-
-
-def main(args):
-    if args.task == "to_conll_format":
-        to_conll_format(args.input_files, args.output_path, multi_label=args.multi_label)
-    if args.task == "train_dev_test_split":
-        train_dev_test_split(args.input_files, args.output_path, args.train_ratio, args.dev_ratio)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-    )
-
-    parser.add_argument(
-        "--input_files",
-        type=str,
-        nargs="+",
-        required=True,
-        help="List of input files",
-    )
-
-    parser.add_argument(
-        "--output_path",
-        type=str,
-        required=True,
-        help="Output path",
-    )
-
-    parser.add_argument(
-        "--train_ratio",
-        type=float,
-        required=False,
-        help="Training data ratio (percent of segments). Required with the task train_dev_test_split. "
-             "Files must in ConLL format",
-    )
-
-    parser.add_argument(
-        "--dev_ratio",
-        type=float,
-        required=False,
-        help="Dev/val data ratio (percent of segments). Required with the task train_dev_test_split. "
-             "Files must in ConLL format",
-    )
-
-    parser.add_argument(
-        "--task", required=True, choices=["to_conll_format", "train_dev_test_split"]
-    )
-
-    parser.add_argument(
-        "--multi_label", action='store_true'
-    )
-
-    args = parser.parse_args()
-    logging_config(os.path.join(args.output_path, "process.log"))
-    main(args)

nlptools/arabiner/bin/train.py

@@ -1,221 +0,0 @@
-import os
-import logging
-import json
-import argparse
-import torch.utils.tensorboard
-from torchvision import *
-import pickle
-from nlptools.arabiner.utils.data import get_dataloaders, parse_conll_files
-from nlptools.arabiner.utils.helpers import logging_config, load_object, make_output_dirs, set_seed
-
-logger = logging.getLogger(__name__)
-
-
-def parse_args():
-    parser = argparse.ArgumentParser(
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-    )
-
-    parser.add_argument(
-        "--output_path",
-        type=str,
-        required=True,
-        help="Output path",
-    )
-
-    parser.add_argument(
-        "--train_path",
-        type=str,
-        required=True,
-        help="Path to training data",
-    )
-
-    parser.add_argument(
-        "--val_path",
-        type=str,
-        required=True,
-        help="Path to training data",
-    )
-
-    parser.add_argument(
-        "--test_path",
-        type=str,
-        required=True,
-        help="Path to training data",
-    )
-
-    parser.add_argument(
-        "--bert_model",
-        type=str,
-        default="aubmindlab/bert-base-arabertv2",
-        help="BERT model",
-    )
-
-    parser.add_argument(
-        "--gpus",
-        type=int,
-        nargs="+",
-        default=[0],
-        help="GPU IDs to train on",
-    )
-
-    parser.add_argument(
-        "--log_interval",
-        type=int,
-        default=10,
-        help="Log results every that many timesteps",
-    )
-
-    parser.add_argument(
-        "--batch_size",
-        type=int,
-        default=32,
-        help="Batch size",
-    )
-
-    parser.add_argument(
-        "--num_workers",
-        type=int,
-        default=0,
-        help="Dataloader number of workers",
-    )
-
-    parser.add_argument(
-        "--data_config",
-        type=json.loads,
-        default='{"fn": "arabiner.data.datasets.DefaultDataset", "kwargs": {"max_seq_len": 512}}',
-        help="Dataset configurations",
-    )
-
-    parser.add_argument(
-        "--trainer_config",
-        type=json.loads,
-        default='{"fn": "arabiner.trainers.BertTrainer", "kwargs": {"max_epochs": 50}}',
-        help="Trainer configurations",
-    )
-
-    parser.add_argument(
-        "--network_config",
-        type=json.loads,
-        default='{"fn": "arabiner.nn.BertSeqTagger", "kwargs": '
-                '{"dropout": 0.1, "bert_model": "aubmindlab/bert-base-arabertv2"}}',
-        help="Network configurations",
-    )
-
-    parser.add_argument(
-        "--optimizer",
-        type=json.loads,
-        default='{"fn": "torch.optim.AdamW", "kwargs": {"lr": 0.0001}}',
-        help="Optimizer configurations",
-    )
-
-    parser.add_argument(
-        "--lr_scheduler",
-        type=json.loads,
-        default='{"fn": "torch.optim.lr_scheduler.ExponentialLR", "kwargs": {"gamma": 1}}',
-        help="Learning rate scheduler configurations",
-    )
-
-    parser.add_argument(
-        "--loss",
-        type=json.loads,
-        default='{"fn": "torch.nn.CrossEntropyLoss", "kwargs": {}}',
-        help="Loss function configurations",
-    )
-
-    parser.add_argument(
-        "--overwrite",
-        action="store_true",
-        help="Overwrite output directory",
-    )
-
-    parser.add_argument(
-        "--seed",
-        type=int,
-        default=1,
-        help="Seed for random initialization",
-    )
-
-    args = parser.parse_args()
-
-    return args
-
-
-def main(args):
-    make_output_dirs(
-        args.output_path,
-        subdirs=("tensorboard", "checkpoints"),
-        overwrite=args.overwrite,
-    )
-
-    # Set the seed for randomization
-    set_seed(args.seed)
-
-    logging_config(os.path.join(args.output_path, "train.log"))
-    summary_writer = torch.utils.tensorboard.SummaryWriter(
-        os.path.join(args.output_path, "tensorboard")
-    )
-    os.environ["CUDA_VISIBLE_DEVICES"] = ",".join([str(gpu) for gpu in args.gpus])
-
-    # Get the datasets and vocab for tags and tokens
-    datasets, vocab = parse_conll_files((args.train_path, args.val_path, args.test_path))
-
-    if "Nested" in args.network_config["fn"]:
-        args.network_config["kwargs"]["num_labels"] = [len(v) for v in vocab.tags[1:]]
-    else:
-        args.network_config["kwargs"]["num_labels"] = len(vocab.tags[0])
-
-    # Save tag vocab to desk
-    with open(os.path.join(args.output_path, "tag_vocab.pkl"), "wb") as fh:
-        pickle.dump(vocab.tags, fh)
-
-    # Write config to file
-    args_file = os.path.join(args.output_path, "args.json")
-    with open(args_file, "w") as fh:
-        logger.info("Writing config to %s", args_file)
-        json.dump(args.__dict__, fh, indent=4)
-
-    # From the datasets generate the dataloaders
-    args.data_config["kwargs"]["bert_model"] = args.network_config["kwargs"]["bert_model"]
-    train_dataloader, val_dataloader, test_dataloader = get_dataloaders(
-        datasets, vocab, args.data_config, args.batch_size, args.num_workers
-    )
-
-    model = load_object(args.network_config["fn"], args.network_config["kwargs"])
-    model = torch.nn.DataParallel(model, device_ids=range(len(args.gpus)))
-
-    if torch.cuda.is_available():
-        model = model.cuda()
-
-    args.optimizer["kwargs"]["params"] = model.parameters()
-    optimizer = load_object(args.optimizer["fn"], args.optimizer["kwargs"])
-
-    args.lr_scheduler["kwargs"]["optimizer"] = optimizer
-    if "num_training_steps" in args.lr_scheduler["kwargs"]:
-        args.lr_scheduler["kwargs"]["num_training_steps"] = args.max_epochs * len(
-            train_dataloader
-        )
-
-    scheduler = load_object(args.lr_scheduler["fn"], args.lr_scheduler["kwargs"])
-    loss = load_object(args.loss["fn"], args.loss["kwargs"])
-
-    args.trainer_config["kwargs"].update({
-        "model": model,
-        "optimizer": optimizer,
-        "scheduler": scheduler,
-        "loss": loss,
-        "train_dataloader": train_dataloader,
-        "val_dataloader": val_dataloader,
-        "test_dataloader": test_dataloader,
-        "log_interval": args.log_interval,
-        "summary_writer": summary_writer,
-        "output_path": args.output_path
-    })
-
-    trainer = load_object(args.trainer_config["fn"], args.trainer_config["kwargs"])
-    trainer.train()
-    return
-
-
-if __name__ == "__main__":
-    main(parse_args())

nlptools/arabiner/data/__init__.py

@@ -1 +0,0 @@
-from nlptools.arabiner.data.datasets import NestedTagsDataset