SinaTools 0.1.1__py2.py3-none-any.whl

SinaTools-0.1.1.data/data/nlptools/environment.yml

@@ -0,0 +1,227 @@
+name: arabicner
+channels:
+  - anaconda
+  - pytorch
+  - nvidia
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - abseil-cpp=20211102.0=h27087fc_1
+  - absl-py=1.3.0=pyhd8ed1ab_0
+  - aiohttp=3.8.1=py310h5764c6d_1
+  - aiosignal=1.2.0=pyhd8ed1ab_0
+  - arrow-cpp=8.0.0=py310h3098874_0
+  - async-timeout=4.0.2=py310h06a4308_0
+  - attrs=22.1.0=pyh71513ae_1
+  - aws-c-common=0.4.57=he6710b0_1
+  - aws-c-event-stream=0.1.6=h2531618_5
+  - aws-checksums=0.1.9=he6710b0_0
+  - aws-sdk-cpp=1.8.185=hce553d0_0
+  - blas=1.0=mkl
+  - blinker=1.5=pyhd8ed1ab_0
+  - boost-cpp=1.78.0=he72f1d9_0
+  - bottleneck=1.3.5=py310ha9d4c09_0
+  - brotli=1.0.9=h166bdaf_7
+  - brotli-bin=1.0.9=h166bdaf_7
+  - brotlipy=0.7.0=py310h7f8727e_1002
+  - bzip2=1.0.8=h7b6447c_0
+  - c-ares=1.18.1=h7f98852_0
+  - ca-certificates=2022.9.24=ha878542_0
+  - cachetools=5.2.0=pyhd8ed1ab_0
+  - certifi=2022.9.24=pyhd8ed1ab_0
+  - cffi=1.15.1=py310h74dc2b5_0
+  - charset-normalizer=2.0.4=pyhd3eb1b0_0
+  - click=8.1.3=unix_pyhd8ed1ab_2
+  - cryptography=38.0.1=py310h9ce1e76_0
+  - cuda=11.7.1=0
+  - cuda-cccl=11.7.91=0
+  - cuda-command-line-tools=11.7.1=0
+  - cuda-compiler=11.7.1=0
+  - cuda-cudart=11.7.99=0
+  - cuda-cudart-dev=11.7.99=0
+  - cuda-cuobjdump=11.7.91=0
+  - cuda-cupti=11.7.101=0
+  - cuda-cuxxfilt=11.7.91=0
+  - cuda-demo-suite=11.8.86=0
+  - cuda-documentation=11.8.86=0
+  - cuda-driver-dev=11.7.99=0
+  - cuda-gdb=11.8.86=0
+  - cuda-libraries=11.7.1=0
+  - cuda-libraries-dev=11.7.1=0
+  - cuda-memcheck=11.8.86=0
+  - cuda-nsight=11.8.86=0
+  - cuda-nsight-compute=11.8.0=0
+  - cuda-nvcc=11.7.99=0  
+  - cuda-nvdisasm=11.8.86=0
+  - cuda-nvml-dev=11.7.91=0
+  - cuda-nvprof=11.8.87=0
+  - cuda-nvprune=11.7.91=0
+  - cuda-nvrtc=11.7.99=0
+  - cuda-nvrtc-dev=11.7.99=0
+  - cuda-nvtx=11.7.91=0
+  - cuda-nvvp=11.8.87=0
+  - cuda-runtime=11.7.1=0
+  - cuda-sanitizer-api=11.8.86=0
+  - cuda-toolkit=11.7.1=0
+  - cuda-tools=11.7.1=0
+  - cuda-visual-tools=11.7.1=0
+  - dataclasses=0.8=pyhc8e2a94_3
+  - datasets=2.6.1=pyhd8ed1ab_0
+  - dill=0.3.5.1=pyhd8ed1ab_0
+  - ffmpeg=4.3=hf484d3e_0
+  - fftw=3.3.10=nompi_h77c792f_102
+  - filelock=3.8.0=pyhd8ed1ab_0
+  - freetype=2.12.1=h4a9f257_0
+  - frozenlist=1.2.0=py310h7f8727e_1
+  - fsspec=2022.10.0=pyhd8ed1ab_0
+  - gds-tools=1.4.0.31=0
+  - gflags=2.2.2=he1b5a44_1004
+  - giflib=5.2.1=h7b6447c_0
+  - glog=0.6.0=h6f12383_0
+  - gmp=6.2.1=h295c915_3
+  - gnutls=3.6.15=he1e5248_0
+  #- google-auth=2.14.0=pyh1a96a4e_0
+  #- google-auth-oauthlib=0.4.6=pyhd8ed1ab_0
+  - grpc-cpp=1.46.1=h33aed49_0
+  - grpcio=1.42.0=py310hce63b2e_0
+  - huggingface_hub=0.10.1=pyhd8ed1ab_0
+  - icu=70.1=h27087fc_0
+  - idna=3.4=py310h06a4308_0
+  - importlib-metadata=5.0.0=pyha770c72_1
+  - importlib_metadata=5.0.0=hd8ed1ab_1
+  - intel-openmp=2021.4.0=h06a4308_3561
+  - joblib=1.2.0=pyhd8ed1ab_0
+  - jpeg=9e=h7f8727e_0
+  - keyutils=1.6.1=h166bdaf_0
+  - krb5=1.19.3=h3790be6_0
+  - lame=3.100=h7b6447c_0
+  - lcms2=2.12=h3be6417_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - lerc=3.0=h295c915_0
+  - libbrotlicommon=1.0.9=h166bdaf_7
+  - libbrotlidec=1.0.9=h166bdaf_7
+  - libbrotlienc=1.0.9=h166bdaf_7
+  - libcublas=11.11.3.6=0
+  - libcublas-dev=11.11.3.6=0
+  - libcufft=10.9.0.58=0
+  - libcufft-dev=10.9.0.58=0
+  - libcufile=1.4.0.31=0
+  - libcufile-dev=1.4.0.31=0
+  - libcurand=10.3.0.86=0    
+  - libcurl=7.85.0=h91b91d3_0
+  - libcusolver=11.4.1.48=0
+  - libcusolver-dev=11.4.1.48=0
+  - libcusparse=11.7.5.86=0
+  - libcusparse-dev=11.7.5.86=0
+  - libdeflate=1.8=h7f8727e_5
+  - libedit=3.1.20191231=he28a2e2_2
+  - libev=4.33=h516909a_1
+  - libevent=2.1.10=h9b69904_4
+  - libffi=3.3=he6710b0_2
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgfortran-ng=12.2.0=h69a702a_19
+  - libgfortran5=12.2.0=h337968e_19
+  - libgomp=11.2.0=h1234567_1
+  - libiconv=1.16=h7f8727e_2
+  - libidn2=2.3.2=h7f8727e_0
+  - libnghttp2=1.46.0=hce63b2e_0
+  - libnpp=11.8.0.86=0
+  - libnpp-dev=11.8.0.86=0
+  - libnvjpeg=11.9.0.86=0
+  - libnvjpeg-dev=11.9.0.86=0
+  - libpng=1.6.37=hbc83047_0
+  - libprotobuf=3.20.1=h4ff587b_0
+  - libssh2=1.10.0=ha56f1ee_2
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - libtasn1=4.16.0=h27cfd23_0
+  - libthrift=0.15.0=he6d91bd_0
+  - libtiff=4.4.0=hecacb30_0  
+  - libunistring=0.9.10=h27cfd23_0
+  - libuuid=1.0.3=h7f8727e_2
+  - libwebp=1.2.4=h11a3e52_0
+  - libwebp-base=1.2.4=h5eee18b_0
+  - lz4-c=1.9.3=h295c915_1
+  - markdown=3.4.1=pyhd8ed1ab_0
+  - markupsafe=2.1.1=py310h5764c6d_1
+  - mkl=2021.4.0=h06a4308_640
+  - mkl-service=2.4.0=py310h7f8727e_0
+  - mkl_fft=1.3.1=py310hd6ae3a3_0
+  - mkl_random=1.2.2=py310h00e6091_0
+  - multidict=6.0.2=py310h5764c6d_1
+  - multiprocess=0.70.12.2=py310h5764c6d_2
+  - natsort=7.1.1=pyhd3eb1b0_0
+  - ncurses=6.3=h5eee18b_3
+  - nettle=3.7.3=hbbd107a_1
+  - nsight-compute=2022.3.0.22=0
+  - numexpr=2.8.3=py310hcea2de6_0
+  - numpy=1.23.3=py310hd5efca6_0
+  #- numpy-base=1.23.3=py310h8e6c178_0
+  - oauthlib=3.2.2=pyhd8ed1ab_0
+  - openh264=2.1.1=h4ff587b_0
+  - openssl=1.1.1s=h7f8727e_0
+  - orc=1.7.4=h07ed6aa_0
+  - packaging=21.3=pyhd8ed1ab_0
+  - pandas=1.4.4=py310h6a678d5_0
+  - pillow=9.2.0=py310hace64e9_1
+  - pip=22.2.2=py310h06a4308_0  
+  - protobuf=3.20.1=py310hd8f1fbe_0
+  - pyarrow=8.0.0=py310h468efa6_0
+  - pyasn1=0.4.8=py_0
+  - pyasn1-modules=0.2.7=py_0
+  - pycparser=2.21=pyhd3eb1b0_0
+  - pyjwt=2.6.0=pyhd8ed1ab_0
+  - pyopenssl=22.0.0=pyhd3eb1b0_0
+  - pyparsing=3.0.9=pyhd8ed1ab_0
+  - pysocks=1.7.1=py310h06a4308_0
+  - python=3.10.6=haa1d7c7_1
+  - python-dateutil=2.8.2=pyhd8ed1ab_0
+  - python-xxhash=3.0.0=py310h5764c6d_1
+  - python_abi=3.10=2_cp310
+  - pytorch=1.13.0=py3.10_cuda11.7_cudnn8.5.0_0
+  - pytorch-cuda=11.7=h67b0de4_0
+  - pytorch-mutex=1.0=cuda
+  - pytz=2022.6=pyhd8ed1ab_0
+  - pyu2f=0.1.5=pyhd8ed1ab_0
+  - pyyaml=6.0=py310h5764c6d_4
+  - re2=2022.04.01=h27087fc_0
+  - readline=8.2=h5eee18b_0
+  - regex=2022.7.9=py310h5eee18b_0
+  - requests=2.28.1=py310h06a4308_0
+  - requests-oauthlib=1.3.1=pyhd8ed1ab_0
+  - responses=0.18.0=pyhd8ed1ab_0
+  - rsa=4.9=pyhd8ed1ab_0
+  - sacremoses=0.0.53=pyhd8ed1ab_0
+  - scikit-learn=1.1.3=py310h6a678d5_0
+  - scipy=1.9.3=py310hd5efca6_0
+  - seqeval=1.2.2=pyhd3deb0d_0
+  - setuptools=65.4.0=py310h06a4308_0
+  - six=1.16.0=pyhd3eb1b0_1
+  - snappy=1.1.9=hbd366e4_1
+  - sqlite=3.39.3=h5082296_0
+  - tensorboard=2.10.1=pyhd8ed1ab_0
+  - tensorboard-data-server=0.6.0=py310h597c629_2
+  - tensorboard-plugin-wit=1.8.1=pyhd8ed1ab_0
+  - threadpoolctl=3.1.0=pyh8a188c0_0
+  - tk=8.6.12=h1ccaba5_0
+  - tokenizers=0.11.4=py310h3dcd8bd_1
+  - torchaudio=0.13.0=py310_cu117
+  - torchtext=0.14.0=py310
+  - torchvision=0.14.0=py310_cu117
+  - tqdm=4.64.1=py310h06a4308_0
+  - transformers=4.24.0=pyhd8ed1ab_0
+  - typing-extensions=4.3.0=py310h06a4308_0
+  - typing_extensions=4.3.0=py310h06a4308_0
+  - tzdata=2022e=h04d1e81_0
+  - urllib3=1.26.12=py310h06a4308_0
+  - utf8proc=2.6.1=h27cfd23_0
+  - werkzeug=2.2.2=pyhd8ed1ab_0
+  - wheel=0.37.1=pyhd3eb1b0_0
+  - xxhash=0.8.0=h7f98852_3
+  - xz=5.2.6=h5eee18b_0
+  - yaml=0.2.5=h7f98852_2
+  - yarl=1.7.2=py310h5764c6d_2    
+  - zipp=3.10.0=pyhd8ed1ab_0
+  - zlib=1.2.13=h5eee18b_0
+  - zstd=1.5.2=ha4553b6_0

SinaTools-0.1.1.dist-info/AUTHORS.rst

@@ -0,0 +1,13 @@
+=======
+Credits
+=======
+
+Development Lead
+----------------
+
+* SinaLab <sina.institute.bzu@gmail.com>
+
+Contributors
+------------
+
+None yet. Why not be the first?

SinaTools-0.1.1.dist-info/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2023, SinaLab
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

SinaTools-0.1.1.dist-info/METADATA

@@ -0,0 +1,72 @@
+Metadata-Version: 2.1
+Name: SinaTools
+Version: 0.1.1
+Summary: UNKNOWN
+Home-page: https://github.com/SinaLab/nlptools
+Author: UNKNOWN
+Author-email: UNKNOWN
+License: MIT license
+Keywords: nlptools
+Platform: UNKNOWN
+Requires-Dist: six
+Requires-Dist: farasapy
+Requires-Dist: tqdm
+Requires-Dist: requests
+Requires-Dist: regex
+Requires-Dist: pathlib
+Requires-Dist: torch (==1.13.0)
+Requires-Dist: transformers (==4.24.0)
+Requires-Dist: torchtext (==0.14.0)
+Requires-Dist: torchvision (==0.14.0)
+Requires-Dist: seqeval (==1.2.2)
+Requires-Dist: natsort (==7.1.1)
+
+========
+nlptools
+========
+
+
+.. image:: https://img.shields.io/pypi/v/nlptools.svg
+        :target: https://pypi.python.org/pypi/SinaTools
+
+.. image:: https://img.shields.io/travis/sina_institute/nlptools.svg
+        :target: https://travis-ci.com/sina_institute/SinaTools
+
+.. image:: https://readthedocs.org/projects/nlptools/badge/?version=latest
+        :target: https://SinaTools.readthedocs.io/en/latest/?version=latest
+        :alt: Documentation Status
+
+
+
+
+Python Boilerplate contains all the boilerplate you need to create a Python package.
+
+
+* Free software: MIT license
+* Documentation: https://SinaTools.readthedocs.io.
+
+
+Features
+--------
+
+* TODO
+
+Credits
+-------
+
+This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template.
+
+.. _Cookiecutter: https://github.com/audreyr/cookiecutter
+.. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage
+
+
+=======
+History
+=======
+
+0.1.0 (2023-04-15)
+------------------
+
+* First release on PyPI.
+
+

SinaTools-0.1.1.dist-info/RECORD

@@ -0,0 +1,122 @@
+SinaTools-0.1.1.data/data/nlptools/environment.yml,sha256=OzilhLjZbo_3nU93EQNUFX-6G5O3newiSWrwxvMH2Os,7231
+nlptools/VERSION,sha256=Ee4juPwvxhnW6rYnettaUnJhBnrAHM_D4RhX9Vvxi80,5
+nlptools/__init__.py,sha256=OoA_p_y2jPjMytcUrG1ED5uJlJemVhSRr9L9Wsym-rQ,134
+nlptools/environment.yml,sha256=OzilhLjZbo_3nU93EQNUFX-6G5O3newiSWrwxvMH2Os,7231
+nlptools/install_env.py,sha256=EODeeE0ZzfM_rz33_JSIruX03Nc4ghyVOM5BHVhsZaQ,404
+nlptools/nlptools.py,sha256=vR5AaF0iel21LvsdcqwheoBz0SIj9K9I_Ub8M8oA98Y,20
+nlptools/CLI/DataDownload/download_files.py,sha256=PMDEPXxZQbrFo-7iyhvrCpzx2RG5T5kPk6NJAwh8RSI,2322
+nlptools/CLI/arabiner/bin/infer.py,sha256=YrNCVro8B3UxpsHjIo_01qiBQURpDNTK7pKTkw1L21Y,4921
+nlptools/CLI/arabiner/bin/infer2.py,sha256=CtR9rwe20ks_qq-l_fQU-ThLqft_1o3Ztmd1my1kHMg,3905
+nlptools/CLI/morphology/ALMA_multi_word.py,sha256=NINts8BtT8BGQPBvs4BJ_y2PsR7czsGPOVAwngaT85A,2644
+nlptools/CLI/morphology/morph_analyzer.py,sha256=39vrFx6ppu7yEITcz8lAJhk3xHweaPWEqL-CcqBM37Q,3565
+nlptools/CLI/salma/salma_tools.py,sha256=7awpCb68QUc3kx-EuwRHxDmItZlX2aSdpukwKF1G3Fo,1999
+nlptools/CLI/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+nlptools/CLI/utils/arStrip.py,sha256=dzy16wZfSznkvGHHBn5P21EvyusKB55dqrZ4zbaa41w,3621
+nlptools/CLI/utils/corpus_tokenizer.py,sha256=S0YG8FRS29K1C8eJVEYuWSV1ABS7PKymlNS7KxvYqxI,2817
+nlptools/CLI/utils/implication.py,sha256=hjYTN0oiLf0bz0bRO_GD4rphZkaB3cH770clFFhuevE,3172
+nlptools/CLI/utils/jaccard.py,sha256=a6oc28yMgm7UewO6Lz25A4Yv8QEzVa85XF-QV9uhMwI,4639
+nlptools/CLI/utils/latin_remove.py,sha256=Xw6PB4GtMLLiYK3zTEwdLhBbivMyy1msD5Ab_QdJoQA,1303
+nlptools/CLI/utils/remove_Punc.py,sha256=dvSiSs9UulhGCogBgtpD8fU860BFuMBTnwa8Ek9aPKQ,1393
+nlptools/CLI/utils/sentence_tokenizer.py,sha256=AcJa_yRdlQqKMwVWWKSv1vRO1Yk-NK75-NpalkHqewc,3469
+nlptools/CLI/utils/text_transliteration.py,sha256=blIGB8FeF10iFeXADM-z01XJ4qeB1qgj6S2Xnk9w5fI,2266
+nlptools/DataDownload/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+nlptools/DataDownload/downloader.py,sha256=yONVa99OtPXD5Lewy4Fm3eUiJMpBt492G1JOPh5sXAU,6523
+nlptools/arabert/__init__.py,sha256=ely2PttjgSv7vKdzskuD1rtK_l_UOpmxJSz8isrveD0,16
+nlptools/arabert/preprocess.py,sha256=qI0FsuMTOzdRlYGCtLrjpXgikNElUZPv9bnjaKDZKJ4,33024
+nlptools/arabert/arabert/__init__.py,sha256=KbSAH-XqbRygn0y59m5-ZYOLXgpT1gSgE3F-qd4rKEc,627
+nlptools/arabert/arabert/create_classification_data.py,sha256=BhemGNRbYz_Pun0Q5WerN2-9n-ILmU3tm4J-OlHw5-A,7678
+nlptools/arabert/arabert/create_pretraining_data.py,sha256=2M-cF3CLHbQ0cdWrzFT6Frg1vVP4Y-CFoq8iEPyxgsE,18924
+nlptools/arabert/arabert/extract_features.py,sha256=C1IzASrlX7u4_M2xdr_PjzWfTRZgklhUXA2WHKgQt-I,15585
+nlptools/arabert/arabert/lamb_optimizer.py,sha256=uN3Dcx-6n2_OwepyymRrGrB4EcSkR8b2ZczZrOr7bpY,6263
+nlptools/arabert/arabert/modeling.py,sha256=KliecCmA1pP3owg0mYge6On3IRHunMF5kMLuEwc0VLw,40896
+nlptools/arabert/arabert/optimization.py,sha256=Wx0Js6Zsfc3iVw-_7Q1SCnxfP_qqbdTAyFD-vZSpOyk,8153
+nlptools/arabert/arabert/run_classifier.py,sha256=AdVGyvidlmbEp12b-PauiBo6EmFLEO7tqeJKuLhK2DA,38777
+nlptools/arabert/arabert/run_pretraining.py,sha256=yO16nKkHDfcYA2Zx7vv8KN4te6_1qFOzyVeDzFT-DQw,21894
+nlptools/arabert/arabert/run_squad.py,sha256=PORxgiByP8L6vZqAFkqgHPJ_ZjAlqlg64gtkdLmDNns,53456
+nlptools/arabert/arabert/tokenization.py,sha256=R6xkyCb8_vgeksXiLeqDvV5vOnLb1cPNsvfDij6YVFk,14132
+nlptools/arabert/araelectra/__init__.py,sha256=ely2PttjgSv7vKdzskuD1rtK_l_UOpmxJSz8isrveD0,16
+nlptools/arabert/araelectra/build_openwebtext_pretraining_dataset.py,sha256=pIo6VFT3XXOYroZaab3msZAP6XjCKu0KcrIZQA0Pj8U,3881
+nlptools/arabert/araelectra/build_pretraining_dataset.py,sha256=Z8ZmKznaE_2SPDRoPYR1SDhjTN_NTpNCFFuhUkykwl8,9041
+nlptools/arabert/araelectra/build_pretraining_dataset_single_file.py,sha256=W7HFr1XoO6bCDR7X7w-bOuwULFtTSjeKbJ2LHzzHf9k,3224
+nlptools/arabert/araelectra/configure_finetuning.py,sha256=YfGLMdgN6Qqm357Mzy5UMjkuLPPWtBs7f4dA-DKE6JM,7768
+nlptools/arabert/araelectra/configure_pretraining.py,sha256=oafQgu4WmVdxBcU5mSfXhPlvCk43CJwAWXC10Q58BlI,5801
+nlptools/arabert/araelectra/flops_computation.py,sha256=krHTeuPH9xQu5ldprBOPJNlJRvC7fmmvXXqUjfWrzPE,9499
+nlptools/arabert/araelectra/run_finetuning.py,sha256=JecbrSmGikBNyid4JKRZ49Rm5xFpt02WfgIIcs3TpcU,12976
+nlptools/arabert/araelectra/run_pretraining.py,sha256=1K2aAFTY0p3iaLY0xkhTlm6v0B-Zun8SwEzz-K6RXM4,20665
+nlptools/arabert/araelectra/finetune/__init__.py,sha256=d55FZ9ZE-_t_WWMnIiRGozkTw50vBZ-s9BMy7l_I-ao,619
+nlptools/arabert/araelectra/finetune/feature_spec.py,sha256=cqNlBa2KK_G1-vkKm1EJUv6BoS3gesCUAHwVagZB6wM,1888
+nlptools/arabert/araelectra/finetune/preprocessing.py,sha256=1mf7-IxknCRsobQZ-VV1zs4Cwt-mfOtoVxysDJa9LZ0,6657
+nlptools/arabert/araelectra/finetune/scorer.py,sha256=PjRg0P5ANCtul2ute7ccq3mRCCoIAoCb-lVLlwd4rVY,1571
+nlptools/arabert/araelectra/finetune/task.py,sha256=zM8M4PGSIrY2u6ytpmkQEXxG-jjoeN9wouEyVR23qeQ,1991
+nlptools/arabert/araelectra/finetune/task_builder.py,sha256=Zsoiuw5M3Ca8QhaZVLVLZyWw09K5R75UeMuPmazMlHI,2768
+nlptools/arabert/araelectra/model/__init__.py,sha256=d55FZ9ZE-_t_WWMnIiRGozkTw50vBZ-s9BMy7l_I-ao,619
+nlptools/arabert/araelectra/model/modeling.py,sha256=5XLIutnmr-SFQOV_XntJ-U5evSCY-J2e9NjvlwVXKkk,40877
+nlptools/arabert/araelectra/model/optimization.py,sha256=BCMb_C5hgBw7wC9ZR8AQ4lwoPopqLIcSiqcCrIjx9XU,7254
+nlptools/arabert/araelectra/model/tokenization.py,sha256=9CkyPzs3L6OEPzN-7EWQDNQmW2mIJoZD4o1rn6xLdL4,11082
+nlptools/arabert/araelectra/pretrain/__init__.py,sha256=d55FZ9ZE-_t_WWMnIiRGozkTw50vBZ-s9BMy7l_I-ao,619
+nlptools/arabert/araelectra/pretrain/pretrain_data.py,sha256=NLgIcLAq1-MgtBNXYu_isDxnOY5k67SyADYy-8nzBok,5413
+nlptools/arabert/araelectra/pretrain/pretrain_helpers.py,sha256=nFl7LEdxAU5kKwiodqJHzi-ty9jMFsCCNYOF__A69j8,9255
+nlptools/arabert/araelectra/util/__init__.py,sha256=d55FZ9ZE-_t_WWMnIiRGozkTw50vBZ-s9BMy7l_I-ao,619
+nlptools/arabert/araelectra/util/training_utils.py,sha256=7h_J1ljUWM0ynBcofEtjZWL_oAfZtTxEemQLkixgn-0,4142
+nlptools/arabert/araelectra/util/utils.py,sha256=G0UAETUCZMlU9R9ASD9AXrWZeodWI1aZJEE9F-goaH4,2591
+nlptools/arabert/aragpt2/__init__.py,sha256=aQkKhQwWaS61wYEeOdx682upeMWFPUjLxXSs7JM1sOA,18
+nlptools/arabert/aragpt2/create_pretraining_data.py,sha256=fFa2_DAyTwc8L2IqQbshsh_Ia26nj1qtVLzC6DooSac,3105
+nlptools/arabert/aragpt2/train_bpe_tokenizer.py,sha256=b-8zHQ02fLmZV4GfjnrPptwjpX259F41SlnWzBrflMA,1888
+nlptools/arabert/aragpt2/gpt2/__init__.py,sha256=aQkKhQwWaS61wYEeOdx682upeMWFPUjLxXSs7JM1sOA,18
+nlptools/arabert/aragpt2/gpt2/lamb_optimizer.py,sha256=uN3Dcx-6n2_OwepyymRrGrB4EcSkR8b2ZczZrOr7bpY,6263
+nlptools/arabert/aragpt2/gpt2/optimization.py,sha256=iqh23cypRSRUt53wt2G5SbNNpJMwERM7gZAOKVh5l4U,8411
+nlptools/arabert/aragpt2/gpt2/run_pretraining.py,sha256=4jjkUbvTO1DHoKJ89yKtlkkofcND_fyAunQ-mlnJhTM,13298
+nlptools/arabert/aragpt2/grover/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+nlptools/arabert/aragpt2/grover/dataloader.py,sha256=-FWPTjtsvweEE1WaWRHBXfOSbsGiUmnXT3qK7KJP8cM,6853
+nlptools/arabert/aragpt2/grover/modeling.py,sha256=XcUvFwqRaxAwWiJstrH2FPBvDJe03pTWIyipdMfWj9g,38280
+nlptools/arabert/aragpt2/grover/modeling_gpt2.py,sha256=WFpCWn1792yATFzt8rZ0rpWvExfbLzV2BqiEs7llFUw,51602
+nlptools/arabert/aragpt2/grover/optimization_adafactor.py,sha256=1geOsCWuv5xxtSnKDz9a8aY5SVwZ1MGq-xVQDBg4Gpg,9765
+nlptools/arabert/aragpt2/grover/train_tpu.py,sha256=qNgLI_j6-KYkTMJfVoFlh4NIKweY1aPz1XPDw6odld0,7102
+nlptools/arabert/aragpt2/grover/utils.py,sha256=V5wMUxK03r5g_pb7R3_uGLOPqQJfbIB0VaJ8ZDM4XAo,8473
+nlptools/arabiner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+nlptools/arabiner/bin/__init__.py,sha256=d1ToN2uheCCVby3TjiSuD1dqo_pvNIuTgz4COFr2Khs,438
+nlptools/arabiner/bin/eval.py,sha256=ihtjJinY1jXpZXW5bQJzTC5MF6_V3GQ5zHzsc691_HQ,2591
+nlptools/arabiner/bin/infer.py,sha256=EZKeq4zucIE-ooHYnegODNxsRiIY_gY5GvDPChH5WRQ,3237
+nlptools/arabiner/bin/process.py,sha256=4QCZjsmYV5lep6waQE37fs7Fe59_1G5seIJLDkArg4s,4698
+nlptools/arabiner/bin/train.py,sha256=hf6ZRhqMZ7bFealMSusBjtWrbzHGHc5HB2Lh4rp2uQA,6390
+nlptools/arabiner/data/__init__.py,sha256=XPic1bPijmZda_LPFL5J6TOps_IHUTiBDJvMx-iJqKo,61
+nlptools/arabiner/data/datasets.py,sha256=p52Uc8Q2D3EzN1OmoHQcWVsJ2oB3TqgTzAcy1B9fJ68,5068
+nlptools/arabiner/data/transforms.py,sha256=KPCDdjZOEvhMC38eiFwJuiQC84cfDrvC0XM4Ye0o3do,4878
+nlptools/arabiner/nn/BaseModel.py,sha256=3GmujQasTZZunOBuFXpY2p1W8W256iI_Uu4hxhOY2Z0,608
+nlptools/arabiner/nn/BertNestedTagger.py,sha256=7vU2tmDSoqSHn6GvMJmyN0hEMLvCkbr_r-AaiAaYdw8,1223
+nlptools/arabiner/nn/BertSeqTagger.py,sha256=dFcBBiMw2QCWsyy7aQDe_PS3aRuNn4DOxKIHgTblFvc,504
+nlptools/arabiner/nn/__init__.py,sha256=ZN7Psm83pysUhGI3ZSaJra2aCYBZb9DZ0UX4CiKGc0A,182
+nlptools/arabiner/trainers/BaseTrainer.py,sha256=oZgFJW-CawfCKT5gtaBHA7Q7XjNfiyqM62KnFsgVzPU,3919
+nlptools/arabiner/trainers/BertNestedTrainer.py,sha256=hVqPRdmaHf2iwftseNpgsAfwGkl6eMHJx1rKunQS_vM,8443
+nlptools/arabiner/trainers/BertTrainer.py,sha256=KkcgZXu6kqsrrnfFtiAQ8ucLsrQtDxLRqdbTiTnRWqI,6447
+nlptools/arabiner/trainers/__init__.py,sha256=kt8WqsaOjX0h1JMa-v7Y9ywT5mfwQIsZTyVWnIAWsEQ,200
+nlptools/arabiner/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+nlptools/arabiner/utils/data.py,sha256=uuPiu-7v0gccNygZjdTKomJGE7X0H9FC24Y9nHZpf4c,4376
+nlptools/arabiner/utils/helpers.py,sha256=PyOOlx5uabvZVmU3SZtZ3ZLA3pliinJ3JXsvos9SUWU,5032
+nlptools/arabiner/utils/metrics.py,sha256=Irz6SsIvpOzGIA2lWxrEV86xnTnm0TzKm9SUVT4SXUU,2734
+nlptools/morphology/ALMA_multi_word.py,sha256=hlzZCk-uUdZ-GbiPsFxDTvoWoIuVof2Sm7NdaxaFipM,1313
+nlptools/morphology/__init__.py,sha256=z6_RGhiyfNHXNKMmhNSI6ObTLmdjQyP58vsFottI8GA,1706
+nlptools/morphology/charsets.py,sha256=7w9OrbnZTnLU3A9q-SUi9GhUN97qNtbYR5T0Pm72cF8,2784
+nlptools/morphology/morph_analyzer.py,sha256=OmCxm4-fM2qfYzKk8yOd6D_T3RsfzZCcd7Oz2V4Advg,6507
+nlptools/morphology/settings.py,sha256=sEZdnA7MiYXHdxrfHWXop1RcKClVzpOYzZwzHC1PxJ8,144
+nlptools/morphology/tokenizers_words.py,sha256=Smtt_KXifl2wRI464Qn07PtUvOsyGBJjZ7E20gd8zVM,602
+nlptools/salma/__init__.py,sha256=pOauGjD-xrGHw05sNx3EiSFc_wpM3bD1vJxQHoDDXOA,376
+nlptools/salma/settings.py,sha256=fqAQg2b22gorzT9Pf_AEJD9p8AlVUaVyKD3FH8g2yUs,1110
+nlptools/salma/views.py,sha256=EH1vc6P88CeAIzQKt7EU_HTI0uJipv4JdXiAX5NjrJY,18416
+nlptools/salma/wsd.py,sha256=kmP5ZvvVMkxApgk91TAGSBkMJZbPPbS0qoNk8OE37og,4434
+nlptools/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+nlptools/utils/corpus_tokenizer.py,sha256=IDWh87XJaFa7V2P4kIxY4QVywPKhz0fIErc_c0gJGUU,4581
+nlptools/utils/implication.py,sha256=Ro1Vw62oOBzELkX-zpHyieq4v2OsoyFrFeTU7BiK7qc,27794
+nlptools/utils/jaccard.py,sha256=TTC5KTVv6kONw5vZtzxEQvv7QM79BCsD0xcJAY0T5tU,10111
+nlptools/utils/parser.py,sha256=0Yd40CZf4wXso2q-d9LULUNAVUAMdiYMImfcVb6i9qQ,6175
+nlptools/utils/readfile.py,sha256=xE4LEaCqXJIk9v37QUSSmWb-aY3UnCFUNb7uVdx3cpM,133
+nlptools/utils/sentence_tokenizer.py,sha256=3C0Wx1ns8ZHiGwKlUkcti-8zA3fB4ju0fIEtGACM7oU,2162
+nlptools/utils/text_transliteration.py,sha256=zhB3sFXSMtkkdqImRMVg415AAB80DOm9lMFKb2IBynw,8765
+nlptools/utils/utils.py,sha256=vKkFOkYclMu8nXS_VZb6Kobx8QGKW9onXkkLCeiRb6g,32
+SinaTools-0.1.1.dist-info/AUTHORS.rst,sha256=aTWeWlIdfLi56iLJfIUAwIrmqDcgxXKLji75_Fjzjyg,174
+SinaTools-0.1.1.dist-info/LICENSE,sha256=uwsKYG4TayHXNANWdpfMN2lVW4dimxQjA_7vuCVhD70,1088
+SinaTools-0.1.1.dist-info/METADATA,sha256=G-49Kky9vazLGwQcFV-lbFQ_tb2PzwidvGdTN3wTG_c,1577
+SinaTools-0.1.1.dist-info/WHEEL,sha256=6T3TYZE4YFi2HTS1BeZHNXAi8N52OZT4O-dJ6-ome_4,116
+SinaTools-0.1.1.dist-info/entry_points.txt,sha256=9-PNkvWGCid8SVN03S2NkJFuxAzvcB22tGpHe-et2q8,951
+SinaTools-0.1.1.dist-info/top_level.txt,sha256=sREDI6iHe4D0BZQmZbZ-LxYIn2cBWUayk9CZwAR9jaE,9
+SinaTools-0.1.1.dist-info/RECORD,,

SinaTools-0.1.1.dist-info/WHEEL

@@ -0,0 +1,6 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.34.2)
+Root-Is-Purelib: true
+Tag: py2-none-any
+Tag: py3-none-any
+

SinaTools-0.1.1.dist-info/entry_points.txt

@@ -0,0 +1,18 @@
+[console_scripts]
+arabi_ner = nlptools.CLI.arabiner.bin.infer:main
+arabi_ner2 = nlptools.CLI.arabiner.bin.infer2:main
+install_env = nlptools.install_env:main
+sina_alma_multi_word = nlptools.CLI.morphology.ALMA_multi_word:main
+sina_appdatadir = nlptools.CLI.DataDownload.get_appdatadir:main
+sina_arStrip = nlptools.CLI.utils.arStrip:main
+sina_corpus_tokenizer = nlptools.CLI.utils.corpus_tokenizer:main
+sina_download_files = nlptools.CLI.DataDownload.download_files:main
+sina_implication = nlptools.CLI.utils.implication:main
+sina_jaccard_similarity = nlptools.CLI.utils.jaccard:main
+sina_morph_analyze = nlptools.CLI.morphology.morph_analyzer:main
+sina_remove_latin = nlptools.CLI.utils.latin_remove:main
+sina_remove_punctuation = nlptools.CLI.utils.remove_Punc:main
+sina_salma = nlptools.CLI.salma.salma_tools:main
+sina_sentence_tokenize = nlptools.CLI.utils.sentence_tokenizer:main
+sina_transliterate = nlptools.CLI.utils.text_transliteration:main
+

SinaTools-0.1.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+nlptools

nlptools/CLI/DataDownload/download_files.py

@@ -0,0 +1,71 @@
+"""
+About:
+------
+
+The sina_download_files tool is a command-line interface for downloading various NLP resources from pre-specified URLs. It is a part of the nlptools package and provides options to choose which files to download and to specify a download directory. The tool automatically handles file extraction for zip and tar.gz files.
+
+Usage:
+------
+
+Below is the usage information that can be generated by running sina_download_files --help.
+
+.. code-block:: none
+
+    Usage:
+     sina_download_files [OPTIONS]
+
+.. code-block:: none
+
+        Options:
+        -f, --files FILES
+            Names of the files to download. Available files are: ner, morph, salma_model, salma_tokenizer, glosses_dic, lemma_dic, five_grams, four_grams, three_grams, two_grams. 
+            If no file is specified, all files will be downloaded.
+
+Examples:
+---------
+
+.. code-block:: none
+
+    sina_download_files -f morph ner 
+    This command will download only the `morph` and `ner` files to the default directory.
+
+Note:
+-----
+
+.. code-block:: none
+
+    - The script automatically handles the extraction of zip and tar.gz files after downloading.
+    - Ensure you have the necessary permissions to write to the specified directory.
+    - The default download directory is based on the operating system and can be obtained using the `get_appdatadir` function.
+
+
+"""
+
+import argparse
+from nlptools.DataDownload.downloader import download_file
+from nlptools.DataDownload.downloader import download_files
+from nlptools.DataDownload.downloader import get_appdatadir
+from nlptools.DataDownload.downloader import urls
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Download files from specified URLs.")
+    parser.add_argument('-f', '--files', nargs="*", choices=urls.keys(),
+                        help="Names of the files to download. Available files are: "
+                             f"{', '.join(urls.keys())}. If no file is specified, all files will be downloaded.")
+    
+    get_appdatadir()
+
+    args = parser.parse_args()
+
+    if args.files:
+        for file in args.files:
+            url = urls[file]
+            download_file(url)
+    else:
+        download_files()
+
+if __name__ == '__main__':
+    main()
+
+#sina_download_files -f morph ner

nlptools/CLI/arabiner/bin/infer.py

@@ -0,0 +1,117 @@
+"""
+About:
+------
+The ArabiNER tool carries out Named Entity Recognition (NER) utilizing the ArabiNER utility from the SinaTools suite. It identifies the named entities and provides a comprehensive analysis in JSON format if the input consists of text, or in a CSV file if the input is a directory of files.
+
+Usage:
+------
+Below is the usage information that can be generated by running arabi_ner --help.
+
+.. code-block:: none
+
+    arabi_ner --text=INPUT_TEXT
+    arabi_ner --dir=INPUT_FILE --output_csv=OUTPUT_FILE_NAME
+
+Options:
+--------
+
+.. code-block:: none
+
+  --text INPUT_TEXT
+        The text that needs to be analyzed for Named Entity Recognition.
+  --file INPUT_FILE
+        File containing the text to be analyzed for Named Entity Recognition.
+  --output_csv OUTPUT_FILE_NAME
+        A file containing the tokenized text and its Named Entity tags.
+Examples:
+---------
+
+.. code-block:: none
+
+    arabi_ner --text "Your text here"
+    arabi_ner --dir "/path/to/your/directory" --output_csv "output.csv"
+
+Note:
+-----
+
+.. code-block:: none
+
+    - Ensure that the text input is appropriately encoded in UTF-8 or compatible formats.
+    - The tool returns results in JSON format with proper indentation for better readability.
+    - The quality and accuracy of the analysis depend on the underlying capabilities of the ArabiNER utility.
+
+"""
+
+import argparse
+import json
+import pandas as pd
+from nlptools.arabiner.bin.infer import ner
+from nlptools.utils.corpus_tokenizer import corpus_tokenizer
+from nlptools.morphology.tokenizers_words import simple_word_tokenize
+
+
+def infer(sentence):
+    # Now infer returns all NER tags for a sentence
+    output = ner(sentence)
+    ##print("ner output : ", output)
+    return [word[1] for word in output]
+
+
+def main():
+    parser = argparse.ArgumentParser(description='NER Analysis using ArabiNER')
+
+    parser.add_argument('--text', type=str, help='Text to be analyzed for Named Entity Recognition')
+    parser.add_argument('--dir', type=str, help='dir containing the text files to be analyzed for Named Entity Recognition')
+    parser.add_argument('--output_csv', type=str, help='Output CSV file to write the results')
+
+    args = parser.parse_args()
+
+    if args.text is not None:
+        results = ner(args.text)
+        # Print the results in JSON format
+        print(json.dumps(results, ensure_ascii=False, indent=4))
+    elif args.dir is not None:
+        corpus_tokenizer(args.dir, args.output_csv)
+        df = pd.read_csv(args.output_csv)
+        df['NER tags'] = None
+        i = 0
+
+        # Use drop_duplicates to get unique values based on Row_ID and Sentence
+        result = df.drop_duplicates(subset=['Global Sentence ID', 'Sentence'])
+        
+        # Get the "Sentence" column as an array
+        unique_sentences = result['Sentence'].to_numpy()
+        
+        # Print the result
+        #print(unique_sentences, len(result['Sentence']))
+        #print("#############")
+
+        for sentence in unique_sentences:  # iterating over unique sentences            
+            #print(" Sentence : ", simple_word_tokenize(sentence), len(simple_word_tokenize(sentence)))
+            ner_tags = infer(sentence)  # getting all NER tags for the sentence            
+            #if len(ner_tags) != len(df[i:i+len(ner_tags)]):
+            #    print("Not Equal...", len(ner_tags) , len(df[i:i+len(ner_tags)]))
+            #    return 
+            if len(simple_word_tokenize(sentence)) > 300:
+                print(" Length of this sentence is more than 300 word:  ", sentence)
+                return
+            #df['NER tags'].iloc[i:i+len(ner_tags)] = ner_tags
+            df.loc[i:i+len(ner_tags)-1, 'NER tags'] = ner_tags  # Use .loc to assign values
+            #print("Exit with ner tags = ", ner_tags, " and length : ", len(ner_tags), type(len(ner_tags)), " and df is " , df[i:i+len(ner_tags)], " with length : ", len(df[i:i+len(ner_tags)]), type(len(df[i:i+len(ner_tags)])),  " i:i+len(ner_tags) : ", i," , ", i+len(ner_tags))
+            i = i + len(ner_tags)
+        
+        df.to_csv(args.output_csv, index=False) 
+    else:    
+        print("Error: Either --text or --file argument must be provided.")
+        return
+
+
+if __name__ == '__main__':
+    main()
+
+#arabi_ner --text "Your text here."
+#arabi_ner --dir /path/to/your/directory --output_csv output.csv
+
+#Each unique sentence in the CSV file is processed once by the infer function to get the NER tags for all the words in the sentence.
+#The current_word_position variable is used to keep track of the position within the list of NER tags returned by infer, ensuring that each word in the CSV file is assigned the correct NER tag.
+#The final CSV file will contain an additional column, NER tags, which contains the NER tag for each word in the Sentence column of the CSV file.