turkish-linguistic-features 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- turkish_linguistic_features-0.2.0/.github/workflows/ci.yml +90 -0
- turkish_linguistic_features-0.2.0/.github/workflows/docs.yml +41 -0
- turkish_linguistic_features-0.2.0/.github/workflows/publish.yml +44 -0
- turkish_linguistic_features-0.2.0/.gitignore +18 -0
- turkish_linguistic_features-0.2.0/.pre-commit-config.yaml +9 -0
- turkish_linguistic_features-0.2.0/CHANGELOG.md +159 -0
- turkish_linguistic_features-0.2.0/CITATION.cff +32 -0
- turkish_linguistic_features-0.2.0/LICENSE +21 -0
- turkish_linguistic_features-0.2.0/PKG-INFO +137 -0
- turkish_linguistic_features-0.2.0/README.md +103 -0
- turkish_linguistic_features-0.2.0/docs/dogrulama-raporu.md +625 -0
- turkish_linguistic_features-0.2.0/docs/en/explanation/concepts.md +180 -0
- turkish_linguistic_features-0.2.0/docs/en/explanation/index.md +12 -0
- turkish_linguistic_features-0.2.0/docs/en/explanation/limitations.md +114 -0
- turkish_linguistic_features-0.2.0/docs/en/explanation/nan.md +97 -0
- turkish_linguistic_features-0.2.0/docs/en/explanation/verification.md +237 -0
- turkish_linguistic_features-0.2.0/docs/en/getting-started.md +202 -0
- turkish_linguistic_features-0.2.0/docs/en/glossary.md +52 -0
- turkish_linguistic_features-0.2.0/docs/en/how-to/citations.md +119 -0
- turkish_linguistic_features-0.2.0/docs/en/how-to/comparison.md +85 -0
- turkish_linguistic_features-0.2.0/docs/en/how-to/corpus.md +164 -0
- turkish_linguistic_features-0.2.0/docs/en/how-to/index.md +17 -0
- turkish_linguistic_features-0.2.0/docs/en/how-to/parameters.md +91 -0
- turkish_linguistic_features-0.2.0/docs/en/how-to/segmenting.md +97 -0
- turkish_linguistic_features-0.2.0/docs/en/how-to/single-text.md +170 -0
- turkish_linguistic_features-0.2.0/docs/en/index.md +49 -0
- turkish_linguistic_features-0.2.0/docs/esik-kalibrasyonu.md +83 -0
- turkish_linguistic_features-0.2.0/docs/index.md +77 -0
- turkish_linguistic_features-0.2.0/docs/reference/api.md +225 -0
- turkish_linguistic_features-0.2.0/docs/reference/features.md +499 -0
- turkish_linguistic_features-0.2.0/docs/reference/index.md +21 -0
- turkish_linguistic_features-0.2.0/docs/threshold-calibration.md +86 -0
- turkish_linguistic_features-0.2.0/docs/tr/aciklama/dogrulama.md +223 -0
- turkish_linguistic_features-0.2.0/docs/tr/aciklama/index.md +12 -0
- turkish_linguistic_features-0.2.0/docs/tr/aciklama/kavramlar.md +182 -0
- turkish_linguistic_features-0.2.0/docs/tr/aciklama/nan.md +96 -0
- turkish_linguistic_features-0.2.0/docs/tr/aciklama/sinirliliklar.md +108 -0
- turkish_linguistic_features-0.2.0/docs/tr/baslangic.md +207 -0
- turkish_linguistic_features-0.2.0/docs/tr/index.md +48 -0
- turkish_linguistic_features-0.2.0/docs/tr/nasil/index.md +17 -0
- turkish_linguistic_features-0.2.0/docs/tr/nasil/karsilastirma.md +81 -0
- turkish_linguistic_features-0.2.0/docs/tr/nasil/korpus.md +163 -0
- turkish_linguistic_features-0.2.0/docs/tr/nasil/kunye.md +117 -0
- turkish_linguistic_features-0.2.0/docs/tr/nasil/parametreler.md +89 -0
- turkish_linguistic_features-0.2.0/docs/tr/nasil/segmentleme.md +97 -0
- turkish_linguistic_features-0.2.0/docs/tr/nasil/tek-metin.md +169 -0
- turkish_linguistic_features-0.2.0/docs/tr/sozluk.md +51 -0
- turkish_linguistic_features-0.2.0/docs/verification-report.md +630 -0
- turkish_linguistic_features-0.2.0/examples/01_hizli_baslangic.py +82 -0
- turkish_linguistic_features-0.2.0/examples/02_korpus_analizi.py +104 -0
- turkish_linguistic_features-0.2.0/examples/03_duzenleme_oncesi_sonrasi.py +86 -0
- turkish_linguistic_features-0.2.0/examples/04_matrisi_sakla.py +89 -0
- turkish_linguistic_features-0.2.0/examples/05_cumle_ritmi.py +89 -0
- turkish_linguistic_features-0.2.0/examples/06_esik_kalibrasyonu.py +116 -0
- turkish_linguistic_features-0.2.0/examples/07_nan_haritasi.py +87 -0
- turkish_linguistic_features-0.2.0/examples/08_tekrarlanabilirlik.py +63 -0
- turkish_linguistic_features-0.2.0/examples/09_uzunluk_duyarliligi.py +77 -0
- turkish_linguistic_features-0.2.0/examples/10_kelime_oruntuleri.py +76 -0
- turkish_linguistic_features-0.2.0/examples/README.md +50 -0
- turkish_linguistic_features-0.2.0/examples/_demo.py +70 -0
- turkish_linguistic_features-0.2.0/mkdocs.yml +106 -0
- turkish_linguistic_features-0.2.0/pyproject.toml +111 -0
- turkish_linguistic_features-0.2.0/requirements.txt +25 -0
- turkish_linguistic_features-0.2.0/scripts/__init__.py +0 -0
- turkish_linguistic_features-0.2.0/scripts/generate_feature_reference.py +118 -0
- turkish_linguistic_features-0.2.0/scripts/generate_verification_report.py +1145 -0
- turkish_linguistic_features-0.2.0/scripts/kincaid_olcum.py +103 -0
- turkish_linguistic_features-0.2.0/tests/__init__.py +0 -0
- turkish_linguistic_features-0.2.0/tests/conftest.py +62 -0
- turkish_linguistic_features-0.2.0/tests/test_analyze.py +231 -0
- turkish_linguistic_features-0.2.0/tests/test_analyze_corpus.py +142 -0
- turkish_linguistic_features-0.2.0/tests/test_cmudict.py +67 -0
- turkish_linguistic_features-0.2.0/tests/test_corpus.py +249 -0
- turkish_linguistic_features-0.2.0/tests/test_dependency.py +96 -0
- turkish_linguistic_features-0.2.0/tests/test_errors.py +64 -0
- turkish_linguistic_features-0.2.0/tests/test_extractor.py +308 -0
- turkish_linguistic_features-0.2.0/tests/test_faz1_kapisi.py +166 -0
- turkish_linguistic_features-0.2.0/tests/test_frequency_structure.py +307 -0
- turkish_linguistic_features-0.2.0/tests/test_import_katmanlari.py +166 -0
- turkish_linguistic_features-0.2.0/tests/test_kaynak_esligi.py +177 -0
- turkish_linguistic_features-0.2.0/tests/test_lexical.py +795 -0
- turkish_linguistic_features-0.2.0/tests/test_morphological.py +323 -0
- turkish_linguistic_features-0.2.0/tests/test_okunus.py +160 -0
- turkish_linguistic_features-0.2.0/tests/test_params.py +103 -0
- turkish_linguistic_features-0.2.0/tests/test_phonetic.py +370 -0
- turkish_linguistic_features-0.2.0/tests/test_pipeline.py +176 -0
- turkish_linguistic_features-0.2.0/tests/test_punctuation.py +205 -0
- turkish_linguistic_features-0.2.0/tests/test_readability.py +451 -0
- turkish_linguistic_features-0.2.0/tests/test_registry.py +487 -0
- turkish_linguistic_features-0.2.0/tests/test_smoke.py +29 -0
- turkish_linguistic_features-0.2.0/tests/test_syntactic.py +483 -0
- turkish_linguistic_features-0.2.0/tests/test_turkce_kucuk_harf.py +67 -0
- turkish_linguistic_features-0.2.0/tests/test_word_alignment.py +74 -0
- turkish_linguistic_features-0.2.0/tests/test_zeyrek.py +119 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/README.md +57 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_01.title.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_01.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_02.title.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_02.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_03.title.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_03.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_04.title.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_04.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_05.title.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_05.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_06.title.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_06.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_07.title.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_07.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_08.title.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_08.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_09.title.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_09.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_10.title.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_10.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_11.title.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_11.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_12.title.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_12.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_13.title.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_13.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_14.title.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_14.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_15.title.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_15.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_16.title.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_16.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_17.title.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_17.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_18.title.txt +1 -0
- turkish_linguistic_features-0.2.0/tests/veri/kincaid/passage_18.txt +1 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/__init__.py +51 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/_analyze.py +164 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/_corpus.py +129 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/_warnings.py +103 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/alfabe.py +23 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/exceptions.py +29 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/features/__init__.py +20 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/features/_registry_definition_texts.py +107 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/features/_registry_definitions.py +253 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/features/_registry_texts.py +1272 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/features/dependency.py +84 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/features/extractor.py +400 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/features/frequency_structure.py +300 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/features/lexical.py +725 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/features/morphological.py +331 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/features/okunus.py +293 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/features/phonetic.py +358 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/features/punctuation.py +213 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/features/readability.py +392 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/features/registry.py +445 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/features/syntactic.py +417 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/features/word_alignment.py +147 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/file_loader.py +261 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/params.py +93 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/pipeline/__init__.py +0 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/pipeline/preprocess.py +56 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/pipeline/spacy_pipeline.py +241 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/pipeline/zeyrek_backend.py +227 -0
- turkish_linguistic_features-0.2.0/turkish_linguistic_features/vocab.py +46 -0
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# CI: her push'ta ve her pull request'te lint + tip denetimi + testler.
|
|
2
|
+
# GitHub bu dosyayı `.github/workflows/` altında bulunca kendiliğinden çalıştırır.
|
|
3
|
+
|
|
4
|
+
name: CI # Actions sekmesinde ve rozette görünen ad
|
|
5
|
+
|
|
6
|
+
on: # ne zaman çalışsın
|
|
7
|
+
push:
|
|
8
|
+
branches: [main] # main'e her push
|
|
9
|
+
pull_request: # her pull request (hedef dal fark etmez)
|
|
10
|
+
|
|
11
|
+
permissions:
|
|
12
|
+
contents: read # yalnız kodu okur; depoya yazamaz
|
|
13
|
+
|
|
14
|
+
concurrency: # aynı dala art arda push'ta eskisini iptal et
|
|
15
|
+
group: ci-${{ github.ref }}
|
|
16
|
+
cancel-in-progress: true
|
|
17
|
+
|
|
18
|
+
defaults:
|
|
19
|
+
run:
|
|
20
|
+
shell: bash # Windows'ta da aynı komut sözdizimi (Git Bash)
|
|
21
|
+
|
|
22
|
+
jobs:
|
|
23
|
+
# ── İş 1: kodu çalıştırmadan denetle ─────────────────────────────────
|
|
24
|
+
lint:
|
|
25
|
+
runs-on: ubuntu-latest # GitHub'ın verdiği temiz bir Linux makinesi
|
|
26
|
+
steps:
|
|
27
|
+
- uses: actions/checkout@v7 # depoyu makineye indir
|
|
28
|
+
- uses: actions/setup-python@v7 # Python'u kur
|
|
29
|
+
with:
|
|
30
|
+
python-version: "3.12"
|
|
31
|
+
cache: pip # pip indirmelerini koşular arası sakla
|
|
32
|
+
cache-dependency-path: | # bu dosyalar değişince önbellek yenilenir
|
|
33
|
+
requirements.txt
|
|
34
|
+
pyproject.toml
|
|
35
|
+
- name: Install dependencies
|
|
36
|
+
run: pip install -r requirements.txt # ruff ve mypy de burada
|
|
37
|
+
- name: Ruff
|
|
38
|
+
run: ruff check .
|
|
39
|
+
- name: Mypy
|
|
40
|
+
run: mypy # ayarı pyproject.toml [tool.mypy]
|
|
41
|
+
|
|
42
|
+
# ── İş 2: testler, birden çok Python sürümü ve işletim sisteminde ─────
|
|
43
|
+
test:
|
|
44
|
+
name: test (${{ matrix.os }}, py${{ matrix.python-version }})
|
|
45
|
+
runs-on: ${{ matrix.os }}
|
|
46
|
+
strategy:
|
|
47
|
+
fail-fast: false # biri kırılsa da ötekiler sonuna kadar koşsun
|
|
48
|
+
matrix: # aşağıdaki her birleşim ayrı bir makinede
|
|
49
|
+
os: [ubuntu-latest]
|
|
50
|
+
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
51
|
+
include: # + Windows: zeyrek/spaCy import sırası Windows'a özgü
|
|
52
|
+
- os: windows-latest
|
|
53
|
+
python-version: "3.11"
|
|
54
|
+
env:
|
|
55
|
+
# Dil verisi eksikse testler ATLANMASIN, başarısız olsun (tests/conftest.py).
|
|
56
|
+
# Kurulum adımı sessizce bozulursa CI yeşil "skipped" ile geçmesin.
|
|
57
|
+
TLF_REQUIRE_MODELS: "1"
|
|
58
|
+
TR_MODEL_URL: https://huggingface.co/turkish-nlp-suite/tr_core_news_md/resolve/main/tr_core_news_md-1.0-py3-none-any.whl
|
|
59
|
+
steps:
|
|
60
|
+
- uses: actions/checkout@v7
|
|
61
|
+
- uses: actions/setup-python@v7
|
|
62
|
+
with:
|
|
63
|
+
python-version: ${{ matrix.python-version }}
|
|
64
|
+
cache: pip
|
|
65
|
+
cache-dependency-path: |
|
|
66
|
+
requirements.txt
|
|
67
|
+
pyproject.toml
|
|
68
|
+
|
|
69
|
+
- name: Install dependencies
|
|
70
|
+
run: pip install -r requirements.txt
|
|
71
|
+
|
|
72
|
+
# Türkçe model 156 MB ve Hugging Face'ten geliyor: wheel dosyası bir kez
|
|
73
|
+
# indirilip önbellekte tutulur. Anahtar model sürümüne bağlı: TR_MODEL_URL'deki
|
|
74
|
+
# sürüm değişirse anahtarı da değiştir, yoksa eski wheel kullanılır.
|
|
75
|
+
- name: Cache Turkish model wheel
|
|
76
|
+
uses: actions/cache@v6
|
|
77
|
+
with:
|
|
78
|
+
path: ~/tlf-models
|
|
79
|
+
key: tr-core-news-md-1.0
|
|
80
|
+
- name: Install language data
|
|
81
|
+
run: |
|
|
82
|
+
mkdir -p ~/tlf-models
|
|
83
|
+
whl=~/tlf-models/tr_core_news_md-1.0-py3-none-any.whl
|
|
84
|
+
[ -f "$whl" ] || curl -fsSL -o "$whl" "$TR_MODEL_URL"
|
|
85
|
+
pip install "$whl"
|
|
86
|
+
python -m spacy download en_core_web_sm
|
|
87
|
+
python -m nltk.downloader cmudict # İngilizce hece sayımı (textstat)
|
|
88
|
+
|
|
89
|
+
- name: Tests
|
|
90
|
+
run: python -m pytest
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# Docs: main'e her push'ta dokümantasyonu MkDocs ile kurar ve gh-pages dalına yazar.
|
|
2
|
+
# GitHub Pages ayarı: Settings → Pages → Source "Deploy from a branch",
|
|
3
|
+
# Branch "gh-pages" / "(root)". ("main /docs" seçilirse GitHub ham Markdown'ı
|
|
4
|
+
# Jekyll ile kurar; menü, sekmeler ve bilgi kutuları bozulur.)
|
|
5
|
+
|
|
6
|
+
name: Docs
|
|
7
|
+
|
|
8
|
+
on:
|
|
9
|
+
push:
|
|
10
|
+
branches: [main]
|
|
11
|
+
paths: # yalnız dokümantasyon değişince
|
|
12
|
+
- "docs/**"
|
|
13
|
+
- "mkdocs.yml"
|
|
14
|
+
- ".github/workflows/docs.yml"
|
|
15
|
+
workflow_dispatch: # Actions sekmesinden elle de çalıştırılabilir
|
|
16
|
+
|
|
17
|
+
permissions:
|
|
18
|
+
contents: write # gh-pages dalına yazabilmek için
|
|
19
|
+
|
|
20
|
+
concurrency: # iki yayın aynı anda gh-pages'e yazmasın
|
|
21
|
+
group: docs
|
|
22
|
+
cancel-in-progress: false
|
|
23
|
+
|
|
24
|
+
jobs:
|
|
25
|
+
deploy:
|
|
26
|
+
runs-on: ubuntu-latest
|
|
27
|
+
steps:
|
|
28
|
+
- uses: actions/checkout@v7
|
|
29
|
+
with:
|
|
30
|
+
fetch-depth: 0 # gh-deploy'un gh-pages dalını görebilmesi için
|
|
31
|
+
- uses: actions/setup-python@v7
|
|
32
|
+
with:
|
|
33
|
+
python-version: "3.12"
|
|
34
|
+
- name: Install MkDocs
|
|
35
|
+
run: pip install "mkdocs-material>=9.5" # requirements.txt ile aynı alt sınır
|
|
36
|
+
- name: Configure git
|
|
37
|
+
run: |
|
|
38
|
+
git config user.name "github-actions[bot]"
|
|
39
|
+
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
|
40
|
+
- name: Build and deploy
|
|
41
|
+
run: mkdocs gh-deploy --strict --force
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Yayın: GitHub'da bir release yayımlanınca paketi derleyip PyPI'a yükler.
|
|
2
|
+
# Actions sekmesinden elle çalıştırılırsa ("Run workflow") TestPyPI'a yükler — deneme için.
|
|
3
|
+
#
|
|
4
|
+
# Token ya da şifre kullanmaz: PyPI'ın "trusted publishing" özelliği (OIDC). PyPI'da
|
|
5
|
+
# (ve TestPyPI'da) proje ayarlarına bir kez şu yayıncı tanımlanır:
|
|
6
|
+
# sahip: efeziya1 · depo: turkish-linguistic-features · iş akışı: publish.yml
|
|
7
|
+
# ortam: pypi (TestPyPI için: testpypi)
|
|
8
|
+
# Sürüm numarası turkish_linguistic_features/__init__.py'deki __version__'dan gelir;
|
|
9
|
+
# PyPI aynı numarayı ikinci kez kabul etmez.
|
|
10
|
+
|
|
11
|
+
name: Publish
|
|
12
|
+
|
|
13
|
+
on:
|
|
14
|
+
release:
|
|
15
|
+
types: [published] # release taslağı değil, yayımlanınca
|
|
16
|
+
workflow_dispatch: # elle: TestPyPI'a deneme yüklemesi
|
|
17
|
+
|
|
18
|
+
permissions:
|
|
19
|
+
contents: read
|
|
20
|
+
|
|
21
|
+
jobs:
|
|
22
|
+
publish:
|
|
23
|
+
runs-on: ubuntu-latest
|
|
24
|
+
# Elle çalıştırılınca testpypi, release'te pypi ortamı.
|
|
25
|
+
environment: ${{ github.event_name == 'release' && 'pypi' || 'testpypi' }}
|
|
26
|
+
permissions:
|
|
27
|
+
id-token: write # trusted publishing için kimlik belirteci
|
|
28
|
+
steps:
|
|
29
|
+
- uses: actions/checkout@v7
|
|
30
|
+
- uses: actions/setup-python@v7
|
|
31
|
+
with:
|
|
32
|
+
python-version: "3.12"
|
|
33
|
+
- name: Build
|
|
34
|
+
run: |
|
|
35
|
+
pip install build
|
|
36
|
+
python -m build # dist/ altına sdist ve wheel
|
|
37
|
+
- name: Publish to TestPyPI
|
|
38
|
+
if: github.event_name != 'release'
|
|
39
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
40
|
+
with:
|
|
41
|
+
repository-url: https://test.pypi.org/legacy/
|
|
42
|
+
- name: Publish to PyPI
|
|
43
|
+
if: github.event_name == 'release'
|
|
44
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.py[cod]
|
|
3
|
+
.venv/
|
|
4
|
+
dist/
|
|
5
|
+
build/
|
|
6
|
+
*.egg-info/
|
|
7
|
+
.pytest_cache/
|
|
8
|
+
.mypy_cache/
|
|
9
|
+
examples/output/
|
|
10
|
+
.vscode/
|
|
11
|
+
|
|
12
|
+
# MkDocs derleme çıktısı — üretilen dosya, depoda durmaz
|
|
13
|
+
site/
|
|
14
|
+
|
|
15
|
+
# Yerel çalışma dosyaları — diskte durur, depoya girmez
|
|
16
|
+
plan/
|
|
17
|
+
_referans/
|
|
18
|
+
scripts/check_plan.py
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [0.2.0] - 2026-10-13
|
|
4
|
+
|
|
5
|
+
This release renames and removes feature keys; tables built with 0.1.0 will not line up
|
|
6
|
+
column for column. Keys: Turkish 208 (the same number, not the same keys), English 180 → 181. Cited features: Turkish
|
|
7
|
+
141 → 197, English 116 → 170; 11 keys in each language are plain definitions without a source.
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- The package is on PyPI: `pip install turkish-linguistic-features`. The language data
|
|
12
|
+
is still installed separately.
|
|
13
|
+
- Nine counts. `lexical`: `word_count` (words by the library's word), `type_count` (V),
|
|
14
|
+
`hapax_count` (V1, words occurring once) and `dislegomena_count` (V2, twice); `sentence`:
|
|
15
|
+
`sent_count`; `paragraph`: `para_count`; `phonetic`: `syllable_count`; `punctuation`:
|
|
16
|
+
`char_count` (whitespace included) and `punct_count` (marks). Divide other counts by them
|
|
17
|
+
to compare texts of different length. Citations: de Vel (2000) Table 2, attributes 1, 3 and
|
|
18
|
+
8; Zheng et al. (2006) Table 3, nos. 1, 54, 60–62, 247 and 248; Tweedie & Baayen (1998)
|
|
19
|
+
pp.325 and 329; de Vel et al. (2001) Table 2; Kincaid et al. (1975) p.38.
|
|
20
|
+
- `ngram_matches(text, phrase, lang="tr")`: what one `custom_ngrams` phrase matched, with
|
|
21
|
+
counts, most frequent first: `{"eski ev": 2, "büyük bahçe": 1}`. The matching is the
|
|
22
|
+
one `analyze` uses, so the counts add up to the phrase's `ngram_{...}_count`. The public API
|
|
23
|
+
has eleven names.
|
|
24
|
+
- Five lexical richness measures: `cttr` (Carroll's corrected TTR), `summer_s` (Summer's S),
|
|
25
|
+
`maas_a2` (Maas' a²), `herdan_vm` (Herdan's Vm) and `honore_r` (Honoré's R, 100 · ln N /
|
|
26
|
+
(1 − V1/V)). Citations: Carroll (1964) and Somers (1966) as cited in Torruella & Capsada
|
|
27
|
+
(2013) p.448; Maas (1972), Herdan (1955) and Honoré (1979) as cited in Tweedie & Baayen
|
|
28
|
+
(1998) eqs. (7), (18) and (11).
|
|
29
|
+
- `describe_feature(key)["definitions"]`: for every term the `formula` uses (`sentence`,
|
|
30
|
+
`word`, `type`, `syllable`, `pos_tag` … 21 terms), the rule the library counts it with, who
|
|
31
|
+
defines that rule (`tlf`, `spacy`, `zeyrek`, `textstat`, `wordfreq`) and one sentence on
|
|
32
|
+
how it is counted. `describe_feature(key, lang=...)` picks the language's entry where a
|
|
33
|
+
definition differs by language (`syllable`, `type`).
|
|
34
|
+
- `examples/10_kelime_oruntuleri.py`: counting your own word and tag phrases and seeing what
|
|
35
|
+
they matched.
|
|
36
|
+
|
|
37
|
+
### Changed
|
|
38
|
+
|
|
39
|
+
- `analyze_corpus` still stops at the first segment it cannot analyse, but the `ValueError` now
|
|
40
|
+
names that segment's `label`, `source` and `segment_id`; the original error is its `__cause__`.
|
|
41
|
+
- Python 3.13 is tested in CI and listed in the package classifiers.
|
|
42
|
+
- **Key names.** A name now says what the feature measures: a share between 0 and 1 ends in
|
|
43
|
+
`_ratio` unless the measure has an established name (`ttr`, `hdd`, `lexical_density`);
|
|
44
|
+
means and medians end the key; one family, one prefix. Values do not change.
|
|
45
|
+
- Punctuation: `punc_,_ratio` → `punct_comma_ratio`, `punc_._ratio` →
|
|
46
|
+
`punct_period_ratio`, `punc_;_ratio` → `punct_semicolon_ratio`, `punc_!_ratio` →
|
|
47
|
+
`punct_exclamation_ratio`, `punc_:_ratio` → `punct_colon_ratio`, `punc_-_ratio` →
|
|
48
|
+
`punct_dash_ratio`, `punc_ellipsis/paren/quote/question_ratio` → `punct_…_ratio`,
|
|
49
|
+
`punct_density` → `punct_char_ratio`, `digit_vs_all` → `digit_ratio`. Column names with
|
|
50
|
+
`.`, `-` or `!` broke attribute access in pandas and were rewritten by R.
|
|
51
|
+
- Means and medians: `avg_sent_len_word` → `sent_len_mean`, `avg_sent_len_char` →
|
|
52
|
+
`sent_len_char_mean`, `med_sent_len` → `sent_len_median`, `avg_word_length` →
|
|
53
|
+
`word_len_mean`, `sentence_syllable_mean` → `sent_syllable_mean`.
|
|
54
|
+
- Shares: `pos_noun` … → `pos_noun_ratio` … (12), `sentfinal_noun` … →
|
|
55
|
+
`sentfinal_noun_ratio` … (14), `char_a` … → `char_a_ratio` …, `hapax_percentage` →
|
|
56
|
+
`hapax_token_ratio` (a share, not a percentage), `question_per_sent` →
|
|
57
|
+
`question_sent_ratio`, `pronoun_freq` → `pronoun_ratio`, `n_lemma_count` → `lemma_count`.
|
|
58
|
+
- Morphology: keys from spaCy's UD tags drop `morph_` (`morph_case_loc` →
|
|
59
|
+
`case_loc_ratio`, 18 keys); keys from Zeyrek take `zeyrek_` (`case_loc_ratio` →
|
|
60
|
+
`zeyrek_case_loc_ratio`, `tense_past_def` → `zeyrek_tense_past_def_ratio`,
|
|
61
|
+
`agglutination_depth` → `zeyrek_agglutination_depth`, 23 keys).
|
|
62
|
+
- Deutsch, Jasbi & Shieber (2020) measures take their source names: `pos_dist_std` →
|
|
63
|
+
`posddev`, `pos_kl_div` → `posdiv`.
|
|
64
|
+
- Feature functions named after a key are renamed the same way.
|
|
65
|
+
- **One word.** Every feature except the dependency group counts the same word: a
|
|
66
|
+
whitespace-separated piece with edge punctuation stripped, containing a letter or digit
|
|
67
|
+
(`e-posta`, `%50` and numbers are one word each). On the TOMA set it equals the expert's
|
|
68
|
+
word count in all 57 texts. Features that need a tag per word take the POS tag,
|
|
69
|
+
morphological tags, lemma and Zeyrek analysis from the first token inside the word; spaCy
|
|
70
|
+
splits 0.3% of Turkish and 2.7% of English words (`Türk-Amerikan`, `it's`). Punctuation no
|
|
71
|
+
longer enters the denominators of the `pos_*_ratio` shares and `pronoun_ratio`.
|
|
72
|
+
`syntactic_dep` keeps spaCy's tokens and parser sentences.
|
|
73
|
+
- **Turkish lemmas come from Zeyrek**, not spaCy: the dictionary entry of Zeyrek's first
|
|
74
|
+
analysis, lowercased, without the infinitive `-mak/-mek`; circumflexes stay (`millî`). A word
|
|
75
|
+
Zeyrek cannot analyse keeps the part before its apostrophe (`Pittsburgh'tan` →
|
|
76
|
+
`pittsburgh`). spaCy's Turkish lemma left inflected forms as lemmas in 15% of words.
|
|
77
|
+
English lemmas still come from spaCy. Affected: `lemma_count`, `*_variation`, `wordfreq_*`,
|
|
78
|
+
`surface_per_lemma` and the `frequency_structure` group.
|
|
79
|
+
- **One sentence rule.** `. ? ! …` end a sentence; `:` only when what follows starts like a new
|
|
80
|
+
sentence (capital letter, quote, dash or opening bracket). The `sentence` group,
|
|
81
|
+
`question_sent_ratio`, `posdiv` and `sent_syllable_mean` use it instead of spaCy's parser
|
|
82
|
+
sentences. `sents_per_para_mean` is now `sent_count` / `para_count`; it counted runs of
|
|
83
|
+
`.!?…` in each paragraph, so `Dr.` ended a sentence. The readability formulas use the same conditional colon (Kincaid's rule too);
|
|
84
|
+
the Çetinkaya rule keeps its source's unconditional `:`, and `syntactic_dep` keeps the
|
|
85
|
+
parser.
|
|
86
|
+
- **Sentence-length thresholds** are calibrated on newspaper columns with the rules above
|
|
87
|
+
(15th/85th percentile; Turkish 162 columnists / 197,990 sentences, English 30 columnists /
|
|
88
|
+
52,745 sentences): Turkish 4/18 → 4/17, English 7/39 → 8/32.
|
|
89
|
+
- **`segment_text` and `analyze_corpus(segment_size=...)` count words**, the same word
|
|
90
|
+
`analyze` counts: a `segment_size=1000` segment is exactly 1000 words. They used to count
|
|
91
|
+
spaCy tokens, punctuation included (100 tokens were about 83 words). Segment boundaries
|
|
92
|
+
change.
|
|
93
|
+
- **`segment_text(size=...)` is now `segment_text(segment_size=...)`**, the name
|
|
94
|
+
`analyze_corpus` already uses for the same thing. Passing `size=` raises `TypeError`.
|
|
95
|
+
- **Punctuation shares.** The ten `punct_*_ratio` keys are each mark type's share of all
|
|
96
|
+
punctuation marks (0-1, summing to 1), no longer marks per word, which could exceed 1. How
|
|
97
|
+
much punctuation a text has is `punct_char_ratio`. A text without punctuation gives `nan`.
|
|
98
|
+
- **`custom_ngrams`** keys are `ngram_{...}_count` and the value is the number of matches, not
|
|
99
|
+
matches per window. A phrase item written as an UPPERCASE UD tag (`NOUN`, `VERB` …) matches
|
|
100
|
+
any word with that tag: `["ADJ", "NOUN"]` counts an adjective followed by a noun. Matches stay
|
|
101
|
+
inside a sentence. An empty text gives `nan`, like every other feature.
|
|
102
|
+
- **Natural logarithm everywhere.** `dugast_u`, `lambda_pa`, `adjusted_modulus` (were log₁₀)
|
|
103
|
+
and `entropy`, `punct_entropy`, `sent_len_entropy`, `zeyrek_suffix_bigram_entropy`, `posdiv`
|
|
104
|
+
(were log₂) change by a constant factor; the ranking of texts does not change. The scale
|
|
105
|
+
`"bits"` is now `"nats"`.
|
|
106
|
+
- **Syllable counts** read Turkish ordinals (`3. kat` → üçüncü), times and scores (`10:30` →
|
|
107
|
+
on otuz) and numbers glued to letters (`3kg` → üç kilogram), and add the reading of a listed
|
|
108
|
+
symbol to the number next to it in both languages (`%50` → yüzde elli, `$5` → five dollars).
|
|
109
|
+
- **Citations.** Features that had no citation now cite a source read in full, among them
|
|
110
|
+
`sent_len_mean`
|
|
111
|
+
(Flesch 1948), `word_len_mean` (Mendenhall 1887), `sent_len_median` (Yule 1939),
|
|
112
|
+
`hapax_ratio` (de Vel 2000), the letter and punctuation shares (Zheng et al. 2006) and
|
|
113
|
+
`yule_k`, which now cites Yule (1944) itself. Where the library's definition differs from
|
|
114
|
+
the source, the citation says so. Bibliography 45 → 56 works.
|
|
115
|
+
- **Verification tolerance** is 1% of the published value instead of a fixed 0.05. One row
|
|
116
|
+
moves from ✅ to 🟡 with its reason written out (`ttr`, QUITA Text 2, a misprint in the
|
|
117
|
+
source). Turkish report: 46 ✅ + 2 🟡; English: 35 ✅ + 3 🟡; no ❌.
|
|
118
|
+
- **Examples.** `05` ranks labels by the share of short plus long sentences; `06` counts
|
|
119
|
+
sentences with the library's own rule; `02` no longer lists correlated feature pairs.
|
|
120
|
+
|
|
121
|
+
### Removed
|
|
122
|
+
|
|
123
|
+
- `pos_punct`: with shares counted per word there is no punctuation token to count;
|
|
124
|
+
punctuation is measured in the `punctuation` group.
|
|
125
|
+
- `ttr_moving_slope` and `FeatureParams.ttr_slope_chunk_size` (passing it now raises
|
|
126
|
+
`TypeError`).
|
|
127
|
+
- The eight coefficients of variation (`word_length_cv`, `sentence_length_cv`,
|
|
128
|
+
`para_len_cv`, `sents_per_para_cv`, `syllable_cv`, `sentence_syllable_cv`,
|
|
129
|
+
`verb_dist_cv`, `suffix_chain_cv`), `entropy_std` and `sent_len_skewness`: spread
|
|
130
|
+
statistics without a source of their own.
|
|
131
|
+
- `para_count_norm`: exactly 1000 / `para_len_mean`. The plain count is `para_count`.
|
|
132
|
+
- `nominal_verbal_ratio`: count `custom_ngrams=[["NOUN"], ["VERB"]]` and divide.
|
|
133
|
+
- The `"cv"` and `"signed"` scales, which no key uses any more.
|
|
134
|
+
- `exceptions.MissingDependencyError`: it was never raised. A missing optional package gives
|
|
135
|
+
`MissingDependencyWarning` and `nan`; missing language data gives `ModelNotFoundError`.
|
|
136
|
+
- The undocumented `LINGUISTIC_FEATURES_NO_ZEYREK_WARMUP` environment variable: skipping the
|
|
137
|
+
Zeyrek warm-up and then analysing Turkish in the same process could crash on Windows.
|
|
138
|
+
|
|
139
|
+
### Fixed
|
|
140
|
+
|
|
141
|
+
- `…` (one character) now ends a sentence like `...` in every rule set; before, the same text
|
|
142
|
+
got a different sentence count and readability score depending on how the ellipsis was
|
|
143
|
+
typed.
|
|
144
|
+
- Words with a curly apostrophe (`Zeynep’i`) were left unanalysed by Zeyrek and dropped out
|
|
145
|
+
of the `morphological_zeyrek` features; `’` and `‘` are turned into `'` first.
|
|
146
|
+
- The minimum spaCy version is 3.8 (`spacy>=3.8,<4`, was 3.5): both language models need it.
|
|
147
|
+
- The install hint for a missing optional package names the package itself
|
|
148
|
+
(`pip install 'wordfreq>=3.0'`), not an extra of a package not yet on PyPI.
|
|
149
|
+
- Registry and documentation texts: three descriptions ran words together, the parameter
|
|
150
|
+
guide showed `params` as a list (it is a tuple), the `analyze` docstring example gave 3.5
|
|
151
|
+
for a mean of 3.0, four bibliography entries had Turkish fragments.
|
|
152
|
+
- The limitations page explains how Zeyrek orders its analyses and that the first one can
|
|
153
|
+
depend on `PYTHONHASHSEED` (200 of 8,215 words on the TOMA set).
|
|
154
|
+
|
|
155
|
+
## [0.1.0] - 2026-09-28
|
|
156
|
+
|
|
157
|
+
### Added
|
|
158
|
+
|
|
159
|
+
- Initial public release.
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use this software in your research, please cite it using this metadata."
|
|
3
|
+
type: software
|
|
4
|
+
title: "turkish-linguistic-features"
|
|
5
|
+
abstract: >-
|
|
6
|
+
A toolkit for extracting quantitative linguistic features from Turkish and
|
|
7
|
+
English text: 208 features for Turkish, 181 for English. 197 of the Turkish
|
|
8
|
+
features cite a source in the literature; the rest are plain definitions.
|
|
9
|
+
authors:
|
|
10
|
+
- given-names: "Efe Ziya"
|
|
11
|
+
family-names: "Okul"
|
|
12
|
+
version: "0.2.0"
|
|
13
|
+
date-released: 2026-10-13
|
|
14
|
+
doi: 10.5281/zenodo.23009540
|
|
15
|
+
license: MIT
|
|
16
|
+
repository-code: "https://github.com/efeziya1/turkish-linguistic-features"
|
|
17
|
+
keywords:
|
|
18
|
+
- quantitative-linguistics
|
|
19
|
+
- computational-linguistics
|
|
20
|
+
- corpus-linguistics
|
|
21
|
+
- stylometry
|
|
22
|
+
- turkish
|
|
23
|
+
- english
|
|
24
|
+
- feature-extraction
|
|
25
|
+
- text-analysis
|
|
26
|
+
- natural-language-processing
|
|
27
|
+
- readability
|
|
28
|
+
- lexical-richness
|
|
29
|
+
- lexical-diversity
|
|
30
|
+
- syntactic-complexity
|
|
31
|
+
- morphology
|
|
32
|
+
- digital-humanities
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Efe Ziya Okul
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: turkish-linguistic-features
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: A comprehensive toolkit for extracting quantitative linguistic features from Turkish and English text.
|
|
5
|
+
Project-URL: Repository, https://github.com/efeziya1/turkish-linguistic-features
|
|
6
|
+
Project-URL: Documentation, https://efeziya1.github.io/turkish-linguistic-features/
|
|
7
|
+
Project-URL: Issues, https://github.com/efeziya1/turkish-linguistic-features/issues
|
|
8
|
+
Author-email: Efe Ziya Okul <efeziya@live.com>
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: computational-linguistics,corpus-linguistics,digital-humanities,english,feature-extraction,lexical-diversity,lexical-richness,morphology,natural-language-processing,quantitative-linguistics,readability,stylometry,syntactic-complexity,text-analysis,turkish
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Requires-Dist: nltk>=3.8
|
|
22
|
+
Requires-Dist: numpy>=1.21
|
|
23
|
+
Requires-Dist: spacy<4,>=3.8
|
|
24
|
+
Requires-Dist: textstat<0.8,>=0.7.10
|
|
25
|
+
Requires-Dist: zeyrek<0.2,>=0.1.3
|
|
26
|
+
Provides-Extra: all
|
|
27
|
+
Requires-Dist: pandas>=1.5; extra == 'all'
|
|
28
|
+
Requires-Dist: wordfreq>=3.0; extra == 'all'
|
|
29
|
+
Provides-Extra: lexical-freq
|
|
30
|
+
Requires-Dist: wordfreq>=3.0; extra == 'lexical-freq'
|
|
31
|
+
Provides-Extra: pandas
|
|
32
|
+
Requires-Dist: pandas>=1.5; extra == 'pandas'
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
|
|
35
|
+
# turkish-linguistic-features
|
|
36
|
+
|
|
37
|
+
[](https://github.com/efeziya1/turkish-linguistic-features/actions/workflows/ci.yml)
|
|
38
|
+
[](https://doi.org/10.5281/zenodo.23009540)
|
|
39
|
+
|
|
40
|
+
Extracts 208 quantitative linguistic features from Turkish text and 181 from
|
|
41
|
+
English. Every feature has its formula written out; 197 of the Turkish features
|
|
42
|
+
and 170 of the English ones cite a source in the literature, and the rest are
|
|
43
|
+
plain definitions (such as the dash's share of punctuation marks). The
|
|
44
|
+
[verification report](https://efeziya1.github.io/turkish-linguistic-features/verification-report/) shows which ones have been
|
|
45
|
+
checked against a number their source published. Documentation:
|
|
46
|
+
https://efeziya1.github.io/turkish-linguistic-features/
|
|
47
|
+
|
|
48
|
+
**Status: early development (0.x).** Changes: [CHANGELOG](https://github.com/efeziya1/turkish-linguistic-features/blob/main/CHANGELOG.md).
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
import turkish_linguistic_features as tlf
|
|
52
|
+
|
|
53
|
+
oz = tlf.analyze("Renksiz yeşil fikirler öfkeyle uyur.", lang="tr")
|
|
54
|
+
oz["word_len_mean"] # 6.2
|
|
55
|
+
oz["atesman"] # 89.355 (Ateşman 1997 readability)
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## Documentation
|
|
59
|
+
|
|
60
|
+
**→ [Full documentation](https://efeziya1.github.io/turkish-linguistic-features/)** — tutorial, how-to
|
|
61
|
+
guides, feature reference and limitations, in **Turkish and English**.
|
|
62
|
+
|
|
63
|
+
| | |
|
|
64
|
+
|---|---|
|
|
65
|
+
| Türkçe | [tr/](https://efeziya1.github.io/turkish-linguistic-features/tr/) |
|
|
66
|
+
| English | [en/](https://efeziya1.github.io/turkish-linguistic-features/en/) |
|
|
67
|
+
| Feature reference | [reference/features/](https://efeziya1.github.io/turkish-linguistic-features/reference/features/) |
|
|
68
|
+
| Verification report | [TR](https://efeziya1.github.io/turkish-linguistic-features/dogrulama-raporu/) · [EN](https://efeziya1.github.io/turkish-linguistic-features/verification-report/) |
|
|
69
|
+
|
|
70
|
+
## Install
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
pip install turkish-linguistic-features
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
### Language data
|
|
77
|
+
|
|
78
|
+
`pip install` sets up the Python dependencies but not the language data.
|
|
79
|
+
Install the data for the language(s) you use, once:
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
# Turkish — 156 MB. The model is by turkish-nlp-suite and is not in spaCy's
|
|
83
|
+
# registry, so install the wheel directly:
|
|
84
|
+
pip install https://huggingface.co/turkish-nlp-suite/tr_core_news_md/resolve/main/tr_core_news_md-1.0-py3-none-any.whl
|
|
85
|
+
|
|
86
|
+
# English — the spaCy model (12 MB) and the CMU Pronouncing Dictionary for
|
|
87
|
+
# syllable counts (1 MB):
|
|
88
|
+
python -m spacy download en_core_web_sm
|
|
89
|
+
python -m nltk.downloader cmudict
|
|
90
|
+
|
|
91
|
+
# Optional — for the two wordfreq_* features:
|
|
92
|
+
pip install 'wordfreq>=3.0'
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
If something is missing, `analyze()` stops with an error that shows the
|
|
96
|
+
command to run.
|
|
97
|
+
|
|
98
|
+
Verified with spaCy 3.8.16, `en_core_web_sm` 3.8.0 and `tr_core_news_md` 1.0.
|
|
99
|
+
The Turkish wheel says `1.0` in its filename but `3.4.2` in its metadata, and
|
|
100
|
+
loading it prints a `W094` warning; both are harmless
|
|
101
|
+
([details](https://efeziya1.github.io/turkish-linguistic-features/en/getting-started/)).
|
|
102
|
+
|
|
103
|
+
## The whole API
|
|
104
|
+
|
|
105
|
+
Eleven public names:
|
|
106
|
+
|
|
107
|
+
```python
|
|
108
|
+
tlf.analyze(text, lang="tr") # 208 features (TR) / 181 (EN)
|
|
109
|
+
tlf.analyze(text, groups=["readability"]) # one group only
|
|
110
|
+
tlf.analyze_corpus("corpus/") # one row per file
|
|
111
|
+
tlf.analyze_corpus("corpus/", segment_size=1000) # or split into chunks
|
|
112
|
+
tlf.save_csv(rows, "features.csv") # write them out
|
|
113
|
+
tlf.segment_text(text, segment_size=1000, lang="tr") # split a single text
|
|
114
|
+
tlf.analyze(text, custom_ngrams=[["ADJ", "NOUN"]]) # count your own phrases
|
|
115
|
+
tlf.ngram_matches(text, ["ADJ", "NOUN"]) # ...and see what they matched
|
|
116
|
+
tlf.describe_feature("mtld") # what a key measures, and its source
|
|
117
|
+
tlf.FeatureParams(mattr_window=100) # metric constants
|
|
118
|
+
|
|
119
|
+
# tlf.LinguisticFeaturesError tlf.ModelNotFoundError
|
|
120
|
+
# tlf.MissingDependencyWarning tlf.ParagraphStructureWarning
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Full signatures: [Public API](https://efeziya1.github.io/turkish-linguistic-features/reference/api/).
|
|
124
|
+
|
|
125
|
+
## Development
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
pip install -r requirements.txt
|
|
129
|
+
pytest # pytest -m "" also runs the slow test
|
|
130
|
+
ruff check .
|
|
131
|
+
python scripts/generate_verification_report.py # regenerate the verification reports
|
|
132
|
+
python scripts/generate_feature_reference.py # regenerate the feature reference
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
## License
|
|
136
|
+
|
|
137
|
+
MIT — see `LICENSE`.
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# turkish-linguistic-features
|
|
2
|
+
|
|
3
|
+
[](https://github.com/efeziya1/turkish-linguistic-features/actions/workflows/ci.yml)
|
|
4
|
+
[](https://doi.org/10.5281/zenodo.23009540)
|
|
5
|
+
|
|
6
|
+
Extracts 208 quantitative linguistic features from Turkish text and 181 from
|
|
7
|
+
English. Every feature has its formula written out; 197 of the Turkish features
|
|
8
|
+
and 170 of the English ones cite a source in the literature, and the rest are
|
|
9
|
+
plain definitions (such as the dash's share of punctuation marks). The
|
|
10
|
+
[verification report](https://efeziya1.github.io/turkish-linguistic-features/verification-report/) shows which ones have been
|
|
11
|
+
checked against a number their source published. Documentation:
|
|
12
|
+
https://efeziya1.github.io/turkish-linguistic-features/
|
|
13
|
+
|
|
14
|
+
**Status: early development (0.x).** Changes: [CHANGELOG](https://github.com/efeziya1/turkish-linguistic-features/blob/main/CHANGELOG.md).
|
|
15
|
+
|
|
16
|
+
```python
|
|
17
|
+
import turkish_linguistic_features as tlf
|
|
18
|
+
|
|
19
|
+
oz = tlf.analyze("Renksiz yeşil fikirler öfkeyle uyur.", lang="tr")
|
|
20
|
+
oz["word_len_mean"] # 6.2
|
|
21
|
+
oz["atesman"] # 89.355 (Ateşman 1997 readability)
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## Documentation
|
|
25
|
+
|
|
26
|
+
**→ [Full documentation](https://efeziya1.github.io/turkish-linguistic-features/)** — tutorial, how-to
|
|
27
|
+
guides, feature reference and limitations, in **Turkish and English**.
|
|
28
|
+
|
|
29
|
+
| | |
|
|
30
|
+
|---|---|
|
|
31
|
+
| Türkçe | [tr/](https://efeziya1.github.io/turkish-linguistic-features/tr/) |
|
|
32
|
+
| English | [en/](https://efeziya1.github.io/turkish-linguistic-features/en/) |
|
|
33
|
+
| Feature reference | [reference/features/](https://efeziya1.github.io/turkish-linguistic-features/reference/features/) |
|
|
34
|
+
| Verification report | [TR](https://efeziya1.github.io/turkish-linguistic-features/dogrulama-raporu/) · [EN](https://efeziya1.github.io/turkish-linguistic-features/verification-report/) |
|
|
35
|
+
|
|
36
|
+
## Install
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install turkish-linguistic-features
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
### Language data
|
|
43
|
+
|
|
44
|
+
`pip install` sets up the Python dependencies but not the language data.
|
|
45
|
+
Install the data for the language(s) you use, once:
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
# Turkish — 156 MB. The model is by turkish-nlp-suite and is not in spaCy's
|
|
49
|
+
# registry, so install the wheel directly:
|
|
50
|
+
pip install https://huggingface.co/turkish-nlp-suite/tr_core_news_md/resolve/main/tr_core_news_md-1.0-py3-none-any.whl
|
|
51
|
+
|
|
52
|
+
# English — the spaCy model (12 MB) and the CMU Pronouncing Dictionary for
|
|
53
|
+
# syllable counts (1 MB):
|
|
54
|
+
python -m spacy download en_core_web_sm
|
|
55
|
+
python -m nltk.downloader cmudict
|
|
56
|
+
|
|
57
|
+
# Optional — for the two wordfreq_* features:
|
|
58
|
+
pip install 'wordfreq>=3.0'
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
If something is missing, `analyze()` stops with an error that shows the
|
|
62
|
+
command to run.
|
|
63
|
+
|
|
64
|
+
Verified with spaCy 3.8.16, `en_core_web_sm` 3.8.0 and `tr_core_news_md` 1.0.
|
|
65
|
+
The Turkish wheel says `1.0` in its filename but `3.4.2` in its metadata, and
|
|
66
|
+
loading it prints a `W094` warning; both are harmless
|
|
67
|
+
([details](https://efeziya1.github.io/turkish-linguistic-features/en/getting-started/)).
|
|
68
|
+
|
|
69
|
+
## The whole API
|
|
70
|
+
|
|
71
|
+
Eleven public names:
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
tlf.analyze(text, lang="tr") # 208 features (TR) / 181 (EN)
|
|
75
|
+
tlf.analyze(text, groups=["readability"]) # one group only
|
|
76
|
+
tlf.analyze_corpus("corpus/") # one row per file
|
|
77
|
+
tlf.analyze_corpus("corpus/", segment_size=1000) # or split into chunks
|
|
78
|
+
tlf.save_csv(rows, "features.csv") # write them out
|
|
79
|
+
tlf.segment_text(text, segment_size=1000, lang="tr") # split a single text
|
|
80
|
+
tlf.analyze(text, custom_ngrams=[["ADJ", "NOUN"]]) # count your own phrases
|
|
81
|
+
tlf.ngram_matches(text, ["ADJ", "NOUN"]) # ...and see what they matched
|
|
82
|
+
tlf.describe_feature("mtld") # what a key measures, and its source
|
|
83
|
+
tlf.FeatureParams(mattr_window=100) # metric constants
|
|
84
|
+
|
|
85
|
+
# tlf.LinguisticFeaturesError tlf.ModelNotFoundError
|
|
86
|
+
# tlf.MissingDependencyWarning tlf.ParagraphStructureWarning
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
Full signatures: [Public API](https://efeziya1.github.io/turkish-linguistic-features/reference/api/).
|
|
90
|
+
|
|
91
|
+
## Development
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
pip install -r requirements.txt
|
|
95
|
+
pytest # pytest -m "" also runs the slow test
|
|
96
|
+
ruff check .
|
|
97
|
+
python scripts/generate_verification_report.py # regenerate the verification reports
|
|
98
|
+
python scripts/generate_feature_reference.py # regenerate the feature reference
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
## License
|
|
102
|
+
|
|
103
|
+
MIT — see `LICENSE`.
|