nltk 3.2__zip → 3.3__zip
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nltk-3.2 → nltk-3.3}/INSTALL.txt +6 -6
- {nltk-3.2 → nltk-3.3}/LICENSE.txt +13 -13
- {nltk-3.2 → nltk-3.3}/MANIFEST.in +6 -6
- {nltk-3.2/nltk.egg-info → nltk-3.3}/PKG-INFO +13 -5
- nltk-3.2/README.txt → nltk-3.3/README.md +49 -48
- nltk-3.3/nltk/VERSION +1 -0
- {nltk-3.2 → nltk-3.3}/nltk/__init__.py +185 -185
- {nltk-3.2 → nltk-3.3}/nltk/app/__init__.py +52 -53
- {nltk-3.2 → nltk-3.3}/nltk/app/chartparser_app.py +2276 -2276
- {nltk-3.2 → nltk-3.3}/nltk/app/chunkparser_app.py +1262 -1264
- {nltk-3.2 → nltk-3.3}/nltk/app/collocations_app.py +347 -350
- {nltk-3.2 → nltk-3.3}/nltk/app/concordance_app.py +568 -570
- {nltk-3.2 → nltk-3.3}/nltk/app/nemo_app.py +156 -155
- {nltk-3.2 → nltk-3.3}/nltk/app/rdparser_app.py +892 -894
- {nltk-3.2 → nltk-3.3}/nltk/app/srparser_app.py +809 -809
- {nltk-3.2 → nltk-3.3}/nltk/app/wordfreq_app.py +32 -32
- {nltk-3.2 → nltk-3.3}/nltk/app/wordnet_app.py +968 -968
- {nltk-3.2 → nltk-3.3}/nltk/book.py +91 -90
- {nltk-3.2 → nltk-3.3}/nltk/ccg/__init__.py +22 -22
- {nltk-3.2 → nltk-3.3}/nltk/ccg/api.py +359 -335
- {nltk-3.2 → nltk-3.3}/nltk/ccg/chart.py +394 -361
- {nltk-3.2 → nltk-3.3}/nltk/ccg/combinator.py +343 -312
- {nltk-3.2 → nltk-3.3}/nltk/ccg/lexicon.py +328 -290
- nltk-3.3/nltk/ccg/logic.py +46 -0
- {nltk-3.2 → nltk-3.3}/nltk/chat/__init__.py +49 -49
- {nltk-3.2 → nltk-3.3}/nltk/chat/eliza.py +244 -244
- {nltk-3.2 → nltk-3.3}/nltk/chat/iesha.py +140 -140
- {nltk-3.2 → nltk-3.3}/nltk/chat/rude.py +92 -92
- {nltk-3.2 → nltk-3.3}/nltk/chat/suntsu.py +117 -117
- {nltk-3.2 → nltk-3.3}/nltk/chat/util.py +122 -120
- {nltk-3.2 → nltk-3.3}/nltk/chat/zen.py +282 -282
- {nltk-3.2 → nltk-3.3}/nltk/chunk/__init__.py +190 -190
- {nltk-3.2 → nltk-3.3}/nltk/chunk/api.py +51 -51
- {nltk-3.2 → nltk-3.3}/nltk/chunk/named_entity.py +331 -331
- {nltk-3.2 → nltk-3.3}/nltk/chunk/regexp.py +1390 -1384
- {nltk-3.2 → nltk-3.3}/nltk/chunk/util.py +600 -600
- {nltk-3.2 → nltk-3.3}/nltk/classify/__init__.py +98 -98
- {nltk-3.2 → nltk-3.3}/nltk/classify/api.py +193 -193
- {nltk-3.2 → nltk-3.3}/nltk/classify/decisiontree.py +295 -295
- {nltk-3.2 → nltk-3.3}/nltk/classify/maxent.py +1510 -1492
- {nltk-3.2 → nltk-3.3}/nltk/classify/megam.py +179 -178
- {nltk-3.2 → nltk-3.3}/nltk/classify/naivebayes.py +245 -242
- {nltk-3.2 → nltk-3.3}/nltk/classify/positivenaivebayes.py +170 -170
- {nltk-3.2 → nltk-3.3}/nltk/classify/rte_classify.py +160 -191
- {nltk-3.2 → nltk-3.3}/nltk/classify/scikitlearn.py +153 -151
- {nltk-3.2 → nltk-3.3}/nltk/classify/senna.py +183 -181
- {nltk-3.2 → nltk-3.3}/nltk/classify/svm.py +15 -15
- {nltk-3.2 → nltk-3.3}/nltk/classify/tadm.py +113 -112
- {nltk-3.2 → nltk-3.3}/nltk/classify/textcat.py +193 -193
- {nltk-3.2 → nltk-3.3}/nltk/classify/util.py +324 -312
- {nltk-3.2 → nltk-3.3}/nltk/classify/weka.py +346 -345
- {nltk-3.2 → nltk-3.3}/nltk/cluster/__init__.py +86 -86
- {nltk-3.2 → nltk-3.3}/nltk/cluster/api.py +74 -70
- {nltk-3.2 → nltk-3.3}/nltk/cluster/em.py +247 -247
- {nltk-3.2 → nltk-3.3}/nltk/cluster/gaac.py +168 -168
- {nltk-3.2 → nltk-3.3}/nltk/cluster/kmeans.py +221 -221
- {nltk-3.2 → nltk-3.3}/nltk/cluster/util.py +304 -289
- nltk-3.3/nltk/collections.py +640 -0
- {nltk-3.2 → nltk-3.3}/nltk/collocations.py +388 -388
- nltk-3.3/nltk/compat.py +365 -0
- {nltk-3.2 → nltk-3.3}/nltk/corpus/__init__.py +341 -320
- {nltk-3.2 → nltk-3.3}/nltk/corpus/europarl_raw.py +44 -44
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/__init__.py +147 -145
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/aligned.py +115 -114
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/api.py +448 -446
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/bnc.py +252 -252
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/bracket_parse.py +227 -227
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/categorized_sents.py +179 -178
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/chasen.py +140 -139
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/childes.py +522 -494
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/chunked.py +212 -212
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/cmudict.py +97 -95
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/comparative_sents.py +280 -278
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/conll.py +528 -521
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/crubadan.py +116 -116
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/dependency.py +101 -101
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/framenet.py +3059 -2093
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ieer.py +112 -111
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/indian.py +85 -86
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ipipan.py +332 -331
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/knbc.py +187 -187
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/lin.py +156 -156
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/mte.py +311 -310
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/nkjp.py +429 -428
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/nombank.py +421 -421
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/nps_chat.py +73 -73
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/opinion_lexicon.py +115 -115
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/panlex_lite.py +165 -165
- nltk-3.3/nltk/corpus/reader/pl196x.py +292 -0
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/plaintext.py +232 -233
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ppattach.py +96 -95
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/propbank.py +479 -481
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/pros_cons.py +128 -126
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/reviews.py +330 -328
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/rte.py +144 -146
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/semcor.py +256 -256
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/senseval.py +203 -201
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/sentiwordnet.py +137 -137
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/sinica_treebank.py +75 -75
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/string_category.py +60 -59
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/switchboard.py +119 -119
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/tagged.py +295 -294
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/timit.py +452 -450
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/toolbox.py +68 -68
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/twitter.py +155 -157
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/udhr.py +80 -80
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/util.py +804 -800
- nltk-3.3/nltk/corpus/reader/verbnet.py +595 -0
- nltk-3.3/nltk/corpus/reader/wordlist.py +134 -0
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/wordnet.py +2108 -2036
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/xmldocs.py +392 -387
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ycoe.py +243 -242
- {nltk-3.2 → nltk-3.3}/nltk/corpus/util.py +144 -127
- {nltk-3.2 → nltk-3.3}/nltk/data.py +1487 -1451
- {nltk-3.2 → nltk-3.3}/nltk/decorators.py +219 -215
- {nltk-3.2 → nltk-3.3}/nltk/downloader.py +2278 -2274
- {nltk-3.2 → nltk-3.3}/nltk/draw/__init__.py +27 -27
- {nltk-3.2 → nltk-3.3}/nltk/draw/cfg.py +774 -777
- {nltk-3.2 → nltk-3.3}/nltk/draw/dispersion.py +58 -58
- {nltk-3.2 → nltk-3.3}/nltk/draw/table.py +1098 -1098
- {nltk-3.2 → nltk-3.3}/nltk/draw/tree.py +963 -966
- {nltk-3.2 → nltk-3.3}/nltk/draw/util.py +2356 -2356
- {nltk-3.2 → nltk-3.3}/nltk/featstruct.py +2617 -2500
- {nltk-3.2 → nltk-3.3}/nltk/grammar.py +1559 -1516
- {nltk-3.2 → nltk-3.3}/nltk/help.py +56 -56
- {nltk-3.2 → nltk-3.3}/nltk/inference/__init__.py +20 -20
- {nltk-3.2 → nltk-3.3}/nltk/inference/api.py +597 -589
- {nltk-3.2 → nltk-3.3}/nltk/inference/discourse.py +612 -608
- {nltk-3.2 → nltk-3.3}/nltk/inference/mace.py +311 -311
- {nltk-3.2 → nltk-3.3}/nltk/inference/nonmonotonic.py +509 -509
- {nltk-3.2 → nltk-3.3}/nltk/inference/prover9.py +431 -431
- {nltk-3.2 → nltk-3.3}/nltk/inference/resolution.py +687 -687
- {nltk-3.2 → nltk-3.3}/nltk/inference/tableau.py +607 -607
- {nltk-3.2 → nltk-3.3}/nltk/internals.py +982 -982
- {nltk-3.2 → nltk-3.3}/nltk/jsontags.py +63 -63
- {nltk-3.2 → nltk-3.3}/nltk/lazyimport.py +142 -142
- {nltk-3.2 → nltk-3.3}/nltk/metrics/__init__.py +30 -29
- {nltk-3.2 → nltk-3.3}/nltk/metrics/agreement.py +421 -401
- nltk-3.3/nltk/metrics/aline.py +607 -0
- {nltk-3.2 → nltk-3.3}/nltk/metrics/association.py +414 -410
- {nltk-3.2 → nltk-3.3}/nltk/metrics/confusionmatrix.py +206 -206
- {nltk-3.2 → nltk-3.3}/nltk/metrics/distance.py +207 -202
- {nltk-3.2 → nltk-3.3}/nltk/metrics/paice.py +381 -381
- {nltk-3.2 → nltk-3.3}/nltk/metrics/scores.py +228 -228
- {nltk-3.2 → nltk-3.3}/nltk/metrics/segmentation.py +231 -232
- {nltk-3.2 → nltk-3.3}/nltk/metrics/spearman.py +69 -69
- {nltk-3.2 → nltk-3.3}/nltk/misc/__init__.py +11 -11
- {nltk-3.2 → nltk-3.3}/nltk/misc/babelfish.py +10 -10
- {nltk-3.2 → nltk-3.3}/nltk/misc/chomsky.py +135 -133
- {nltk-3.2 → nltk-3.3}/nltk/misc/minimalset.py +83 -83
- {nltk-3.2 → nltk-3.3}/nltk/misc/sort.py +157 -157
- {nltk-3.2 → nltk-3.3}/nltk/misc/wordfinder.py +129 -129
- {nltk-3.2 → nltk-3.3}/nltk/parse/__init__.py +81 -80
- {nltk-3.2 → nltk-3.3}/nltk/parse/api.py +66 -66
- {nltk-3.2 → nltk-3.3}/nltk/parse/bllip.py +282 -282
- {nltk-3.2 → nltk-3.3}/nltk/parse/chart.py +1682 -1681
- nltk-3.3/nltk/parse/corenlp.py +780 -0
- {nltk-3.2 → nltk-3.3}/nltk/parse/dependencygraph.py +760 -758
- {nltk-3.2 → nltk-3.3}/nltk/parse/earleychart.py +452 -451
- {nltk-3.2 → nltk-3.3}/nltk/parse/evaluate.py +129 -129
- {nltk-3.2 → nltk-3.3}/nltk/parse/featurechart.py +580 -578
- {nltk-3.2 → nltk-3.3}/nltk/parse/generate.py +87 -76
- {nltk-3.2 → nltk-3.3}/nltk/parse/malt.py +356 -356
- {nltk-3.2 → nltk-3.3}/nltk/parse/nonprojectivedependencyparser.py +783 -783
- {nltk-3.2 → nltk-3.3}/nltk/parse/pchart.py +522 -522
- {nltk-3.2 → nltk-3.3}/nltk/parse/projectivedependencyparser.py +579 -578
- {nltk-3.2 → nltk-3.3}/nltk/parse/recursivedescent.py +655 -656
- {nltk-3.2 → nltk-3.3}/nltk/parse/shiftreduce.py +458 -459
- {nltk-3.2 → nltk-3.3}/nltk/parse/stanford.py +431 -406
- {nltk-3.2 → nltk-3.3}/nltk/parse/transitionparser.py +774 -772
- {nltk-3.2 → nltk-3.3}/nltk/parse/util.py +231 -231
- {nltk-3.2 → nltk-3.3}/nltk/parse/viterbi.py +401 -401
- {nltk-3.2 → nltk-3.3}/nltk/probability.py +2410 -2355
- {nltk-3.2 → nltk-3.3}/nltk/sem/__init__.py +61 -61
- {nltk-3.2 → nltk-3.3}/nltk/sem/boxer.py +1261 -1261
- {nltk-3.2 → nltk-3.3}/nltk/sem/chat80.py +782 -782
- {nltk-3.2 → nltk-3.3}/nltk/sem/cooper_storage.py +118 -118
- {nltk-3.2 → nltk-3.3}/nltk/sem/drt.py +1258 -1257
- {nltk-3.2 → nltk-3.3}/nltk/sem/drt_glue_demo.py +485 -489
- {nltk-3.2 → nltk-3.3}/nltk/sem/evaluate.py +790 -784
- {nltk-3.2 → nltk-3.3}/nltk/sem/glue.py +696 -694
- {nltk-3.2 → nltk-3.3}/nltk/sem/hole.py +386 -384
- {nltk-3.2 → nltk-3.3}/nltk/sem/lfg.py +210 -208
- {nltk-3.2 → nltk-3.3}/nltk/sem/linearlogic.py +451 -449
- {nltk-3.2 → nltk-3.3}/nltk/sem/logic.py +1910 -1910
- {nltk-3.2 → nltk-3.3}/nltk/sem/relextract.py +476 -483
- {nltk-3.2 → nltk-3.3}/nltk/sem/skolemize.py +101 -101
- {nltk-3.2 → nltk-3.3}/nltk/sem/util.py +249 -249
- {nltk-3.2 → nltk-3.3}/nltk/sentiment/__init__.py +14 -14
- {nltk-3.2 → nltk-3.3}/nltk/sentiment/sentiment_analyzer.py +229 -229
- {nltk-3.2 → nltk-3.3}/nltk/sentiment/util.py +762 -752
- {nltk-3.2 → nltk-3.3}/nltk/sentiment/vader.py +443 -445
- {nltk-3.2 → nltk-3.3}/nltk/stem/__init__.py +31 -33
- {nltk-3.2 → nltk-3.3}/nltk/stem/api.py +28 -25
- nltk-3.3/nltk/stem/arlstem.py +369 -0
- {nltk-3.2 → nltk-3.3}/nltk/stem/isri.py +345 -345
- {nltk-3.2 → nltk-3.3}/nltk/stem/lancaster.py +337 -310
- nltk-3.3/nltk/stem/porter.py +710 -0
- {nltk-3.2 → nltk-3.3}/nltk/stem/regexp.py +61 -61
- {nltk-3.2 → nltk-3.3}/nltk/stem/rslp.py +142 -142
- {nltk-3.2 → nltk-3.3}/nltk/stem/snowball.py +4236 -3712
- {nltk-3.2 → nltk-3.3}/nltk/stem/util.py +22 -12
- {nltk-3.2 → nltk-3.3}/nltk/stem/wordnet.py +51 -51
- {nltk-3.2 → nltk-3.3}/nltk/tag/__init__.py +152 -127
- {nltk-3.2 → nltk-3.3}/nltk/tag/api.py +86 -81
- {nltk-3.2 → nltk-3.3}/nltk/tag/brill.py +424 -426
- {nltk-3.2 → nltk-3.3}/nltk/tag/brill_trainer.py +608 -608
- {nltk-3.2 → nltk-3.3}/nltk/tag/crf.py +204 -200
- {nltk-3.2 → nltk-3.3}/nltk/tag/hmm.py +1277 -1277
- {nltk-3.2 → nltk-3.3}/nltk/tag/hunpos.py +132 -132
- {nltk-3.2 → nltk-3.3}/nltk/tag/mapping.py +101 -101
- {nltk-3.2 → nltk-3.3}/nltk/tag/perceptron.py +329 -314
- {nltk-3.2 → nltk-3.3}/nltk/tag/senna.py +147 -145
- {nltk-3.2 → nltk-3.3}/nltk/tag/sequential.py +740 -736
- {nltk-3.2 → nltk-3.3}/nltk/tag/stanford.py +219 -196
- {nltk-3.2 → nltk-3.3}/nltk/tag/tnt.py +595 -604
- {nltk-3.2 → nltk-3.3}/nltk/tag/util.py +72 -72
- {nltk-3.2 → nltk-3.3}/nltk/tbl/__init__.py +28 -28
- {nltk-3.2 → nltk-3.3}/nltk/tbl/api.py +1 -1
- {nltk-3.2 → nltk-3.3}/nltk/tbl/demo.py +366 -366
- {nltk-3.2 → nltk-3.3}/nltk/tbl/erroranalysis.py +39 -39
- {nltk-3.2 → nltk-3.3}/nltk/tbl/feature.py +259 -261
- {nltk-3.2 → nltk-3.3}/nltk/tbl/rule.py +321 -321
- {nltk-3.2 → nltk-3.3}/nltk/tbl/template.py +303 -308
- {nltk-3.2 → nltk-3.3}/nltk/test/__init__.py +18 -18
- {nltk-3.2 → nltk-3.3}/nltk/test/all.py +24 -23
- {nltk-3.2 → nltk-3.3}/nltk/test/bleu.doctest +14 -14
- {nltk-3.2 → nltk-3.3}/nltk/test/bnc.doctest +60 -60
- {nltk-3.2 → nltk-3.3}/nltk/test/ccg.doctest +376 -395
- nltk-3.3/nltk/test/ccg_semantics.doctest +553 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/chat80.doctest +234 -234
- {nltk-3.2 → nltk-3.3}/nltk/test/childes.doctest +184 -184
- {nltk-3.2 → nltk-3.3}/nltk/test/childes_fixt.py +14 -13
- {nltk-3.2 → nltk-3.3}/nltk/test/chunk.doctest +373 -373
- {nltk-3.2 → nltk-3.3}/nltk/test/classify.doctest +185 -185
- {nltk-3.2 → nltk-3.3}/nltk/test/classify_fixt.py +11 -10
- nltk-3.3/nltk/test/collections.doctest +20 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/collocations.doctest +276 -276
- {nltk-3.2 → nltk-3.3}/nltk/test/compat.doctest +134 -138
- {nltk-3.2 → nltk-3.3}/nltk/test/compat_fixt.py +9 -8
- nltk-3.3/nltk/test/concordance.doctest +55 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/corpus.doctest +2200 -2212
- {nltk-3.2 → nltk-3.3}/nltk/test/corpus_fixt.py +3 -3
- {nltk-3.2 → nltk-3.3}/nltk/test/crubadan.doctest +65 -65
- {nltk-3.2 → nltk-3.3}/nltk/test/data.doctest +379 -379
- {nltk-3.2 → nltk-3.3}/nltk/test/dependency.doctest +241 -241
- {nltk-3.2 → nltk-3.3}/nltk/test/discourse.doctest +546 -546
- {nltk-3.2 → nltk-3.3}/nltk/test/discourse_fixt.py +14 -13
- {nltk-3.2 → nltk-3.3}/nltk/test/doctest_nose_plugin.py +159 -156
- {nltk-3.2 → nltk-3.3}/nltk/test/drt.doctest +517 -517
- {nltk-3.2 → nltk-3.3}/nltk/test/featgram.doctest +607 -607
- {nltk-3.2 → nltk-3.3}/nltk/test/featstruct.doctest +1229 -1230
- {nltk-3.2 → nltk-3.3}/nltk/test/framenet.doctest +288 -239
- {nltk-3.2 → nltk-3.3}/nltk/test/generate.doctest +67 -67
- {nltk-3.2 → nltk-3.3}/nltk/test/gensim.doctest +140 -141
- {nltk-3.2 → nltk-3.3}/nltk/test/gensim_fixt.py +10 -9
- {nltk-3.2 → nltk-3.3}/nltk/test/gluesemantics.doctest +384 -386
- {nltk-3.2 → nltk-3.3}/nltk/test/gluesemantics_malt.doctest +68 -68
- {nltk-3.2 → nltk-3.3}/nltk/test/gluesemantics_malt_fixt.py +12 -11
- {nltk-3.2 → nltk-3.3}/nltk/test/grammar.doctest +48 -48
- {nltk-3.2 → nltk-3.3}/nltk/test/grammartestsuites.doctest +109 -109
- {nltk-3.2 → nltk-3.3}/nltk/test/index.doctest +100 -100
- {nltk-3.2 → nltk-3.3}/nltk/test/inference.doctest +534 -534
- {nltk-3.2 → nltk-3.3}/nltk/test/inference_fixt.py +12 -11
- {nltk-3.2 → nltk-3.3}/nltk/test/internals.doctest +140 -140
- {nltk-3.2 → nltk-3.3}/nltk/test/japanese.doctest +48 -48
- {nltk-3.2 → nltk-3.3}/nltk/test/logic.doctest +1098 -1098
- {nltk-3.2 → nltk-3.3}/nltk/test/metrics.doctest +270 -270
- {nltk-3.2 → nltk-3.3}/nltk/test/misc.doctest +118 -118
- {nltk-3.2 → nltk-3.3}/nltk/test/nonmonotonic.doctest +286 -286
- {nltk-3.2 → nltk-3.3}/nltk/test/nonmonotonic_fixt.py +12 -11
- {nltk-3.2 → nltk-3.3}/nltk/test/paice.doctest +35 -35
- {nltk-3.2 → nltk-3.3}/nltk/test/parse.doctest +884 -884
- {nltk-3.2 → nltk-3.3}/nltk/test/portuguese_en.doctest +565 -565
- {nltk-3.2 → nltk-3.3}/nltk/test/portuguese_en_fixt.py +16 -16
- {nltk-3.2 → nltk-3.3}/nltk/test/probability.doctest +304 -304
- {nltk-3.2 → nltk-3.3}/nltk/test/probability_fixt.py +13 -12
- {nltk-3.2 → nltk-3.3}/nltk/test/propbank.doctest +176 -176
- {nltk-3.2 → nltk-3.3}/nltk/test/relextract.doctest +263 -263
- {nltk-3.2 → nltk-3.3}/nltk/test/resolution.doctest +221 -221
- {nltk-3.2 → nltk-3.3}/nltk/test/runtests.py +86 -64
- {nltk-3.2 → nltk-3.3}/nltk/test/segmentation_fixt.py +10 -10
- {nltk-3.2 → nltk-3.3}/nltk/test/semantics.doctest +665 -665
- {nltk-3.2 → nltk-3.3}/nltk/test/semantics_fixt.py +7 -7
- {nltk-3.2 → nltk-3.3}/nltk/test/sentiment.doctest +233 -233
- {nltk-3.2 → nltk-3.3}/nltk/test/sentiwordnet.doctest +41 -41
- {nltk-3.2 → nltk-3.3}/nltk/test/simple.doctest +85 -85
- {nltk-3.2 → nltk-3.3}/nltk/test/stem.doctest +78 -78
- {nltk-3.2 → nltk-3.3}/nltk/test/tag.doctest +33 -33
- {nltk-3.2 → nltk-3.3}/nltk/test/tokenize.doctest +273 -166
- {nltk-3.2 → nltk-3.3}/nltk/test/toolbox.doctest +307 -307
- {nltk-3.2 → nltk-3.3}/nltk/test/translate.doctest +242 -242
- {nltk-3.2 → nltk-3.3}/nltk/test/translate_fixt.py +3 -3
- {nltk-3.2 → nltk-3.3}/nltk/test/tree.doctest +1101 -1101
- {nltk-3.2 → nltk-3.3}/nltk/test/treeprettyprinter.doctest +127 -127
- {nltk-3.2 → nltk-3.3}/nltk/test/treetransforms.doctest +156 -156
- nltk-3.3/nltk/test/unit/test_2x_compat.py +63 -0
- nltk-3.3/nltk/test/unit/test_aline.py +52 -0
- nltk-3.3/nltk/test/unit/test_chunk.py +56 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_classify.py +55 -53
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_collocations.py +88 -79
- nltk-3.3/nltk/test/unit/test_concordance.py +98 -0
- nltk-3.3/nltk/test/unit/test_corenlp.py +1000 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_corpora.py +200 -183
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_corpus_views.py +45 -45
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_hmm.py +88 -88
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_json2csv_corpus.py +188 -187
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_naivebayes.py +26 -26
- nltk-3.3/nltk/test/unit/test_rte_classify.py +86 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_seekable_unicode_stream_reader.py +124 -124
- nltk-3.3/nltk/test/unit/test_senna.py +74 -0
- nltk-3.3/nltk/test/unit/test_stem.py +121 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_tag.py +20 -20
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_tgrep.py +628 -626
- nltk-3.3/nltk/test/unit/test_tokenize.py +168 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_twitter_auth.py +185 -185
- nltk-3.3/nltk/test/unit/test_wordnet.py +134 -0
- nltk-3.3/nltk/test/unit/translate/test_bleu.py +244 -0
- nltk-3.3/nltk/test/unit/translate/test_gdfa.py +57 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm1.py +75 -75
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm2.py +87 -87
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm3.py +105 -105
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm4.py +126 -126
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm5.py +166 -166
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm_model.py +270 -270
- nltk-3.3/nltk/test/unit/translate/test_nist.py +37 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_stack_decoder.py +300 -300
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/utils.py +41 -41
- {nltk-3.2 → nltk-3.3}/nltk/test/util.doctest +48 -48
- {nltk-3.2 → nltk-3.3}/nltk/test/wordnet.doctest +595 -581
- {nltk-3.2 → nltk-3.3}/nltk/test/wordnet_fixt.py +7 -6
- {nltk-3.2 → nltk-3.3}/nltk/test/wordnet_lch.doctest +53 -53
- {nltk-3.2 → nltk-3.3}/nltk/test/wsd.doctest +68 -68
- {nltk-3.2 → nltk-3.3}/nltk/text.py +663 -610
- {nltk-3.2 → nltk-3.3}/nltk/tgrep.py +938 -937
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/__init__.py +130 -107
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/api.py +77 -78
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/casual.py +343 -342
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/mwe.py +120 -120
- nltk-3.3/nltk/tokenize/nist.py +173 -0
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/punkt.py +1620 -1601
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/regexp.py +208 -202
- nltk-3.3/nltk/tokenize/repp.py +151 -0
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/sexpr.py +140 -142
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/simple.py +138 -136
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/stanford.py +117 -113
- nltk-3.3/nltk/tokenize/stanford_segmenter.py +246 -0
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/texttiling.py +458 -460
- nltk-3.3/nltk/tokenize/toktok.py +155 -0
- nltk-3.3/nltk/tokenize/treebank.py +363 -0
- nltk-3.3/nltk/tokenize/util.py +261 -0
- {nltk-3.2 → nltk-3.3}/nltk/toolbox.py +499 -497
- {nltk-3.2 → nltk-3.3}/nltk/translate/__init__.py +24 -24
- {nltk-3.2 → nltk-3.3}/nltk/translate/api.py +331 -321
- {nltk-3.2 → nltk-3.3}/nltk/translate/bleu_score.py +615 -556
- nltk-3.3/nltk/translate/chrf_score.py +137 -0
- {nltk-3.2 → nltk-3.3}/nltk/translate/gale_church.py +248 -235
- {nltk-3.2 → nltk-3.3}/nltk/translate/gdfa.py +135 -131
- nltk-3.3/nltk/translate/gleu_score.py +193 -0
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm1.py +250 -241
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm2.py +308 -308
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm3.py +337 -337
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm4.py +467 -467
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm5.py +639 -639
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm_model.py +536 -533
- {nltk-3.2 → nltk-3.3}/nltk/translate/metrics.py +40 -40
- nltk-3.3/nltk/translate/nist_score.py +178 -0
- {nltk-3.2 → nltk-3.3}/nltk/translate/phrase_based.py +175 -175
- {nltk-3.2 → nltk-3.3}/nltk/translate/ribes_score.py +325 -325
- {nltk-3.2 → nltk-3.3}/nltk/translate/stack_decoder.py +499 -495
- {nltk-3.2 → nltk-3.3}/nltk/tree.py +1605 -1603
- {nltk-3.2 → nltk-3.3}/nltk/treeprettyprinter.py +564 -564
- {nltk-3.2 → nltk-3.3}/nltk/treetransforms.py +309 -309
- {nltk-3.2 → nltk-3.3}/nltk/twitter/__init__.py +28 -28
- {nltk-3.2 → nltk-3.3}/nltk/twitter/api.py +141 -136
- {nltk-3.2 → nltk-3.3}/nltk/twitter/common.py +264 -258
- {nltk-3.2 → nltk-3.3}/nltk/twitter/twitter_demo.py +288 -288
- {nltk-3.2 → nltk-3.3}/nltk/twitter/twitterclient.py +530 -530
- {nltk-3.2 → nltk-3.3}/nltk/twitter/util.py +147 -146
- nltk-3.3/nltk/util.py +751 -0
- {nltk-3.2 → nltk-3.3}/nltk/wsd.py +53 -53
- {nltk-3.2 → nltk-3.3/nltk.egg-info}/PKG-INFO +13 -5
- {nltk-3.2 → nltk-3.3}/nltk.egg-info/SOURCES.txt +26 -3
- nltk-3.3/nltk.egg-info/requires.txt +31 -0
- {nltk-3.2 → nltk-3.3}/setup.cfg +0 -1
- {nltk-3.2 → nltk-3.3}/setup.py +102 -75
- nltk-3.2/nltk/VERSION +0 -1
- nltk-3.2/nltk/compat.py +0 -680
- nltk-3.2/nltk/corpus/reader/pl196x.py +0 -278
- nltk-3.2/nltk/corpus/reader/verbnet.py +0 -408
- nltk-3.2/nltk/corpus/reader/wordlist.py +0 -37
- nltk-3.2/nltk/sentiment/vader_lexicon.txt +0 -7517
- nltk-3.2/nltk/six.py +0 -868
- nltk-3.2/nltk/stem/porter.py +0 -692
- nltk-3.2/nltk/test/unit/test_2x_compat.py +0 -26
- nltk-3.2/nltk/test/unit/test_stem.py +0 -36
- nltk-3.2/nltk/test/unit/test_tokenize.py +0 -23
- nltk-3.2/nltk/test/unit/translate/test_bleu.py +0 -84
- nltk-3.2/nltk/tokenize/treebank.py +0 -124
- nltk-3.2/nltk/tokenize/util.py +0 -90
- nltk-3.2/nltk/util.py +0 -1364
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/__init__.py +0 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/__init__.py +0 -0
- {nltk-3.2 → nltk-3.3}/nltk.egg-info/dependency_links.txt +0 -0
- {nltk-3.2 → nltk-3.3}/nltk.egg-info/not-zip-safe +0 -0
- {nltk-3.2 → nltk-3.3}/nltk.egg-info/top_level.txt +0 -0
|
@@ -1,421 +1,421 @@
|
|
|
1
|
-
# Natural Language Toolkit: NomBank Corpus Reader
|
|
2
|
-
#
|
|
3
|
-
# Copyright (C) 2001-
|
|
4
|
-
# Authors: Paul Bedaride <paul.bedaride@gmail.com>
|
|
5
|
-
# Edward Loper <edloper@gmail.com>
|
|
6
|
-
# URL: <http://nltk.org/>
|
|
7
|
-
# For license information, see LICENSE.TXT
|
|
8
|
-
|
|
9
|
-
from __future__ import unicode_literals
|
|
10
|
-
|
|
11
|
-
from
|
|
12
|
-
|
|
13
|
-
from
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
from nltk.
|
|
17
|
-
from nltk.
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
""
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
:param
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
framefiles
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
self.
|
|
62
|
-
self.
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
"""
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
"""
|
|
89
|
-
return
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
for
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
#
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
self.
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
r =
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
if self.
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
- ``
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
``
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
@
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
wordnum:height
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
self.
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
#print
|
|
400
|
-
#
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
1
|
+
# Natural Language Toolkit: NomBank Corpus Reader
|
|
2
|
+
#
|
|
3
|
+
# Copyright (C) 2001-2018 NLTK Project
|
|
4
|
+
# Authors: Paul Bedaride <paul.bedaride@gmail.com>
|
|
5
|
+
# Edward Loper <edloper@gmail.com>
|
|
6
|
+
# URL: <http://nltk.org/>
|
|
7
|
+
# For license information, see LICENSE.TXT
|
|
8
|
+
|
|
9
|
+
from __future__ import unicode_literals
|
|
10
|
+
from xml.etree import ElementTree
|
|
11
|
+
from functools import total_ordering
|
|
12
|
+
|
|
13
|
+
from six import string_types
|
|
14
|
+
|
|
15
|
+
from nltk.tree import Tree
|
|
16
|
+
from nltk.internals import raise_unorderable_types
|
|
17
|
+
from nltk.compat import python_2_unicode_compatible
|
|
18
|
+
|
|
19
|
+
from nltk.corpus.reader.util import *
|
|
20
|
+
from nltk.corpus.reader.api import *
|
|
21
|
+
|
|
22
|
+
class NombankCorpusReader(CorpusReader):
|
|
23
|
+
"""
|
|
24
|
+
Corpus reader for the nombank corpus, which augments the Penn
|
|
25
|
+
Treebank with information about the predicate argument structure
|
|
26
|
+
of every noun instance. The corpus consists of two parts: the
|
|
27
|
+
predicate-argument annotations themselves, and a set of "frameset
|
|
28
|
+
files" which define the argument labels used by the annotations,
|
|
29
|
+
on a per-noun basis. Each "frameset file" contains one or more
|
|
30
|
+
predicates, such as ``'turn'`` or ``'turn_on'``, each of which is
|
|
31
|
+
divided into coarse-grained word senses called "rolesets". For
|
|
32
|
+
each "roleset", the frameset file provides descriptions of the
|
|
33
|
+
argument roles, along with examples.
|
|
34
|
+
"""
|
|
35
|
+
def __init__(self, root, nomfile, framefiles='',
|
|
36
|
+
nounsfile=None, parse_fileid_xform=None,
|
|
37
|
+
parse_corpus=None, encoding='utf8'):
|
|
38
|
+
"""
|
|
39
|
+
:param root: The root directory for this corpus.
|
|
40
|
+
:param nomfile: The name of the file containing the predicate-
|
|
41
|
+
argument annotations (relative to ``root``).
|
|
42
|
+
:param framefiles: A list or regexp specifying the frameset
|
|
43
|
+
fileids for this corpus.
|
|
44
|
+
:param parse_fileid_xform: A transform that should be applied
|
|
45
|
+
to the fileids in this corpus. This should be a function
|
|
46
|
+
of one argument (a fileid) that returns a string (the new
|
|
47
|
+
fileid).
|
|
48
|
+
:param parse_corpus: The corpus containing the parse trees
|
|
49
|
+
corresponding to this corpus. These parse trees are
|
|
50
|
+
necessary to resolve the tree pointers used by nombank.
|
|
51
|
+
"""
|
|
52
|
+
# If framefiles is specified as a regexp, expand it.
|
|
53
|
+
if isinstance(framefiles, string_types):
|
|
54
|
+
framefiles = find_corpus_fileids(root, framefiles)
|
|
55
|
+
framefiles = list(framefiles)
|
|
56
|
+
# Initialze the corpus reader.
|
|
57
|
+
CorpusReader.__init__(self, root, [nomfile, nounsfile] + framefiles,
|
|
58
|
+
encoding)
|
|
59
|
+
|
|
60
|
+
# Record our frame fileids & nom file.
|
|
61
|
+
self._nomfile = nomfile
|
|
62
|
+
self._framefiles = framefiles
|
|
63
|
+
self._nounsfile = nounsfile
|
|
64
|
+
self._parse_fileid_xform = parse_fileid_xform
|
|
65
|
+
self._parse_corpus = parse_corpus
|
|
66
|
+
|
|
67
|
+
def raw(self, fileids=None):
|
|
68
|
+
"""
|
|
69
|
+
:return: the text contents of the given fileids, as a single string.
|
|
70
|
+
"""
|
|
71
|
+
if fileids is None: fileids = self._fileids
|
|
72
|
+
elif isinstance(fileids, string_types): fileids = [fileids]
|
|
73
|
+
return concat([self.open(f).read() for f in fileids])
|
|
74
|
+
|
|
75
|
+
def instances(self, baseform=None):
|
|
76
|
+
"""
|
|
77
|
+
:return: a corpus view that acts as a list of
|
|
78
|
+
``NombankInstance`` objects, one for each noun in the corpus.
|
|
79
|
+
"""
|
|
80
|
+
kwargs = {}
|
|
81
|
+
if baseform is not None:
|
|
82
|
+
kwargs['instance_filter'] = lambda inst: inst.baseform==baseform
|
|
83
|
+
return StreamBackedCorpusView(self.abspath(self._nomfile),
|
|
84
|
+
lambda stream: self._read_instance_block(stream, **kwargs),
|
|
85
|
+
encoding=self.encoding(self._nomfile))
|
|
86
|
+
|
|
87
|
+
def lines(self):
|
|
88
|
+
"""
|
|
89
|
+
:return: a corpus view that acts as a list of strings, one for
|
|
90
|
+
each line in the predicate-argument annotation file.
|
|
91
|
+
"""
|
|
92
|
+
return StreamBackedCorpusView(self.abspath(self._nomfile),
|
|
93
|
+
read_line_block,
|
|
94
|
+
encoding=self.encoding(self._nomfile))
|
|
95
|
+
|
|
96
|
+
def roleset(self, roleset_id):
|
|
97
|
+
"""
|
|
98
|
+
:return: the xml description for the given roleset.
|
|
99
|
+
"""
|
|
100
|
+
baseform = roleset_id.split('.')[0]
|
|
101
|
+
baseform = baseform.replace('perc-sign','%')
|
|
102
|
+
baseform = baseform.replace('oneslashonezero', '1/10').replace('1/10','1-slash-10')
|
|
103
|
+
framefile = 'frames/%s.xml' % baseform
|
|
104
|
+
if framefile not in self._framefiles:
|
|
105
|
+
raise ValueError('Frameset file for %s not found' %
|
|
106
|
+
roleset_id)
|
|
107
|
+
|
|
108
|
+
# n.b.: The encoding for XML fileids is specified by the file
|
|
109
|
+
# itself; so we ignore self._encoding here.
|
|
110
|
+
etree = ElementTree.parse(self.abspath(framefile).open()).getroot()
|
|
111
|
+
for roleset in etree.findall('predicate/roleset'):
|
|
112
|
+
if roleset.attrib['id'] == roleset_id:
|
|
113
|
+
return roleset
|
|
114
|
+
raise ValueError('Roleset %s not found in %s' % (roleset_id, framefile))
|
|
115
|
+
|
|
116
|
+
def rolesets(self, baseform=None):
|
|
117
|
+
"""
|
|
118
|
+
:return: list of xml descriptions for rolesets.
|
|
119
|
+
"""
|
|
120
|
+
if baseform is not None:
|
|
121
|
+
framefile = 'frames/%s.xml' % baseform
|
|
122
|
+
if framefile not in self._framefiles:
|
|
123
|
+
raise ValueError('Frameset file for %s not found' %
|
|
124
|
+
baseform)
|
|
125
|
+
framefiles = [framefile]
|
|
126
|
+
else:
|
|
127
|
+
framefiles = self._framefiles
|
|
128
|
+
|
|
129
|
+
rsets = []
|
|
130
|
+
for framefile in framefiles:
|
|
131
|
+
# n.b.: The encoding for XML fileids is specified by the file
|
|
132
|
+
# itself; so we ignore self._encoding here.
|
|
133
|
+
etree = ElementTree.parse(self.abspath(framefile).open()).getroot()
|
|
134
|
+
rsets.append(etree.findall('predicate/roleset'))
|
|
135
|
+
return LazyConcatenation(rsets)
|
|
136
|
+
|
|
137
|
+
def nouns(self):
|
|
138
|
+
"""
|
|
139
|
+
:return: a corpus view that acts as a list of all noun lemmas
|
|
140
|
+
in this corpus (from the nombank.1.0.words file).
|
|
141
|
+
"""
|
|
142
|
+
return StreamBackedCorpusView(self.abspath(self._nounsfile),
|
|
143
|
+
read_line_block,
|
|
144
|
+
encoding=self.encoding(self._nounsfile))
|
|
145
|
+
|
|
146
|
+
def _read_instance_block(self, stream, instance_filter=lambda inst: True):
|
|
147
|
+
block = []
|
|
148
|
+
|
|
149
|
+
# Read 100 at a time.
|
|
150
|
+
for i in range(100):
|
|
151
|
+
line = stream.readline().strip()
|
|
152
|
+
if line:
|
|
153
|
+
inst = NombankInstance.parse(
|
|
154
|
+
line, self._parse_fileid_xform,
|
|
155
|
+
self._parse_corpus)
|
|
156
|
+
if instance_filter(inst):
|
|
157
|
+
block.append(inst)
|
|
158
|
+
|
|
159
|
+
return block
|
|
160
|
+
|
|
161
|
+
######################################################################
|
|
162
|
+
#{ Nombank Instance & related datatypes
|
|
163
|
+
######################################################################
|
|
164
|
+
|
|
165
|
+
@python_2_unicode_compatible
|
|
166
|
+
class NombankInstance(object):
|
|
167
|
+
|
|
168
|
+
def __init__(self, fileid, sentnum, wordnum, baseform, sensenumber,
|
|
169
|
+
predicate, predid, arguments, parse_corpus=None):
|
|
170
|
+
|
|
171
|
+
self.fileid = fileid
|
|
172
|
+
"""The name of the file containing the parse tree for this
|
|
173
|
+
instance's sentence."""
|
|
174
|
+
|
|
175
|
+
self.sentnum = sentnum
|
|
176
|
+
"""The sentence number of this sentence within ``fileid``.
|
|
177
|
+
Indexing starts from zero."""
|
|
178
|
+
|
|
179
|
+
self.wordnum = wordnum
|
|
180
|
+
"""The word number of this instance's predicate within its
|
|
181
|
+
containing sentence. Word numbers are indexed starting from
|
|
182
|
+
zero, and include traces and other empty parse elements."""
|
|
183
|
+
|
|
184
|
+
self.baseform = baseform
|
|
185
|
+
"""The baseform of the predicate."""
|
|
186
|
+
|
|
187
|
+
self.sensenumber = sensenumber
|
|
188
|
+
"""The sense number of the predicate."""
|
|
189
|
+
|
|
190
|
+
self.predicate = predicate
|
|
191
|
+
"""A ``NombankTreePointer`` indicating the position of this
|
|
192
|
+
instance's predicate within its containing sentence."""
|
|
193
|
+
|
|
194
|
+
self.predid = predid
|
|
195
|
+
"""Identifier of the predicate."""
|
|
196
|
+
|
|
197
|
+
self.arguments = tuple(arguments)
|
|
198
|
+
"""A list of tuples (argloc, argid), specifying the location
|
|
199
|
+
and identifier for each of the predicate's argument in the
|
|
200
|
+
containing sentence. Argument identifiers are strings such as
|
|
201
|
+
``'ARG0'`` or ``'ARGM-TMP'``. This list does *not* contain
|
|
202
|
+
the predicate."""
|
|
203
|
+
|
|
204
|
+
self.parse_corpus = parse_corpus
|
|
205
|
+
"""A corpus reader for the parse trees corresponding to the
|
|
206
|
+
instances in this nombank corpus."""
|
|
207
|
+
|
|
208
|
+
@property
|
|
209
|
+
def roleset(self):
|
|
210
|
+
"""The name of the roleset used by this instance's predicate.
|
|
211
|
+
Use ``nombank.roleset() <NombankCorpusReader.roleset>`` to
|
|
212
|
+
look up information about the roleset."""
|
|
213
|
+
r = self.baseform.replace('%', 'perc-sign')
|
|
214
|
+
r = r.replace('1/10', '1-slash-10').replace('1-slash-10', 'oneslashonezero')
|
|
215
|
+
return '%s.%s'%(r, self.sensenumber)
|
|
216
|
+
|
|
217
|
+
def __repr__(self):
|
|
218
|
+
return ('<NombankInstance: %s, sent %s, word %s>' %
|
|
219
|
+
(self.fileid, self.sentnum, self.wordnum))
|
|
220
|
+
|
|
221
|
+
def __str__(self):
|
|
222
|
+
s = '%s %s %s %s %s' % (self.fileid, self.sentnum, self.wordnum,
|
|
223
|
+
self.baseform, self.sensenumber)
|
|
224
|
+
items = self.arguments + ((self.predicate, 'rel'),)
|
|
225
|
+
for (argloc, argid) in sorted(items):
|
|
226
|
+
s += ' %s-%s' % (argloc, argid)
|
|
227
|
+
return s
|
|
228
|
+
|
|
229
|
+
def _get_tree(self):
|
|
230
|
+
if self.parse_corpus is None: return None
|
|
231
|
+
if self.fileid not in self.parse_corpus.fileids(): return None
|
|
232
|
+
return self.parse_corpus.parsed_sents(self.fileid)[self.sentnum]
|
|
233
|
+
tree = property(_get_tree, doc="""
|
|
234
|
+
The parse tree corresponding to this instance, or None if
|
|
235
|
+
the corresponding tree is not available.""")
|
|
236
|
+
|
|
237
|
+
@staticmethod
|
|
238
|
+
def parse(s, parse_fileid_xform=None, parse_corpus=None):
|
|
239
|
+
pieces = s.split()
|
|
240
|
+
if len(pieces) < 6:
|
|
241
|
+
raise ValueError('Badly formatted nombank line: %r' % s)
|
|
242
|
+
|
|
243
|
+
# Divide the line into its basic pieces.
|
|
244
|
+
(fileid, sentnum, wordnum,
|
|
245
|
+
baseform, sensenumber) = pieces[:5]
|
|
246
|
+
|
|
247
|
+
args = pieces[5:]
|
|
248
|
+
rel = [args.pop(i) for i,p in enumerate(args) if '-rel' in p]
|
|
249
|
+
if len(rel) != 1:
|
|
250
|
+
raise ValueError('Badly formatted nombank line: %r' % s)
|
|
251
|
+
|
|
252
|
+
# Apply the fileid selector, if any.
|
|
253
|
+
if parse_fileid_xform is not None:
|
|
254
|
+
fileid = parse_fileid_xform(fileid)
|
|
255
|
+
|
|
256
|
+
# Convert sentence & word numbers to ints.
|
|
257
|
+
sentnum = int(sentnum)
|
|
258
|
+
wordnum = int(wordnum)
|
|
259
|
+
|
|
260
|
+
# Parse the predicate location.
|
|
261
|
+
|
|
262
|
+
predloc, predid = rel[0].split('-', 1)
|
|
263
|
+
predicate = NombankTreePointer.parse(predloc)
|
|
264
|
+
|
|
265
|
+
# Parse the arguments.
|
|
266
|
+
arguments = []
|
|
267
|
+
for arg in args:
|
|
268
|
+
argloc, argid = arg.split('-', 1)
|
|
269
|
+
arguments.append( (NombankTreePointer.parse(argloc), argid) )
|
|
270
|
+
|
|
271
|
+
# Put it all together.
|
|
272
|
+
return NombankInstance(fileid, sentnum, wordnum, baseform, sensenumber,
|
|
273
|
+
predicate, predid, arguments, parse_corpus)
|
|
274
|
+
|
|
275
|
+
class NombankPointer(object):
|
|
276
|
+
"""
|
|
277
|
+
A pointer used by nombank to identify one or more constituents in
|
|
278
|
+
a parse tree. ``NombankPointer`` is an abstract base class with
|
|
279
|
+
three concrete subclasses:
|
|
280
|
+
|
|
281
|
+
- ``NombankTreePointer`` is used to point to single constituents.
|
|
282
|
+
- ``NombankSplitTreePointer`` is used to point to 'split'
|
|
283
|
+
constituents, which consist of a sequence of two or more
|
|
284
|
+
``NombankTreePointer`` pointers.
|
|
285
|
+
- ``NombankChainTreePointer`` is used to point to entire trace
|
|
286
|
+
chains in a tree. It consists of a sequence of pieces, which
|
|
287
|
+
can be ``NombankTreePointer`` or ``NombankSplitTreePointer`` pointers.
|
|
288
|
+
"""
|
|
289
|
+
def __init__(self):
|
|
290
|
+
if self.__class__ == NombankPointer:
|
|
291
|
+
raise NotImplementedError()
|
|
292
|
+
|
|
293
|
+
@python_2_unicode_compatible
|
|
294
|
+
class NombankChainTreePointer(NombankPointer):
|
|
295
|
+
def __init__(self, pieces):
|
|
296
|
+
self.pieces = pieces
|
|
297
|
+
"""A list of the pieces that make up this chain. Elements may
|
|
298
|
+
be either ``NombankSplitTreePointer`` or
|
|
299
|
+
``NombankTreePointer`` pointers."""
|
|
300
|
+
|
|
301
|
+
def __str__(self):
|
|
302
|
+
return '*'.join('%s' % p for p in self.pieces)
|
|
303
|
+
def __repr__(self):
|
|
304
|
+
return '<NombankChainTreePointer: %s>' % self
|
|
305
|
+
def select(self, tree):
|
|
306
|
+
if tree is None: raise ValueError('Parse tree not avaialable')
|
|
307
|
+
return Tree('*CHAIN*', [p.select(tree) for p in self.pieces])
|
|
308
|
+
|
|
309
|
+
@python_2_unicode_compatible
|
|
310
|
+
class NombankSplitTreePointer(NombankPointer):
|
|
311
|
+
def __init__(self, pieces):
|
|
312
|
+
self.pieces = pieces
|
|
313
|
+
"""A list of the pieces that make up this chain. Elements are
|
|
314
|
+
all ``NombankTreePointer`` pointers."""
|
|
315
|
+
|
|
316
|
+
def __str__(self):
|
|
317
|
+
return ','.join('%s' % p for p in self.pieces)
|
|
318
|
+
def __repr__(self):
|
|
319
|
+
return '<NombankSplitTreePointer: %s>' % self
|
|
320
|
+
def select(self, tree):
|
|
321
|
+
if tree is None: raise ValueError('Parse tree not avaialable')
|
|
322
|
+
return Tree('*SPLIT*', [p.select(tree) for p in self.pieces])
|
|
323
|
+
|
|
324
|
+
@total_ordering
|
|
325
|
+
@python_2_unicode_compatible
|
|
326
|
+
class NombankTreePointer(NombankPointer):
|
|
327
|
+
"""
|
|
328
|
+
wordnum:height*wordnum:height*...
|
|
329
|
+
wordnum:height,
|
|
330
|
+
|
|
331
|
+
"""
|
|
332
|
+
def __init__(self, wordnum, height):
|
|
333
|
+
self.wordnum = wordnum
|
|
334
|
+
self.height = height
|
|
335
|
+
|
|
336
|
+
@staticmethod
|
|
337
|
+
def parse(s):
|
|
338
|
+
# Deal with chains (xx*yy*zz)
|
|
339
|
+
pieces = s.split('*')
|
|
340
|
+
if len(pieces) > 1:
|
|
341
|
+
return NombankChainTreePointer([NombankTreePointer.parse(elt)
|
|
342
|
+
for elt in pieces])
|
|
343
|
+
|
|
344
|
+
# Deal with split args (xx,yy,zz)
|
|
345
|
+
pieces = s.split(',')
|
|
346
|
+
if len(pieces) > 1:
|
|
347
|
+
return NombankSplitTreePointer([NombankTreePointer.parse(elt)
|
|
348
|
+
for elt in pieces])
|
|
349
|
+
|
|
350
|
+
# Deal with normal pointers.
|
|
351
|
+
pieces = s.split(':')
|
|
352
|
+
if len(pieces) != 2: raise ValueError('bad nombank pointer %r' % s)
|
|
353
|
+
return NombankTreePointer(int(pieces[0]), int(pieces[1]))
|
|
354
|
+
|
|
355
|
+
def __str__(self):
|
|
356
|
+
return '%s:%s' % (self.wordnum, self.height)
|
|
357
|
+
|
|
358
|
+
def __repr__(self):
|
|
359
|
+
return 'NombankTreePointer(%d, %d)' % (self.wordnum, self.height)
|
|
360
|
+
|
|
361
|
+
def __eq__(self, other):
|
|
362
|
+
while isinstance(other, (NombankChainTreePointer,
|
|
363
|
+
NombankSplitTreePointer)):
|
|
364
|
+
other = other.pieces[0]
|
|
365
|
+
|
|
366
|
+
if not isinstance(other, NombankTreePointer):
|
|
367
|
+
return self is other
|
|
368
|
+
|
|
369
|
+
return (self.wordnum == other.wordnum and self.height == other.height)
|
|
370
|
+
|
|
371
|
+
def __ne__(self, other):
|
|
372
|
+
return not self == other
|
|
373
|
+
|
|
374
|
+
def __lt__(self, other):
|
|
375
|
+
while isinstance(other, (NombankChainTreePointer,
|
|
376
|
+
NombankSplitTreePointer)):
|
|
377
|
+
other = other.pieces[0]
|
|
378
|
+
|
|
379
|
+
if not isinstance(other, NombankTreePointer):
|
|
380
|
+
return id(self) < id(other)
|
|
381
|
+
|
|
382
|
+
return (self.wordnum, -self.height) < (other.wordnum, -other.height)
|
|
383
|
+
|
|
384
|
+
def select(self, tree):
|
|
385
|
+
if tree is None: raise ValueError('Parse tree not avaialable')
|
|
386
|
+
return tree[self.treepos(tree)]
|
|
387
|
+
|
|
388
|
+
def treepos(self, tree):
|
|
389
|
+
"""
|
|
390
|
+
Convert this pointer to a standard 'tree position' pointer,
|
|
391
|
+
given that it points to the given tree.
|
|
392
|
+
"""
|
|
393
|
+
if tree is None: raise ValueError('Parse tree not avaialable')
|
|
394
|
+
stack = [tree]
|
|
395
|
+
treepos = []
|
|
396
|
+
|
|
397
|
+
wordnum = 0
|
|
398
|
+
while True:
|
|
399
|
+
#print treepos
|
|
400
|
+
#print stack[-1]
|
|
401
|
+
# tree node:
|
|
402
|
+
if isinstance(stack[-1], Tree):
|
|
403
|
+
# Select the next child.
|
|
404
|
+
if len(treepos) < len(stack):
|
|
405
|
+
treepos.append(0)
|
|
406
|
+
else:
|
|
407
|
+
treepos[-1] += 1
|
|
408
|
+
# Update the stack.
|
|
409
|
+
if treepos[-1] < len(stack[-1]):
|
|
410
|
+
stack.append(stack[-1][treepos[-1]])
|
|
411
|
+
else:
|
|
412
|
+
# End of node's child list: pop up a level.
|
|
413
|
+
stack.pop()
|
|
414
|
+
treepos.pop()
|
|
415
|
+
# word node:
|
|
416
|
+
else:
|
|
417
|
+
if wordnum == self.wordnum:
|
|
418
|
+
return tuple(treepos[:len(treepos)-self.height-1])
|
|
419
|
+
else:
|
|
420
|
+
wordnum += 1
|
|
421
|
+
stack.pop()
|