nltk 3.2__zip → 3.3__zip
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nltk-3.2 → nltk-3.3}/INSTALL.txt +6 -6
- {nltk-3.2 → nltk-3.3}/LICENSE.txt +13 -13
- {nltk-3.2 → nltk-3.3}/MANIFEST.in +6 -6
- {nltk-3.2/nltk.egg-info → nltk-3.3}/PKG-INFO +13 -5
- nltk-3.2/README.txt → nltk-3.3/README.md +49 -48
- nltk-3.3/nltk/VERSION +1 -0
- {nltk-3.2 → nltk-3.3}/nltk/__init__.py +185 -185
- {nltk-3.2 → nltk-3.3}/nltk/app/__init__.py +52 -53
- {nltk-3.2 → nltk-3.3}/nltk/app/chartparser_app.py +2276 -2276
- {nltk-3.2 → nltk-3.3}/nltk/app/chunkparser_app.py +1262 -1264
- {nltk-3.2 → nltk-3.3}/nltk/app/collocations_app.py +347 -350
- {nltk-3.2 → nltk-3.3}/nltk/app/concordance_app.py +568 -570
- {nltk-3.2 → nltk-3.3}/nltk/app/nemo_app.py +156 -155
- {nltk-3.2 → nltk-3.3}/nltk/app/rdparser_app.py +892 -894
- {nltk-3.2 → nltk-3.3}/nltk/app/srparser_app.py +809 -809
- {nltk-3.2 → nltk-3.3}/nltk/app/wordfreq_app.py +32 -32
- {nltk-3.2 → nltk-3.3}/nltk/app/wordnet_app.py +968 -968
- {nltk-3.2 → nltk-3.3}/nltk/book.py +91 -90
- {nltk-3.2 → nltk-3.3}/nltk/ccg/__init__.py +22 -22
- {nltk-3.2 → nltk-3.3}/nltk/ccg/api.py +359 -335
- {nltk-3.2 → nltk-3.3}/nltk/ccg/chart.py +394 -361
- {nltk-3.2 → nltk-3.3}/nltk/ccg/combinator.py +343 -312
- {nltk-3.2 → nltk-3.3}/nltk/ccg/lexicon.py +328 -290
- nltk-3.3/nltk/ccg/logic.py +46 -0
- {nltk-3.2 → nltk-3.3}/nltk/chat/__init__.py +49 -49
- {nltk-3.2 → nltk-3.3}/nltk/chat/eliza.py +244 -244
- {nltk-3.2 → nltk-3.3}/nltk/chat/iesha.py +140 -140
- {nltk-3.2 → nltk-3.3}/nltk/chat/rude.py +92 -92
- {nltk-3.2 → nltk-3.3}/nltk/chat/suntsu.py +117 -117
- {nltk-3.2 → nltk-3.3}/nltk/chat/util.py +122 -120
- {nltk-3.2 → nltk-3.3}/nltk/chat/zen.py +282 -282
- {nltk-3.2 → nltk-3.3}/nltk/chunk/__init__.py +190 -190
- {nltk-3.2 → nltk-3.3}/nltk/chunk/api.py +51 -51
- {nltk-3.2 → nltk-3.3}/nltk/chunk/named_entity.py +331 -331
- {nltk-3.2 → nltk-3.3}/nltk/chunk/regexp.py +1390 -1384
- {nltk-3.2 → nltk-3.3}/nltk/chunk/util.py +600 -600
- {nltk-3.2 → nltk-3.3}/nltk/classify/__init__.py +98 -98
- {nltk-3.2 → nltk-3.3}/nltk/classify/api.py +193 -193
- {nltk-3.2 → nltk-3.3}/nltk/classify/decisiontree.py +295 -295
- {nltk-3.2 → nltk-3.3}/nltk/classify/maxent.py +1510 -1492
- {nltk-3.2 → nltk-3.3}/nltk/classify/megam.py +179 -178
- {nltk-3.2 → nltk-3.3}/nltk/classify/naivebayes.py +245 -242
- {nltk-3.2 → nltk-3.3}/nltk/classify/positivenaivebayes.py +170 -170
- {nltk-3.2 → nltk-3.3}/nltk/classify/rte_classify.py +160 -191
- {nltk-3.2 → nltk-3.3}/nltk/classify/scikitlearn.py +153 -151
- {nltk-3.2 → nltk-3.3}/nltk/classify/senna.py +183 -181
- {nltk-3.2 → nltk-3.3}/nltk/classify/svm.py +15 -15
- {nltk-3.2 → nltk-3.3}/nltk/classify/tadm.py +113 -112
- {nltk-3.2 → nltk-3.3}/nltk/classify/textcat.py +193 -193
- {nltk-3.2 → nltk-3.3}/nltk/classify/util.py +324 -312
- {nltk-3.2 → nltk-3.3}/nltk/classify/weka.py +346 -345
- {nltk-3.2 → nltk-3.3}/nltk/cluster/__init__.py +86 -86
- {nltk-3.2 → nltk-3.3}/nltk/cluster/api.py +74 -70
- {nltk-3.2 → nltk-3.3}/nltk/cluster/em.py +247 -247
- {nltk-3.2 → nltk-3.3}/nltk/cluster/gaac.py +168 -168
- {nltk-3.2 → nltk-3.3}/nltk/cluster/kmeans.py +221 -221
- {nltk-3.2 → nltk-3.3}/nltk/cluster/util.py +304 -289
- nltk-3.3/nltk/collections.py +640 -0
- {nltk-3.2 → nltk-3.3}/nltk/collocations.py +388 -388
- nltk-3.3/nltk/compat.py +365 -0
- {nltk-3.2 → nltk-3.3}/nltk/corpus/__init__.py +341 -320
- {nltk-3.2 → nltk-3.3}/nltk/corpus/europarl_raw.py +44 -44
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/__init__.py +147 -145
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/aligned.py +115 -114
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/api.py +448 -446
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/bnc.py +252 -252
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/bracket_parse.py +227 -227
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/categorized_sents.py +179 -178
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/chasen.py +140 -139
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/childes.py +522 -494
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/chunked.py +212 -212
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/cmudict.py +97 -95
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/comparative_sents.py +280 -278
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/conll.py +528 -521
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/crubadan.py +116 -116
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/dependency.py +101 -101
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/framenet.py +3059 -2093
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ieer.py +112 -111
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/indian.py +85 -86
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ipipan.py +332 -331
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/knbc.py +187 -187
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/lin.py +156 -156
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/mte.py +311 -310
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/nkjp.py +429 -428
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/nombank.py +421 -421
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/nps_chat.py +73 -73
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/opinion_lexicon.py +115 -115
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/panlex_lite.py +165 -165
- nltk-3.3/nltk/corpus/reader/pl196x.py +292 -0
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/plaintext.py +232 -233
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ppattach.py +96 -95
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/propbank.py +479 -481
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/pros_cons.py +128 -126
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/reviews.py +330 -328
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/rte.py +144 -146
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/semcor.py +256 -256
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/senseval.py +203 -201
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/sentiwordnet.py +137 -137
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/sinica_treebank.py +75 -75
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/string_category.py +60 -59
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/switchboard.py +119 -119
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/tagged.py +295 -294
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/timit.py +452 -450
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/toolbox.py +68 -68
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/twitter.py +155 -157
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/udhr.py +80 -80
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/util.py +804 -800
- nltk-3.3/nltk/corpus/reader/verbnet.py +595 -0
- nltk-3.3/nltk/corpus/reader/wordlist.py +134 -0
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/wordnet.py +2108 -2036
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/xmldocs.py +392 -387
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ycoe.py +243 -242
- {nltk-3.2 → nltk-3.3}/nltk/corpus/util.py +144 -127
- {nltk-3.2 → nltk-3.3}/nltk/data.py +1487 -1451
- {nltk-3.2 → nltk-3.3}/nltk/decorators.py +219 -215
- {nltk-3.2 → nltk-3.3}/nltk/downloader.py +2278 -2274
- {nltk-3.2 → nltk-3.3}/nltk/draw/__init__.py +27 -27
- {nltk-3.2 → nltk-3.3}/nltk/draw/cfg.py +774 -777
- {nltk-3.2 → nltk-3.3}/nltk/draw/dispersion.py +58 -58
- {nltk-3.2 → nltk-3.3}/nltk/draw/table.py +1098 -1098
- {nltk-3.2 → nltk-3.3}/nltk/draw/tree.py +963 -966
- {nltk-3.2 → nltk-3.3}/nltk/draw/util.py +2356 -2356
- {nltk-3.2 → nltk-3.3}/nltk/featstruct.py +2617 -2500
- {nltk-3.2 → nltk-3.3}/nltk/grammar.py +1559 -1516
- {nltk-3.2 → nltk-3.3}/nltk/help.py +56 -56
- {nltk-3.2 → nltk-3.3}/nltk/inference/__init__.py +20 -20
- {nltk-3.2 → nltk-3.3}/nltk/inference/api.py +597 -589
- {nltk-3.2 → nltk-3.3}/nltk/inference/discourse.py +612 -608
- {nltk-3.2 → nltk-3.3}/nltk/inference/mace.py +311 -311
- {nltk-3.2 → nltk-3.3}/nltk/inference/nonmonotonic.py +509 -509
- {nltk-3.2 → nltk-3.3}/nltk/inference/prover9.py +431 -431
- {nltk-3.2 → nltk-3.3}/nltk/inference/resolution.py +687 -687
- {nltk-3.2 → nltk-3.3}/nltk/inference/tableau.py +607 -607
- {nltk-3.2 → nltk-3.3}/nltk/internals.py +982 -982
- {nltk-3.2 → nltk-3.3}/nltk/jsontags.py +63 -63
- {nltk-3.2 → nltk-3.3}/nltk/lazyimport.py +142 -142
- {nltk-3.2 → nltk-3.3}/nltk/metrics/__init__.py +30 -29
- {nltk-3.2 → nltk-3.3}/nltk/metrics/agreement.py +421 -401
- nltk-3.3/nltk/metrics/aline.py +607 -0
- {nltk-3.2 → nltk-3.3}/nltk/metrics/association.py +414 -410
- {nltk-3.2 → nltk-3.3}/nltk/metrics/confusionmatrix.py +206 -206
- {nltk-3.2 → nltk-3.3}/nltk/metrics/distance.py +207 -202
- {nltk-3.2 → nltk-3.3}/nltk/metrics/paice.py +381 -381
- {nltk-3.2 → nltk-3.3}/nltk/metrics/scores.py +228 -228
- {nltk-3.2 → nltk-3.3}/nltk/metrics/segmentation.py +231 -232
- {nltk-3.2 → nltk-3.3}/nltk/metrics/spearman.py +69 -69
- {nltk-3.2 → nltk-3.3}/nltk/misc/__init__.py +11 -11
- {nltk-3.2 → nltk-3.3}/nltk/misc/babelfish.py +10 -10
- {nltk-3.2 → nltk-3.3}/nltk/misc/chomsky.py +135 -133
- {nltk-3.2 → nltk-3.3}/nltk/misc/minimalset.py +83 -83
- {nltk-3.2 → nltk-3.3}/nltk/misc/sort.py +157 -157
- {nltk-3.2 → nltk-3.3}/nltk/misc/wordfinder.py +129 -129
- {nltk-3.2 → nltk-3.3}/nltk/parse/__init__.py +81 -80
- {nltk-3.2 → nltk-3.3}/nltk/parse/api.py +66 -66
- {nltk-3.2 → nltk-3.3}/nltk/parse/bllip.py +282 -282
- {nltk-3.2 → nltk-3.3}/nltk/parse/chart.py +1682 -1681
- nltk-3.3/nltk/parse/corenlp.py +780 -0
- {nltk-3.2 → nltk-3.3}/nltk/parse/dependencygraph.py +760 -758
- {nltk-3.2 → nltk-3.3}/nltk/parse/earleychart.py +452 -451
- {nltk-3.2 → nltk-3.3}/nltk/parse/evaluate.py +129 -129
- {nltk-3.2 → nltk-3.3}/nltk/parse/featurechart.py +580 -578
- {nltk-3.2 → nltk-3.3}/nltk/parse/generate.py +87 -76
- {nltk-3.2 → nltk-3.3}/nltk/parse/malt.py +356 -356
- {nltk-3.2 → nltk-3.3}/nltk/parse/nonprojectivedependencyparser.py +783 -783
- {nltk-3.2 → nltk-3.3}/nltk/parse/pchart.py +522 -522
- {nltk-3.2 → nltk-3.3}/nltk/parse/projectivedependencyparser.py +579 -578
- {nltk-3.2 → nltk-3.3}/nltk/parse/recursivedescent.py +655 -656
- {nltk-3.2 → nltk-3.3}/nltk/parse/shiftreduce.py +458 -459
- {nltk-3.2 → nltk-3.3}/nltk/parse/stanford.py +431 -406
- {nltk-3.2 → nltk-3.3}/nltk/parse/transitionparser.py +774 -772
- {nltk-3.2 → nltk-3.3}/nltk/parse/util.py +231 -231
- {nltk-3.2 → nltk-3.3}/nltk/parse/viterbi.py +401 -401
- {nltk-3.2 → nltk-3.3}/nltk/probability.py +2410 -2355
- {nltk-3.2 → nltk-3.3}/nltk/sem/__init__.py +61 -61
- {nltk-3.2 → nltk-3.3}/nltk/sem/boxer.py +1261 -1261
- {nltk-3.2 → nltk-3.3}/nltk/sem/chat80.py +782 -782
- {nltk-3.2 → nltk-3.3}/nltk/sem/cooper_storage.py +118 -118
- {nltk-3.2 → nltk-3.3}/nltk/sem/drt.py +1258 -1257
- {nltk-3.2 → nltk-3.3}/nltk/sem/drt_glue_demo.py +485 -489
- {nltk-3.2 → nltk-3.3}/nltk/sem/evaluate.py +790 -784
- {nltk-3.2 → nltk-3.3}/nltk/sem/glue.py +696 -694
- {nltk-3.2 → nltk-3.3}/nltk/sem/hole.py +386 -384
- {nltk-3.2 → nltk-3.3}/nltk/sem/lfg.py +210 -208
- {nltk-3.2 → nltk-3.3}/nltk/sem/linearlogic.py +451 -449
- {nltk-3.2 → nltk-3.3}/nltk/sem/logic.py +1910 -1910
- {nltk-3.2 → nltk-3.3}/nltk/sem/relextract.py +476 -483
- {nltk-3.2 → nltk-3.3}/nltk/sem/skolemize.py +101 -101
- {nltk-3.2 → nltk-3.3}/nltk/sem/util.py +249 -249
- {nltk-3.2 → nltk-3.3}/nltk/sentiment/__init__.py +14 -14
- {nltk-3.2 → nltk-3.3}/nltk/sentiment/sentiment_analyzer.py +229 -229
- {nltk-3.2 → nltk-3.3}/nltk/sentiment/util.py +762 -752
- {nltk-3.2 → nltk-3.3}/nltk/sentiment/vader.py +443 -445
- {nltk-3.2 → nltk-3.3}/nltk/stem/__init__.py +31 -33
- {nltk-3.2 → nltk-3.3}/nltk/stem/api.py +28 -25
- nltk-3.3/nltk/stem/arlstem.py +369 -0
- {nltk-3.2 → nltk-3.3}/nltk/stem/isri.py +345 -345
- {nltk-3.2 → nltk-3.3}/nltk/stem/lancaster.py +337 -310
- nltk-3.3/nltk/stem/porter.py +710 -0
- {nltk-3.2 → nltk-3.3}/nltk/stem/regexp.py +61 -61
- {nltk-3.2 → nltk-3.3}/nltk/stem/rslp.py +142 -142
- {nltk-3.2 → nltk-3.3}/nltk/stem/snowball.py +4236 -3712
- {nltk-3.2 → nltk-3.3}/nltk/stem/util.py +22 -12
- {nltk-3.2 → nltk-3.3}/nltk/stem/wordnet.py +51 -51
- {nltk-3.2 → nltk-3.3}/nltk/tag/__init__.py +152 -127
- {nltk-3.2 → nltk-3.3}/nltk/tag/api.py +86 -81
- {nltk-3.2 → nltk-3.3}/nltk/tag/brill.py +424 -426
- {nltk-3.2 → nltk-3.3}/nltk/tag/brill_trainer.py +608 -608
- {nltk-3.2 → nltk-3.3}/nltk/tag/crf.py +204 -200
- {nltk-3.2 → nltk-3.3}/nltk/tag/hmm.py +1277 -1277
- {nltk-3.2 → nltk-3.3}/nltk/tag/hunpos.py +132 -132
- {nltk-3.2 → nltk-3.3}/nltk/tag/mapping.py +101 -101
- {nltk-3.2 → nltk-3.3}/nltk/tag/perceptron.py +329 -314
- {nltk-3.2 → nltk-3.3}/nltk/tag/senna.py +147 -145
- {nltk-3.2 → nltk-3.3}/nltk/tag/sequential.py +740 -736
- {nltk-3.2 → nltk-3.3}/nltk/tag/stanford.py +219 -196
- {nltk-3.2 → nltk-3.3}/nltk/tag/tnt.py +595 -604
- {nltk-3.2 → nltk-3.3}/nltk/tag/util.py +72 -72
- {nltk-3.2 → nltk-3.3}/nltk/tbl/__init__.py +28 -28
- {nltk-3.2 → nltk-3.3}/nltk/tbl/api.py +1 -1
- {nltk-3.2 → nltk-3.3}/nltk/tbl/demo.py +366 -366
- {nltk-3.2 → nltk-3.3}/nltk/tbl/erroranalysis.py +39 -39
- {nltk-3.2 → nltk-3.3}/nltk/tbl/feature.py +259 -261
- {nltk-3.2 → nltk-3.3}/nltk/tbl/rule.py +321 -321
- {nltk-3.2 → nltk-3.3}/nltk/tbl/template.py +303 -308
- {nltk-3.2 → nltk-3.3}/nltk/test/__init__.py +18 -18
- {nltk-3.2 → nltk-3.3}/nltk/test/all.py +24 -23
- {nltk-3.2 → nltk-3.3}/nltk/test/bleu.doctest +14 -14
- {nltk-3.2 → nltk-3.3}/nltk/test/bnc.doctest +60 -60
- {nltk-3.2 → nltk-3.3}/nltk/test/ccg.doctest +376 -395
- nltk-3.3/nltk/test/ccg_semantics.doctest +553 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/chat80.doctest +234 -234
- {nltk-3.2 → nltk-3.3}/nltk/test/childes.doctest +184 -184
- {nltk-3.2 → nltk-3.3}/nltk/test/childes_fixt.py +14 -13
- {nltk-3.2 → nltk-3.3}/nltk/test/chunk.doctest +373 -373
- {nltk-3.2 → nltk-3.3}/nltk/test/classify.doctest +185 -185
- {nltk-3.2 → nltk-3.3}/nltk/test/classify_fixt.py +11 -10
- nltk-3.3/nltk/test/collections.doctest +20 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/collocations.doctest +276 -276
- {nltk-3.2 → nltk-3.3}/nltk/test/compat.doctest +134 -138
- {nltk-3.2 → nltk-3.3}/nltk/test/compat_fixt.py +9 -8
- nltk-3.3/nltk/test/concordance.doctest +55 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/corpus.doctest +2200 -2212
- {nltk-3.2 → nltk-3.3}/nltk/test/corpus_fixt.py +3 -3
- {nltk-3.2 → nltk-3.3}/nltk/test/crubadan.doctest +65 -65
- {nltk-3.2 → nltk-3.3}/nltk/test/data.doctest +379 -379
- {nltk-3.2 → nltk-3.3}/nltk/test/dependency.doctest +241 -241
- {nltk-3.2 → nltk-3.3}/nltk/test/discourse.doctest +546 -546
- {nltk-3.2 → nltk-3.3}/nltk/test/discourse_fixt.py +14 -13
- {nltk-3.2 → nltk-3.3}/nltk/test/doctest_nose_plugin.py +159 -156
- {nltk-3.2 → nltk-3.3}/nltk/test/drt.doctest +517 -517
- {nltk-3.2 → nltk-3.3}/nltk/test/featgram.doctest +607 -607
- {nltk-3.2 → nltk-3.3}/nltk/test/featstruct.doctest +1229 -1230
- {nltk-3.2 → nltk-3.3}/nltk/test/framenet.doctest +288 -239
- {nltk-3.2 → nltk-3.3}/nltk/test/generate.doctest +67 -67
- {nltk-3.2 → nltk-3.3}/nltk/test/gensim.doctest +140 -141
- {nltk-3.2 → nltk-3.3}/nltk/test/gensim_fixt.py +10 -9
- {nltk-3.2 → nltk-3.3}/nltk/test/gluesemantics.doctest +384 -386
- {nltk-3.2 → nltk-3.3}/nltk/test/gluesemantics_malt.doctest +68 -68
- {nltk-3.2 → nltk-3.3}/nltk/test/gluesemantics_malt_fixt.py +12 -11
- {nltk-3.2 → nltk-3.3}/nltk/test/grammar.doctest +48 -48
- {nltk-3.2 → nltk-3.3}/nltk/test/grammartestsuites.doctest +109 -109
- {nltk-3.2 → nltk-3.3}/nltk/test/index.doctest +100 -100
- {nltk-3.2 → nltk-3.3}/nltk/test/inference.doctest +534 -534
- {nltk-3.2 → nltk-3.3}/nltk/test/inference_fixt.py +12 -11
- {nltk-3.2 → nltk-3.3}/nltk/test/internals.doctest +140 -140
- {nltk-3.2 → nltk-3.3}/nltk/test/japanese.doctest +48 -48
- {nltk-3.2 → nltk-3.3}/nltk/test/logic.doctest +1098 -1098
- {nltk-3.2 → nltk-3.3}/nltk/test/metrics.doctest +270 -270
- {nltk-3.2 → nltk-3.3}/nltk/test/misc.doctest +118 -118
- {nltk-3.2 → nltk-3.3}/nltk/test/nonmonotonic.doctest +286 -286
- {nltk-3.2 → nltk-3.3}/nltk/test/nonmonotonic_fixt.py +12 -11
- {nltk-3.2 → nltk-3.3}/nltk/test/paice.doctest +35 -35
- {nltk-3.2 → nltk-3.3}/nltk/test/parse.doctest +884 -884
- {nltk-3.2 → nltk-3.3}/nltk/test/portuguese_en.doctest +565 -565
- {nltk-3.2 → nltk-3.3}/nltk/test/portuguese_en_fixt.py +16 -16
- {nltk-3.2 → nltk-3.3}/nltk/test/probability.doctest +304 -304
- {nltk-3.2 → nltk-3.3}/nltk/test/probability_fixt.py +13 -12
- {nltk-3.2 → nltk-3.3}/nltk/test/propbank.doctest +176 -176
- {nltk-3.2 → nltk-3.3}/nltk/test/relextract.doctest +263 -263
- {nltk-3.2 → nltk-3.3}/nltk/test/resolution.doctest +221 -221
- {nltk-3.2 → nltk-3.3}/nltk/test/runtests.py +86 -64
- {nltk-3.2 → nltk-3.3}/nltk/test/segmentation_fixt.py +10 -10
- {nltk-3.2 → nltk-3.3}/nltk/test/semantics.doctest +665 -665
- {nltk-3.2 → nltk-3.3}/nltk/test/semantics_fixt.py +7 -7
- {nltk-3.2 → nltk-3.3}/nltk/test/sentiment.doctest +233 -233
- {nltk-3.2 → nltk-3.3}/nltk/test/sentiwordnet.doctest +41 -41
- {nltk-3.2 → nltk-3.3}/nltk/test/simple.doctest +85 -85
- {nltk-3.2 → nltk-3.3}/nltk/test/stem.doctest +78 -78
- {nltk-3.2 → nltk-3.3}/nltk/test/tag.doctest +33 -33
- {nltk-3.2 → nltk-3.3}/nltk/test/tokenize.doctest +273 -166
- {nltk-3.2 → nltk-3.3}/nltk/test/toolbox.doctest +307 -307
- {nltk-3.2 → nltk-3.3}/nltk/test/translate.doctest +242 -242
- {nltk-3.2 → nltk-3.3}/nltk/test/translate_fixt.py +3 -3
- {nltk-3.2 → nltk-3.3}/nltk/test/tree.doctest +1101 -1101
- {nltk-3.2 → nltk-3.3}/nltk/test/treeprettyprinter.doctest +127 -127
- {nltk-3.2 → nltk-3.3}/nltk/test/treetransforms.doctest +156 -156
- nltk-3.3/nltk/test/unit/test_2x_compat.py +63 -0
- nltk-3.3/nltk/test/unit/test_aline.py +52 -0
- nltk-3.3/nltk/test/unit/test_chunk.py +56 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_classify.py +55 -53
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_collocations.py +88 -79
- nltk-3.3/nltk/test/unit/test_concordance.py +98 -0
- nltk-3.3/nltk/test/unit/test_corenlp.py +1000 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_corpora.py +200 -183
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_corpus_views.py +45 -45
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_hmm.py +88 -88
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_json2csv_corpus.py +188 -187
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_naivebayes.py +26 -26
- nltk-3.3/nltk/test/unit/test_rte_classify.py +86 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_seekable_unicode_stream_reader.py +124 -124
- nltk-3.3/nltk/test/unit/test_senna.py +74 -0
- nltk-3.3/nltk/test/unit/test_stem.py +121 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_tag.py +20 -20
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_tgrep.py +628 -626
- nltk-3.3/nltk/test/unit/test_tokenize.py +168 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_twitter_auth.py +185 -185
- nltk-3.3/nltk/test/unit/test_wordnet.py +134 -0
- nltk-3.3/nltk/test/unit/translate/test_bleu.py +244 -0
- nltk-3.3/nltk/test/unit/translate/test_gdfa.py +57 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm1.py +75 -75
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm2.py +87 -87
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm3.py +105 -105
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm4.py +126 -126
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm5.py +166 -166
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm_model.py +270 -270
- nltk-3.3/nltk/test/unit/translate/test_nist.py +37 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_stack_decoder.py +300 -300
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/utils.py +41 -41
- {nltk-3.2 → nltk-3.3}/nltk/test/util.doctest +48 -48
- {nltk-3.2 → nltk-3.3}/nltk/test/wordnet.doctest +595 -581
- {nltk-3.2 → nltk-3.3}/nltk/test/wordnet_fixt.py +7 -6
- {nltk-3.2 → nltk-3.3}/nltk/test/wordnet_lch.doctest +53 -53
- {nltk-3.2 → nltk-3.3}/nltk/test/wsd.doctest +68 -68
- {nltk-3.2 → nltk-3.3}/nltk/text.py +663 -610
- {nltk-3.2 → nltk-3.3}/nltk/tgrep.py +938 -937
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/__init__.py +130 -107
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/api.py +77 -78
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/casual.py +343 -342
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/mwe.py +120 -120
- nltk-3.3/nltk/tokenize/nist.py +173 -0
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/punkt.py +1620 -1601
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/regexp.py +208 -202
- nltk-3.3/nltk/tokenize/repp.py +151 -0
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/sexpr.py +140 -142
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/simple.py +138 -136
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/stanford.py +117 -113
- nltk-3.3/nltk/tokenize/stanford_segmenter.py +246 -0
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/texttiling.py +458 -460
- nltk-3.3/nltk/tokenize/toktok.py +155 -0
- nltk-3.3/nltk/tokenize/treebank.py +363 -0
- nltk-3.3/nltk/tokenize/util.py +261 -0
- {nltk-3.2 → nltk-3.3}/nltk/toolbox.py +499 -497
- {nltk-3.2 → nltk-3.3}/nltk/translate/__init__.py +24 -24
- {nltk-3.2 → nltk-3.3}/nltk/translate/api.py +331 -321
- {nltk-3.2 → nltk-3.3}/nltk/translate/bleu_score.py +615 -556
- nltk-3.3/nltk/translate/chrf_score.py +137 -0
- {nltk-3.2 → nltk-3.3}/nltk/translate/gale_church.py +248 -235
- {nltk-3.2 → nltk-3.3}/nltk/translate/gdfa.py +135 -131
- nltk-3.3/nltk/translate/gleu_score.py +193 -0
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm1.py +250 -241
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm2.py +308 -308
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm3.py +337 -337
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm4.py +467 -467
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm5.py +639 -639
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm_model.py +536 -533
- {nltk-3.2 → nltk-3.3}/nltk/translate/metrics.py +40 -40
- nltk-3.3/nltk/translate/nist_score.py +178 -0
- {nltk-3.2 → nltk-3.3}/nltk/translate/phrase_based.py +175 -175
- {nltk-3.2 → nltk-3.3}/nltk/translate/ribes_score.py +325 -325
- {nltk-3.2 → nltk-3.3}/nltk/translate/stack_decoder.py +499 -495
- {nltk-3.2 → nltk-3.3}/nltk/tree.py +1605 -1603
- {nltk-3.2 → nltk-3.3}/nltk/treeprettyprinter.py +564 -564
- {nltk-3.2 → nltk-3.3}/nltk/treetransforms.py +309 -309
- {nltk-3.2 → nltk-3.3}/nltk/twitter/__init__.py +28 -28
- {nltk-3.2 → nltk-3.3}/nltk/twitter/api.py +141 -136
- {nltk-3.2 → nltk-3.3}/nltk/twitter/common.py +264 -258
- {nltk-3.2 → nltk-3.3}/nltk/twitter/twitter_demo.py +288 -288
- {nltk-3.2 → nltk-3.3}/nltk/twitter/twitterclient.py +530 -530
- {nltk-3.2 → nltk-3.3}/nltk/twitter/util.py +147 -146
- nltk-3.3/nltk/util.py +751 -0
- {nltk-3.2 → nltk-3.3}/nltk/wsd.py +53 -53
- {nltk-3.2 → nltk-3.3/nltk.egg-info}/PKG-INFO +13 -5
- {nltk-3.2 → nltk-3.3}/nltk.egg-info/SOURCES.txt +26 -3
- nltk-3.3/nltk.egg-info/requires.txt +31 -0
- {nltk-3.2 → nltk-3.3}/setup.cfg +0 -1
- {nltk-3.2 → nltk-3.3}/setup.py +102 -75
- nltk-3.2/nltk/VERSION +0 -1
- nltk-3.2/nltk/compat.py +0 -680
- nltk-3.2/nltk/corpus/reader/pl196x.py +0 -278
- nltk-3.2/nltk/corpus/reader/verbnet.py +0 -408
- nltk-3.2/nltk/corpus/reader/wordlist.py +0 -37
- nltk-3.2/nltk/sentiment/vader_lexicon.txt +0 -7517
- nltk-3.2/nltk/six.py +0 -868
- nltk-3.2/nltk/stem/porter.py +0 -692
- nltk-3.2/nltk/test/unit/test_2x_compat.py +0 -26
- nltk-3.2/nltk/test/unit/test_stem.py +0 -36
- nltk-3.2/nltk/test/unit/test_tokenize.py +0 -23
- nltk-3.2/nltk/test/unit/translate/test_bleu.py +0 -84
- nltk-3.2/nltk/tokenize/treebank.py +0 -124
- nltk-3.2/nltk/tokenize/util.py +0 -90
- nltk-3.2/nltk/util.py +0 -1364
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/__init__.py +0 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/__init__.py +0 -0
- {nltk-3.2 → nltk-3.3}/nltk.egg-info/dependency_links.txt +0 -0
- {nltk-3.2 → nltk-3.3}/nltk.egg-info/not-zip-safe +0 -0
- {nltk-3.2 → nltk-3.3}/nltk.egg-info/top_level.txt +0 -0
|
@@ -1,694 +1,696 @@
|
|
|
1
|
-
# Natural Language Toolkit: Glue Semantics
|
|
2
|
-
#
|
|
3
|
-
# Author: Dan Garrette <dhgarrette@gmail.com>
|
|
4
|
-
#
|
|
5
|
-
# Copyright (C) 2001-
|
|
6
|
-
# URL: <http://nltk.org/>
|
|
7
|
-
# For license information, see LICENSE.TXT
|
|
8
|
-
from __future__ import print_function, division, unicode_literals
|
|
9
|
-
|
|
10
|
-
import os
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
from
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
from nltk.
|
|
17
|
-
|
|
18
|
-
from nltk.
|
|
19
|
-
|
|
20
|
-
from nltk.
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
'
|
|
26
|
-
'
|
|
27
|
-
'
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
self.meaning = meaning
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
self.glue = glue
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
self.
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
paren_count
|
|
162
|
-
|
|
163
|
-
paren_count
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
relationships = frozenset(
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
self[
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
self[sem]
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
if
|
|
213
|
-
self[sem][relationships]
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
accum += ' '
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
#
|
|
298
|
-
#
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
return [SPEC_SEMTYPES[
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
return [node['tag']]
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
gf.word =
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
:param
|
|
352
|
-
""
|
|
353
|
-
word
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
return linearlogic.
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
return self.find_label_name(after_dot,
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
return
|
|
385
|
-
elif name == '
|
|
386
|
-
return '%
|
|
387
|
-
elif name == '
|
|
388
|
-
return
|
|
389
|
-
elif name == '
|
|
390
|
-
return '
|
|
391
|
-
elif name == '
|
|
392
|
-
return
|
|
393
|
-
elif name == '
|
|
394
|
-
return self.get_label(self.lookup_unique('
|
|
395
|
-
|
|
396
|
-
return self.get_label(self.lookup_unique(
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
value
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
return letter
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
"""
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
raise KeyError("'%s'
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
self.
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
self.semtype_file =
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
self.depparser.
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
bindings =
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
nonatomics[glue_simp.antecedent]
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
bindings =
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
atomics[glue_simp]
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
#
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
:
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
self.
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
(r'
|
|
589
|
-
(r'.*
|
|
590
|
-
(r'.*
|
|
591
|
-
(r'.*
|
|
592
|
-
(r'.*
|
|
593
|
-
(r'.*', '
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
self.meaning = meaning
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
self.glue = glue
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
'
|
|
657
|
-
'
|
|
658
|
-
'
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
# 'John
|
|
662
|
-
# 'John
|
|
663
|
-
# '
|
|
664
|
-
# '
|
|
665
|
-
# '
|
|
666
|
-
# 'every big
|
|
667
|
-
# '
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
('^(
|
|
676
|
-
('^(
|
|
677
|
-
('^(
|
|
678
|
-
('^(
|
|
679
|
-
('^(
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
1
|
+
# Natural Language Toolkit: Glue Semantics
|
|
2
|
+
#
|
|
3
|
+
# Author: Dan Garrette <dhgarrette@gmail.com>
|
|
4
|
+
#
|
|
5
|
+
# Copyright (C) 2001-2018 NLTK Project
|
|
6
|
+
# URL: <http://nltk.org/>
|
|
7
|
+
# For license information, see LICENSE.TXT
|
|
8
|
+
from __future__ import print_function, division, unicode_literals
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
from itertools import chain
|
|
12
|
+
|
|
13
|
+
from six import string_types
|
|
14
|
+
|
|
15
|
+
import nltk
|
|
16
|
+
from nltk.internals import Counter
|
|
17
|
+
from nltk.tag import UnigramTagger, BigramTagger, TrigramTagger, RegexpTagger
|
|
18
|
+
from nltk.sem.logic import (Expression, Variable, VariableExpression,
|
|
19
|
+
LambdaExpression, AbstractVariableExpression)
|
|
20
|
+
from nltk.compat import python_2_unicode_compatible
|
|
21
|
+
from nltk.sem import drt
|
|
22
|
+
from nltk.sem import linearlogic
|
|
23
|
+
|
|
24
|
+
SPEC_SEMTYPES = {'a' : 'ex_quant',
|
|
25
|
+
'an' : 'ex_quant',
|
|
26
|
+
'every' : 'univ_quant',
|
|
27
|
+
'the' : 'def_art',
|
|
28
|
+
'no' : 'no_quant',
|
|
29
|
+
'default' : 'ex_quant'}
|
|
30
|
+
|
|
31
|
+
OPTIONAL_RELATIONSHIPS = ['nmod', 'vmod', 'punct']
|
|
32
|
+
|
|
33
|
+
@python_2_unicode_compatible
|
|
34
|
+
class GlueFormula(object):
|
|
35
|
+
def __init__(self, meaning, glue, indices=None):
|
|
36
|
+
if not indices:
|
|
37
|
+
indices = set()
|
|
38
|
+
|
|
39
|
+
if isinstance(meaning, string_types):
|
|
40
|
+
self.meaning = Expression.fromstring(meaning)
|
|
41
|
+
elif isinstance(meaning, Expression):
|
|
42
|
+
self.meaning = meaning
|
|
43
|
+
else:
|
|
44
|
+
raise RuntimeError('Meaning term neither string or expression: %s, %s' % (meaning, meaning.__class__))
|
|
45
|
+
|
|
46
|
+
if isinstance(glue, string_types):
|
|
47
|
+
self.glue = linearlogic.LinearLogicParser().parse(glue)
|
|
48
|
+
elif isinstance(glue, linearlogic.Expression):
|
|
49
|
+
self.glue = glue
|
|
50
|
+
else:
|
|
51
|
+
raise RuntimeError('Glue term neither string or expression: %s, %s' % (glue, glue.__class__))
|
|
52
|
+
|
|
53
|
+
self.indices = indices
|
|
54
|
+
|
|
55
|
+
def applyto(self, arg):
|
|
56
|
+
""" self = (\\x.(walk x), (subj -o f))
|
|
57
|
+
arg = (john , subj)
|
|
58
|
+
returns ((walk john), f)
|
|
59
|
+
"""
|
|
60
|
+
if self.indices & arg.indices: # if the sets are NOT disjoint
|
|
61
|
+
raise linearlogic.LinearLogicApplicationException("'%s' applied to '%s'. Indices are not disjoint." % (self, arg))
|
|
62
|
+
else: # if the sets ARE disjoint
|
|
63
|
+
return_indices = (self.indices | arg.indices)
|
|
64
|
+
|
|
65
|
+
try:
|
|
66
|
+
return_glue = linearlogic.ApplicationExpression(self.glue, arg.glue, arg.indices)
|
|
67
|
+
except linearlogic.LinearLogicApplicationException:
|
|
68
|
+
raise linearlogic.LinearLogicApplicationException("'%s' applied to '%s'" % (self.simplify(), arg.simplify()))
|
|
69
|
+
|
|
70
|
+
arg_meaning_abstracted = arg.meaning
|
|
71
|
+
if return_indices:
|
|
72
|
+
for dep in self.glue.simplify().antecedent.dependencies[::-1]: # if self.glue is (A -o B), dep is in A.dependencies
|
|
73
|
+
arg_meaning_abstracted = self.make_LambdaExpression(Variable('v%s' % dep),
|
|
74
|
+
arg_meaning_abstracted)
|
|
75
|
+
return_meaning = self.meaning.applyto(arg_meaning_abstracted)
|
|
76
|
+
|
|
77
|
+
return self.__class__(return_meaning, return_glue, return_indices)
|
|
78
|
+
|
|
79
|
+
def make_VariableExpression(self, name):
|
|
80
|
+
return VariableExpression(name)
|
|
81
|
+
|
|
82
|
+
def make_LambdaExpression(self, variable, term):
|
|
83
|
+
return LambdaExpression(variable, term)
|
|
84
|
+
|
|
85
|
+
def lambda_abstract(self, other):
|
|
86
|
+
assert isinstance(other, GlueFormula)
|
|
87
|
+
assert isinstance(other.meaning, AbstractVariableExpression)
|
|
88
|
+
return self.__class__(self.make_LambdaExpression(other.meaning.variable,
|
|
89
|
+
self.meaning),
|
|
90
|
+
linearlogic.ImpExpression(other.glue, self.glue))
|
|
91
|
+
|
|
92
|
+
def compile(self, counter=None):
|
|
93
|
+
"""From Iddo Lev's PhD Dissertation p108-109"""
|
|
94
|
+
if not counter:
|
|
95
|
+
counter = Counter()
|
|
96
|
+
(compiled_glue, new_forms) = self.glue.simplify().compile_pos(counter, self.__class__)
|
|
97
|
+
return new_forms + [self.__class__(self.meaning, compiled_glue, set([counter.get()]))]
|
|
98
|
+
|
|
99
|
+
def simplify(self):
|
|
100
|
+
return self.__class__(self.meaning.simplify(), self.glue.simplify(), self.indices)
|
|
101
|
+
|
|
102
|
+
def __eq__(self, other):
|
|
103
|
+
return self.__class__ == other.__class__ and self.meaning == other.meaning and self.glue == other.glue
|
|
104
|
+
|
|
105
|
+
def __ne__(self, other):
|
|
106
|
+
return not self == other
|
|
107
|
+
|
|
108
|
+
# sorting for use in doctests which must be deterministic
|
|
109
|
+
def __lt__(self, other):
|
|
110
|
+
return str(self) < str(other)
|
|
111
|
+
|
|
112
|
+
def __str__(self):
|
|
113
|
+
assert isinstance(self.indices, set)
|
|
114
|
+
accum = '%s : %s' % (self.meaning, self.glue)
|
|
115
|
+
if self.indices:
|
|
116
|
+
accum += ' : {' + ', '.join(str(index) for index in self.indices) + '}'
|
|
117
|
+
return accum
|
|
118
|
+
|
|
119
|
+
def __repr__(self):
|
|
120
|
+
return "%s" % self
|
|
121
|
+
|
|
122
|
+
@python_2_unicode_compatible
|
|
123
|
+
class GlueDict(dict):
|
|
124
|
+
def __init__(self, filename, encoding=None):
|
|
125
|
+
self.filename = filename
|
|
126
|
+
self.file_encoding = encoding
|
|
127
|
+
self.read_file()
|
|
128
|
+
|
|
129
|
+
def read_file(self, empty_first=True):
|
|
130
|
+
if empty_first:
|
|
131
|
+
self.clear()
|
|
132
|
+
|
|
133
|
+
try:
|
|
134
|
+
contents = nltk.data.load(self.filename, format='text', encoding=self.file_encoding)
|
|
135
|
+
# TODO: the above can't handle zip files, but this should anyway be fixed in nltk.data.load()
|
|
136
|
+
except LookupError as e:
|
|
137
|
+
try:
|
|
138
|
+
contents = nltk.data.load('file:' + self.filename, format='text', encoding=self.file_encoding)
|
|
139
|
+
except LookupError:
|
|
140
|
+
raise e
|
|
141
|
+
lines = contents.splitlines()
|
|
142
|
+
|
|
143
|
+
for line in lines: # example: 'n : (\\x.(<word> x), (v-or))'
|
|
144
|
+
# lambdacalc -^ linear logic -^
|
|
145
|
+
line = line.strip() # remove trailing newline
|
|
146
|
+
if not len(line): continue # skip empty lines
|
|
147
|
+
if line[0] == '#': continue # skip commented out lines
|
|
148
|
+
|
|
149
|
+
parts = line.split(' : ', 2) # ['verb', '(\\x.(<word> x), ( subj -o f ))', '[subj]']
|
|
150
|
+
|
|
151
|
+
glue_formulas = []
|
|
152
|
+
paren_count = 0
|
|
153
|
+
tuple_start = 0
|
|
154
|
+
tuple_comma = 0
|
|
155
|
+
|
|
156
|
+
relationships = None
|
|
157
|
+
|
|
158
|
+
if len(parts) > 1:
|
|
159
|
+
for (i, c) in enumerate(parts[1]):
|
|
160
|
+
if c == '(':
|
|
161
|
+
if paren_count == 0: # if it's the first '(' of a tuple
|
|
162
|
+
tuple_start = i+1 # then save the index
|
|
163
|
+
paren_count += 1
|
|
164
|
+
elif c == ')':
|
|
165
|
+
paren_count -= 1
|
|
166
|
+
if paren_count == 0: # if it's the last ')' of a tuple
|
|
167
|
+
meaning_term = parts[1][tuple_start:tuple_comma] # '\\x.(<word> x)'
|
|
168
|
+
glue_term = parts[1][tuple_comma+1:i] # '(v-r)'
|
|
169
|
+
glue_formulas.append([meaning_term, glue_term]) # add the GlueFormula to the list
|
|
170
|
+
elif c == ',':
|
|
171
|
+
if paren_count == 1: # if it's a comma separating the parts of the tuple
|
|
172
|
+
tuple_comma = i # then save the index
|
|
173
|
+
elif c == '#': # skip comments at the ends of lines
|
|
174
|
+
if paren_count != 0: # if the line hasn't parsed correctly so far
|
|
175
|
+
raise RuntimeError('Formula syntax is incorrect for entry ' + line)
|
|
176
|
+
break # break to the next line
|
|
177
|
+
|
|
178
|
+
if len(parts) > 2: #if there is a relationship entry at the end
|
|
179
|
+
rel_start = parts[2].index('[')+1
|
|
180
|
+
rel_end = parts[2].index(']')
|
|
181
|
+
if rel_start == rel_end:
|
|
182
|
+
relationships = frozenset()
|
|
183
|
+
else:
|
|
184
|
+
relationships = frozenset(r.strip() for r in parts[2][rel_start:rel_end].split(','))
|
|
185
|
+
|
|
186
|
+
try:
|
|
187
|
+
start_inheritance = parts[0].index('(')
|
|
188
|
+
end_inheritance = parts[0].index(')')
|
|
189
|
+
sem = parts[0][:start_inheritance].strip()
|
|
190
|
+
supertype = parts[0][start_inheritance+1:end_inheritance]
|
|
191
|
+
except:
|
|
192
|
+
sem = parts[0].strip()
|
|
193
|
+
supertype = None
|
|
194
|
+
|
|
195
|
+
if sem not in self:
|
|
196
|
+
self[sem] = {}
|
|
197
|
+
|
|
198
|
+
if relationships is None: #if not specified for a specific relationship set
|
|
199
|
+
#add all relationship entries for parents
|
|
200
|
+
if supertype:
|
|
201
|
+
for rels in self[supertype]:
|
|
202
|
+
if rels not in self[sem]:
|
|
203
|
+
self[sem][rels] = []
|
|
204
|
+
glue = self[supertype][rels]
|
|
205
|
+
self[sem][rels].extend(glue)
|
|
206
|
+
self[sem][rels].extend(glue_formulas) # add the glue formulas to every rel entry
|
|
207
|
+
else:
|
|
208
|
+
if None not in self[sem]:
|
|
209
|
+
self[sem][None] = []
|
|
210
|
+
self[sem][None].extend(glue_formulas) # add the glue formulas to every rel entry
|
|
211
|
+
else:
|
|
212
|
+
if relationships not in self[sem]:
|
|
213
|
+
self[sem][relationships] = []
|
|
214
|
+
if supertype:
|
|
215
|
+
self[sem][relationships].extend(self[supertype][relationships])
|
|
216
|
+
self[sem][relationships].extend(glue_formulas) # add the glue entry to the dictionary
|
|
217
|
+
|
|
218
|
+
def __str__(self):
|
|
219
|
+
accum = ''
|
|
220
|
+
for pos in self:
|
|
221
|
+
str_pos = "%s" % pos
|
|
222
|
+
for relset in self[pos]:
|
|
223
|
+
i = 1
|
|
224
|
+
for gf in self[pos][relset]:
|
|
225
|
+
if i == 1:
|
|
226
|
+
accum += str_pos + ': '
|
|
227
|
+
else:
|
|
228
|
+
accum += ' '*(len(str_pos)+2)
|
|
229
|
+
accum += "%s" % gf
|
|
230
|
+
if relset and i == len(self[pos][relset]):
|
|
231
|
+
accum += ' : %s' % relset
|
|
232
|
+
accum += '\n'
|
|
233
|
+
i += 1
|
|
234
|
+
return accum
|
|
235
|
+
|
|
236
|
+
def to_glueformula_list(self, depgraph, node=None, counter=None, verbose=False):
|
|
237
|
+
if node is None:
|
|
238
|
+
# TODO: should it be depgraph.root? Is this code tested?
|
|
239
|
+
top = depgraph.nodes[0]
|
|
240
|
+
depList = list(chain(*top['deps'].values()))
|
|
241
|
+
root = depgraph.nodes[depList[0]]
|
|
242
|
+
|
|
243
|
+
return self.to_glueformula_list(depgraph, root, Counter(), verbose)
|
|
244
|
+
|
|
245
|
+
glueformulas = self.lookup(node, depgraph, counter)
|
|
246
|
+
for dep_idx in chain(*node['deps'].values()):
|
|
247
|
+
dep = depgraph.nodes[dep_idx]
|
|
248
|
+
glueformulas.extend(self.to_glueformula_list(depgraph, dep, counter, verbose))
|
|
249
|
+
return glueformulas
|
|
250
|
+
|
|
251
|
+
def lookup(self, node, depgraph, counter):
|
|
252
|
+
semtype_names = self.get_semtypes(node)
|
|
253
|
+
|
|
254
|
+
semtype = None
|
|
255
|
+
for name in semtype_names:
|
|
256
|
+
if name in self:
|
|
257
|
+
semtype = self[name]
|
|
258
|
+
break
|
|
259
|
+
if semtype is None:
|
|
260
|
+
# raise KeyError, "There is no GlueDict entry for sem type '%s' (for '%s')" % (sem, word)
|
|
261
|
+
return []
|
|
262
|
+
|
|
263
|
+
self.add_missing_dependencies(node, depgraph)
|
|
264
|
+
|
|
265
|
+
lookup = self._lookup_semtype_option(semtype, node, depgraph)
|
|
266
|
+
|
|
267
|
+
if not len(lookup):
|
|
268
|
+
raise KeyError(
|
|
269
|
+
"There is no GlueDict entry for sem type of '%s' "
|
|
270
|
+
"with tag '%s', and rel '%s'" %
|
|
271
|
+
(node['word'], node['tag'], node['rel'])
|
|
272
|
+
)
|
|
273
|
+
|
|
274
|
+
return self.get_glueformulas_from_semtype_entry(lookup, node['word'], node, depgraph, counter)
|
|
275
|
+
|
|
276
|
+
def add_missing_dependencies(self, node, depgraph):
|
|
277
|
+
rel = node['rel'].lower()
|
|
278
|
+
|
|
279
|
+
if rel == 'main':
|
|
280
|
+
headnode = depgraph.nodes[node['head']]
|
|
281
|
+
subj = self.lookup_unique('subj', headnode, depgraph)
|
|
282
|
+
relation = subj['rel']
|
|
283
|
+
node['deps'].setdefault(relation,[])
|
|
284
|
+
node['deps'][relation].append(subj['address'])
|
|
285
|
+
#node['deps'].append(subj['address'])
|
|
286
|
+
|
|
287
|
+
def _lookup_semtype_option(self, semtype, node, depgraph):
|
|
288
|
+
relationships = frozenset(
|
|
289
|
+
depgraph.nodes[dep]['rel'].lower()
|
|
290
|
+
for dep in chain(*node['deps'].values())
|
|
291
|
+
if depgraph.nodes[dep]['rel'].lower() not in OPTIONAL_RELATIONSHIPS
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
try:
|
|
295
|
+
lookup = semtype[relationships]
|
|
296
|
+
except KeyError:
|
|
297
|
+
# An exact match is not found, so find the best match where
|
|
298
|
+
# 'best' is defined as the glue entry whose relationship set has the
|
|
299
|
+
# most relations of any possible relationship set that is a subset
|
|
300
|
+
# of the actual depgraph
|
|
301
|
+
best_match = frozenset()
|
|
302
|
+
for relset_option in set(semtype)-set([None]):
|
|
303
|
+
if len(relset_option) > len(best_match) and \
|
|
304
|
+
relset_option < relationships:
|
|
305
|
+
best_match = relset_option
|
|
306
|
+
if not best_match:
|
|
307
|
+
if None in semtype:
|
|
308
|
+
best_match = None
|
|
309
|
+
else:
|
|
310
|
+
return None
|
|
311
|
+
lookup = semtype[best_match]
|
|
312
|
+
|
|
313
|
+
return lookup
|
|
314
|
+
|
|
315
|
+
def get_semtypes(self, node):
|
|
316
|
+
"""
|
|
317
|
+
Based on the node, return a list of plausible semtypes in order of
|
|
318
|
+
plausibility.
|
|
319
|
+
"""
|
|
320
|
+
rel = node['rel'].lower()
|
|
321
|
+
word = node['word'].lower()
|
|
322
|
+
|
|
323
|
+
if rel == 'spec':
|
|
324
|
+
if word in SPEC_SEMTYPES:
|
|
325
|
+
return [SPEC_SEMTYPES[word]]
|
|
326
|
+
else:
|
|
327
|
+
return [SPEC_SEMTYPES['default']]
|
|
328
|
+
elif rel in ['nmod', 'vmod']:
|
|
329
|
+
return [node['tag'], rel]
|
|
330
|
+
else:
|
|
331
|
+
return [node['tag']]
|
|
332
|
+
|
|
333
|
+
def get_glueformulas_from_semtype_entry(self, lookup, word, node, depgraph, counter):
|
|
334
|
+
glueformulas = []
|
|
335
|
+
|
|
336
|
+
glueFormulaFactory = self.get_GlueFormula_factory()
|
|
337
|
+
for meaning, glue in lookup:
|
|
338
|
+
gf = glueFormulaFactory(self.get_meaning_formula(meaning, word), glue)
|
|
339
|
+
if not len(glueformulas):
|
|
340
|
+
gf.word = word
|
|
341
|
+
else:
|
|
342
|
+
gf.word = '%s%s' % (word, len(glueformulas)+1)
|
|
343
|
+
|
|
344
|
+
gf.glue = self.initialize_labels(gf.glue, node, depgraph, counter.get())
|
|
345
|
+
|
|
346
|
+
glueformulas.append(gf)
|
|
347
|
+
return glueformulas
|
|
348
|
+
|
|
349
|
+
def get_meaning_formula(self, generic, word):
|
|
350
|
+
"""
|
|
351
|
+
:param generic: A meaning formula string containing the
|
|
352
|
+
parameter "<word>"
|
|
353
|
+
:param word: The actual word to be replace "<word>"
|
|
354
|
+
"""
|
|
355
|
+
word = word.replace('.', '')
|
|
356
|
+
return generic.replace('<word>', word)
|
|
357
|
+
|
|
358
|
+
def initialize_labels(self, expr, node, depgraph, unique_index):
|
|
359
|
+
if isinstance(expr, linearlogic.AtomicExpression):
|
|
360
|
+
name = self.find_label_name(expr.name.lower(), node, depgraph, unique_index)
|
|
361
|
+
if name[0].isupper():
|
|
362
|
+
return linearlogic.VariableExpression(name)
|
|
363
|
+
else:
|
|
364
|
+
return linearlogic.ConstantExpression(name)
|
|
365
|
+
else:
|
|
366
|
+
return linearlogic.ImpExpression(
|
|
367
|
+
self.initialize_labels(expr.antecedent, node, depgraph, unique_index),
|
|
368
|
+
self.initialize_labels(expr.consequent, node, depgraph, unique_index)
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
def find_label_name(self, name, node, depgraph, unique_index):
|
|
372
|
+
try:
|
|
373
|
+
dot = name.index('.')
|
|
374
|
+
|
|
375
|
+
before_dot = name[:dot]
|
|
376
|
+
after_dot = name[dot+1:]
|
|
377
|
+
if before_dot == 'super':
|
|
378
|
+
return self.find_label_name(after_dot, depgraph.nodes[node['head']], depgraph, unique_index)
|
|
379
|
+
else:
|
|
380
|
+
return self.find_label_name(after_dot, self.lookup_unique(before_dot, node, depgraph), depgraph, unique_index)
|
|
381
|
+
except ValueError:
|
|
382
|
+
lbl = self.get_label(node)
|
|
383
|
+
if name == 'f':
|
|
384
|
+
return lbl
|
|
385
|
+
elif name == 'v':
|
|
386
|
+
return '%sv' % lbl
|
|
387
|
+
elif name == 'r':
|
|
388
|
+
return '%sr' % lbl
|
|
389
|
+
elif name == 'super':
|
|
390
|
+
return self.get_label(depgraph.nodes[node['head']])
|
|
391
|
+
elif name == 'var':
|
|
392
|
+
return '%s%s' % (lbl.upper(), unique_index)
|
|
393
|
+
elif name == 'a':
|
|
394
|
+
return self.get_label(self.lookup_unique('conja', node, depgraph))
|
|
395
|
+
elif name == 'b':
|
|
396
|
+
return self.get_label(self.lookup_unique('conjb', node, depgraph))
|
|
397
|
+
else:
|
|
398
|
+
return self.get_label(self.lookup_unique(name, node, depgraph))
|
|
399
|
+
|
|
400
|
+
def get_label(self, node):
|
|
401
|
+
"""
|
|
402
|
+
Pick an alphabetic character as identifier for an entity in the model.
|
|
403
|
+
|
|
404
|
+
:param value: where to index into the list of characters
|
|
405
|
+
:type value: int
|
|
406
|
+
"""
|
|
407
|
+
value = node['address']
|
|
408
|
+
|
|
409
|
+
letter = ['f','g','h','i','j','k','l','m','n','o','p','q','r','s',
|
|
410
|
+
't','u','v','w','x','y','z','a','b','c','d','e'][value-1]
|
|
411
|
+
num = int(value) // 26
|
|
412
|
+
if num > 0:
|
|
413
|
+
return letter + str(num)
|
|
414
|
+
else:
|
|
415
|
+
return letter
|
|
416
|
+
|
|
417
|
+
def lookup_unique(self, rel, node, depgraph):
|
|
418
|
+
"""
|
|
419
|
+
Lookup 'key'. There should be exactly one item in the associated relation.
|
|
420
|
+
"""
|
|
421
|
+
deps = [
|
|
422
|
+
depgraph.nodes[dep]
|
|
423
|
+
for dep in chain(*node['deps'].values())
|
|
424
|
+
if depgraph.nodes[dep]['rel'].lower() == rel.lower()
|
|
425
|
+
]
|
|
426
|
+
|
|
427
|
+
if len(deps) == 0:
|
|
428
|
+
raise KeyError("'%s' doesn't contain a feature '%s'" % (node['word'], rel))
|
|
429
|
+
elif len(deps) > 1:
|
|
430
|
+
raise KeyError("'%s' should only have one feature '%s'" % (node['word'], rel))
|
|
431
|
+
else:
|
|
432
|
+
return deps[0]
|
|
433
|
+
|
|
434
|
+
def get_GlueFormula_factory(self):
|
|
435
|
+
return GlueFormula
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
class Glue(object):
|
|
439
|
+
def __init__(self, semtype_file=None, remove_duplicates=False,
|
|
440
|
+
depparser=None, verbose=False):
|
|
441
|
+
self.verbose = verbose
|
|
442
|
+
self.remove_duplicates = remove_duplicates
|
|
443
|
+
self.depparser = depparser
|
|
444
|
+
|
|
445
|
+
from nltk import Prover9
|
|
446
|
+
self.prover = Prover9()
|
|
447
|
+
|
|
448
|
+
if semtype_file:
|
|
449
|
+
self.semtype_file = semtype_file
|
|
450
|
+
else:
|
|
451
|
+
self.semtype_file = os.path.join('grammars', 'sample_grammars','glue.semtype')
|
|
452
|
+
|
|
453
|
+
def train_depparser(self, depgraphs=None):
|
|
454
|
+
if depgraphs:
|
|
455
|
+
self.depparser.train(depgraphs)
|
|
456
|
+
else:
|
|
457
|
+
self.depparser.train_from_file(nltk.data.find(
|
|
458
|
+
os.path.join('grammars', 'sample_grammars',
|
|
459
|
+
'glue_train.conll')))
|
|
460
|
+
|
|
461
|
+
def parse_to_meaning(self, sentence):
|
|
462
|
+
readings = []
|
|
463
|
+
for agenda in self.parse_to_compiled(sentence):
|
|
464
|
+
readings.extend(self.get_readings(agenda))
|
|
465
|
+
return readings
|
|
466
|
+
|
|
467
|
+
def get_readings(self, agenda):
|
|
468
|
+
readings = []
|
|
469
|
+
agenda_length = len(agenda)
|
|
470
|
+
atomics = dict()
|
|
471
|
+
nonatomics = dict()
|
|
472
|
+
while agenda: # is not empty
|
|
473
|
+
cur = agenda.pop()
|
|
474
|
+
glue_simp = cur.glue.simplify()
|
|
475
|
+
if isinstance(glue_simp, linearlogic.ImpExpression): # if cur.glue is non-atomic
|
|
476
|
+
for key in atomics:
|
|
477
|
+
try:
|
|
478
|
+
if isinstance(cur.glue, linearlogic.ApplicationExpression):
|
|
479
|
+
bindings = cur.glue.bindings
|
|
480
|
+
else:
|
|
481
|
+
bindings = linearlogic.BindingDict()
|
|
482
|
+
glue_simp.antecedent.unify(key, bindings)
|
|
483
|
+
for atomic in atomics[key]:
|
|
484
|
+
if not (cur.indices & atomic.indices): # if the sets of indices are disjoint
|
|
485
|
+
try:
|
|
486
|
+
agenda.append(cur.applyto(atomic))
|
|
487
|
+
except linearlogic.LinearLogicApplicationException:
|
|
488
|
+
pass
|
|
489
|
+
except linearlogic.UnificationException:
|
|
490
|
+
pass
|
|
491
|
+
try:
|
|
492
|
+
nonatomics[glue_simp.antecedent].append(cur)
|
|
493
|
+
except KeyError:
|
|
494
|
+
nonatomics[glue_simp.antecedent] = [cur]
|
|
495
|
+
|
|
496
|
+
else: # else cur.glue is atomic
|
|
497
|
+
for key in nonatomics:
|
|
498
|
+
for nonatomic in nonatomics[key]:
|
|
499
|
+
try:
|
|
500
|
+
if isinstance(nonatomic.glue, linearlogic.ApplicationExpression):
|
|
501
|
+
bindings = nonatomic.glue.bindings
|
|
502
|
+
else:
|
|
503
|
+
bindings = linearlogic.BindingDict()
|
|
504
|
+
glue_simp.unify(key, bindings)
|
|
505
|
+
if not (cur.indices & nonatomic.indices): # if the sets of indices are disjoint
|
|
506
|
+
try:
|
|
507
|
+
agenda.append(nonatomic.applyto(cur))
|
|
508
|
+
except linearlogic.LinearLogicApplicationException:
|
|
509
|
+
pass
|
|
510
|
+
except linearlogic.UnificationException:
|
|
511
|
+
pass
|
|
512
|
+
try:
|
|
513
|
+
atomics[glue_simp].append(cur)
|
|
514
|
+
except KeyError:
|
|
515
|
+
atomics[glue_simp] = [cur]
|
|
516
|
+
|
|
517
|
+
for entry in atomics:
|
|
518
|
+
for gf in atomics[entry]:
|
|
519
|
+
if len(gf.indices) == agenda_length:
|
|
520
|
+
self._add_to_reading_list(gf, readings)
|
|
521
|
+
for entry in nonatomics:
|
|
522
|
+
for gf in nonatomics[entry]:
|
|
523
|
+
if len(gf.indices) == agenda_length:
|
|
524
|
+
self._add_to_reading_list(gf, readings)
|
|
525
|
+
return readings
|
|
526
|
+
|
|
527
|
+
def _add_to_reading_list(self, glueformula, reading_list):
|
|
528
|
+
add_reading = True
|
|
529
|
+
if self.remove_duplicates:
|
|
530
|
+
for reading in reading_list:
|
|
531
|
+
try:
|
|
532
|
+
if reading.equiv(glueformula.meaning, self.prover):
|
|
533
|
+
add_reading = False
|
|
534
|
+
break
|
|
535
|
+
except Exception as e:
|
|
536
|
+
#if there is an exception, the syntax of the formula
|
|
537
|
+
#may not be understandable by the prover, so don't
|
|
538
|
+
#throw out the reading.
|
|
539
|
+
print('Error when checking logical equality of statements', e)
|
|
540
|
+
pass
|
|
541
|
+
if add_reading:
|
|
542
|
+
reading_list.append(glueformula.meaning)
|
|
543
|
+
|
|
544
|
+
def parse_to_compiled(self, sentence):
|
|
545
|
+
gfls = [self.depgraph_to_glue(dg) for dg in self.dep_parse(sentence)]
|
|
546
|
+
return [self.gfl_to_compiled(gfl) for gfl in gfls]
|
|
547
|
+
|
|
548
|
+
def dep_parse(self, sentence):
|
|
549
|
+
"""
|
|
550
|
+
Return a dependency graph for the sentence.
|
|
551
|
+
|
|
552
|
+
:param sentence: the sentence to be parsed
|
|
553
|
+
:type sentence: list(str)
|
|
554
|
+
:rtype: DependencyGraph
|
|
555
|
+
"""
|
|
556
|
+
|
|
557
|
+
#Lazy-initialize the depparser
|
|
558
|
+
if self.depparser is None:
|
|
559
|
+
from nltk.parse import MaltParser
|
|
560
|
+
self.depparser = MaltParser(tagger=self.get_pos_tagger())
|
|
561
|
+
if not self.depparser._trained:
|
|
562
|
+
self.train_depparser()
|
|
563
|
+
return self.depparser.parse(sentence, verbose=self.verbose)
|
|
564
|
+
|
|
565
|
+
def depgraph_to_glue(self, depgraph):
|
|
566
|
+
return self.get_glue_dict().to_glueformula_list(depgraph)
|
|
567
|
+
|
|
568
|
+
def get_glue_dict(self):
|
|
569
|
+
return GlueDict(self.semtype_file)
|
|
570
|
+
|
|
571
|
+
def gfl_to_compiled(self, gfl):
|
|
572
|
+
index_counter = Counter()
|
|
573
|
+
return_list = []
|
|
574
|
+
for gf in gfl:
|
|
575
|
+
return_list.extend(gf.compile(index_counter))
|
|
576
|
+
|
|
577
|
+
if self.verbose:
|
|
578
|
+
print('Compiled Glue Premises:')
|
|
579
|
+
for cgf in return_list:
|
|
580
|
+
print(cgf)
|
|
581
|
+
|
|
582
|
+
return return_list
|
|
583
|
+
|
|
584
|
+
def get_pos_tagger(self):
|
|
585
|
+
from nltk.corpus import brown
|
|
586
|
+
regexp_tagger = RegexpTagger(
|
|
587
|
+
[(r'^-?[0-9]+(.[0-9]+)?$', 'CD'), # cardinal numbers
|
|
588
|
+
(r'(The|the|A|a|An|an)$', 'AT'), # articles
|
|
589
|
+
(r'.*able$', 'JJ'), # adjectives
|
|
590
|
+
(r'.*ness$', 'NN'), # nouns formed from adjectives
|
|
591
|
+
(r'.*ly$', 'RB'), # adverbs
|
|
592
|
+
(r'.*s$', 'NNS'), # plural nouns
|
|
593
|
+
(r'.*ing$', 'VBG'), # gerunds
|
|
594
|
+
(r'.*ed$', 'VBD'), # past tense verbs
|
|
595
|
+
(r'.*', 'NN') # nouns (default)
|
|
596
|
+
])
|
|
597
|
+
brown_train = brown.tagged_sents(categories='news')
|
|
598
|
+
unigram_tagger = UnigramTagger(brown_train, backoff=regexp_tagger)
|
|
599
|
+
bigram_tagger = BigramTagger(brown_train, backoff=unigram_tagger)
|
|
600
|
+
trigram_tagger = TrigramTagger(brown_train, backoff=bigram_tagger)
|
|
601
|
+
|
|
602
|
+
#Override particular words
|
|
603
|
+
main_tagger = RegexpTagger(
|
|
604
|
+
[(r'(A|a|An|an)$', 'ex_quant'),
|
|
605
|
+
(r'(Every|every|All|all)$', 'univ_quant')
|
|
606
|
+
], backoff=trigram_tagger)
|
|
607
|
+
|
|
608
|
+
return main_tagger
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
class DrtGlueFormula(GlueFormula):
|
|
612
|
+
def __init__(self, meaning, glue, indices=None):
|
|
613
|
+
if not indices:
|
|
614
|
+
indices = set()
|
|
615
|
+
|
|
616
|
+
if isinstance(meaning, string_types):
|
|
617
|
+
self.meaning = drt.DrtExpression.fromstring(meaning)
|
|
618
|
+
elif isinstance(meaning, drt.DrtExpression):
|
|
619
|
+
self.meaning = meaning
|
|
620
|
+
else:
|
|
621
|
+
raise RuntimeError('Meaning term neither string or expression: %s, %s' % (meaning, meaning.__class__))
|
|
622
|
+
|
|
623
|
+
if isinstance(glue, string_types):
|
|
624
|
+
self.glue = linearlogic.LinearLogicParser().parse(glue)
|
|
625
|
+
elif isinstance(glue, linearlogic.Expression):
|
|
626
|
+
self.glue = glue
|
|
627
|
+
else:
|
|
628
|
+
raise RuntimeError('Glue term neither string or expression: %s, %s' % (glue, glue.__class__))
|
|
629
|
+
|
|
630
|
+
self.indices = indices
|
|
631
|
+
|
|
632
|
+
def make_VariableExpression(self, name):
|
|
633
|
+
return drt.DrtVariableExpression(name)
|
|
634
|
+
|
|
635
|
+
def make_LambdaExpression(self, variable, term):
|
|
636
|
+
return drt.DrtLambdaExpression(variable, term)
|
|
637
|
+
|
|
638
|
+
class DrtGlueDict(GlueDict):
|
|
639
|
+
def get_GlueFormula_factory(self):
|
|
640
|
+
return DrtGlueFormula
|
|
641
|
+
|
|
642
|
+
class DrtGlue(Glue):
|
|
643
|
+
def __init__(self, semtype_file=None, remove_duplicates=False,
|
|
644
|
+
depparser=None, verbose=False):
|
|
645
|
+
if not semtype_file:
|
|
646
|
+
semtype_file = os.path.join('grammars', 'sample_grammars','drt_glue.semtype')
|
|
647
|
+
Glue.__init__(self, semtype_file, remove_duplicates, depparser, verbose)
|
|
648
|
+
|
|
649
|
+
def get_glue_dict(self):
|
|
650
|
+
return DrtGlueDict(self.semtype_file)
|
|
651
|
+
|
|
652
|
+
|
|
653
|
+
def demo(show_example=-1):
|
|
654
|
+
from nltk.parse import MaltParser
|
|
655
|
+
examples = ['David sees Mary',
|
|
656
|
+
'David eats a sandwich',
|
|
657
|
+
'every man chases a dog',
|
|
658
|
+
'every man believes a dog sleeps',
|
|
659
|
+
'John gives David a sandwich',
|
|
660
|
+
'John chases himself']
|
|
661
|
+
# 'John persuades David to order a pizza',
|
|
662
|
+
# 'John tries to go',
|
|
663
|
+
# 'John tries to find a unicorn',
|
|
664
|
+
# 'John seems to vanish',
|
|
665
|
+
# 'a unicorn seems to approach',
|
|
666
|
+
# 'every big cat leaves',
|
|
667
|
+
# 'every gray cat leaves',
|
|
668
|
+
# 'every big gray cat leaves',
|
|
669
|
+
# 'a former senator leaves',
|
|
670
|
+
|
|
671
|
+
print('============== DEMO ==============')
|
|
672
|
+
|
|
673
|
+
tagger = RegexpTagger(
|
|
674
|
+
[('^(David|Mary|John)$', 'NNP'),
|
|
675
|
+
('^(sees|eats|chases|believes|gives|sleeps|chases|persuades|tries|seems|leaves)$', 'VB'),
|
|
676
|
+
('^(go|order|vanish|find|approach)$', 'VB'),
|
|
677
|
+
('^(a)$', 'ex_quant'),
|
|
678
|
+
('^(every)$', 'univ_quant'),
|
|
679
|
+
('^(sandwich|man|dog|pizza|unicorn|cat|senator)$', 'NN'),
|
|
680
|
+
('^(big|gray|former)$', 'JJ'),
|
|
681
|
+
('^(him|himself)$', 'PRP')
|
|
682
|
+
])
|
|
683
|
+
|
|
684
|
+
depparser = MaltParser(tagger=tagger)
|
|
685
|
+
glue = Glue(depparser=depparser, verbose=False)
|
|
686
|
+
|
|
687
|
+
for (i, sentence) in enumerate(examples):
|
|
688
|
+
if i==show_example or show_example==-1:
|
|
689
|
+
print('[[[Example %s]]] %s' % (i, sentence))
|
|
690
|
+
for reading in glue.parse_to_meaning(sentence.split()):
|
|
691
|
+
print(reading.simplify())
|
|
692
|
+
print('')
|
|
693
|
+
|
|
694
|
+
|
|
695
|
+
if __name__ == '__main__':
|
|
696
|
+
demo()
|