nltk 3.2__zip → 3.3__zip

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (406) hide show
  1. {nltk-3.2 → nltk-3.3}/INSTALL.txt +6 -6
  2. {nltk-3.2 → nltk-3.3}/LICENSE.txt +13 -13
  3. {nltk-3.2 → nltk-3.3}/MANIFEST.in +6 -6
  4. {nltk-3.2/nltk.egg-info → nltk-3.3}/PKG-INFO +13 -5
  5. nltk-3.2/README.txt → nltk-3.3/README.md +49 -48
  6. nltk-3.3/nltk/VERSION +1 -0
  7. {nltk-3.2 → nltk-3.3}/nltk/__init__.py +185 -185
  8. {nltk-3.2 → nltk-3.3}/nltk/app/__init__.py +52 -53
  9. {nltk-3.2 → nltk-3.3}/nltk/app/chartparser_app.py +2276 -2276
  10. {nltk-3.2 → nltk-3.3}/nltk/app/chunkparser_app.py +1262 -1264
  11. {nltk-3.2 → nltk-3.3}/nltk/app/collocations_app.py +347 -350
  12. {nltk-3.2 → nltk-3.3}/nltk/app/concordance_app.py +568 -570
  13. {nltk-3.2 → nltk-3.3}/nltk/app/nemo_app.py +156 -155
  14. {nltk-3.2 → nltk-3.3}/nltk/app/rdparser_app.py +892 -894
  15. {nltk-3.2 → nltk-3.3}/nltk/app/srparser_app.py +809 -809
  16. {nltk-3.2 → nltk-3.3}/nltk/app/wordfreq_app.py +32 -32
  17. {nltk-3.2 → nltk-3.3}/nltk/app/wordnet_app.py +968 -968
  18. {nltk-3.2 → nltk-3.3}/nltk/book.py +91 -90
  19. {nltk-3.2 → nltk-3.3}/nltk/ccg/__init__.py +22 -22
  20. {nltk-3.2 → nltk-3.3}/nltk/ccg/api.py +359 -335
  21. {nltk-3.2 → nltk-3.3}/nltk/ccg/chart.py +394 -361
  22. {nltk-3.2 → nltk-3.3}/nltk/ccg/combinator.py +343 -312
  23. {nltk-3.2 → nltk-3.3}/nltk/ccg/lexicon.py +328 -290
  24. nltk-3.3/nltk/ccg/logic.py +46 -0
  25. {nltk-3.2 → nltk-3.3}/nltk/chat/__init__.py +49 -49
  26. {nltk-3.2 → nltk-3.3}/nltk/chat/eliza.py +244 -244
  27. {nltk-3.2 → nltk-3.3}/nltk/chat/iesha.py +140 -140
  28. {nltk-3.2 → nltk-3.3}/nltk/chat/rude.py +92 -92
  29. {nltk-3.2 → nltk-3.3}/nltk/chat/suntsu.py +117 -117
  30. {nltk-3.2 → nltk-3.3}/nltk/chat/util.py +122 -120
  31. {nltk-3.2 → nltk-3.3}/nltk/chat/zen.py +282 -282
  32. {nltk-3.2 → nltk-3.3}/nltk/chunk/__init__.py +190 -190
  33. {nltk-3.2 → nltk-3.3}/nltk/chunk/api.py +51 -51
  34. {nltk-3.2 → nltk-3.3}/nltk/chunk/named_entity.py +331 -331
  35. {nltk-3.2 → nltk-3.3}/nltk/chunk/regexp.py +1390 -1384
  36. {nltk-3.2 → nltk-3.3}/nltk/chunk/util.py +600 -600
  37. {nltk-3.2 → nltk-3.3}/nltk/classify/__init__.py +98 -98
  38. {nltk-3.2 → nltk-3.3}/nltk/classify/api.py +193 -193
  39. {nltk-3.2 → nltk-3.3}/nltk/classify/decisiontree.py +295 -295
  40. {nltk-3.2 → nltk-3.3}/nltk/classify/maxent.py +1510 -1492
  41. {nltk-3.2 → nltk-3.3}/nltk/classify/megam.py +179 -178
  42. {nltk-3.2 → nltk-3.3}/nltk/classify/naivebayes.py +245 -242
  43. {nltk-3.2 → nltk-3.3}/nltk/classify/positivenaivebayes.py +170 -170
  44. {nltk-3.2 → nltk-3.3}/nltk/classify/rte_classify.py +160 -191
  45. {nltk-3.2 → nltk-3.3}/nltk/classify/scikitlearn.py +153 -151
  46. {nltk-3.2 → nltk-3.3}/nltk/classify/senna.py +183 -181
  47. {nltk-3.2 → nltk-3.3}/nltk/classify/svm.py +15 -15
  48. {nltk-3.2 → nltk-3.3}/nltk/classify/tadm.py +113 -112
  49. {nltk-3.2 → nltk-3.3}/nltk/classify/textcat.py +193 -193
  50. {nltk-3.2 → nltk-3.3}/nltk/classify/util.py +324 -312
  51. {nltk-3.2 → nltk-3.3}/nltk/classify/weka.py +346 -345
  52. {nltk-3.2 → nltk-3.3}/nltk/cluster/__init__.py +86 -86
  53. {nltk-3.2 → nltk-3.3}/nltk/cluster/api.py +74 -70
  54. {nltk-3.2 → nltk-3.3}/nltk/cluster/em.py +247 -247
  55. {nltk-3.2 → nltk-3.3}/nltk/cluster/gaac.py +168 -168
  56. {nltk-3.2 → nltk-3.3}/nltk/cluster/kmeans.py +221 -221
  57. {nltk-3.2 → nltk-3.3}/nltk/cluster/util.py +304 -289
  58. nltk-3.3/nltk/collections.py +640 -0
  59. {nltk-3.2 → nltk-3.3}/nltk/collocations.py +388 -388
  60. nltk-3.3/nltk/compat.py +365 -0
  61. {nltk-3.2 → nltk-3.3}/nltk/corpus/__init__.py +341 -320
  62. {nltk-3.2 → nltk-3.3}/nltk/corpus/europarl_raw.py +44 -44
  63. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/__init__.py +147 -145
  64. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/aligned.py +115 -114
  65. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/api.py +448 -446
  66. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/bnc.py +252 -252
  67. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/bracket_parse.py +227 -227
  68. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/categorized_sents.py +179 -178
  69. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/chasen.py +140 -139
  70. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/childes.py +522 -494
  71. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/chunked.py +212 -212
  72. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/cmudict.py +97 -95
  73. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/comparative_sents.py +280 -278
  74. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/conll.py +528 -521
  75. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/crubadan.py +116 -116
  76. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/dependency.py +101 -101
  77. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/framenet.py +3059 -2093
  78. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ieer.py +112 -111
  79. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/indian.py +85 -86
  80. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ipipan.py +332 -331
  81. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/knbc.py +187 -187
  82. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/lin.py +156 -156
  83. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/mte.py +311 -310
  84. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/nkjp.py +429 -428
  85. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/nombank.py +421 -421
  86. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/nps_chat.py +73 -73
  87. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/opinion_lexicon.py +115 -115
  88. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/panlex_lite.py +165 -165
  89. nltk-3.3/nltk/corpus/reader/pl196x.py +292 -0
  90. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/plaintext.py +232 -233
  91. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ppattach.py +96 -95
  92. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/propbank.py +479 -481
  93. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/pros_cons.py +128 -126
  94. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/reviews.py +330 -328
  95. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/rte.py +144 -146
  96. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/semcor.py +256 -256
  97. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/senseval.py +203 -201
  98. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/sentiwordnet.py +137 -137
  99. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/sinica_treebank.py +75 -75
  100. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/string_category.py +60 -59
  101. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/switchboard.py +119 -119
  102. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/tagged.py +295 -294
  103. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/timit.py +452 -450
  104. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/toolbox.py +68 -68
  105. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/twitter.py +155 -157
  106. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/udhr.py +80 -80
  107. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/util.py +804 -800
  108. nltk-3.3/nltk/corpus/reader/verbnet.py +595 -0
  109. nltk-3.3/nltk/corpus/reader/wordlist.py +134 -0
  110. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/wordnet.py +2108 -2036
  111. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/xmldocs.py +392 -387
  112. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ycoe.py +243 -242
  113. {nltk-3.2 → nltk-3.3}/nltk/corpus/util.py +144 -127
  114. {nltk-3.2 → nltk-3.3}/nltk/data.py +1487 -1451
  115. {nltk-3.2 → nltk-3.3}/nltk/decorators.py +219 -215
  116. {nltk-3.2 → nltk-3.3}/nltk/downloader.py +2278 -2274
  117. {nltk-3.2 → nltk-3.3}/nltk/draw/__init__.py +27 -27
  118. {nltk-3.2 → nltk-3.3}/nltk/draw/cfg.py +774 -777
  119. {nltk-3.2 → nltk-3.3}/nltk/draw/dispersion.py +58 -58
  120. {nltk-3.2 → nltk-3.3}/nltk/draw/table.py +1098 -1098
  121. {nltk-3.2 → nltk-3.3}/nltk/draw/tree.py +963 -966
  122. {nltk-3.2 → nltk-3.3}/nltk/draw/util.py +2356 -2356
  123. {nltk-3.2 → nltk-3.3}/nltk/featstruct.py +2617 -2500
  124. {nltk-3.2 → nltk-3.3}/nltk/grammar.py +1559 -1516
  125. {nltk-3.2 → nltk-3.3}/nltk/help.py +56 -56
  126. {nltk-3.2 → nltk-3.3}/nltk/inference/__init__.py +20 -20
  127. {nltk-3.2 → nltk-3.3}/nltk/inference/api.py +597 -589
  128. {nltk-3.2 → nltk-3.3}/nltk/inference/discourse.py +612 -608
  129. {nltk-3.2 → nltk-3.3}/nltk/inference/mace.py +311 -311
  130. {nltk-3.2 → nltk-3.3}/nltk/inference/nonmonotonic.py +509 -509
  131. {nltk-3.2 → nltk-3.3}/nltk/inference/prover9.py +431 -431
  132. {nltk-3.2 → nltk-3.3}/nltk/inference/resolution.py +687 -687
  133. {nltk-3.2 → nltk-3.3}/nltk/inference/tableau.py +607 -607
  134. {nltk-3.2 → nltk-3.3}/nltk/internals.py +982 -982
  135. {nltk-3.2 → nltk-3.3}/nltk/jsontags.py +63 -63
  136. {nltk-3.2 → nltk-3.3}/nltk/lazyimport.py +142 -142
  137. {nltk-3.2 → nltk-3.3}/nltk/metrics/__init__.py +30 -29
  138. {nltk-3.2 → nltk-3.3}/nltk/metrics/agreement.py +421 -401
  139. nltk-3.3/nltk/metrics/aline.py +607 -0
  140. {nltk-3.2 → nltk-3.3}/nltk/metrics/association.py +414 -410
  141. {nltk-3.2 → nltk-3.3}/nltk/metrics/confusionmatrix.py +206 -206
  142. {nltk-3.2 → nltk-3.3}/nltk/metrics/distance.py +207 -202
  143. {nltk-3.2 → nltk-3.3}/nltk/metrics/paice.py +381 -381
  144. {nltk-3.2 → nltk-3.3}/nltk/metrics/scores.py +228 -228
  145. {nltk-3.2 → nltk-3.3}/nltk/metrics/segmentation.py +231 -232
  146. {nltk-3.2 → nltk-3.3}/nltk/metrics/spearman.py +69 -69
  147. {nltk-3.2 → nltk-3.3}/nltk/misc/__init__.py +11 -11
  148. {nltk-3.2 → nltk-3.3}/nltk/misc/babelfish.py +10 -10
  149. {nltk-3.2 → nltk-3.3}/nltk/misc/chomsky.py +135 -133
  150. {nltk-3.2 → nltk-3.3}/nltk/misc/minimalset.py +83 -83
  151. {nltk-3.2 → nltk-3.3}/nltk/misc/sort.py +157 -157
  152. {nltk-3.2 → nltk-3.3}/nltk/misc/wordfinder.py +129 -129
  153. {nltk-3.2 → nltk-3.3}/nltk/parse/__init__.py +81 -80
  154. {nltk-3.2 → nltk-3.3}/nltk/parse/api.py +66 -66
  155. {nltk-3.2 → nltk-3.3}/nltk/parse/bllip.py +282 -282
  156. {nltk-3.2 → nltk-3.3}/nltk/parse/chart.py +1682 -1681
  157. nltk-3.3/nltk/parse/corenlp.py +780 -0
  158. {nltk-3.2 → nltk-3.3}/nltk/parse/dependencygraph.py +760 -758
  159. {nltk-3.2 → nltk-3.3}/nltk/parse/earleychart.py +452 -451
  160. {nltk-3.2 → nltk-3.3}/nltk/parse/evaluate.py +129 -129
  161. {nltk-3.2 → nltk-3.3}/nltk/parse/featurechart.py +580 -578
  162. {nltk-3.2 → nltk-3.3}/nltk/parse/generate.py +87 -76
  163. {nltk-3.2 → nltk-3.3}/nltk/parse/malt.py +356 -356
  164. {nltk-3.2 → nltk-3.3}/nltk/parse/nonprojectivedependencyparser.py +783 -783
  165. {nltk-3.2 → nltk-3.3}/nltk/parse/pchart.py +522 -522
  166. {nltk-3.2 → nltk-3.3}/nltk/parse/projectivedependencyparser.py +579 -578
  167. {nltk-3.2 → nltk-3.3}/nltk/parse/recursivedescent.py +655 -656
  168. {nltk-3.2 → nltk-3.3}/nltk/parse/shiftreduce.py +458 -459
  169. {nltk-3.2 → nltk-3.3}/nltk/parse/stanford.py +431 -406
  170. {nltk-3.2 → nltk-3.3}/nltk/parse/transitionparser.py +774 -772
  171. {nltk-3.2 → nltk-3.3}/nltk/parse/util.py +231 -231
  172. {nltk-3.2 → nltk-3.3}/nltk/parse/viterbi.py +401 -401
  173. {nltk-3.2 → nltk-3.3}/nltk/probability.py +2410 -2355
  174. {nltk-3.2 → nltk-3.3}/nltk/sem/__init__.py +61 -61
  175. {nltk-3.2 → nltk-3.3}/nltk/sem/boxer.py +1261 -1261
  176. {nltk-3.2 → nltk-3.3}/nltk/sem/chat80.py +782 -782
  177. {nltk-3.2 → nltk-3.3}/nltk/sem/cooper_storage.py +118 -118
  178. {nltk-3.2 → nltk-3.3}/nltk/sem/drt.py +1258 -1257
  179. {nltk-3.2 → nltk-3.3}/nltk/sem/drt_glue_demo.py +485 -489
  180. {nltk-3.2 → nltk-3.3}/nltk/sem/evaluate.py +790 -784
  181. {nltk-3.2 → nltk-3.3}/nltk/sem/glue.py +696 -694
  182. {nltk-3.2 → nltk-3.3}/nltk/sem/hole.py +386 -384
  183. {nltk-3.2 → nltk-3.3}/nltk/sem/lfg.py +210 -208
  184. {nltk-3.2 → nltk-3.3}/nltk/sem/linearlogic.py +451 -449
  185. {nltk-3.2 → nltk-3.3}/nltk/sem/logic.py +1910 -1910
  186. {nltk-3.2 → nltk-3.3}/nltk/sem/relextract.py +476 -483
  187. {nltk-3.2 → nltk-3.3}/nltk/sem/skolemize.py +101 -101
  188. {nltk-3.2 → nltk-3.3}/nltk/sem/util.py +249 -249
  189. {nltk-3.2 → nltk-3.3}/nltk/sentiment/__init__.py +14 -14
  190. {nltk-3.2 → nltk-3.3}/nltk/sentiment/sentiment_analyzer.py +229 -229
  191. {nltk-3.2 → nltk-3.3}/nltk/sentiment/util.py +762 -752
  192. {nltk-3.2 → nltk-3.3}/nltk/sentiment/vader.py +443 -445
  193. {nltk-3.2 → nltk-3.3}/nltk/stem/__init__.py +31 -33
  194. {nltk-3.2 → nltk-3.3}/nltk/stem/api.py +28 -25
  195. nltk-3.3/nltk/stem/arlstem.py +369 -0
  196. {nltk-3.2 → nltk-3.3}/nltk/stem/isri.py +345 -345
  197. {nltk-3.2 → nltk-3.3}/nltk/stem/lancaster.py +337 -310
  198. nltk-3.3/nltk/stem/porter.py +710 -0
  199. {nltk-3.2 → nltk-3.3}/nltk/stem/regexp.py +61 -61
  200. {nltk-3.2 → nltk-3.3}/nltk/stem/rslp.py +142 -142
  201. {nltk-3.2 → nltk-3.3}/nltk/stem/snowball.py +4236 -3712
  202. {nltk-3.2 → nltk-3.3}/nltk/stem/util.py +22 -12
  203. {nltk-3.2 → nltk-3.3}/nltk/stem/wordnet.py +51 -51
  204. {nltk-3.2 → nltk-3.3}/nltk/tag/__init__.py +152 -127
  205. {nltk-3.2 → nltk-3.3}/nltk/tag/api.py +86 -81
  206. {nltk-3.2 → nltk-3.3}/nltk/tag/brill.py +424 -426
  207. {nltk-3.2 → nltk-3.3}/nltk/tag/brill_trainer.py +608 -608
  208. {nltk-3.2 → nltk-3.3}/nltk/tag/crf.py +204 -200
  209. {nltk-3.2 → nltk-3.3}/nltk/tag/hmm.py +1277 -1277
  210. {nltk-3.2 → nltk-3.3}/nltk/tag/hunpos.py +132 -132
  211. {nltk-3.2 → nltk-3.3}/nltk/tag/mapping.py +101 -101
  212. {nltk-3.2 → nltk-3.3}/nltk/tag/perceptron.py +329 -314
  213. {nltk-3.2 → nltk-3.3}/nltk/tag/senna.py +147 -145
  214. {nltk-3.2 → nltk-3.3}/nltk/tag/sequential.py +740 -736
  215. {nltk-3.2 → nltk-3.3}/nltk/tag/stanford.py +219 -196
  216. {nltk-3.2 → nltk-3.3}/nltk/tag/tnt.py +595 -604
  217. {nltk-3.2 → nltk-3.3}/nltk/tag/util.py +72 -72
  218. {nltk-3.2 → nltk-3.3}/nltk/tbl/__init__.py +28 -28
  219. {nltk-3.2 → nltk-3.3}/nltk/tbl/api.py +1 -1
  220. {nltk-3.2 → nltk-3.3}/nltk/tbl/demo.py +366 -366
  221. {nltk-3.2 → nltk-3.3}/nltk/tbl/erroranalysis.py +39 -39
  222. {nltk-3.2 → nltk-3.3}/nltk/tbl/feature.py +259 -261
  223. {nltk-3.2 → nltk-3.3}/nltk/tbl/rule.py +321 -321
  224. {nltk-3.2 → nltk-3.3}/nltk/tbl/template.py +303 -308
  225. {nltk-3.2 → nltk-3.3}/nltk/test/__init__.py +18 -18
  226. {nltk-3.2 → nltk-3.3}/nltk/test/all.py +24 -23
  227. {nltk-3.2 → nltk-3.3}/nltk/test/bleu.doctest +14 -14
  228. {nltk-3.2 → nltk-3.3}/nltk/test/bnc.doctest +60 -60
  229. {nltk-3.2 → nltk-3.3}/nltk/test/ccg.doctest +376 -395
  230. nltk-3.3/nltk/test/ccg_semantics.doctest +553 -0
  231. {nltk-3.2 → nltk-3.3}/nltk/test/chat80.doctest +234 -234
  232. {nltk-3.2 → nltk-3.3}/nltk/test/childes.doctest +184 -184
  233. {nltk-3.2 → nltk-3.3}/nltk/test/childes_fixt.py +14 -13
  234. {nltk-3.2 → nltk-3.3}/nltk/test/chunk.doctest +373 -373
  235. {nltk-3.2 → nltk-3.3}/nltk/test/classify.doctest +185 -185
  236. {nltk-3.2 → nltk-3.3}/nltk/test/classify_fixt.py +11 -10
  237. nltk-3.3/nltk/test/collections.doctest +20 -0
  238. {nltk-3.2 → nltk-3.3}/nltk/test/collocations.doctest +276 -276
  239. {nltk-3.2 → nltk-3.3}/nltk/test/compat.doctest +134 -138
  240. {nltk-3.2 → nltk-3.3}/nltk/test/compat_fixt.py +9 -8
  241. nltk-3.3/nltk/test/concordance.doctest +55 -0
  242. {nltk-3.2 → nltk-3.3}/nltk/test/corpus.doctest +2200 -2212
  243. {nltk-3.2 → nltk-3.3}/nltk/test/corpus_fixt.py +3 -3
  244. {nltk-3.2 → nltk-3.3}/nltk/test/crubadan.doctest +65 -65
  245. {nltk-3.2 → nltk-3.3}/nltk/test/data.doctest +379 -379
  246. {nltk-3.2 → nltk-3.3}/nltk/test/dependency.doctest +241 -241
  247. {nltk-3.2 → nltk-3.3}/nltk/test/discourse.doctest +546 -546
  248. {nltk-3.2 → nltk-3.3}/nltk/test/discourse_fixt.py +14 -13
  249. {nltk-3.2 → nltk-3.3}/nltk/test/doctest_nose_plugin.py +159 -156
  250. {nltk-3.2 → nltk-3.3}/nltk/test/drt.doctest +517 -517
  251. {nltk-3.2 → nltk-3.3}/nltk/test/featgram.doctest +607 -607
  252. {nltk-3.2 → nltk-3.3}/nltk/test/featstruct.doctest +1229 -1230
  253. {nltk-3.2 → nltk-3.3}/nltk/test/framenet.doctest +288 -239
  254. {nltk-3.2 → nltk-3.3}/nltk/test/generate.doctest +67 -67
  255. {nltk-3.2 → nltk-3.3}/nltk/test/gensim.doctest +140 -141
  256. {nltk-3.2 → nltk-3.3}/nltk/test/gensim_fixt.py +10 -9
  257. {nltk-3.2 → nltk-3.3}/nltk/test/gluesemantics.doctest +384 -386
  258. {nltk-3.2 → nltk-3.3}/nltk/test/gluesemantics_malt.doctest +68 -68
  259. {nltk-3.2 → nltk-3.3}/nltk/test/gluesemantics_malt_fixt.py +12 -11
  260. {nltk-3.2 → nltk-3.3}/nltk/test/grammar.doctest +48 -48
  261. {nltk-3.2 → nltk-3.3}/nltk/test/grammartestsuites.doctest +109 -109
  262. {nltk-3.2 → nltk-3.3}/nltk/test/index.doctest +100 -100
  263. {nltk-3.2 → nltk-3.3}/nltk/test/inference.doctest +534 -534
  264. {nltk-3.2 → nltk-3.3}/nltk/test/inference_fixt.py +12 -11
  265. {nltk-3.2 → nltk-3.3}/nltk/test/internals.doctest +140 -140
  266. {nltk-3.2 → nltk-3.3}/nltk/test/japanese.doctest +48 -48
  267. {nltk-3.2 → nltk-3.3}/nltk/test/logic.doctest +1098 -1098
  268. {nltk-3.2 → nltk-3.3}/nltk/test/metrics.doctest +270 -270
  269. {nltk-3.2 → nltk-3.3}/nltk/test/misc.doctest +118 -118
  270. {nltk-3.2 → nltk-3.3}/nltk/test/nonmonotonic.doctest +286 -286
  271. {nltk-3.2 → nltk-3.3}/nltk/test/nonmonotonic_fixt.py +12 -11
  272. {nltk-3.2 → nltk-3.3}/nltk/test/paice.doctest +35 -35
  273. {nltk-3.2 → nltk-3.3}/nltk/test/parse.doctest +884 -884
  274. {nltk-3.2 → nltk-3.3}/nltk/test/portuguese_en.doctest +565 -565
  275. {nltk-3.2 → nltk-3.3}/nltk/test/portuguese_en_fixt.py +16 -16
  276. {nltk-3.2 → nltk-3.3}/nltk/test/probability.doctest +304 -304
  277. {nltk-3.2 → nltk-3.3}/nltk/test/probability_fixt.py +13 -12
  278. {nltk-3.2 → nltk-3.3}/nltk/test/propbank.doctest +176 -176
  279. {nltk-3.2 → nltk-3.3}/nltk/test/relextract.doctest +263 -263
  280. {nltk-3.2 → nltk-3.3}/nltk/test/resolution.doctest +221 -221
  281. {nltk-3.2 → nltk-3.3}/nltk/test/runtests.py +86 -64
  282. {nltk-3.2 → nltk-3.3}/nltk/test/segmentation_fixt.py +10 -10
  283. {nltk-3.2 → nltk-3.3}/nltk/test/semantics.doctest +665 -665
  284. {nltk-3.2 → nltk-3.3}/nltk/test/semantics_fixt.py +7 -7
  285. {nltk-3.2 → nltk-3.3}/nltk/test/sentiment.doctest +233 -233
  286. {nltk-3.2 → nltk-3.3}/nltk/test/sentiwordnet.doctest +41 -41
  287. {nltk-3.2 → nltk-3.3}/nltk/test/simple.doctest +85 -85
  288. {nltk-3.2 → nltk-3.3}/nltk/test/stem.doctest +78 -78
  289. {nltk-3.2 → nltk-3.3}/nltk/test/tag.doctest +33 -33
  290. {nltk-3.2 → nltk-3.3}/nltk/test/tokenize.doctest +273 -166
  291. {nltk-3.2 → nltk-3.3}/nltk/test/toolbox.doctest +307 -307
  292. {nltk-3.2 → nltk-3.3}/nltk/test/translate.doctest +242 -242
  293. {nltk-3.2 → nltk-3.3}/nltk/test/translate_fixt.py +3 -3
  294. {nltk-3.2 → nltk-3.3}/nltk/test/tree.doctest +1101 -1101
  295. {nltk-3.2 → nltk-3.3}/nltk/test/treeprettyprinter.doctest +127 -127
  296. {nltk-3.2 → nltk-3.3}/nltk/test/treetransforms.doctest +156 -156
  297. nltk-3.3/nltk/test/unit/test_2x_compat.py +63 -0
  298. nltk-3.3/nltk/test/unit/test_aline.py +52 -0
  299. nltk-3.3/nltk/test/unit/test_chunk.py +56 -0
  300. {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_classify.py +55 -53
  301. {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_collocations.py +88 -79
  302. nltk-3.3/nltk/test/unit/test_concordance.py +98 -0
  303. nltk-3.3/nltk/test/unit/test_corenlp.py +1000 -0
  304. {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_corpora.py +200 -183
  305. {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_corpus_views.py +45 -45
  306. {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_hmm.py +88 -88
  307. {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_json2csv_corpus.py +188 -187
  308. {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_naivebayes.py +26 -26
  309. nltk-3.3/nltk/test/unit/test_rte_classify.py +86 -0
  310. {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_seekable_unicode_stream_reader.py +124 -124
  311. nltk-3.3/nltk/test/unit/test_senna.py +74 -0
  312. nltk-3.3/nltk/test/unit/test_stem.py +121 -0
  313. {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_tag.py +20 -20
  314. {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_tgrep.py +628 -626
  315. nltk-3.3/nltk/test/unit/test_tokenize.py +168 -0
  316. {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_twitter_auth.py +185 -185
  317. nltk-3.3/nltk/test/unit/test_wordnet.py +134 -0
  318. nltk-3.3/nltk/test/unit/translate/test_bleu.py +244 -0
  319. nltk-3.3/nltk/test/unit/translate/test_gdfa.py +57 -0
  320. {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm1.py +75 -75
  321. {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm2.py +87 -87
  322. {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm3.py +105 -105
  323. {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm4.py +126 -126
  324. {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm5.py +166 -166
  325. {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm_model.py +270 -270
  326. nltk-3.3/nltk/test/unit/translate/test_nist.py +37 -0
  327. {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_stack_decoder.py +300 -300
  328. {nltk-3.2 → nltk-3.3}/nltk/test/unit/utils.py +41 -41
  329. {nltk-3.2 → nltk-3.3}/nltk/test/util.doctest +48 -48
  330. {nltk-3.2 → nltk-3.3}/nltk/test/wordnet.doctest +595 -581
  331. {nltk-3.2 → nltk-3.3}/nltk/test/wordnet_fixt.py +7 -6
  332. {nltk-3.2 → nltk-3.3}/nltk/test/wordnet_lch.doctest +53 -53
  333. {nltk-3.2 → nltk-3.3}/nltk/test/wsd.doctest +68 -68
  334. {nltk-3.2 → nltk-3.3}/nltk/text.py +663 -610
  335. {nltk-3.2 → nltk-3.3}/nltk/tgrep.py +938 -937
  336. {nltk-3.2 → nltk-3.3}/nltk/tokenize/__init__.py +130 -107
  337. {nltk-3.2 → nltk-3.3}/nltk/tokenize/api.py +77 -78
  338. {nltk-3.2 → nltk-3.3}/nltk/tokenize/casual.py +343 -342
  339. {nltk-3.2 → nltk-3.3}/nltk/tokenize/mwe.py +120 -120
  340. nltk-3.3/nltk/tokenize/nist.py +173 -0
  341. {nltk-3.2 → nltk-3.3}/nltk/tokenize/punkt.py +1620 -1601
  342. {nltk-3.2 → nltk-3.3}/nltk/tokenize/regexp.py +208 -202
  343. nltk-3.3/nltk/tokenize/repp.py +151 -0
  344. {nltk-3.2 → nltk-3.3}/nltk/tokenize/sexpr.py +140 -142
  345. {nltk-3.2 → nltk-3.3}/nltk/tokenize/simple.py +138 -136
  346. {nltk-3.2 → nltk-3.3}/nltk/tokenize/stanford.py +117 -113
  347. nltk-3.3/nltk/tokenize/stanford_segmenter.py +246 -0
  348. {nltk-3.2 → nltk-3.3}/nltk/tokenize/texttiling.py +458 -460
  349. nltk-3.3/nltk/tokenize/toktok.py +155 -0
  350. nltk-3.3/nltk/tokenize/treebank.py +363 -0
  351. nltk-3.3/nltk/tokenize/util.py +261 -0
  352. {nltk-3.2 → nltk-3.3}/nltk/toolbox.py +499 -497
  353. {nltk-3.2 → nltk-3.3}/nltk/translate/__init__.py +24 -24
  354. {nltk-3.2 → nltk-3.3}/nltk/translate/api.py +331 -321
  355. {nltk-3.2 → nltk-3.3}/nltk/translate/bleu_score.py +615 -556
  356. nltk-3.3/nltk/translate/chrf_score.py +137 -0
  357. {nltk-3.2 → nltk-3.3}/nltk/translate/gale_church.py +248 -235
  358. {nltk-3.2 → nltk-3.3}/nltk/translate/gdfa.py +135 -131
  359. nltk-3.3/nltk/translate/gleu_score.py +193 -0
  360. {nltk-3.2 → nltk-3.3}/nltk/translate/ibm1.py +250 -241
  361. {nltk-3.2 → nltk-3.3}/nltk/translate/ibm2.py +308 -308
  362. {nltk-3.2 → nltk-3.3}/nltk/translate/ibm3.py +337 -337
  363. {nltk-3.2 → nltk-3.3}/nltk/translate/ibm4.py +467 -467
  364. {nltk-3.2 → nltk-3.3}/nltk/translate/ibm5.py +639 -639
  365. {nltk-3.2 → nltk-3.3}/nltk/translate/ibm_model.py +536 -533
  366. {nltk-3.2 → nltk-3.3}/nltk/translate/metrics.py +40 -40
  367. nltk-3.3/nltk/translate/nist_score.py +178 -0
  368. {nltk-3.2 → nltk-3.3}/nltk/translate/phrase_based.py +175 -175
  369. {nltk-3.2 → nltk-3.3}/nltk/translate/ribes_score.py +325 -325
  370. {nltk-3.2 → nltk-3.3}/nltk/translate/stack_decoder.py +499 -495
  371. {nltk-3.2 → nltk-3.3}/nltk/tree.py +1605 -1603
  372. {nltk-3.2 → nltk-3.3}/nltk/treeprettyprinter.py +564 -564
  373. {nltk-3.2 → nltk-3.3}/nltk/treetransforms.py +309 -309
  374. {nltk-3.2 → nltk-3.3}/nltk/twitter/__init__.py +28 -28
  375. {nltk-3.2 → nltk-3.3}/nltk/twitter/api.py +141 -136
  376. {nltk-3.2 → nltk-3.3}/nltk/twitter/common.py +264 -258
  377. {nltk-3.2 → nltk-3.3}/nltk/twitter/twitter_demo.py +288 -288
  378. {nltk-3.2 → nltk-3.3}/nltk/twitter/twitterclient.py +530 -530
  379. {nltk-3.2 → nltk-3.3}/nltk/twitter/util.py +147 -146
  380. nltk-3.3/nltk/util.py +751 -0
  381. {nltk-3.2 → nltk-3.3}/nltk/wsd.py +53 -53
  382. {nltk-3.2 → nltk-3.3/nltk.egg-info}/PKG-INFO +13 -5
  383. {nltk-3.2 → nltk-3.3}/nltk.egg-info/SOURCES.txt +26 -3
  384. nltk-3.3/nltk.egg-info/requires.txt +31 -0
  385. {nltk-3.2 → nltk-3.3}/setup.cfg +0 -1
  386. {nltk-3.2 → nltk-3.3}/setup.py +102 -75
  387. nltk-3.2/nltk/VERSION +0 -1
  388. nltk-3.2/nltk/compat.py +0 -680
  389. nltk-3.2/nltk/corpus/reader/pl196x.py +0 -278
  390. nltk-3.2/nltk/corpus/reader/verbnet.py +0 -408
  391. nltk-3.2/nltk/corpus/reader/wordlist.py +0 -37
  392. nltk-3.2/nltk/sentiment/vader_lexicon.txt +0 -7517
  393. nltk-3.2/nltk/six.py +0 -868
  394. nltk-3.2/nltk/stem/porter.py +0 -692
  395. nltk-3.2/nltk/test/unit/test_2x_compat.py +0 -26
  396. nltk-3.2/nltk/test/unit/test_stem.py +0 -36
  397. nltk-3.2/nltk/test/unit/test_tokenize.py +0 -23
  398. nltk-3.2/nltk/test/unit/translate/test_bleu.py +0 -84
  399. nltk-3.2/nltk/tokenize/treebank.py +0 -124
  400. nltk-3.2/nltk/tokenize/util.py +0 -90
  401. nltk-3.2/nltk/util.py +0 -1364
  402. {nltk-3.2 → nltk-3.3}/nltk/test/unit/__init__.py +0 -0
  403. {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/__init__.py +0 -0
  404. {nltk-3.2 → nltk-3.3}/nltk.egg-info/dependency_links.txt +0 -0
  405. {nltk-3.2 → nltk-3.3}/nltk.egg-info/not-zip-safe +0 -0
  406. {nltk-3.2 → nltk-3.3}/nltk.egg-info/top_level.txt +0 -0
@@ -1,694 +1,696 @@
1
- # Natural Language Toolkit: Glue Semantics
2
- #
3
- # Author: Dan Garrette <dhgarrette@gmail.com>
4
- #
5
- # Copyright (C) 2001-2015 NLTK Project
6
- # URL: <http://nltk.org/>
7
- # For license information, see LICENSE.TXT
8
- from __future__ import print_function, division, unicode_literals
9
-
10
- import os
11
-
12
- import nltk
13
- from nltk.internals import Counter
14
- from nltk.compat import string_types
15
- from nltk.tag import UnigramTagger, BigramTagger, TrigramTagger, RegexpTagger
16
- from nltk.sem.logic import (Expression, Variable, VariableExpression,
17
- LambdaExpression, AbstractVariableExpression)
18
- from nltk.compat import python_2_unicode_compatible
19
- from nltk.sem import drt
20
- from nltk.sem import linearlogic
21
-
22
- SPEC_SEMTYPES = {'a' : 'ex_quant',
23
- 'an' : 'ex_quant',
24
- 'every' : 'univ_quant',
25
- 'the' : 'def_art',
26
- 'no' : 'no_quant',
27
- 'default' : 'ex_quant'}
28
-
29
- OPTIONAL_RELATIONSHIPS = ['nmod', 'vmod', 'punct']
30
-
31
- @python_2_unicode_compatible
32
- class GlueFormula(object):
33
- def __init__(self, meaning, glue, indices=None):
34
- if not indices:
35
- indices = set()
36
-
37
- if isinstance(meaning, string_types):
38
- self.meaning = Expression.fromstring(meaning)
39
- elif isinstance(meaning, Expression):
40
- self.meaning = meaning
41
- else:
42
- raise RuntimeError('Meaning term neither string or expression: %s, %s' % (meaning, meaning.__class__))
43
-
44
- if isinstance(glue, string_types):
45
- self.glue = linearlogic.LinearLogicParser().parse(glue)
46
- elif isinstance(glue, linearlogic.Expression):
47
- self.glue = glue
48
- else:
49
- raise RuntimeError('Glue term neither string or expression: %s, %s' % (glue, glue.__class__))
50
-
51
- self.indices = indices
52
-
53
- def applyto(self, arg):
54
- """ self = (\\x.(walk x), (subj -o f))
55
- arg = (john , subj)
56
- returns ((walk john), f)
57
- """
58
- if self.indices & arg.indices: # if the sets are NOT disjoint
59
- raise linearlogic.LinearLogicApplicationException("'%s' applied to '%s'. Indices are not disjoint." % (self, arg))
60
- else: # if the sets ARE disjoint
61
- return_indices = (self.indices | arg.indices)
62
-
63
- try:
64
- return_glue = linearlogic.ApplicationExpression(self.glue, arg.glue, arg.indices)
65
- except linearlogic.LinearLogicApplicationException:
66
- raise linearlogic.LinearLogicApplicationException("'%s' applied to '%s'" % (self.simplify(), arg.simplify()))
67
-
68
- arg_meaning_abstracted = arg.meaning
69
- if return_indices:
70
- for dep in self.glue.simplify().antecedent.dependencies[::-1]: # if self.glue is (A -o B), dep is in A.dependencies
71
- arg_meaning_abstracted = self.make_LambdaExpression(Variable('v%s' % dep),
72
- arg_meaning_abstracted)
73
- return_meaning = self.meaning.applyto(arg_meaning_abstracted)
74
-
75
- return self.__class__(return_meaning, return_glue, return_indices)
76
-
77
- def make_VariableExpression(self, name):
78
- return VariableExpression(name)
79
-
80
- def make_LambdaExpression(self, variable, term):
81
- return LambdaExpression(variable, term)
82
-
83
- def lambda_abstract(self, other):
84
- assert isinstance(other, GlueFormula)
85
- assert isinstance(other.meaning, AbstractVariableExpression)
86
- return self.__class__(self.make_LambdaExpression(other.meaning.variable,
87
- self.meaning),
88
- linearlogic.ImpExpression(other.glue, self.glue))
89
-
90
- def compile(self, counter=None):
91
- """From Iddo Lev's PhD Dissertation p108-109"""
92
- if not counter:
93
- counter = Counter()
94
- (compiled_glue, new_forms) = self.glue.simplify().compile_pos(counter, self.__class__)
95
- return new_forms + [self.__class__(self.meaning, compiled_glue, set([counter.get()]))]
96
-
97
- def simplify(self):
98
- return self.__class__(self.meaning.simplify(), self.glue.simplify(), self.indices)
99
-
100
- def __eq__(self, other):
101
- return self.__class__ == other.__class__ and self.meaning == other.meaning and self.glue == other.glue
102
-
103
- def __ne__(self, other):
104
- return not self == other
105
-
106
- # sorting for use in doctests which must be deterministic
107
- def __lt__(self, other):
108
- return str(self) < str(other)
109
-
110
- def __str__(self):
111
- assert isinstance(self.indices, set)
112
- accum = '%s : %s' % (self.meaning, self.glue)
113
- if self.indices:
114
- accum += ' : {' + ', '.join(str(index) for index in self.indices) + '}'
115
- return accum
116
-
117
- def __repr__(self):
118
- return "%s" % self
119
-
120
- @python_2_unicode_compatible
121
- class GlueDict(dict):
122
- def __init__(self, filename, encoding=None):
123
- self.filename = filename
124
- self.file_encoding = encoding
125
- self.read_file()
126
-
127
- def read_file(self, empty_first=True):
128
- if empty_first:
129
- self.clear()
130
-
131
- try:
132
- contents = nltk.data.load(self.filename, format='text', encoding=self.file_encoding)
133
- # TODO: the above can't handle zip files, but this should anyway be fixed in nltk.data.load()
134
- except LookupError as e:
135
- try:
136
- contents = nltk.data.load('file:' + self.filename, format='text', encoding=self.file_encoding)
137
- except LookupError:
138
- raise e
139
- lines = contents.splitlines()
140
-
141
- for line in lines: # example: 'n : (\\x.(<word> x), (v-or))'
142
- # lambdacalc -^ linear logic -^
143
- line = line.strip() # remove trailing newline
144
- if not len(line): continue # skip empty lines
145
- if line[0] == '#': continue # skip commented out lines
146
-
147
- parts = line.split(' : ', 2) # ['verb', '(\\x.(<word> x), ( subj -o f ))', '[subj]']
148
-
149
- glue_formulas = []
150
- paren_count = 0
151
- tuple_start = 0
152
- tuple_comma = 0
153
-
154
- relationships = None
155
-
156
- if len(parts) > 1:
157
- for (i, c) in enumerate(parts[1]):
158
- if c == '(':
159
- if paren_count == 0: # if it's the first '(' of a tuple
160
- tuple_start = i+1 # then save the index
161
- paren_count += 1
162
- elif c == ')':
163
- paren_count -= 1
164
- if paren_count == 0: # if it's the last ')' of a tuple
165
- meaning_term = parts[1][tuple_start:tuple_comma] # '\\x.(<word> x)'
166
- glue_term = parts[1][tuple_comma+1:i] # '(v-r)'
167
- glue_formulas.append([meaning_term, glue_term]) # add the GlueFormula to the list
168
- elif c == ',':
169
- if paren_count == 1: # if it's a comma separating the parts of the tuple
170
- tuple_comma = i # then save the index
171
- elif c == '#': # skip comments at the ends of lines
172
- if paren_count != 0: # if the line hasn't parsed correctly so far
173
- raise RuntimeError('Formula syntax is incorrect for entry ' + line)
174
- break # break to the next line
175
-
176
- if len(parts) > 2: #if there is a relationship entry at the end
177
- rel_start = parts[2].index('[')+1
178
- rel_end = parts[2].index(']')
179
- if rel_start == rel_end:
180
- relationships = frozenset()
181
- else:
182
- relationships = frozenset(r.strip() for r in parts[2][rel_start:rel_end].split(','))
183
-
184
- try:
185
- start_inheritance = parts[0].index('(')
186
- end_inheritance = parts[0].index(')')
187
- sem = parts[0][:start_inheritance].strip()
188
- supertype = parts[0][start_inheritance+1:end_inheritance]
189
- except:
190
- sem = parts[0].strip()
191
- supertype = None
192
-
193
- if sem not in self:
194
- self[sem] = {}
195
-
196
- if relationships is None: #if not specified for a specific relationship set
197
- #add all relationship entries for parents
198
- if supertype:
199
- for rels in self[supertype]:
200
- if rels not in self[sem]:
201
- self[sem][rels] = []
202
- glue = self[supertype][rels]
203
- self[sem][rels].extend(glue)
204
- self[sem][rels].extend(glue_formulas) # add the glue formulas to every rel entry
205
- else:
206
- if None not in self[sem]:
207
- self[sem][None] = []
208
- self[sem][None].extend(glue_formulas) # add the glue formulas to every rel entry
209
- else:
210
- if relationships not in self[sem]:
211
- self[sem][relationships] = []
212
- if supertype:
213
- self[sem][relationships].extend(self[supertype][relationships])
214
- self[sem][relationships].extend(glue_formulas) # add the glue entry to the dictionary
215
-
216
- def __str__(self):
217
- accum = ''
218
- for pos in self:
219
- str_pos = "%s" % pos
220
- for relset in self[pos]:
221
- i = 1
222
- for gf in self[pos][relset]:
223
- if i == 1:
224
- accum += str_pos + ': '
225
- else:
226
- accum += ' '*(len(str_pos)+2)
227
- accum += "%s" % gf
228
- if relset and i == len(self[pos][relset]):
229
- accum += ' : %s' % relset
230
- accum += '\n'
231
- i += 1
232
- return accum
233
-
234
- def to_glueformula_list(self, depgraph, node=None, counter=None, verbose=False):
235
- if node is None:
236
- # TODO: should it be depgraph.root? Is this code tested?
237
- top = depgraph.nodes[0]
238
- depList = sum(list(top['deps'].values()), [])
239
- root = depgraph.nodes[depList[0]]
240
-
241
- return self.to_glueformula_list(depgraph, root, Counter(), verbose)
242
-
243
- glueformulas = self.lookup(node, depgraph, counter)
244
- for dep_idx in sum(list(node['deps'].values()), []):
245
- dep = depgraph.nodes[dep_idx]
246
- glueformulas.extend(self.to_glueformula_list(depgraph, dep, counter, verbose))
247
- return glueformulas
248
-
249
- def lookup(self, node, depgraph, counter):
250
- semtype_names = self.get_semtypes(node)
251
-
252
- semtype = None
253
- for name in semtype_names:
254
- if name in self:
255
- semtype = self[name]
256
- break
257
- if semtype is None:
258
- # raise KeyError, "There is no GlueDict entry for sem type '%s' (for '%s')" % (sem, word)
259
- return []
260
-
261
- self.add_missing_dependencies(node, depgraph)
262
-
263
- lookup = self._lookup_semtype_option(semtype, node, depgraph)
264
-
265
- if not len(lookup):
266
- raise KeyError(
267
- "There is no GlueDict entry for sem type of '%s' "
268
- "with tag '%s', and rel '%s'" %
269
- (node['word'], node['tag'], node['rel'])
270
- )
271
-
272
- return self.get_glueformulas_from_semtype_entry(lookup, node['word'], node, depgraph, counter)
273
-
274
- def add_missing_dependencies(self, node, depgraph):
275
- rel = node['rel'].lower()
276
-
277
- if rel == 'main':
278
- headnode = depgraph.nodes[node['head']]
279
- subj = self.lookup_unique('subj', headnode, depgraph)
280
- relation = subj['rel']
281
- node['deps'].setdefault(relation,[])
282
- node['deps'][relation].append(subj['address'])
283
- #node['deps'].append(subj['address'])
284
-
285
- def _lookup_semtype_option(self, semtype, node, depgraph):
286
- relationships = frozenset(
287
- depgraph.nodes[dep]['rel'].lower()
288
- for dep in sum(list(node['deps'].values()), [])
289
- if depgraph.nodes[dep]['rel'].lower() not in OPTIONAL_RELATIONSHIPS
290
- )
291
-
292
- try:
293
- lookup = semtype[relationships]
294
- except KeyError:
295
- # An exact match is not found, so find the best match where
296
- # 'best' is defined as the glue entry whose relationship set has the
297
- # most relations of any possible relationship set that is a subset
298
- # of the actual depgraph
299
- best_match = frozenset()
300
- for relset_option in set(semtype)-set([None]):
301
- if len(relset_option) > len(best_match) and \
302
- relset_option < relationships:
303
- best_match = relset_option
304
- if not best_match:
305
- if None in semtype:
306
- best_match = None
307
- else:
308
- return None
309
- lookup = semtype[best_match]
310
-
311
- return lookup
312
-
313
- def get_semtypes(self, node):
314
- """
315
- Based on the node, return a list of plausible semtypes in order of
316
- plausibility.
317
- """
318
- rel = node['rel'].lower()
319
- word = node['word'].lower()
320
-
321
- if rel == 'spec':
322
- if word in SPEC_SEMTYPES:
323
- return [SPEC_SEMTYPES[word]]
324
- else:
325
- return [SPEC_SEMTYPES['default']]
326
- elif rel in ['nmod', 'vmod']:
327
- return [node['tag'], rel]
328
- else:
329
- return [node['tag']]
330
-
331
- def get_glueformulas_from_semtype_entry(self, lookup, word, node, depgraph, counter):
332
- glueformulas = []
333
-
334
- glueFormulaFactory = self.get_GlueFormula_factory()
335
- for meaning, glue in lookup:
336
- gf = glueFormulaFactory(self.get_meaning_formula(meaning, word), glue)
337
- if not len(glueformulas):
338
- gf.word = word
339
- else:
340
- gf.word = '%s%s' % (word, len(glueformulas)+1)
341
-
342
- gf.glue = self.initialize_labels(gf.glue, node, depgraph, counter.get())
343
-
344
- glueformulas.append(gf)
345
- return glueformulas
346
-
347
- def get_meaning_formula(self, generic, word):
348
- """
349
- :param generic: A meaning formula string containing the
350
- parameter "<word>"
351
- :param word: The actual word to be replace "<word>"
352
- """
353
- word = word.replace('.', '')
354
- return generic.replace('<word>', word)
355
-
356
- def initialize_labels(self, expr, node, depgraph, unique_index):
357
- if isinstance(expr, linearlogic.AtomicExpression):
358
- name = self.find_label_name(expr.name.lower(), node, depgraph, unique_index)
359
- if name[0].isupper():
360
- return linearlogic.VariableExpression(name)
361
- else:
362
- return linearlogic.ConstantExpression(name)
363
- else:
364
- return linearlogic.ImpExpression(
365
- self.initialize_labels(expr.antecedent, node, depgraph, unique_index),
366
- self.initialize_labels(expr.consequent, node, depgraph, unique_index)
367
- )
368
-
369
- def find_label_name(self, name, node, depgraph, unique_index):
370
- try:
371
- dot = name.index('.')
372
-
373
- before_dot = name[:dot]
374
- after_dot = name[dot+1:]
375
- if before_dot == 'super':
376
- return self.find_label_name(after_dot, depgraph.nodes[node['head']], depgraph, unique_index)
377
- else:
378
- return self.find_label_name(after_dot, self.lookup_unique(before_dot, node, depgraph), depgraph, unique_index)
379
- except ValueError:
380
- lbl = self.get_label(node)
381
- if name == 'f':
382
- return lbl
383
- elif name == 'v':
384
- return '%sv' % lbl
385
- elif name == 'r':
386
- return '%sr' % lbl
387
- elif name == 'super':
388
- return self.get_label(depgraph.nodes[node['head']])
389
- elif name == 'var':
390
- return '%s%s' % (lbl.upper(), unique_index)
391
- elif name == 'a':
392
- return self.get_label(self.lookup_unique('conja', node, depgraph))
393
- elif name == 'b':
394
- return self.get_label(self.lookup_unique('conjb', node, depgraph))
395
- else:
396
- return self.get_label(self.lookup_unique(name, node, depgraph))
397
-
398
- def get_label(self, node):
399
- """
400
- Pick an alphabetic character as identifier for an entity in the model.
401
-
402
- :param value: where to index into the list of characters
403
- :type value: int
404
- """
405
- value = node['address']
406
-
407
- letter = ['f','g','h','i','j','k','l','m','n','o','p','q','r','s',
408
- 't','u','v','w','x','y','z','a','b','c','d','e'][value-1]
409
- num = int(value) // 26
410
- if num > 0:
411
- return letter + str(num)
412
- else:
413
- return letter
414
-
415
- def lookup_unique(self, rel, node, depgraph):
416
- """
417
- Lookup 'key'. There should be exactly one item in the associated relation.
418
- """
419
- deps = [
420
- depgraph.nodes[dep]
421
- for dep in sum(list(node['deps'].values()), [])
422
- if depgraph.nodes[dep]['rel'].lower() == rel.lower()
423
- ]
424
-
425
- if len(deps) == 0:
426
- raise KeyError("'%s' doesn't contain a feature '%s'" % (node['word'], rel))
427
- elif len(deps) > 1:
428
- raise KeyError("'%s' should only have one feature '%s'" % (node['word'], rel))
429
- else:
430
- return deps[0]
431
-
432
- def get_GlueFormula_factory(self):
433
- return GlueFormula
434
-
435
-
436
- class Glue(object):
437
- def __init__(self, semtype_file=None, remove_duplicates=False,
438
- depparser=None, verbose=False):
439
- self.verbose = verbose
440
- self.remove_duplicates = remove_duplicates
441
- self.depparser = depparser
442
-
443
- from nltk import Prover9
444
- self.prover = Prover9()
445
-
446
- if semtype_file:
447
- self.semtype_file = semtype_file
448
- else:
449
- self.semtype_file = os.path.join('grammars', 'sample_grammars','glue.semtype')
450
-
451
- def train_depparser(self, depgraphs=None):
452
- if depgraphs:
453
- self.depparser.train(depgraphs)
454
- else:
455
- self.depparser.train_from_file(nltk.data.find(
456
- os.path.join('grammars', 'sample_grammars',
457
- 'glue_train.conll')))
458
-
459
- def parse_to_meaning(self, sentence):
460
- readings = []
461
- for agenda in self.parse_to_compiled(sentence):
462
- readings.extend(self.get_readings(agenda))
463
- return readings
464
-
465
- def get_readings(self, agenda):
466
- readings = []
467
- agenda_length = len(agenda)
468
- atomics = dict()
469
- nonatomics = dict()
470
- while agenda: # is not empty
471
- cur = agenda.pop()
472
- glue_simp = cur.glue.simplify()
473
- if isinstance(glue_simp, linearlogic.ImpExpression): # if cur.glue is non-atomic
474
- for key in atomics:
475
- try:
476
- if isinstance(cur.glue, linearlogic.ApplicationExpression):
477
- bindings = cur.glue.bindings
478
- else:
479
- bindings = linearlogic.BindingDict()
480
- glue_simp.antecedent.unify(key, bindings)
481
- for atomic in atomics[key]:
482
- if not (cur.indices & atomic.indices): # if the sets of indices are disjoint
483
- try:
484
- agenda.append(cur.applyto(atomic))
485
- except linearlogic.LinearLogicApplicationException:
486
- pass
487
- except linearlogic.UnificationException:
488
- pass
489
- try:
490
- nonatomics[glue_simp.antecedent].append(cur)
491
- except KeyError:
492
- nonatomics[glue_simp.antecedent] = [cur]
493
-
494
- else: # else cur.glue is atomic
495
- for key in nonatomics:
496
- for nonatomic in nonatomics[key]:
497
- try:
498
- if isinstance(nonatomic.glue, linearlogic.ApplicationExpression):
499
- bindings = nonatomic.glue.bindings
500
- else:
501
- bindings = linearlogic.BindingDict()
502
- glue_simp.unify(key, bindings)
503
- if not (cur.indices & nonatomic.indices): # if the sets of indices are disjoint
504
- try:
505
- agenda.append(nonatomic.applyto(cur))
506
- except linearlogic.LinearLogicApplicationException:
507
- pass
508
- except linearlogic.UnificationException:
509
- pass
510
- try:
511
- atomics[glue_simp].append(cur)
512
- except KeyError:
513
- atomics[glue_simp] = [cur]
514
-
515
- for entry in atomics:
516
- for gf in atomics[entry]:
517
- if len(gf.indices) == agenda_length:
518
- self._add_to_reading_list(gf, readings)
519
- for entry in nonatomics:
520
- for gf in nonatomics[entry]:
521
- if len(gf.indices) == agenda_length:
522
- self._add_to_reading_list(gf, readings)
523
- return readings
524
-
525
- def _add_to_reading_list(self, glueformula, reading_list):
526
- add_reading = True
527
- if self.remove_duplicates:
528
- for reading in reading_list:
529
- try:
530
- if reading.equiv(glueformula.meaning, self.prover):
531
- add_reading = False
532
- break
533
- except Exception as e:
534
- #if there is an exception, the syntax of the formula
535
- #may not be understandable by the prover, so don't
536
- #throw out the reading.
537
- print('Error when checking logical equality of statements', e)
538
- pass
539
- if add_reading:
540
- reading_list.append(glueformula.meaning)
541
-
542
- def parse_to_compiled(self, sentence):
543
- gfls = [self.depgraph_to_glue(dg) for dg in self.dep_parse(sentence)]
544
- return [self.gfl_to_compiled(gfl) for gfl in gfls]
545
-
546
- def dep_parse(self, sentence):
547
- """
548
- Return a dependency graph for the sentence.
549
-
550
- :param sentence: the sentence to be parsed
551
- :type sentence: list(str)
552
- :rtype: DependencyGraph
553
- """
554
-
555
- #Lazy-initialize the depparser
556
- if self.depparser is None:
557
- from nltk.parse import MaltParser
558
- self.depparser = MaltParser(tagger=self.get_pos_tagger())
559
- if not self.depparser._trained:
560
- self.train_depparser()
561
- return self.depparser.parse(sentence, verbose=self.verbose)
562
-
563
- def depgraph_to_glue(self, depgraph):
564
- return self.get_glue_dict().to_glueformula_list(depgraph)
565
-
566
- def get_glue_dict(self):
567
- return GlueDict(self.semtype_file)
568
-
569
- def gfl_to_compiled(self, gfl):
570
- index_counter = Counter()
571
- return_list = []
572
- for gf in gfl:
573
- return_list.extend(gf.compile(index_counter))
574
-
575
- if self.verbose:
576
- print('Compiled Glue Premises:')
577
- for cgf in return_list:
578
- print(cgf)
579
-
580
- return return_list
581
-
582
- def get_pos_tagger(self):
583
- from nltk.corpus import brown
584
- regexp_tagger = RegexpTagger(
585
- [(r'^-?[0-9]+(.[0-9]+)?$', 'CD'), # cardinal numbers
586
- (r'(The|the|A|a|An|an)$', 'AT'), # articles
587
- (r'.*able$', 'JJ'), # adjectives
588
- (r'.*ness$', 'NN'), # nouns formed from adjectives
589
- (r'.*ly$', 'RB'), # adverbs
590
- (r'.*s$', 'NNS'), # plural nouns
591
- (r'.*ing$', 'VBG'), # gerunds
592
- (r'.*ed$', 'VBD'), # past tense verbs
593
- (r'.*', 'NN') # nouns (default)
594
- ])
595
- brown_train = brown.tagged_sents(categories='news')
596
- unigram_tagger = UnigramTagger(brown_train, backoff=regexp_tagger)
597
- bigram_tagger = BigramTagger(brown_train, backoff=unigram_tagger)
598
- trigram_tagger = TrigramTagger(brown_train, backoff=bigram_tagger)
599
-
600
- #Override particular words
601
- main_tagger = RegexpTagger(
602
- [(r'(A|a|An|an)$', 'ex_quant'),
603
- (r'(Every|every|All|all)$', 'univ_quant')
604
- ], backoff=trigram_tagger)
605
-
606
- return main_tagger
607
-
608
-
609
- class DrtGlueFormula(GlueFormula):
610
- def __init__(self, meaning, glue, indices=None):
611
- if not indices:
612
- indices = set()
613
-
614
- if isinstance(meaning, string_types):
615
- self.meaning = drt.DrtExpression.fromstring(meaning)
616
- elif isinstance(meaning, drt.DrtExpression):
617
- self.meaning = meaning
618
- else:
619
- raise RuntimeError('Meaning term neither string or expression: %s, %s' % (meaning, meaning.__class__))
620
-
621
- if isinstance(glue, string_types):
622
- self.glue = linearlogic.LinearLogicParser().parse(glue)
623
- elif isinstance(glue, linearlogic.Expression):
624
- self.glue = glue
625
- else:
626
- raise RuntimeError('Glue term neither string or expression: %s, %s' % (glue, glue.__class__))
627
-
628
- self.indices = indices
629
-
630
- def make_VariableExpression(self, name):
631
- return drt.DrtVariableExpression(name)
632
-
633
- def make_LambdaExpression(self, variable, term):
634
- return drt.DrtLambdaExpression(variable, term)
635
-
636
- class DrtGlueDict(GlueDict):
637
- def get_GlueFormula_factory(self):
638
- return DrtGlueFormula
639
-
640
- class DrtGlue(Glue):
641
- def __init__(self, semtype_file=None, remove_duplicates=False,
642
- depparser=None, verbose=False):
643
- if not semtype_file:
644
- semtype_file = os.path.join('grammars', 'sample_grammars','drt_glue.semtype')
645
- Glue.__init__(self, semtype_file, remove_duplicates, depparser, verbose)
646
-
647
- def get_glue_dict(self):
648
- return DrtGlueDict(self.semtype_file)
649
-
650
-
651
- def demo(show_example=-1):
652
- from nltk.parse import MaltParser
653
- examples = ['David sees Mary',
654
- 'David eats a sandwich',
655
- 'every man chases a dog',
656
- 'every man believes a dog sleeps',
657
- 'John gives David a sandwich',
658
- 'John chases himself']
659
- # 'John persuades David to order a pizza',
660
- # 'John tries to go',
661
- # 'John tries to find a unicorn',
662
- # 'John seems to vanish',
663
- # 'a unicorn seems to approach',
664
- # 'every big cat leaves',
665
- # 'every gray cat leaves',
666
- # 'every big gray cat leaves',
667
- # 'a former senator leaves',
668
-
669
- print('============== DEMO ==============')
670
-
671
- tagger = RegexpTagger(
672
- [('^(David|Mary|John)$', 'NNP'),
673
- ('^(sees|eats|chases|believes|gives|sleeps|chases|persuades|tries|seems|leaves)$', 'VB'),
674
- ('^(go|order|vanish|find|approach)$', 'VB'),
675
- ('^(a)$', 'ex_quant'),
676
- ('^(every)$', 'univ_quant'),
677
- ('^(sandwich|man|dog|pizza|unicorn|cat|senator)$', 'NN'),
678
- ('^(big|gray|former)$', 'JJ'),
679
- ('^(him|himself)$', 'PRP')
680
- ])
681
-
682
- depparser = MaltParser(tagger=tagger)
683
- glue = Glue(depparser=depparser, verbose=False)
684
-
685
- for (i, sentence) in enumerate(examples):
686
- if i==show_example or show_example==-1:
687
- print('[[[Example %s]]] %s' % (i, sentence))
688
- for reading in glue.parse_to_meaning(sentence.split()):
689
- print(reading.simplify())
690
- print('')
691
-
692
-
693
- if __name__ == '__main__':
694
- demo()
1
+ # Natural Language Toolkit: Glue Semantics
2
+ #
3
+ # Author: Dan Garrette <dhgarrette@gmail.com>
4
+ #
5
+ # Copyright (C) 2001-2018 NLTK Project
6
+ # URL: <http://nltk.org/>
7
+ # For license information, see LICENSE.TXT
8
+ from __future__ import print_function, division, unicode_literals
9
+
10
+ import os
11
+ from itertools import chain
12
+
13
+ from six import string_types
14
+
15
+ import nltk
16
+ from nltk.internals import Counter
17
+ from nltk.tag import UnigramTagger, BigramTagger, TrigramTagger, RegexpTagger
18
+ from nltk.sem.logic import (Expression, Variable, VariableExpression,
19
+ LambdaExpression, AbstractVariableExpression)
20
+ from nltk.compat import python_2_unicode_compatible
21
+ from nltk.sem import drt
22
+ from nltk.sem import linearlogic
23
+
24
+ SPEC_SEMTYPES = {'a' : 'ex_quant',
25
+ 'an' : 'ex_quant',
26
+ 'every' : 'univ_quant',
27
+ 'the' : 'def_art',
28
+ 'no' : 'no_quant',
29
+ 'default' : 'ex_quant'}
30
+
31
+ OPTIONAL_RELATIONSHIPS = ['nmod', 'vmod', 'punct']
32
+
33
+ @python_2_unicode_compatible
34
+ class GlueFormula(object):
35
+ def __init__(self, meaning, glue, indices=None):
36
+ if not indices:
37
+ indices = set()
38
+
39
+ if isinstance(meaning, string_types):
40
+ self.meaning = Expression.fromstring(meaning)
41
+ elif isinstance(meaning, Expression):
42
+ self.meaning = meaning
43
+ else:
44
+ raise RuntimeError('Meaning term neither string or expression: %s, %s' % (meaning, meaning.__class__))
45
+
46
+ if isinstance(glue, string_types):
47
+ self.glue = linearlogic.LinearLogicParser().parse(glue)
48
+ elif isinstance(glue, linearlogic.Expression):
49
+ self.glue = glue
50
+ else:
51
+ raise RuntimeError('Glue term neither string or expression: %s, %s' % (glue, glue.__class__))
52
+
53
+ self.indices = indices
54
+
55
+ def applyto(self, arg):
56
+ """ self = (\\x.(walk x), (subj -o f))
57
+ arg = (john , subj)
58
+ returns ((walk john), f)
59
+ """
60
+ if self.indices & arg.indices: # if the sets are NOT disjoint
61
+ raise linearlogic.LinearLogicApplicationException("'%s' applied to '%s'. Indices are not disjoint." % (self, arg))
62
+ else: # if the sets ARE disjoint
63
+ return_indices = (self.indices | arg.indices)
64
+
65
+ try:
66
+ return_glue = linearlogic.ApplicationExpression(self.glue, arg.glue, arg.indices)
67
+ except linearlogic.LinearLogicApplicationException:
68
+ raise linearlogic.LinearLogicApplicationException("'%s' applied to '%s'" % (self.simplify(), arg.simplify()))
69
+
70
+ arg_meaning_abstracted = arg.meaning
71
+ if return_indices:
72
+ for dep in self.glue.simplify().antecedent.dependencies[::-1]: # if self.glue is (A -o B), dep is in A.dependencies
73
+ arg_meaning_abstracted = self.make_LambdaExpression(Variable('v%s' % dep),
74
+ arg_meaning_abstracted)
75
+ return_meaning = self.meaning.applyto(arg_meaning_abstracted)
76
+
77
+ return self.__class__(return_meaning, return_glue, return_indices)
78
+
79
+ def make_VariableExpression(self, name):
80
+ return VariableExpression(name)
81
+
82
+ def make_LambdaExpression(self, variable, term):
83
+ return LambdaExpression(variable, term)
84
+
85
+ def lambda_abstract(self, other):
86
+ assert isinstance(other, GlueFormula)
87
+ assert isinstance(other.meaning, AbstractVariableExpression)
88
+ return self.__class__(self.make_LambdaExpression(other.meaning.variable,
89
+ self.meaning),
90
+ linearlogic.ImpExpression(other.glue, self.glue))
91
+
92
+ def compile(self, counter=None):
93
+ """From Iddo Lev's PhD Dissertation p108-109"""
94
+ if not counter:
95
+ counter = Counter()
96
+ (compiled_glue, new_forms) = self.glue.simplify().compile_pos(counter, self.__class__)
97
+ return new_forms + [self.__class__(self.meaning, compiled_glue, set([counter.get()]))]
98
+
99
+ def simplify(self):
100
+ return self.__class__(self.meaning.simplify(), self.glue.simplify(), self.indices)
101
+
102
+ def __eq__(self, other):
103
+ return self.__class__ == other.__class__ and self.meaning == other.meaning and self.glue == other.glue
104
+
105
+ def __ne__(self, other):
106
+ return not self == other
107
+
108
+ # sorting for use in doctests which must be deterministic
109
+ def __lt__(self, other):
110
+ return str(self) < str(other)
111
+
112
+ def __str__(self):
113
+ assert isinstance(self.indices, set)
114
+ accum = '%s : %s' % (self.meaning, self.glue)
115
+ if self.indices:
116
+ accum += ' : {' + ', '.join(str(index) for index in self.indices) + '}'
117
+ return accum
118
+
119
+ def __repr__(self):
120
+ return "%s" % self
121
+
122
+ @python_2_unicode_compatible
123
+ class GlueDict(dict):
124
+ def __init__(self, filename, encoding=None):
125
+ self.filename = filename
126
+ self.file_encoding = encoding
127
+ self.read_file()
128
+
129
+ def read_file(self, empty_first=True):
130
+ if empty_first:
131
+ self.clear()
132
+
133
+ try:
134
+ contents = nltk.data.load(self.filename, format='text', encoding=self.file_encoding)
135
+ # TODO: the above can't handle zip files, but this should anyway be fixed in nltk.data.load()
136
+ except LookupError as e:
137
+ try:
138
+ contents = nltk.data.load('file:' + self.filename, format='text', encoding=self.file_encoding)
139
+ except LookupError:
140
+ raise e
141
+ lines = contents.splitlines()
142
+
143
+ for line in lines: # example: 'n : (\\x.(<word> x), (v-or))'
144
+ # lambdacalc -^ linear logic -^
145
+ line = line.strip() # remove trailing newline
146
+ if not len(line): continue # skip empty lines
147
+ if line[0] == '#': continue # skip commented out lines
148
+
149
+ parts = line.split(' : ', 2) # ['verb', '(\\x.(<word> x), ( subj -o f ))', '[subj]']
150
+
151
+ glue_formulas = []
152
+ paren_count = 0
153
+ tuple_start = 0
154
+ tuple_comma = 0
155
+
156
+ relationships = None
157
+
158
+ if len(parts) > 1:
159
+ for (i, c) in enumerate(parts[1]):
160
+ if c == '(':
161
+ if paren_count == 0: # if it's the first '(' of a tuple
162
+ tuple_start = i+1 # then save the index
163
+ paren_count += 1
164
+ elif c == ')':
165
+ paren_count -= 1
166
+ if paren_count == 0: # if it's the last ')' of a tuple
167
+ meaning_term = parts[1][tuple_start:tuple_comma] # '\\x.(<word> x)'
168
+ glue_term = parts[1][tuple_comma+1:i] # '(v-r)'
169
+ glue_formulas.append([meaning_term, glue_term]) # add the GlueFormula to the list
170
+ elif c == ',':
171
+ if paren_count == 1: # if it's a comma separating the parts of the tuple
172
+ tuple_comma = i # then save the index
173
+ elif c == '#': # skip comments at the ends of lines
174
+ if paren_count != 0: # if the line hasn't parsed correctly so far
175
+ raise RuntimeError('Formula syntax is incorrect for entry ' + line)
176
+ break # break to the next line
177
+
178
+ if len(parts) > 2: #if there is a relationship entry at the end
179
+ rel_start = parts[2].index('[')+1
180
+ rel_end = parts[2].index(']')
181
+ if rel_start == rel_end:
182
+ relationships = frozenset()
183
+ else:
184
+ relationships = frozenset(r.strip() for r in parts[2][rel_start:rel_end].split(','))
185
+
186
+ try:
187
+ start_inheritance = parts[0].index('(')
188
+ end_inheritance = parts[0].index(')')
189
+ sem = parts[0][:start_inheritance].strip()
190
+ supertype = parts[0][start_inheritance+1:end_inheritance]
191
+ except:
192
+ sem = parts[0].strip()
193
+ supertype = None
194
+
195
+ if sem not in self:
196
+ self[sem] = {}
197
+
198
+ if relationships is None: #if not specified for a specific relationship set
199
+ #add all relationship entries for parents
200
+ if supertype:
201
+ for rels in self[supertype]:
202
+ if rels not in self[sem]:
203
+ self[sem][rels] = []
204
+ glue = self[supertype][rels]
205
+ self[sem][rels].extend(glue)
206
+ self[sem][rels].extend(glue_formulas) # add the glue formulas to every rel entry
207
+ else:
208
+ if None not in self[sem]:
209
+ self[sem][None] = []
210
+ self[sem][None].extend(glue_formulas) # add the glue formulas to every rel entry
211
+ else:
212
+ if relationships not in self[sem]:
213
+ self[sem][relationships] = []
214
+ if supertype:
215
+ self[sem][relationships].extend(self[supertype][relationships])
216
+ self[sem][relationships].extend(glue_formulas) # add the glue entry to the dictionary
217
+
218
+ def __str__(self):
219
+ accum = ''
220
+ for pos in self:
221
+ str_pos = "%s" % pos
222
+ for relset in self[pos]:
223
+ i = 1
224
+ for gf in self[pos][relset]:
225
+ if i == 1:
226
+ accum += str_pos + ': '
227
+ else:
228
+ accum += ' '*(len(str_pos)+2)
229
+ accum += "%s" % gf
230
+ if relset and i == len(self[pos][relset]):
231
+ accum += ' : %s' % relset
232
+ accum += '\n'
233
+ i += 1
234
+ return accum
235
+
236
+ def to_glueformula_list(self, depgraph, node=None, counter=None, verbose=False):
237
+ if node is None:
238
+ # TODO: should it be depgraph.root? Is this code tested?
239
+ top = depgraph.nodes[0]
240
+ depList = list(chain(*top['deps'].values()))
241
+ root = depgraph.nodes[depList[0]]
242
+
243
+ return self.to_glueformula_list(depgraph, root, Counter(), verbose)
244
+
245
+ glueformulas = self.lookup(node, depgraph, counter)
246
+ for dep_idx in chain(*node['deps'].values()):
247
+ dep = depgraph.nodes[dep_idx]
248
+ glueformulas.extend(self.to_glueformula_list(depgraph, dep, counter, verbose))
249
+ return glueformulas
250
+
251
+ def lookup(self, node, depgraph, counter):
252
+ semtype_names = self.get_semtypes(node)
253
+
254
+ semtype = None
255
+ for name in semtype_names:
256
+ if name in self:
257
+ semtype = self[name]
258
+ break
259
+ if semtype is None:
260
+ # raise KeyError, "There is no GlueDict entry for sem type '%s' (for '%s')" % (sem, word)
261
+ return []
262
+
263
+ self.add_missing_dependencies(node, depgraph)
264
+
265
+ lookup = self._lookup_semtype_option(semtype, node, depgraph)
266
+
267
+ if not len(lookup):
268
+ raise KeyError(
269
+ "There is no GlueDict entry for sem type of '%s' "
270
+ "with tag '%s', and rel '%s'" %
271
+ (node['word'], node['tag'], node['rel'])
272
+ )
273
+
274
+ return self.get_glueformulas_from_semtype_entry(lookup, node['word'], node, depgraph, counter)
275
+
276
+ def add_missing_dependencies(self, node, depgraph):
277
+ rel = node['rel'].lower()
278
+
279
+ if rel == 'main':
280
+ headnode = depgraph.nodes[node['head']]
281
+ subj = self.lookup_unique('subj', headnode, depgraph)
282
+ relation = subj['rel']
283
+ node['deps'].setdefault(relation,[])
284
+ node['deps'][relation].append(subj['address'])
285
+ #node['deps'].append(subj['address'])
286
+
287
+ def _lookup_semtype_option(self, semtype, node, depgraph):
288
+ relationships = frozenset(
289
+ depgraph.nodes[dep]['rel'].lower()
290
+ for dep in chain(*node['deps'].values())
291
+ if depgraph.nodes[dep]['rel'].lower() not in OPTIONAL_RELATIONSHIPS
292
+ )
293
+
294
+ try:
295
+ lookup = semtype[relationships]
296
+ except KeyError:
297
+ # An exact match is not found, so find the best match where
298
+ # 'best' is defined as the glue entry whose relationship set has the
299
+ # most relations of any possible relationship set that is a subset
300
+ # of the actual depgraph
301
+ best_match = frozenset()
302
+ for relset_option in set(semtype)-set([None]):
303
+ if len(relset_option) > len(best_match) and \
304
+ relset_option < relationships:
305
+ best_match = relset_option
306
+ if not best_match:
307
+ if None in semtype:
308
+ best_match = None
309
+ else:
310
+ return None
311
+ lookup = semtype[best_match]
312
+
313
+ return lookup
314
+
315
+ def get_semtypes(self, node):
316
+ """
317
+ Based on the node, return a list of plausible semtypes in order of
318
+ plausibility.
319
+ """
320
+ rel = node['rel'].lower()
321
+ word = node['word'].lower()
322
+
323
+ if rel == 'spec':
324
+ if word in SPEC_SEMTYPES:
325
+ return [SPEC_SEMTYPES[word]]
326
+ else:
327
+ return [SPEC_SEMTYPES['default']]
328
+ elif rel in ['nmod', 'vmod']:
329
+ return [node['tag'], rel]
330
+ else:
331
+ return [node['tag']]
332
+
333
+ def get_glueformulas_from_semtype_entry(self, lookup, word, node, depgraph, counter):
334
+ glueformulas = []
335
+
336
+ glueFormulaFactory = self.get_GlueFormula_factory()
337
+ for meaning, glue in lookup:
338
+ gf = glueFormulaFactory(self.get_meaning_formula(meaning, word), glue)
339
+ if not len(glueformulas):
340
+ gf.word = word
341
+ else:
342
+ gf.word = '%s%s' % (word, len(glueformulas)+1)
343
+
344
+ gf.glue = self.initialize_labels(gf.glue, node, depgraph, counter.get())
345
+
346
+ glueformulas.append(gf)
347
+ return glueformulas
348
+
349
+ def get_meaning_formula(self, generic, word):
350
+ """
351
+ :param generic: A meaning formula string containing the
352
+ parameter "<word>"
353
+ :param word: The actual word to be replace "<word>"
354
+ """
355
+ word = word.replace('.', '')
356
+ return generic.replace('<word>', word)
357
+
358
+ def initialize_labels(self, expr, node, depgraph, unique_index):
359
+ if isinstance(expr, linearlogic.AtomicExpression):
360
+ name = self.find_label_name(expr.name.lower(), node, depgraph, unique_index)
361
+ if name[0].isupper():
362
+ return linearlogic.VariableExpression(name)
363
+ else:
364
+ return linearlogic.ConstantExpression(name)
365
+ else:
366
+ return linearlogic.ImpExpression(
367
+ self.initialize_labels(expr.antecedent, node, depgraph, unique_index),
368
+ self.initialize_labels(expr.consequent, node, depgraph, unique_index)
369
+ )
370
+
371
+ def find_label_name(self, name, node, depgraph, unique_index):
372
+ try:
373
+ dot = name.index('.')
374
+
375
+ before_dot = name[:dot]
376
+ after_dot = name[dot+1:]
377
+ if before_dot == 'super':
378
+ return self.find_label_name(after_dot, depgraph.nodes[node['head']], depgraph, unique_index)
379
+ else:
380
+ return self.find_label_name(after_dot, self.lookup_unique(before_dot, node, depgraph), depgraph, unique_index)
381
+ except ValueError:
382
+ lbl = self.get_label(node)
383
+ if name == 'f':
384
+ return lbl
385
+ elif name == 'v':
386
+ return '%sv' % lbl
387
+ elif name == 'r':
388
+ return '%sr' % lbl
389
+ elif name == 'super':
390
+ return self.get_label(depgraph.nodes[node['head']])
391
+ elif name == 'var':
392
+ return '%s%s' % (lbl.upper(), unique_index)
393
+ elif name == 'a':
394
+ return self.get_label(self.lookup_unique('conja', node, depgraph))
395
+ elif name == 'b':
396
+ return self.get_label(self.lookup_unique('conjb', node, depgraph))
397
+ else:
398
+ return self.get_label(self.lookup_unique(name, node, depgraph))
399
+
400
+ def get_label(self, node):
401
+ """
402
+ Pick an alphabetic character as identifier for an entity in the model.
403
+
404
+ :param value: where to index into the list of characters
405
+ :type value: int
406
+ """
407
+ value = node['address']
408
+
409
+ letter = ['f','g','h','i','j','k','l','m','n','o','p','q','r','s',
410
+ 't','u','v','w','x','y','z','a','b','c','d','e'][value-1]
411
+ num = int(value) // 26
412
+ if num > 0:
413
+ return letter + str(num)
414
+ else:
415
+ return letter
416
+
417
+ def lookup_unique(self, rel, node, depgraph):
418
+ """
419
+ Lookup 'key'. There should be exactly one item in the associated relation.
420
+ """
421
+ deps = [
422
+ depgraph.nodes[dep]
423
+ for dep in chain(*node['deps'].values())
424
+ if depgraph.nodes[dep]['rel'].lower() == rel.lower()
425
+ ]
426
+
427
+ if len(deps) == 0:
428
+ raise KeyError("'%s' doesn't contain a feature '%s'" % (node['word'], rel))
429
+ elif len(deps) > 1:
430
+ raise KeyError("'%s' should only have one feature '%s'" % (node['word'], rel))
431
+ else:
432
+ return deps[0]
433
+
434
+ def get_GlueFormula_factory(self):
435
+ return GlueFormula
436
+
437
+
438
+ class Glue(object):
439
+ def __init__(self, semtype_file=None, remove_duplicates=False,
440
+ depparser=None, verbose=False):
441
+ self.verbose = verbose
442
+ self.remove_duplicates = remove_duplicates
443
+ self.depparser = depparser
444
+
445
+ from nltk import Prover9
446
+ self.prover = Prover9()
447
+
448
+ if semtype_file:
449
+ self.semtype_file = semtype_file
450
+ else:
451
+ self.semtype_file = os.path.join('grammars', 'sample_grammars','glue.semtype')
452
+
453
+ def train_depparser(self, depgraphs=None):
454
+ if depgraphs:
455
+ self.depparser.train(depgraphs)
456
+ else:
457
+ self.depparser.train_from_file(nltk.data.find(
458
+ os.path.join('grammars', 'sample_grammars',
459
+ 'glue_train.conll')))
460
+
461
+ def parse_to_meaning(self, sentence):
462
+ readings = []
463
+ for agenda in self.parse_to_compiled(sentence):
464
+ readings.extend(self.get_readings(agenda))
465
+ return readings
466
+
467
+ def get_readings(self, agenda):
468
+ readings = []
469
+ agenda_length = len(agenda)
470
+ atomics = dict()
471
+ nonatomics = dict()
472
+ while agenda: # is not empty
473
+ cur = agenda.pop()
474
+ glue_simp = cur.glue.simplify()
475
+ if isinstance(glue_simp, linearlogic.ImpExpression): # if cur.glue is non-atomic
476
+ for key in atomics:
477
+ try:
478
+ if isinstance(cur.glue, linearlogic.ApplicationExpression):
479
+ bindings = cur.glue.bindings
480
+ else:
481
+ bindings = linearlogic.BindingDict()
482
+ glue_simp.antecedent.unify(key, bindings)
483
+ for atomic in atomics[key]:
484
+ if not (cur.indices & atomic.indices): # if the sets of indices are disjoint
485
+ try:
486
+ agenda.append(cur.applyto(atomic))
487
+ except linearlogic.LinearLogicApplicationException:
488
+ pass
489
+ except linearlogic.UnificationException:
490
+ pass
491
+ try:
492
+ nonatomics[glue_simp.antecedent].append(cur)
493
+ except KeyError:
494
+ nonatomics[glue_simp.antecedent] = [cur]
495
+
496
+ else: # else cur.glue is atomic
497
+ for key in nonatomics:
498
+ for nonatomic in nonatomics[key]:
499
+ try:
500
+ if isinstance(nonatomic.glue, linearlogic.ApplicationExpression):
501
+ bindings = nonatomic.glue.bindings
502
+ else:
503
+ bindings = linearlogic.BindingDict()
504
+ glue_simp.unify(key, bindings)
505
+ if not (cur.indices & nonatomic.indices): # if the sets of indices are disjoint
506
+ try:
507
+ agenda.append(nonatomic.applyto(cur))
508
+ except linearlogic.LinearLogicApplicationException:
509
+ pass
510
+ except linearlogic.UnificationException:
511
+ pass
512
+ try:
513
+ atomics[glue_simp].append(cur)
514
+ except KeyError:
515
+ atomics[glue_simp] = [cur]
516
+
517
+ for entry in atomics:
518
+ for gf in atomics[entry]:
519
+ if len(gf.indices) == agenda_length:
520
+ self._add_to_reading_list(gf, readings)
521
+ for entry in nonatomics:
522
+ for gf in nonatomics[entry]:
523
+ if len(gf.indices) == agenda_length:
524
+ self._add_to_reading_list(gf, readings)
525
+ return readings
526
+
527
+ def _add_to_reading_list(self, glueformula, reading_list):
528
+ add_reading = True
529
+ if self.remove_duplicates:
530
+ for reading in reading_list:
531
+ try:
532
+ if reading.equiv(glueformula.meaning, self.prover):
533
+ add_reading = False
534
+ break
535
+ except Exception as e:
536
+ #if there is an exception, the syntax of the formula
537
+ #may not be understandable by the prover, so don't
538
+ #throw out the reading.
539
+ print('Error when checking logical equality of statements', e)
540
+ pass
541
+ if add_reading:
542
+ reading_list.append(glueformula.meaning)
543
+
544
+ def parse_to_compiled(self, sentence):
545
+ gfls = [self.depgraph_to_glue(dg) for dg in self.dep_parse(sentence)]
546
+ return [self.gfl_to_compiled(gfl) for gfl in gfls]
547
+
548
+ def dep_parse(self, sentence):
549
+ """
550
+ Return a dependency graph for the sentence.
551
+
552
+ :param sentence: the sentence to be parsed
553
+ :type sentence: list(str)
554
+ :rtype: DependencyGraph
555
+ """
556
+
557
+ #Lazy-initialize the depparser
558
+ if self.depparser is None:
559
+ from nltk.parse import MaltParser
560
+ self.depparser = MaltParser(tagger=self.get_pos_tagger())
561
+ if not self.depparser._trained:
562
+ self.train_depparser()
563
+ return self.depparser.parse(sentence, verbose=self.verbose)
564
+
565
+ def depgraph_to_glue(self, depgraph):
566
+ return self.get_glue_dict().to_glueformula_list(depgraph)
567
+
568
+ def get_glue_dict(self):
569
+ return GlueDict(self.semtype_file)
570
+
571
+ def gfl_to_compiled(self, gfl):
572
+ index_counter = Counter()
573
+ return_list = []
574
+ for gf in gfl:
575
+ return_list.extend(gf.compile(index_counter))
576
+
577
+ if self.verbose:
578
+ print('Compiled Glue Premises:')
579
+ for cgf in return_list:
580
+ print(cgf)
581
+
582
+ return return_list
583
+
584
+ def get_pos_tagger(self):
585
+ from nltk.corpus import brown
586
+ regexp_tagger = RegexpTagger(
587
+ [(r'^-?[0-9]+(.[0-9]+)?$', 'CD'), # cardinal numbers
588
+ (r'(The|the|A|a|An|an)$', 'AT'), # articles
589
+ (r'.*able$', 'JJ'), # adjectives
590
+ (r'.*ness$', 'NN'), # nouns formed from adjectives
591
+ (r'.*ly$', 'RB'), # adverbs
592
+ (r'.*s$', 'NNS'), # plural nouns
593
+ (r'.*ing$', 'VBG'), # gerunds
594
+ (r'.*ed$', 'VBD'), # past tense verbs
595
+ (r'.*', 'NN') # nouns (default)
596
+ ])
597
+ brown_train = brown.tagged_sents(categories='news')
598
+ unigram_tagger = UnigramTagger(brown_train, backoff=regexp_tagger)
599
+ bigram_tagger = BigramTagger(brown_train, backoff=unigram_tagger)
600
+ trigram_tagger = TrigramTagger(brown_train, backoff=bigram_tagger)
601
+
602
+ #Override particular words
603
+ main_tagger = RegexpTagger(
604
+ [(r'(A|a|An|an)$', 'ex_quant'),
605
+ (r'(Every|every|All|all)$', 'univ_quant')
606
+ ], backoff=trigram_tagger)
607
+
608
+ return main_tagger
609
+
610
+
611
+ class DrtGlueFormula(GlueFormula):
612
+ def __init__(self, meaning, glue, indices=None):
613
+ if not indices:
614
+ indices = set()
615
+
616
+ if isinstance(meaning, string_types):
617
+ self.meaning = drt.DrtExpression.fromstring(meaning)
618
+ elif isinstance(meaning, drt.DrtExpression):
619
+ self.meaning = meaning
620
+ else:
621
+ raise RuntimeError('Meaning term neither string or expression: %s, %s' % (meaning, meaning.__class__))
622
+
623
+ if isinstance(glue, string_types):
624
+ self.glue = linearlogic.LinearLogicParser().parse(glue)
625
+ elif isinstance(glue, linearlogic.Expression):
626
+ self.glue = glue
627
+ else:
628
+ raise RuntimeError('Glue term neither string or expression: %s, %s' % (glue, glue.__class__))
629
+
630
+ self.indices = indices
631
+
632
+ def make_VariableExpression(self, name):
633
+ return drt.DrtVariableExpression(name)
634
+
635
+ def make_LambdaExpression(self, variable, term):
636
+ return drt.DrtLambdaExpression(variable, term)
637
+
638
+ class DrtGlueDict(GlueDict):
639
+ def get_GlueFormula_factory(self):
640
+ return DrtGlueFormula
641
+
642
+ class DrtGlue(Glue):
643
+ def __init__(self, semtype_file=None, remove_duplicates=False,
644
+ depparser=None, verbose=False):
645
+ if not semtype_file:
646
+ semtype_file = os.path.join('grammars', 'sample_grammars','drt_glue.semtype')
647
+ Glue.__init__(self, semtype_file, remove_duplicates, depparser, verbose)
648
+
649
+ def get_glue_dict(self):
650
+ return DrtGlueDict(self.semtype_file)
651
+
652
+
653
+ def demo(show_example=-1):
654
+ from nltk.parse import MaltParser
655
+ examples = ['David sees Mary',
656
+ 'David eats a sandwich',
657
+ 'every man chases a dog',
658
+ 'every man believes a dog sleeps',
659
+ 'John gives David a sandwich',
660
+ 'John chases himself']
661
+ # 'John persuades David to order a pizza',
662
+ # 'John tries to go',
663
+ # 'John tries to find a unicorn',
664
+ # 'John seems to vanish',
665
+ # 'a unicorn seems to approach',
666
+ # 'every big cat leaves',
667
+ # 'every gray cat leaves',
668
+ # 'every big gray cat leaves',
669
+ # 'a former senator leaves',
670
+
671
+ print('============== DEMO ==============')
672
+
673
+ tagger = RegexpTagger(
674
+ [('^(David|Mary|John)$', 'NNP'),
675
+ ('^(sees|eats|chases|believes|gives|sleeps|chases|persuades|tries|seems|leaves)$', 'VB'),
676
+ ('^(go|order|vanish|find|approach)$', 'VB'),
677
+ ('^(a)$', 'ex_quant'),
678
+ ('^(every)$', 'univ_quant'),
679
+ ('^(sandwich|man|dog|pizza|unicorn|cat|senator)$', 'NN'),
680
+ ('^(big|gray|former)$', 'JJ'),
681
+ ('^(him|himself)$', 'PRP')
682
+ ])
683
+
684
+ depparser = MaltParser(tagger=tagger)
685
+ glue = Glue(depparser=depparser, verbose=False)
686
+
687
+ for (i, sentence) in enumerate(examples):
688
+ if i==show_example or show_example==-1:
689
+ print('[[[Example %s]]] %s' % (i, sentence))
690
+ for reading in glue.parse_to_meaning(sentence.split()):
691
+ print(reading.simplify())
692
+ print('')
693
+
694
+
695
+ if __name__ == '__main__':
696
+ demo()