nltk 3.2__zip → 3.3__zip

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (406) hide show
  1. {nltk-3.2 → nltk-3.3}/INSTALL.txt +6 -6
  2. {nltk-3.2 → nltk-3.3}/LICENSE.txt +13 -13
  3. {nltk-3.2 → nltk-3.3}/MANIFEST.in +6 -6
  4. {nltk-3.2/nltk.egg-info → nltk-3.3}/PKG-INFO +13 -5
  5. nltk-3.2/README.txt → nltk-3.3/README.md +49 -48
  6. nltk-3.3/nltk/VERSION +1 -0
  7. {nltk-3.2 → nltk-3.3}/nltk/__init__.py +185 -185
  8. {nltk-3.2 → nltk-3.3}/nltk/app/__init__.py +52 -53
  9. {nltk-3.2 → nltk-3.3}/nltk/app/chartparser_app.py +2276 -2276
  10. {nltk-3.2 → nltk-3.3}/nltk/app/chunkparser_app.py +1262 -1264
  11. {nltk-3.2 → nltk-3.3}/nltk/app/collocations_app.py +347 -350
  12. {nltk-3.2 → nltk-3.3}/nltk/app/concordance_app.py +568 -570
  13. {nltk-3.2 → nltk-3.3}/nltk/app/nemo_app.py +156 -155
  14. {nltk-3.2 → nltk-3.3}/nltk/app/rdparser_app.py +892 -894
  15. {nltk-3.2 → nltk-3.3}/nltk/app/srparser_app.py +809 -809
  16. {nltk-3.2 → nltk-3.3}/nltk/app/wordfreq_app.py +32 -32
  17. {nltk-3.2 → nltk-3.3}/nltk/app/wordnet_app.py +968 -968
  18. {nltk-3.2 → nltk-3.3}/nltk/book.py +91 -90
  19. {nltk-3.2 → nltk-3.3}/nltk/ccg/__init__.py +22 -22
  20. {nltk-3.2 → nltk-3.3}/nltk/ccg/api.py +359 -335
  21. {nltk-3.2 → nltk-3.3}/nltk/ccg/chart.py +394 -361
  22. {nltk-3.2 → nltk-3.3}/nltk/ccg/combinator.py +343 -312
  23. {nltk-3.2 → nltk-3.3}/nltk/ccg/lexicon.py +328 -290
  24. nltk-3.3/nltk/ccg/logic.py +46 -0
  25. {nltk-3.2 → nltk-3.3}/nltk/chat/__init__.py +49 -49
  26. {nltk-3.2 → nltk-3.3}/nltk/chat/eliza.py +244 -244
  27. {nltk-3.2 → nltk-3.3}/nltk/chat/iesha.py +140 -140
  28. {nltk-3.2 → nltk-3.3}/nltk/chat/rude.py +92 -92
  29. {nltk-3.2 → nltk-3.3}/nltk/chat/suntsu.py +117 -117
  30. {nltk-3.2 → nltk-3.3}/nltk/chat/util.py +122 -120
  31. {nltk-3.2 → nltk-3.3}/nltk/chat/zen.py +282 -282
  32. {nltk-3.2 → nltk-3.3}/nltk/chunk/__init__.py +190 -190
  33. {nltk-3.2 → nltk-3.3}/nltk/chunk/api.py +51 -51
  34. {nltk-3.2 → nltk-3.3}/nltk/chunk/named_entity.py +331 -331
  35. {nltk-3.2 → nltk-3.3}/nltk/chunk/regexp.py +1390 -1384
  36. {nltk-3.2 → nltk-3.3}/nltk/chunk/util.py +600 -600
  37. {nltk-3.2 → nltk-3.3}/nltk/classify/__init__.py +98 -98
  38. {nltk-3.2 → nltk-3.3}/nltk/classify/api.py +193 -193
  39. {nltk-3.2 → nltk-3.3}/nltk/classify/decisiontree.py +295 -295
  40. {nltk-3.2 → nltk-3.3}/nltk/classify/maxent.py +1510 -1492
  41. {nltk-3.2 → nltk-3.3}/nltk/classify/megam.py +179 -178
  42. {nltk-3.2 → nltk-3.3}/nltk/classify/naivebayes.py +245 -242
  43. {nltk-3.2 → nltk-3.3}/nltk/classify/positivenaivebayes.py +170 -170
  44. {nltk-3.2 → nltk-3.3}/nltk/classify/rte_classify.py +160 -191
  45. {nltk-3.2 → nltk-3.3}/nltk/classify/scikitlearn.py +153 -151
  46. {nltk-3.2 → nltk-3.3}/nltk/classify/senna.py +183 -181
  47. {nltk-3.2 → nltk-3.3}/nltk/classify/svm.py +15 -15
  48. {nltk-3.2 → nltk-3.3}/nltk/classify/tadm.py +113 -112
  49. {nltk-3.2 → nltk-3.3}/nltk/classify/textcat.py +193 -193
  50. {nltk-3.2 → nltk-3.3}/nltk/classify/util.py +324 -312
  51. {nltk-3.2 → nltk-3.3}/nltk/classify/weka.py +346 -345
  52. {nltk-3.2 → nltk-3.3}/nltk/cluster/__init__.py +86 -86
  53. {nltk-3.2 → nltk-3.3}/nltk/cluster/api.py +74 -70
  54. {nltk-3.2 → nltk-3.3}/nltk/cluster/em.py +247 -247
  55. {nltk-3.2 → nltk-3.3}/nltk/cluster/gaac.py +168 -168
  56. {nltk-3.2 → nltk-3.3}/nltk/cluster/kmeans.py +221 -221
  57. {nltk-3.2 → nltk-3.3}/nltk/cluster/util.py +304 -289
  58. nltk-3.3/nltk/collections.py +640 -0
  59. {nltk-3.2 → nltk-3.3}/nltk/collocations.py +388 -388
  60. nltk-3.3/nltk/compat.py +365 -0
  61. {nltk-3.2 → nltk-3.3}/nltk/corpus/__init__.py +341 -320
  62. {nltk-3.2 → nltk-3.3}/nltk/corpus/europarl_raw.py +44 -44
  63. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/__init__.py +147 -145
  64. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/aligned.py +115 -114
  65. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/api.py +448 -446
  66. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/bnc.py +252 -252
  67. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/bracket_parse.py +227 -227
  68. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/categorized_sents.py +179 -178
  69. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/chasen.py +140 -139
  70. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/childes.py +522 -494
  71. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/chunked.py +212 -212
  72. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/cmudict.py +97 -95
  73. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/comparative_sents.py +280 -278
  74. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/conll.py +528 -521
  75. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/crubadan.py +116 -116
  76. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/dependency.py +101 -101
  77. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/framenet.py +3059 -2093
  78. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ieer.py +112 -111
  79. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/indian.py +85 -86
  80. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ipipan.py +332 -331
  81. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/knbc.py +187 -187
  82. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/lin.py +156 -156
  83. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/mte.py +311 -310
  84. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/nkjp.py +429 -428
  85. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/nombank.py +421 -421
  86. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/nps_chat.py +73 -73
  87. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/opinion_lexicon.py +115 -115
  88. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/panlex_lite.py +165 -165
  89. nltk-3.3/nltk/corpus/reader/pl196x.py +292 -0
  90. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/plaintext.py +232 -233
  91. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ppattach.py +96 -95
  92. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/propbank.py +479 -481
  93. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/pros_cons.py +128 -126
  94. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/reviews.py +330 -328
  95. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/rte.py +144 -146
  96. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/semcor.py +256 -256
  97. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/senseval.py +203 -201
  98. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/sentiwordnet.py +137 -137
  99. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/sinica_treebank.py +75 -75
  100. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/string_category.py +60 -59
  101. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/switchboard.py +119 -119
  102. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/tagged.py +295 -294
  103. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/timit.py +452 -450
  104. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/toolbox.py +68 -68
  105. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/twitter.py +155 -157
  106. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/udhr.py +80 -80
  107. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/util.py +804 -800
  108. nltk-3.3/nltk/corpus/reader/verbnet.py +595 -0
  109. nltk-3.3/nltk/corpus/reader/wordlist.py +134 -0
  110. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/wordnet.py +2108 -2036
  111. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/xmldocs.py +392 -387
  112. {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ycoe.py +243 -242
  113. {nltk-3.2 → nltk-3.3}/nltk/corpus/util.py +144 -127
  114. {nltk-3.2 → nltk-3.3}/nltk/data.py +1487 -1451
  115. {nltk-3.2 → nltk-3.3}/nltk/decorators.py +219 -215
  116. {nltk-3.2 → nltk-3.3}/nltk/downloader.py +2278 -2274
  117. {nltk-3.2 → nltk-3.3}/nltk/draw/__init__.py +27 -27
  118. {nltk-3.2 → nltk-3.3}/nltk/draw/cfg.py +774 -777
  119. {nltk-3.2 → nltk-3.3}/nltk/draw/dispersion.py +58 -58
  120. {nltk-3.2 → nltk-3.3}/nltk/draw/table.py +1098 -1098
  121. {nltk-3.2 → nltk-3.3}/nltk/draw/tree.py +963 -966
  122. {nltk-3.2 → nltk-3.3}/nltk/draw/util.py +2356 -2356
  123. {nltk-3.2 → nltk-3.3}/nltk/featstruct.py +2617 -2500
  124. {nltk-3.2 → nltk-3.3}/nltk/grammar.py +1559 -1516
  125. {nltk-3.2 → nltk-3.3}/nltk/help.py +56 -56
  126. {nltk-3.2 → nltk-3.3}/nltk/inference/__init__.py +20 -20
  127. {nltk-3.2 → nltk-3.3}/nltk/inference/api.py +597 -589
  128. {nltk-3.2 → nltk-3.3}/nltk/inference/discourse.py +612 -608
  129. {nltk-3.2 → nltk-3.3}/nltk/inference/mace.py +311 -311
  130. {nltk-3.2 → nltk-3.3}/nltk/inference/nonmonotonic.py +509 -509
  131. {nltk-3.2 → nltk-3.3}/nltk/inference/prover9.py +431 -431
  132. {nltk-3.2 → nltk-3.3}/nltk/inference/resolution.py +687 -687
  133. {nltk-3.2 → nltk-3.3}/nltk/inference/tableau.py +607 -607
  134. {nltk-3.2 → nltk-3.3}/nltk/internals.py +982 -982
  135. {nltk-3.2 → nltk-3.3}/nltk/jsontags.py +63 -63
  136. {nltk-3.2 → nltk-3.3}/nltk/lazyimport.py +142 -142
  137. {nltk-3.2 → nltk-3.3}/nltk/metrics/__init__.py +30 -29
  138. {nltk-3.2 → nltk-3.3}/nltk/metrics/agreement.py +421 -401
  139. nltk-3.3/nltk/metrics/aline.py +607 -0
  140. {nltk-3.2 → nltk-3.3}/nltk/metrics/association.py +414 -410
  141. {nltk-3.2 → nltk-3.3}/nltk/metrics/confusionmatrix.py +206 -206
  142. {nltk-3.2 → nltk-3.3}/nltk/metrics/distance.py +207 -202
  143. {nltk-3.2 → nltk-3.3}/nltk/metrics/paice.py +381 -381
  144. {nltk-3.2 → nltk-3.3}/nltk/metrics/scores.py +228 -228
  145. {nltk-3.2 → nltk-3.3}/nltk/metrics/segmentation.py +231 -232
  146. {nltk-3.2 → nltk-3.3}/nltk/metrics/spearman.py +69 -69
  147. {nltk-3.2 → nltk-3.3}/nltk/misc/__init__.py +11 -11
  148. {nltk-3.2 → nltk-3.3}/nltk/misc/babelfish.py +10 -10
  149. {nltk-3.2 → nltk-3.3}/nltk/misc/chomsky.py +135 -133
  150. {nltk-3.2 → nltk-3.3}/nltk/misc/minimalset.py +83 -83
  151. {nltk-3.2 → nltk-3.3}/nltk/misc/sort.py +157 -157
  152. {nltk-3.2 → nltk-3.3}/nltk/misc/wordfinder.py +129 -129
  153. {nltk-3.2 → nltk-3.3}/nltk/parse/__init__.py +81 -80
  154. {nltk-3.2 → nltk-3.3}/nltk/parse/api.py +66 -66
  155. {nltk-3.2 → nltk-3.3}/nltk/parse/bllip.py +282 -282
  156. {nltk-3.2 → nltk-3.3}/nltk/parse/chart.py +1682 -1681
  157. nltk-3.3/nltk/parse/corenlp.py +780 -0
  158. {nltk-3.2 → nltk-3.3}/nltk/parse/dependencygraph.py +760 -758
  159. {nltk-3.2 → nltk-3.3}/nltk/parse/earleychart.py +452 -451
  160. {nltk-3.2 → nltk-3.3}/nltk/parse/evaluate.py +129 -129
  161. {nltk-3.2 → nltk-3.3}/nltk/parse/featurechart.py +580 -578
  162. {nltk-3.2 → nltk-3.3}/nltk/parse/generate.py +87 -76
  163. {nltk-3.2 → nltk-3.3}/nltk/parse/malt.py +356 -356
  164. {nltk-3.2 → nltk-3.3}/nltk/parse/nonprojectivedependencyparser.py +783 -783
  165. {nltk-3.2 → nltk-3.3}/nltk/parse/pchart.py +522 -522
  166. {nltk-3.2 → nltk-3.3}/nltk/parse/projectivedependencyparser.py +579 -578
  167. {nltk-3.2 → nltk-3.3}/nltk/parse/recursivedescent.py +655 -656
  168. {nltk-3.2 → nltk-3.3}/nltk/parse/shiftreduce.py +458 -459
  169. {nltk-3.2 → nltk-3.3}/nltk/parse/stanford.py +431 -406
  170. {nltk-3.2 → nltk-3.3}/nltk/parse/transitionparser.py +774 -772
  171. {nltk-3.2 → nltk-3.3}/nltk/parse/util.py +231 -231
  172. {nltk-3.2 → nltk-3.3}/nltk/parse/viterbi.py +401 -401
  173. {nltk-3.2 → nltk-3.3}/nltk/probability.py +2410 -2355
  174. {nltk-3.2 → nltk-3.3}/nltk/sem/__init__.py +61 -61
  175. {nltk-3.2 → nltk-3.3}/nltk/sem/boxer.py +1261 -1261
  176. {nltk-3.2 → nltk-3.3}/nltk/sem/chat80.py +782 -782
  177. {nltk-3.2 → nltk-3.3}/nltk/sem/cooper_storage.py +118 -118
  178. {nltk-3.2 → nltk-3.3}/nltk/sem/drt.py +1258 -1257
  179. {nltk-3.2 → nltk-3.3}/nltk/sem/drt_glue_demo.py +485 -489
  180. {nltk-3.2 → nltk-3.3}/nltk/sem/evaluate.py +790 -784
  181. {nltk-3.2 → nltk-3.3}/nltk/sem/glue.py +696 -694
  182. {nltk-3.2 → nltk-3.3}/nltk/sem/hole.py +386 -384
  183. {nltk-3.2 → nltk-3.3}/nltk/sem/lfg.py +210 -208
  184. {nltk-3.2 → nltk-3.3}/nltk/sem/linearlogic.py +451 -449
  185. {nltk-3.2 → nltk-3.3}/nltk/sem/logic.py +1910 -1910
  186. {nltk-3.2 → nltk-3.3}/nltk/sem/relextract.py +476 -483
  187. {nltk-3.2 → nltk-3.3}/nltk/sem/skolemize.py +101 -101
  188. {nltk-3.2 → nltk-3.3}/nltk/sem/util.py +249 -249
  189. {nltk-3.2 → nltk-3.3}/nltk/sentiment/__init__.py +14 -14
  190. {nltk-3.2 → nltk-3.3}/nltk/sentiment/sentiment_analyzer.py +229 -229
  191. {nltk-3.2 → nltk-3.3}/nltk/sentiment/util.py +762 -752
  192. {nltk-3.2 → nltk-3.3}/nltk/sentiment/vader.py +443 -445
  193. {nltk-3.2 → nltk-3.3}/nltk/stem/__init__.py +31 -33
  194. {nltk-3.2 → nltk-3.3}/nltk/stem/api.py +28 -25
  195. nltk-3.3/nltk/stem/arlstem.py +369 -0
  196. {nltk-3.2 → nltk-3.3}/nltk/stem/isri.py +345 -345
  197. {nltk-3.2 → nltk-3.3}/nltk/stem/lancaster.py +337 -310
  198. nltk-3.3/nltk/stem/porter.py +710 -0
  199. {nltk-3.2 → nltk-3.3}/nltk/stem/regexp.py +61 -61
  200. {nltk-3.2 → nltk-3.3}/nltk/stem/rslp.py +142 -142
  201. {nltk-3.2 → nltk-3.3}/nltk/stem/snowball.py +4236 -3712
  202. {nltk-3.2 → nltk-3.3}/nltk/stem/util.py +22 -12
  203. {nltk-3.2 → nltk-3.3}/nltk/stem/wordnet.py +51 -51
  204. {nltk-3.2 → nltk-3.3}/nltk/tag/__init__.py +152 -127
  205. {nltk-3.2 → nltk-3.3}/nltk/tag/api.py +86 -81
  206. {nltk-3.2 → nltk-3.3}/nltk/tag/brill.py +424 -426
  207. {nltk-3.2 → nltk-3.3}/nltk/tag/brill_trainer.py +608 -608
  208. {nltk-3.2 → nltk-3.3}/nltk/tag/crf.py +204 -200
  209. {nltk-3.2 → nltk-3.3}/nltk/tag/hmm.py +1277 -1277
  210. {nltk-3.2 → nltk-3.3}/nltk/tag/hunpos.py +132 -132
  211. {nltk-3.2 → nltk-3.3}/nltk/tag/mapping.py +101 -101
  212. {nltk-3.2 → nltk-3.3}/nltk/tag/perceptron.py +329 -314
  213. {nltk-3.2 → nltk-3.3}/nltk/tag/senna.py +147 -145
  214. {nltk-3.2 → nltk-3.3}/nltk/tag/sequential.py +740 -736
  215. {nltk-3.2 → nltk-3.3}/nltk/tag/stanford.py +219 -196
  216. {nltk-3.2 → nltk-3.3}/nltk/tag/tnt.py +595 -604
  217. {nltk-3.2 → nltk-3.3}/nltk/tag/util.py +72 -72
  218. {nltk-3.2 → nltk-3.3}/nltk/tbl/__init__.py +28 -28
  219. {nltk-3.2 → nltk-3.3}/nltk/tbl/api.py +1 -1
  220. {nltk-3.2 → nltk-3.3}/nltk/tbl/demo.py +366 -366
  221. {nltk-3.2 → nltk-3.3}/nltk/tbl/erroranalysis.py +39 -39
  222. {nltk-3.2 → nltk-3.3}/nltk/tbl/feature.py +259 -261
  223. {nltk-3.2 → nltk-3.3}/nltk/tbl/rule.py +321 -321
  224. {nltk-3.2 → nltk-3.3}/nltk/tbl/template.py +303 -308
  225. {nltk-3.2 → nltk-3.3}/nltk/test/__init__.py +18 -18
  226. {nltk-3.2 → nltk-3.3}/nltk/test/all.py +24 -23
  227. {nltk-3.2 → nltk-3.3}/nltk/test/bleu.doctest +14 -14
  228. {nltk-3.2 → nltk-3.3}/nltk/test/bnc.doctest +60 -60
  229. {nltk-3.2 → nltk-3.3}/nltk/test/ccg.doctest +376 -395
  230. nltk-3.3/nltk/test/ccg_semantics.doctest +553 -0
  231. {nltk-3.2 → nltk-3.3}/nltk/test/chat80.doctest +234 -234
  232. {nltk-3.2 → nltk-3.3}/nltk/test/childes.doctest +184 -184
  233. {nltk-3.2 → nltk-3.3}/nltk/test/childes_fixt.py +14 -13
  234. {nltk-3.2 → nltk-3.3}/nltk/test/chunk.doctest +373 -373
  235. {nltk-3.2 → nltk-3.3}/nltk/test/classify.doctest +185 -185
  236. {nltk-3.2 → nltk-3.3}/nltk/test/classify_fixt.py +11 -10
  237. nltk-3.3/nltk/test/collections.doctest +20 -0
  238. {nltk-3.2 → nltk-3.3}/nltk/test/collocations.doctest +276 -276
  239. {nltk-3.2 → nltk-3.3}/nltk/test/compat.doctest +134 -138
  240. {nltk-3.2 → nltk-3.3}/nltk/test/compat_fixt.py +9 -8
  241. nltk-3.3/nltk/test/concordance.doctest +55 -0
  242. {nltk-3.2 → nltk-3.3}/nltk/test/corpus.doctest +2200 -2212
  243. {nltk-3.2 → nltk-3.3}/nltk/test/corpus_fixt.py +3 -3
  244. {nltk-3.2 → nltk-3.3}/nltk/test/crubadan.doctest +65 -65
  245. {nltk-3.2 → nltk-3.3}/nltk/test/data.doctest +379 -379
  246. {nltk-3.2 → nltk-3.3}/nltk/test/dependency.doctest +241 -241
  247. {nltk-3.2 → nltk-3.3}/nltk/test/discourse.doctest +546 -546
  248. {nltk-3.2 → nltk-3.3}/nltk/test/discourse_fixt.py +14 -13
  249. {nltk-3.2 → nltk-3.3}/nltk/test/doctest_nose_plugin.py +159 -156
  250. {nltk-3.2 → nltk-3.3}/nltk/test/drt.doctest +517 -517
  251. {nltk-3.2 → nltk-3.3}/nltk/test/featgram.doctest +607 -607
  252. {nltk-3.2 → nltk-3.3}/nltk/test/featstruct.doctest +1229 -1230
  253. {nltk-3.2 → nltk-3.3}/nltk/test/framenet.doctest +288 -239
  254. {nltk-3.2 → nltk-3.3}/nltk/test/generate.doctest +67 -67
  255. {nltk-3.2 → nltk-3.3}/nltk/test/gensim.doctest +140 -141
  256. {nltk-3.2 → nltk-3.3}/nltk/test/gensim_fixt.py +10 -9
  257. {nltk-3.2 → nltk-3.3}/nltk/test/gluesemantics.doctest +384 -386
  258. {nltk-3.2 → nltk-3.3}/nltk/test/gluesemantics_malt.doctest +68 -68
  259. {nltk-3.2 → nltk-3.3}/nltk/test/gluesemantics_malt_fixt.py +12 -11
  260. {nltk-3.2 → nltk-3.3}/nltk/test/grammar.doctest +48 -48
  261. {nltk-3.2 → nltk-3.3}/nltk/test/grammartestsuites.doctest +109 -109
  262. {nltk-3.2 → nltk-3.3}/nltk/test/index.doctest +100 -100
  263. {nltk-3.2 → nltk-3.3}/nltk/test/inference.doctest +534 -534
  264. {nltk-3.2 → nltk-3.3}/nltk/test/inference_fixt.py +12 -11
  265. {nltk-3.2 → nltk-3.3}/nltk/test/internals.doctest +140 -140
  266. {nltk-3.2 → nltk-3.3}/nltk/test/japanese.doctest +48 -48
  267. {nltk-3.2 → nltk-3.3}/nltk/test/logic.doctest +1098 -1098
  268. {nltk-3.2 → nltk-3.3}/nltk/test/metrics.doctest +270 -270
  269. {nltk-3.2 → nltk-3.3}/nltk/test/misc.doctest +118 -118
  270. {nltk-3.2 → nltk-3.3}/nltk/test/nonmonotonic.doctest +286 -286
  271. {nltk-3.2 → nltk-3.3}/nltk/test/nonmonotonic_fixt.py +12 -11
  272. {nltk-3.2 → nltk-3.3}/nltk/test/paice.doctest +35 -35
  273. {nltk-3.2 → nltk-3.3}/nltk/test/parse.doctest +884 -884
  274. {nltk-3.2 → nltk-3.3}/nltk/test/portuguese_en.doctest +565 -565
  275. {nltk-3.2 → nltk-3.3}/nltk/test/portuguese_en_fixt.py +16 -16
  276. {nltk-3.2 → nltk-3.3}/nltk/test/probability.doctest +304 -304
  277. {nltk-3.2 → nltk-3.3}/nltk/test/probability_fixt.py +13 -12
  278. {nltk-3.2 → nltk-3.3}/nltk/test/propbank.doctest +176 -176
  279. {nltk-3.2 → nltk-3.3}/nltk/test/relextract.doctest +263 -263
  280. {nltk-3.2 → nltk-3.3}/nltk/test/resolution.doctest +221 -221
  281. {nltk-3.2 → nltk-3.3}/nltk/test/runtests.py +86 -64
  282. {nltk-3.2 → nltk-3.3}/nltk/test/segmentation_fixt.py +10 -10
  283. {nltk-3.2 → nltk-3.3}/nltk/test/semantics.doctest +665 -665
  284. {nltk-3.2 → nltk-3.3}/nltk/test/semantics_fixt.py +7 -7
  285. {nltk-3.2 → nltk-3.3}/nltk/test/sentiment.doctest +233 -233
  286. {nltk-3.2 → nltk-3.3}/nltk/test/sentiwordnet.doctest +41 -41
  287. {nltk-3.2 → nltk-3.3}/nltk/test/simple.doctest +85 -85
  288. {nltk-3.2 → nltk-3.3}/nltk/test/stem.doctest +78 -78
  289. {nltk-3.2 → nltk-3.3}/nltk/test/tag.doctest +33 -33
  290. {nltk-3.2 → nltk-3.3}/nltk/test/tokenize.doctest +273 -166
  291. {nltk-3.2 → nltk-3.3}/nltk/test/toolbox.doctest +307 -307
  292. {nltk-3.2 → nltk-3.3}/nltk/test/translate.doctest +242 -242
  293. {nltk-3.2 → nltk-3.3}/nltk/test/translate_fixt.py +3 -3
  294. {nltk-3.2 → nltk-3.3}/nltk/test/tree.doctest +1101 -1101
  295. {nltk-3.2 → nltk-3.3}/nltk/test/treeprettyprinter.doctest +127 -127
  296. {nltk-3.2 → nltk-3.3}/nltk/test/treetransforms.doctest +156 -156
  297. nltk-3.3/nltk/test/unit/test_2x_compat.py +63 -0
  298. nltk-3.3/nltk/test/unit/test_aline.py +52 -0
  299. nltk-3.3/nltk/test/unit/test_chunk.py +56 -0
  300. {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_classify.py +55 -53
  301. {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_collocations.py +88 -79
  302. nltk-3.3/nltk/test/unit/test_concordance.py +98 -0
  303. nltk-3.3/nltk/test/unit/test_corenlp.py +1000 -0
  304. {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_corpora.py +200 -183
  305. {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_corpus_views.py +45 -45
  306. {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_hmm.py +88 -88
  307. {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_json2csv_corpus.py +188 -187
  308. {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_naivebayes.py +26 -26
  309. nltk-3.3/nltk/test/unit/test_rte_classify.py +86 -0
  310. {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_seekable_unicode_stream_reader.py +124 -124
  311. nltk-3.3/nltk/test/unit/test_senna.py +74 -0
  312. nltk-3.3/nltk/test/unit/test_stem.py +121 -0
  313. {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_tag.py +20 -20
  314. {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_tgrep.py +628 -626
  315. nltk-3.3/nltk/test/unit/test_tokenize.py +168 -0
  316. {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_twitter_auth.py +185 -185
  317. nltk-3.3/nltk/test/unit/test_wordnet.py +134 -0
  318. nltk-3.3/nltk/test/unit/translate/test_bleu.py +244 -0
  319. nltk-3.3/nltk/test/unit/translate/test_gdfa.py +57 -0
  320. {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm1.py +75 -75
  321. {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm2.py +87 -87
  322. {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm3.py +105 -105
  323. {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm4.py +126 -126
  324. {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm5.py +166 -166
  325. {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm_model.py +270 -270
  326. nltk-3.3/nltk/test/unit/translate/test_nist.py +37 -0
  327. {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_stack_decoder.py +300 -300
  328. {nltk-3.2 → nltk-3.3}/nltk/test/unit/utils.py +41 -41
  329. {nltk-3.2 → nltk-3.3}/nltk/test/util.doctest +48 -48
  330. {nltk-3.2 → nltk-3.3}/nltk/test/wordnet.doctest +595 -581
  331. {nltk-3.2 → nltk-3.3}/nltk/test/wordnet_fixt.py +7 -6
  332. {nltk-3.2 → nltk-3.3}/nltk/test/wordnet_lch.doctest +53 -53
  333. {nltk-3.2 → nltk-3.3}/nltk/test/wsd.doctest +68 -68
  334. {nltk-3.2 → nltk-3.3}/nltk/text.py +663 -610
  335. {nltk-3.2 → nltk-3.3}/nltk/tgrep.py +938 -937
  336. {nltk-3.2 → nltk-3.3}/nltk/tokenize/__init__.py +130 -107
  337. {nltk-3.2 → nltk-3.3}/nltk/tokenize/api.py +77 -78
  338. {nltk-3.2 → nltk-3.3}/nltk/tokenize/casual.py +343 -342
  339. {nltk-3.2 → nltk-3.3}/nltk/tokenize/mwe.py +120 -120
  340. nltk-3.3/nltk/tokenize/nist.py +173 -0
  341. {nltk-3.2 → nltk-3.3}/nltk/tokenize/punkt.py +1620 -1601
  342. {nltk-3.2 → nltk-3.3}/nltk/tokenize/regexp.py +208 -202
  343. nltk-3.3/nltk/tokenize/repp.py +151 -0
  344. {nltk-3.2 → nltk-3.3}/nltk/tokenize/sexpr.py +140 -142
  345. {nltk-3.2 → nltk-3.3}/nltk/tokenize/simple.py +138 -136
  346. {nltk-3.2 → nltk-3.3}/nltk/tokenize/stanford.py +117 -113
  347. nltk-3.3/nltk/tokenize/stanford_segmenter.py +246 -0
  348. {nltk-3.2 → nltk-3.3}/nltk/tokenize/texttiling.py +458 -460
  349. nltk-3.3/nltk/tokenize/toktok.py +155 -0
  350. nltk-3.3/nltk/tokenize/treebank.py +363 -0
  351. nltk-3.3/nltk/tokenize/util.py +261 -0
  352. {nltk-3.2 → nltk-3.3}/nltk/toolbox.py +499 -497
  353. {nltk-3.2 → nltk-3.3}/nltk/translate/__init__.py +24 -24
  354. {nltk-3.2 → nltk-3.3}/nltk/translate/api.py +331 -321
  355. {nltk-3.2 → nltk-3.3}/nltk/translate/bleu_score.py +615 -556
  356. nltk-3.3/nltk/translate/chrf_score.py +137 -0
  357. {nltk-3.2 → nltk-3.3}/nltk/translate/gale_church.py +248 -235
  358. {nltk-3.2 → nltk-3.3}/nltk/translate/gdfa.py +135 -131
  359. nltk-3.3/nltk/translate/gleu_score.py +193 -0
  360. {nltk-3.2 → nltk-3.3}/nltk/translate/ibm1.py +250 -241
  361. {nltk-3.2 → nltk-3.3}/nltk/translate/ibm2.py +308 -308
  362. {nltk-3.2 → nltk-3.3}/nltk/translate/ibm3.py +337 -337
  363. {nltk-3.2 → nltk-3.3}/nltk/translate/ibm4.py +467 -467
  364. {nltk-3.2 → nltk-3.3}/nltk/translate/ibm5.py +639 -639
  365. {nltk-3.2 → nltk-3.3}/nltk/translate/ibm_model.py +536 -533
  366. {nltk-3.2 → nltk-3.3}/nltk/translate/metrics.py +40 -40
  367. nltk-3.3/nltk/translate/nist_score.py +178 -0
  368. {nltk-3.2 → nltk-3.3}/nltk/translate/phrase_based.py +175 -175
  369. {nltk-3.2 → nltk-3.3}/nltk/translate/ribes_score.py +325 -325
  370. {nltk-3.2 → nltk-3.3}/nltk/translate/stack_decoder.py +499 -495
  371. {nltk-3.2 → nltk-3.3}/nltk/tree.py +1605 -1603
  372. {nltk-3.2 → nltk-3.3}/nltk/treeprettyprinter.py +564 -564
  373. {nltk-3.2 → nltk-3.3}/nltk/treetransforms.py +309 -309
  374. {nltk-3.2 → nltk-3.3}/nltk/twitter/__init__.py +28 -28
  375. {nltk-3.2 → nltk-3.3}/nltk/twitter/api.py +141 -136
  376. {nltk-3.2 → nltk-3.3}/nltk/twitter/common.py +264 -258
  377. {nltk-3.2 → nltk-3.3}/nltk/twitter/twitter_demo.py +288 -288
  378. {nltk-3.2 → nltk-3.3}/nltk/twitter/twitterclient.py +530 -530
  379. {nltk-3.2 → nltk-3.3}/nltk/twitter/util.py +147 -146
  380. nltk-3.3/nltk/util.py +751 -0
  381. {nltk-3.2 → nltk-3.3}/nltk/wsd.py +53 -53
  382. {nltk-3.2 → nltk-3.3/nltk.egg-info}/PKG-INFO +13 -5
  383. {nltk-3.2 → nltk-3.3}/nltk.egg-info/SOURCES.txt +26 -3
  384. nltk-3.3/nltk.egg-info/requires.txt +31 -0
  385. {nltk-3.2 → nltk-3.3}/setup.cfg +0 -1
  386. {nltk-3.2 → nltk-3.3}/setup.py +102 -75
  387. nltk-3.2/nltk/VERSION +0 -1
  388. nltk-3.2/nltk/compat.py +0 -680
  389. nltk-3.2/nltk/corpus/reader/pl196x.py +0 -278
  390. nltk-3.2/nltk/corpus/reader/verbnet.py +0 -408
  391. nltk-3.2/nltk/corpus/reader/wordlist.py +0 -37
  392. nltk-3.2/nltk/sentiment/vader_lexicon.txt +0 -7517
  393. nltk-3.2/nltk/six.py +0 -868
  394. nltk-3.2/nltk/stem/porter.py +0 -692
  395. nltk-3.2/nltk/test/unit/test_2x_compat.py +0 -26
  396. nltk-3.2/nltk/test/unit/test_stem.py +0 -36
  397. nltk-3.2/nltk/test/unit/test_tokenize.py +0 -23
  398. nltk-3.2/nltk/test/unit/translate/test_bleu.py +0 -84
  399. nltk-3.2/nltk/tokenize/treebank.py +0 -124
  400. nltk-3.2/nltk/tokenize/util.py +0 -90
  401. nltk-3.2/nltk/util.py +0 -1364
  402. {nltk-3.2 → nltk-3.3}/nltk/test/unit/__init__.py +0 -0
  403. {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/__init__.py +0 -0
  404. {nltk-3.2 → nltk-3.3}/nltk.egg-info/dependency_links.txt +0 -0
  405. {nltk-3.2 → nltk-3.3}/nltk.egg-info/not-zip-safe +0 -0
  406. {nltk-3.2 → nltk-3.3}/nltk.egg-info/top_level.txt +0 -0
@@ -1,421 +1,421 @@
1
- # Natural Language Toolkit: NomBank Corpus Reader
2
- #
3
- # Copyright (C) 2001-2015 NLTK Project
4
- # Authors: Paul Bedaride <paul.bedaride@gmail.com>
5
- # Edward Loper <edloper@gmail.com>
6
- # URL: <http://nltk.org/>
7
- # For license information, see LICENSE.TXT
8
-
9
- from __future__ import unicode_literals
10
-
11
- from nltk.tree import Tree
12
- from xml.etree import ElementTree
13
- from nltk.internals import raise_unorderable_types
14
- from nltk.compat import total_ordering, python_2_unicode_compatible, string_types
15
-
16
- from nltk.corpus.reader.util import *
17
- from nltk.corpus.reader.api import *
18
-
19
- class NombankCorpusReader(CorpusReader):
20
- """
21
- Corpus reader for the nombank corpus, which augments the Penn
22
- Treebank with information about the predicate argument structure
23
- of every noun instance. The corpus consists of two parts: the
24
- predicate-argument annotations themselves, and a set of "frameset
25
- files" which define the argument labels used by the annotations,
26
- on a per-noun basis. Each "frameset file" contains one or more
27
- predicates, such as ``'turn'`` or ``'turn_on'``, each of which is
28
- divided into coarse-grained word senses called "rolesets". For
29
- each "roleset", the frameset file provides descriptions of the
30
- argument roles, along with examples.
31
- """
32
- def __init__(self, root, nomfile, framefiles='',
33
- nounsfile=None, parse_fileid_xform=None,
34
- parse_corpus=None, encoding='utf8'):
35
- """
36
- :param root: The root directory for this corpus.
37
- :param nomfile: The name of the file containing the predicate-
38
- argument annotations (relative to ``root``).
39
- :param framefiles: A list or regexp specifying the frameset
40
- fileids for this corpus.
41
- :param parse_fileid_xform: A transform that should be applied
42
- to the fileids in this corpus. This should be a function
43
- of one argument (a fileid) that returns a string (the new
44
- fileid).
45
- :param parse_corpus: The corpus containing the parse trees
46
- corresponding to this corpus. These parse trees are
47
- necessary to resolve the tree pointers used by nombank.
48
- """
49
- # If framefiles is specified as a regexp, expand it.
50
- if isinstance(framefiles, string_types):
51
- framefiles = find_corpus_fileids(root, framefiles)
52
- framefiles = list(framefiles)
53
- # Initialze the corpus reader.
54
- CorpusReader.__init__(self, root, [nomfile, nounsfile] + framefiles,
55
- encoding)
56
-
57
- # Record our frame fileids & nom file.
58
- self._nomfile = nomfile
59
- self._framefiles = framefiles
60
- self._nounsfile = nounsfile
61
- self._parse_fileid_xform = parse_fileid_xform
62
- self._parse_corpus = parse_corpus
63
-
64
- def raw(self, fileids=None):
65
- """
66
- :return: the text contents of the given fileids, as a single string.
67
- """
68
- if fileids is None: fileids = self._fileids
69
- elif isinstance(fileids, compat.string_types): fileids = [fileids]
70
- return concat([self.open(f).read() for f in fileids])
71
-
72
- def instances(self, baseform=None):
73
- """
74
- :return: a corpus view that acts as a list of
75
- ``NombankInstance`` objects, one for each noun in the corpus.
76
- """
77
- kwargs = {}
78
- if baseform is not None:
79
- kwargs['instance_filter'] = lambda inst: inst.baseform==baseform
80
- return StreamBackedCorpusView(self.abspath(self._nomfile),
81
- lambda stream: self._read_instance_block(stream, **kwargs),
82
- encoding=self.encoding(self._nomfile))
83
-
84
- def lines(self):
85
- """
86
- :return: a corpus view that acts as a list of strings, one for
87
- each line in the predicate-argument annotation file.
88
- """
89
- return StreamBackedCorpusView(self.abspath(self._nomfile),
90
- read_line_block,
91
- encoding=self.encoding(self._nomfile))
92
-
93
- def roleset(self, roleset_id):
94
- """
95
- :return: the xml description for the given roleset.
96
- """
97
- baseform = roleset_id.split('.')[0]
98
- baseform = baseform.replace('perc-sign','%')
99
- baseform = baseform.replace('oneslashonezero', '1/10').replace('1/10','1-slash-10')
100
- framefile = 'frames/%s.xml' % baseform
101
- if framefile not in self._framefiles:
102
- raise ValueError('Frameset file for %s not found' %
103
- roleset_id)
104
-
105
- # n.b.: The encoding for XML fileids is specified by the file
106
- # itself; so we ignore self._encoding here.
107
- etree = ElementTree.parse(self.abspath(framefile).open()).getroot()
108
- for roleset in etree.findall('predicate/roleset'):
109
- if roleset.attrib['id'] == roleset_id:
110
- return roleset
111
- else:
112
- raise ValueError('Roleset %s not found in %s' %
113
- (roleset_id, framefile))
114
-
115
- def rolesets(self, baseform=None):
116
- """
117
- :return: list of xml descriptions for rolesets.
118
- """
119
- if baseform is not None:
120
- framefile = 'frames/%s.xml' % baseform
121
- if framefile not in self._framefiles:
122
- raise ValueError('Frameset file for %s not found' %
123
- baseform)
124
- framefiles = [framefile]
125
- else:
126
- framefiles = self._framefiles
127
-
128
- rsets = []
129
- for framefile in framefiles:
130
- # n.b.: The encoding for XML fileids is specified by the file
131
- # itself; so we ignore self._encoding here.
132
- etree = ElementTree.parse(self.abspath(framefile).open()).getroot()
133
- rsets.append(etree.findall('predicate/roleset'))
134
- return LazyConcatenation(rsets)
135
-
136
- def nouns(self):
137
- """
138
- :return: a corpus view that acts as a list of all noun lemmas
139
- in this corpus (from the nombank.1.0.words file).
140
- """
141
- return StreamBackedCorpusView(self.abspath(self._nounsfile),
142
- read_line_block,
143
- encoding=self.encoding(self._nounsfile))
144
-
145
- def _read_instance_block(self, stream, instance_filter=lambda inst: True):
146
- block = []
147
-
148
- # Read 100 at a time.
149
- for i in range(100):
150
- line = stream.readline().strip()
151
- if line:
152
- inst = NombankInstance.parse(
153
- line, self._parse_fileid_xform,
154
- self._parse_corpus)
155
- if instance_filter(inst):
156
- block.append(inst)
157
-
158
- return block
159
-
160
- ######################################################################
161
- #{ Nombank Instance & related datatypes
162
- ######################################################################
163
-
164
- @python_2_unicode_compatible
165
- class NombankInstance(object):
166
-
167
- def __init__(self, fileid, sentnum, wordnum, baseform, sensenumber,
168
- predicate, predid, arguments, parse_corpus=None):
169
-
170
- self.fileid = fileid
171
- """The name of the file containing the parse tree for this
172
- instance's sentence."""
173
-
174
- self.sentnum = sentnum
175
- """The sentence number of this sentence within ``fileid``.
176
- Indexing starts from zero."""
177
-
178
- self.wordnum = wordnum
179
- """The word number of this instance's predicate within its
180
- containing sentence. Word numbers are indexed starting from
181
- zero, and include traces and other empty parse elements."""
182
-
183
- self.baseform = baseform
184
- """The baseform of the predicate."""
185
-
186
- self.sensenumber = sensenumber
187
- """The sense number of the predicate."""
188
-
189
- self.predicate = predicate
190
- """A ``NombankTreePointer`` indicating the position of this
191
- instance's predicate within its containing sentence."""
192
-
193
- self.predid = predid
194
- """Identifier of the predicate."""
195
-
196
- self.arguments = tuple(arguments)
197
- """A list of tuples (argloc, argid), specifying the location
198
- and identifier for each of the predicate's argument in the
199
- containing sentence. Argument identifiers are strings such as
200
- ``'ARG0'`` or ``'ARGM-TMP'``. This list does *not* contain
201
- the predicate."""
202
-
203
- self.parse_corpus = parse_corpus
204
- """A corpus reader for the parse trees corresponding to the
205
- instances in this nombank corpus."""
206
-
207
- @property
208
- def roleset(self):
209
- """The name of the roleset used by this instance's predicate.
210
- Use ``nombank.roleset() <NombankCorpusReader.roleset>`` to
211
- look up information about the roleset."""
212
- r = self.baseform.replace('%', 'perc-sign')
213
- r = r.replace('1/10', '1-slash-10').replace('1-slash-10', 'oneslashonezero')
214
- return '%s.%s'%(r, self.sensenumber)
215
-
216
- def __repr__(self):
217
- return ('<NombankInstance: %s, sent %s, word %s>' %
218
- (self.fileid, self.sentnum, self.wordnum))
219
-
220
- def __str__(self):
221
- s = '%s %s %s %s %s' % (self.fileid, self.sentnum, self.wordnum,
222
- self.baseform, self.sensenumber)
223
- items = self.arguments + ((self.predicate, 'rel'),)
224
- for (argloc, argid) in sorted(items):
225
- s += ' %s-%s' % (argloc, argid)
226
- return s
227
-
228
- def _get_tree(self):
229
- if self.parse_corpus is None: return None
230
- if self.fileid not in self.parse_corpus.fileids(): return None
231
- return self.parse_corpus.parsed_sents(self.fileid)[self.sentnum]
232
- tree = property(_get_tree, doc="""
233
- The parse tree corresponding to this instance, or None if
234
- the corresponding tree is not available.""")
235
-
236
- @staticmethod
237
- def parse(s, parse_fileid_xform=None, parse_corpus=None):
238
- pieces = s.split()
239
- if len(pieces) < 6:
240
- raise ValueError('Badly formatted nombank line: %r' % s)
241
-
242
- # Divide the line into its basic pieces.
243
- (fileid, sentnum, wordnum,
244
- baseform, sensenumber) = pieces[:5]
245
-
246
- args = pieces[5:]
247
- rel = [args.pop(i) for i,p in enumerate(args) if '-rel' in p]
248
- if len(rel) != 1:
249
- raise ValueError('Badly formatted nombank line: %r' % s)
250
-
251
- # Apply the fileid selector, if any.
252
- if parse_fileid_xform is not None:
253
- fileid = parse_fileid_xform(fileid)
254
-
255
- # Convert sentence & word numbers to ints.
256
- sentnum = int(sentnum)
257
- wordnum = int(wordnum)
258
-
259
- # Parse the predicate location.
260
-
261
- predloc, predid = rel[0].split('-', 1)
262
- predicate = NombankTreePointer.parse(predloc)
263
-
264
- # Parse the arguments.
265
- arguments = []
266
- for arg in args:
267
- argloc, argid = arg.split('-', 1)
268
- arguments.append( (NombankTreePointer.parse(argloc), argid) )
269
-
270
- # Put it all together.
271
- return NombankInstance(fileid, sentnum, wordnum, baseform, sensenumber,
272
- predicate, predid, arguments, parse_corpus)
273
-
274
- class NombankPointer(object):
275
- """
276
- A pointer used by nombank to identify one or more constituents in
277
- a parse tree. ``NombankPointer`` is an abstract base class with
278
- three concrete subclasses:
279
-
280
- - ``NombankTreePointer`` is used to point to single constituents.
281
- - ``NombankSplitTreePointer`` is used to point to 'split'
282
- constituents, which consist of a sequence of two or more
283
- ``NombankTreePointer`` pointers.
284
- - ``NombankChainTreePointer`` is used to point to entire trace
285
- chains in a tree. It consists of a sequence of pieces, which
286
- can be ``NombankTreePointer`` or ``NombankSplitTreePointer`` pointers.
287
- """
288
- def __init__(self):
289
- if self.__class__ == NombankPointer:
290
- raise NotImplementedError()
291
-
292
- @python_2_unicode_compatible
293
- class NombankChainTreePointer(NombankPointer):
294
- def __init__(self, pieces):
295
- self.pieces = pieces
296
- """A list of the pieces that make up this chain. Elements may
297
- be either ``NombankSplitTreePointer`` or
298
- ``NombankTreePointer`` pointers."""
299
-
300
- def __str__(self):
301
- return '*'.join('%s' % p for p in self.pieces)
302
- def __repr__(self):
303
- return '<NombankChainTreePointer: %s>' % self
304
- def select(self, tree):
305
- if tree is None: raise ValueError('Parse tree not avaialable')
306
- return Tree('*CHAIN*', [p.select(tree) for p in self.pieces])
307
-
308
- @python_2_unicode_compatible
309
- class NombankSplitTreePointer(NombankPointer):
310
- def __init__(self, pieces):
311
- self.pieces = pieces
312
- """A list of the pieces that make up this chain. Elements are
313
- all ``NombankTreePointer`` pointers."""
314
-
315
- def __str__(self):
316
- return ','.join('%s' % p for p in self.pieces)
317
- def __repr__(self):
318
- return '<NombankSplitTreePointer: %s>' % self
319
- def select(self, tree):
320
- if tree is None: raise ValueError('Parse tree not avaialable')
321
- return Tree('*SPLIT*', [p.select(tree) for p in self.pieces])
322
-
323
- @total_ordering
324
- @python_2_unicode_compatible
325
- class NombankTreePointer(NombankPointer):
326
- """
327
- wordnum:height*wordnum:height*...
328
- wordnum:height,
329
-
330
- """
331
- def __init__(self, wordnum, height):
332
- self.wordnum = wordnum
333
- self.height = height
334
-
335
- @staticmethod
336
- def parse(s):
337
- # Deal with chains (xx*yy*zz)
338
- pieces = s.split('*')
339
- if len(pieces) > 1:
340
- return NombankChainTreePointer([NombankTreePointer.parse(elt)
341
- for elt in pieces])
342
-
343
- # Deal with split args (xx,yy,zz)
344
- pieces = s.split(',')
345
- if len(pieces) > 1:
346
- return NombankSplitTreePointer([NombankTreePointer.parse(elt)
347
- for elt in pieces])
348
-
349
- # Deal with normal pointers.
350
- pieces = s.split(':')
351
- if len(pieces) != 2: raise ValueError('bad nombank pointer %r' % s)
352
- return NombankTreePointer(int(pieces[0]), int(pieces[1]))
353
-
354
- def __str__(self):
355
- return '%s:%s' % (self.wordnum, self.height)
356
-
357
- def __repr__(self):
358
- return 'NombankTreePointer(%d, %d)' % (self.wordnum, self.height)
359
-
360
- def __eq__(self, other):
361
- while isinstance(other, (NombankChainTreePointer,
362
- NombankSplitTreePointer)):
363
- other = other.pieces[0]
364
-
365
- if not isinstance(other, NombankTreePointer):
366
- return self is other
367
-
368
- return (self.wordnum == other.wordnum and self.height == other.height)
369
-
370
- def __ne__(self, other):
371
- return not self == other
372
-
373
- def __lt__(self, other):
374
- while isinstance(other, (NombankChainTreePointer,
375
- NombankSplitTreePointer)):
376
- other = other.pieces[0]
377
-
378
- if not isinstance(other, NombankTreePointer):
379
- return id(self) < id(other)
380
-
381
- return (self.wordnum, -self.height) < (other.wordnum, -other.height)
382
-
383
- def select(self, tree):
384
- if tree is None: raise ValueError('Parse tree not avaialable')
385
- return tree[self.treepos(tree)]
386
-
387
- def treepos(self, tree):
388
- """
389
- Convert this pointer to a standard 'tree position' pointer,
390
- given that it points to the given tree.
391
- """
392
- if tree is None: raise ValueError('Parse tree not avaialable')
393
- stack = [tree]
394
- treepos = []
395
-
396
- wordnum = 0
397
- while True:
398
- #print treepos
399
- #print stack[-1]
400
- # tree node:
401
- if isinstance(stack[-1], Tree):
402
- # Select the next child.
403
- if len(treepos) < len(stack):
404
- treepos.append(0)
405
- else:
406
- treepos[-1] += 1
407
- # Update the stack.
408
- if treepos[-1] < len(stack[-1]):
409
- stack.append(stack[-1][treepos[-1]])
410
- else:
411
- # End of node's child list: pop up a level.
412
- stack.pop()
413
- treepos.pop()
414
- # word node:
415
- else:
416
- if wordnum == self.wordnum:
417
- return tuple(treepos[:len(treepos)-self.height-1])
418
- else:
419
- wordnum += 1
420
- stack.pop()
421
-
1
+ # Natural Language Toolkit: NomBank Corpus Reader
2
+ #
3
+ # Copyright (C) 2001-2018 NLTK Project
4
+ # Authors: Paul Bedaride <paul.bedaride@gmail.com>
5
+ # Edward Loper <edloper@gmail.com>
6
+ # URL: <http://nltk.org/>
7
+ # For license information, see LICENSE.TXT
8
+
9
+ from __future__ import unicode_literals
10
+ from xml.etree import ElementTree
11
+ from functools import total_ordering
12
+
13
+ from six import string_types
14
+
15
+ from nltk.tree import Tree
16
+ from nltk.internals import raise_unorderable_types
17
+ from nltk.compat import python_2_unicode_compatible
18
+
19
+ from nltk.corpus.reader.util import *
20
+ from nltk.corpus.reader.api import *
21
+
22
+ class NombankCorpusReader(CorpusReader):
23
+ """
24
+ Corpus reader for the nombank corpus, which augments the Penn
25
+ Treebank with information about the predicate argument structure
26
+ of every noun instance. The corpus consists of two parts: the
27
+ predicate-argument annotations themselves, and a set of "frameset
28
+ files" which define the argument labels used by the annotations,
29
+ on a per-noun basis. Each "frameset file" contains one or more
30
+ predicates, such as ``'turn'`` or ``'turn_on'``, each of which is
31
+ divided into coarse-grained word senses called "rolesets". For
32
+ each "roleset", the frameset file provides descriptions of the
33
+ argument roles, along with examples.
34
+ """
35
+ def __init__(self, root, nomfile, framefiles='',
36
+ nounsfile=None, parse_fileid_xform=None,
37
+ parse_corpus=None, encoding='utf8'):
38
+ """
39
+ :param root: The root directory for this corpus.
40
+ :param nomfile: The name of the file containing the predicate-
41
+ argument annotations (relative to ``root``).
42
+ :param framefiles: A list or regexp specifying the frameset
43
+ fileids for this corpus.
44
+ :param parse_fileid_xform: A transform that should be applied
45
+ to the fileids in this corpus. This should be a function
46
+ of one argument (a fileid) that returns a string (the new
47
+ fileid).
48
+ :param parse_corpus: The corpus containing the parse trees
49
+ corresponding to this corpus. These parse trees are
50
+ necessary to resolve the tree pointers used by nombank.
51
+ """
52
+ # If framefiles is specified as a regexp, expand it.
53
+ if isinstance(framefiles, string_types):
54
+ framefiles = find_corpus_fileids(root, framefiles)
55
+ framefiles = list(framefiles)
56
+ # Initialze the corpus reader.
57
+ CorpusReader.__init__(self, root, [nomfile, nounsfile] + framefiles,
58
+ encoding)
59
+
60
+ # Record our frame fileids & nom file.
61
+ self._nomfile = nomfile
62
+ self._framefiles = framefiles
63
+ self._nounsfile = nounsfile
64
+ self._parse_fileid_xform = parse_fileid_xform
65
+ self._parse_corpus = parse_corpus
66
+
67
+ def raw(self, fileids=None):
68
+ """
69
+ :return: the text contents of the given fileids, as a single string.
70
+ """
71
+ if fileids is None: fileids = self._fileids
72
+ elif isinstance(fileids, string_types): fileids = [fileids]
73
+ return concat([self.open(f).read() for f in fileids])
74
+
75
+ def instances(self, baseform=None):
76
+ """
77
+ :return: a corpus view that acts as a list of
78
+ ``NombankInstance`` objects, one for each noun in the corpus.
79
+ """
80
+ kwargs = {}
81
+ if baseform is not None:
82
+ kwargs['instance_filter'] = lambda inst: inst.baseform==baseform
83
+ return StreamBackedCorpusView(self.abspath(self._nomfile),
84
+ lambda stream: self._read_instance_block(stream, **kwargs),
85
+ encoding=self.encoding(self._nomfile))
86
+
87
+ def lines(self):
88
+ """
89
+ :return: a corpus view that acts as a list of strings, one for
90
+ each line in the predicate-argument annotation file.
91
+ """
92
+ return StreamBackedCorpusView(self.abspath(self._nomfile),
93
+ read_line_block,
94
+ encoding=self.encoding(self._nomfile))
95
+
96
+ def roleset(self, roleset_id):
97
+ """
98
+ :return: the xml description for the given roleset.
99
+ """
100
+ baseform = roleset_id.split('.')[0]
101
+ baseform = baseform.replace('perc-sign','%')
102
+ baseform = baseform.replace('oneslashonezero', '1/10').replace('1/10','1-slash-10')
103
+ framefile = 'frames/%s.xml' % baseform
104
+ if framefile not in self._framefiles:
105
+ raise ValueError('Frameset file for %s not found' %
106
+ roleset_id)
107
+
108
+ # n.b.: The encoding for XML fileids is specified by the file
109
+ # itself; so we ignore self._encoding here.
110
+ etree = ElementTree.parse(self.abspath(framefile).open()).getroot()
111
+ for roleset in etree.findall('predicate/roleset'):
112
+ if roleset.attrib['id'] == roleset_id:
113
+ return roleset
114
+ raise ValueError('Roleset %s not found in %s' % (roleset_id, framefile))
115
+
116
+ def rolesets(self, baseform=None):
117
+ """
118
+ :return: list of xml descriptions for rolesets.
119
+ """
120
+ if baseform is not None:
121
+ framefile = 'frames/%s.xml' % baseform
122
+ if framefile not in self._framefiles:
123
+ raise ValueError('Frameset file for %s not found' %
124
+ baseform)
125
+ framefiles = [framefile]
126
+ else:
127
+ framefiles = self._framefiles
128
+
129
+ rsets = []
130
+ for framefile in framefiles:
131
+ # n.b.: The encoding for XML fileids is specified by the file
132
+ # itself; so we ignore self._encoding here.
133
+ etree = ElementTree.parse(self.abspath(framefile).open()).getroot()
134
+ rsets.append(etree.findall('predicate/roleset'))
135
+ return LazyConcatenation(rsets)
136
+
137
+ def nouns(self):
138
+ """
139
+ :return: a corpus view that acts as a list of all noun lemmas
140
+ in this corpus (from the nombank.1.0.words file).
141
+ """
142
+ return StreamBackedCorpusView(self.abspath(self._nounsfile),
143
+ read_line_block,
144
+ encoding=self.encoding(self._nounsfile))
145
+
146
+ def _read_instance_block(self, stream, instance_filter=lambda inst: True):
147
+ block = []
148
+
149
+ # Read 100 at a time.
150
+ for i in range(100):
151
+ line = stream.readline().strip()
152
+ if line:
153
+ inst = NombankInstance.parse(
154
+ line, self._parse_fileid_xform,
155
+ self._parse_corpus)
156
+ if instance_filter(inst):
157
+ block.append(inst)
158
+
159
+ return block
160
+
161
+ ######################################################################
162
+ #{ Nombank Instance & related datatypes
163
+ ######################################################################
164
+
165
+ @python_2_unicode_compatible
166
+ class NombankInstance(object):
167
+
168
+ def __init__(self, fileid, sentnum, wordnum, baseform, sensenumber,
169
+ predicate, predid, arguments, parse_corpus=None):
170
+
171
+ self.fileid = fileid
172
+ """The name of the file containing the parse tree for this
173
+ instance's sentence."""
174
+
175
+ self.sentnum = sentnum
176
+ """The sentence number of this sentence within ``fileid``.
177
+ Indexing starts from zero."""
178
+
179
+ self.wordnum = wordnum
180
+ """The word number of this instance's predicate within its
181
+ containing sentence. Word numbers are indexed starting from
182
+ zero, and include traces and other empty parse elements."""
183
+
184
+ self.baseform = baseform
185
+ """The baseform of the predicate."""
186
+
187
+ self.sensenumber = sensenumber
188
+ """The sense number of the predicate."""
189
+
190
+ self.predicate = predicate
191
+ """A ``NombankTreePointer`` indicating the position of this
192
+ instance's predicate within its containing sentence."""
193
+
194
+ self.predid = predid
195
+ """Identifier of the predicate."""
196
+
197
+ self.arguments = tuple(arguments)
198
+ """A list of tuples (argloc, argid), specifying the location
199
+ and identifier for each of the predicate's argument in the
200
+ containing sentence. Argument identifiers are strings such as
201
+ ``'ARG0'`` or ``'ARGM-TMP'``. This list does *not* contain
202
+ the predicate."""
203
+
204
+ self.parse_corpus = parse_corpus
205
+ """A corpus reader for the parse trees corresponding to the
206
+ instances in this nombank corpus."""
207
+
208
+ @property
209
+ def roleset(self):
210
+ """The name of the roleset used by this instance's predicate.
211
+ Use ``nombank.roleset() <NombankCorpusReader.roleset>`` to
212
+ look up information about the roleset."""
213
+ r = self.baseform.replace('%', 'perc-sign')
214
+ r = r.replace('1/10', '1-slash-10').replace('1-slash-10', 'oneslashonezero')
215
+ return '%s.%s'%(r, self.sensenumber)
216
+
217
+ def __repr__(self):
218
+ return ('<NombankInstance: %s, sent %s, word %s>' %
219
+ (self.fileid, self.sentnum, self.wordnum))
220
+
221
+ def __str__(self):
222
+ s = '%s %s %s %s %s' % (self.fileid, self.sentnum, self.wordnum,
223
+ self.baseform, self.sensenumber)
224
+ items = self.arguments + ((self.predicate, 'rel'),)
225
+ for (argloc, argid) in sorted(items):
226
+ s += ' %s-%s' % (argloc, argid)
227
+ return s
228
+
229
+ def _get_tree(self):
230
+ if self.parse_corpus is None: return None
231
+ if self.fileid not in self.parse_corpus.fileids(): return None
232
+ return self.parse_corpus.parsed_sents(self.fileid)[self.sentnum]
233
+ tree = property(_get_tree, doc="""
234
+ The parse tree corresponding to this instance, or None if
235
+ the corresponding tree is not available.""")
236
+
237
+ @staticmethod
238
+ def parse(s, parse_fileid_xform=None, parse_corpus=None):
239
+ pieces = s.split()
240
+ if len(pieces) < 6:
241
+ raise ValueError('Badly formatted nombank line: %r' % s)
242
+
243
+ # Divide the line into its basic pieces.
244
+ (fileid, sentnum, wordnum,
245
+ baseform, sensenumber) = pieces[:5]
246
+
247
+ args = pieces[5:]
248
+ rel = [args.pop(i) for i,p in enumerate(args) if '-rel' in p]
249
+ if len(rel) != 1:
250
+ raise ValueError('Badly formatted nombank line: %r' % s)
251
+
252
+ # Apply the fileid selector, if any.
253
+ if parse_fileid_xform is not None:
254
+ fileid = parse_fileid_xform(fileid)
255
+
256
+ # Convert sentence & word numbers to ints.
257
+ sentnum = int(sentnum)
258
+ wordnum = int(wordnum)
259
+
260
+ # Parse the predicate location.
261
+
262
+ predloc, predid = rel[0].split('-', 1)
263
+ predicate = NombankTreePointer.parse(predloc)
264
+
265
+ # Parse the arguments.
266
+ arguments = []
267
+ for arg in args:
268
+ argloc, argid = arg.split('-', 1)
269
+ arguments.append( (NombankTreePointer.parse(argloc), argid) )
270
+
271
+ # Put it all together.
272
+ return NombankInstance(fileid, sentnum, wordnum, baseform, sensenumber,
273
+ predicate, predid, arguments, parse_corpus)
274
+
275
+ class NombankPointer(object):
276
+ """
277
+ A pointer used by nombank to identify one or more constituents in
278
+ a parse tree. ``NombankPointer`` is an abstract base class with
279
+ three concrete subclasses:
280
+
281
+ - ``NombankTreePointer`` is used to point to single constituents.
282
+ - ``NombankSplitTreePointer`` is used to point to 'split'
283
+ constituents, which consist of a sequence of two or more
284
+ ``NombankTreePointer`` pointers.
285
+ - ``NombankChainTreePointer`` is used to point to entire trace
286
+ chains in a tree. It consists of a sequence of pieces, which
287
+ can be ``NombankTreePointer`` or ``NombankSplitTreePointer`` pointers.
288
+ """
289
+ def __init__(self):
290
+ if self.__class__ == NombankPointer:
291
+ raise NotImplementedError()
292
+
293
+ @python_2_unicode_compatible
294
+ class NombankChainTreePointer(NombankPointer):
295
+ def __init__(self, pieces):
296
+ self.pieces = pieces
297
+ """A list of the pieces that make up this chain. Elements may
298
+ be either ``NombankSplitTreePointer`` or
299
+ ``NombankTreePointer`` pointers."""
300
+
301
+ def __str__(self):
302
+ return '*'.join('%s' % p for p in self.pieces)
303
+ def __repr__(self):
304
+ return '<NombankChainTreePointer: %s>' % self
305
+ def select(self, tree):
306
+ if tree is None: raise ValueError('Parse tree not avaialable')
307
+ return Tree('*CHAIN*', [p.select(tree) for p in self.pieces])
308
+
309
+ @python_2_unicode_compatible
310
+ class NombankSplitTreePointer(NombankPointer):
311
+ def __init__(self, pieces):
312
+ self.pieces = pieces
313
+ """A list of the pieces that make up this chain. Elements are
314
+ all ``NombankTreePointer`` pointers."""
315
+
316
+ def __str__(self):
317
+ return ','.join('%s' % p for p in self.pieces)
318
+ def __repr__(self):
319
+ return '<NombankSplitTreePointer: %s>' % self
320
+ def select(self, tree):
321
+ if tree is None: raise ValueError('Parse tree not avaialable')
322
+ return Tree('*SPLIT*', [p.select(tree) for p in self.pieces])
323
+
324
+ @total_ordering
325
+ @python_2_unicode_compatible
326
+ class NombankTreePointer(NombankPointer):
327
+ """
328
+ wordnum:height*wordnum:height*...
329
+ wordnum:height,
330
+
331
+ """
332
+ def __init__(self, wordnum, height):
333
+ self.wordnum = wordnum
334
+ self.height = height
335
+
336
+ @staticmethod
337
+ def parse(s):
338
+ # Deal with chains (xx*yy*zz)
339
+ pieces = s.split('*')
340
+ if len(pieces) > 1:
341
+ return NombankChainTreePointer([NombankTreePointer.parse(elt)
342
+ for elt in pieces])
343
+
344
+ # Deal with split args (xx,yy,zz)
345
+ pieces = s.split(',')
346
+ if len(pieces) > 1:
347
+ return NombankSplitTreePointer([NombankTreePointer.parse(elt)
348
+ for elt in pieces])
349
+
350
+ # Deal with normal pointers.
351
+ pieces = s.split(':')
352
+ if len(pieces) != 2: raise ValueError('bad nombank pointer %r' % s)
353
+ return NombankTreePointer(int(pieces[0]), int(pieces[1]))
354
+
355
+ def __str__(self):
356
+ return '%s:%s' % (self.wordnum, self.height)
357
+
358
+ def __repr__(self):
359
+ return 'NombankTreePointer(%d, %d)' % (self.wordnum, self.height)
360
+
361
+ def __eq__(self, other):
362
+ while isinstance(other, (NombankChainTreePointer,
363
+ NombankSplitTreePointer)):
364
+ other = other.pieces[0]
365
+
366
+ if not isinstance(other, NombankTreePointer):
367
+ return self is other
368
+
369
+ return (self.wordnum == other.wordnum and self.height == other.height)
370
+
371
+ def __ne__(self, other):
372
+ return not self == other
373
+
374
+ def __lt__(self, other):
375
+ while isinstance(other, (NombankChainTreePointer,
376
+ NombankSplitTreePointer)):
377
+ other = other.pieces[0]
378
+
379
+ if not isinstance(other, NombankTreePointer):
380
+ return id(self) < id(other)
381
+
382
+ return (self.wordnum, -self.height) < (other.wordnum, -other.height)
383
+
384
+ def select(self, tree):
385
+ if tree is None: raise ValueError('Parse tree not avaialable')
386
+ return tree[self.treepos(tree)]
387
+
388
+ def treepos(self, tree):
389
+ """
390
+ Convert this pointer to a standard 'tree position' pointer,
391
+ given that it points to the given tree.
392
+ """
393
+ if tree is None: raise ValueError('Parse tree not avaialable')
394
+ stack = [tree]
395
+ treepos = []
396
+
397
+ wordnum = 0
398
+ while True:
399
+ #print treepos
400
+ #print stack[-1]
401
+ # tree node:
402
+ if isinstance(stack[-1], Tree):
403
+ # Select the next child.
404
+ if len(treepos) < len(stack):
405
+ treepos.append(0)
406
+ else:
407
+ treepos[-1] += 1
408
+ # Update the stack.
409
+ if treepos[-1] < len(stack[-1]):
410
+ stack.append(stack[-1][treepos[-1]])
411
+ else:
412
+ # End of node's child list: pop up a level.
413
+ stack.pop()
414
+ treepos.pop()
415
+ # word node:
416
+ else:
417
+ if wordnum == self.wordnum:
418
+ return tuple(treepos[:len(treepos)-self.height-1])
419
+ else:
420
+ wordnum += 1
421
+ stack.pop()