nltk 3.2__zip → 3.3__zip
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nltk-3.2 → nltk-3.3}/INSTALL.txt +6 -6
- {nltk-3.2 → nltk-3.3}/LICENSE.txt +13 -13
- {nltk-3.2 → nltk-3.3}/MANIFEST.in +6 -6
- {nltk-3.2/nltk.egg-info → nltk-3.3}/PKG-INFO +13 -5
- nltk-3.2/README.txt → nltk-3.3/README.md +49 -48
- nltk-3.3/nltk/VERSION +1 -0
- {nltk-3.2 → nltk-3.3}/nltk/__init__.py +185 -185
- {nltk-3.2 → nltk-3.3}/nltk/app/__init__.py +52 -53
- {nltk-3.2 → nltk-3.3}/nltk/app/chartparser_app.py +2276 -2276
- {nltk-3.2 → nltk-3.3}/nltk/app/chunkparser_app.py +1262 -1264
- {nltk-3.2 → nltk-3.3}/nltk/app/collocations_app.py +347 -350
- {nltk-3.2 → nltk-3.3}/nltk/app/concordance_app.py +568 -570
- {nltk-3.2 → nltk-3.3}/nltk/app/nemo_app.py +156 -155
- {nltk-3.2 → nltk-3.3}/nltk/app/rdparser_app.py +892 -894
- {nltk-3.2 → nltk-3.3}/nltk/app/srparser_app.py +809 -809
- {nltk-3.2 → nltk-3.3}/nltk/app/wordfreq_app.py +32 -32
- {nltk-3.2 → nltk-3.3}/nltk/app/wordnet_app.py +968 -968
- {nltk-3.2 → nltk-3.3}/nltk/book.py +91 -90
- {nltk-3.2 → nltk-3.3}/nltk/ccg/__init__.py +22 -22
- {nltk-3.2 → nltk-3.3}/nltk/ccg/api.py +359 -335
- {nltk-3.2 → nltk-3.3}/nltk/ccg/chart.py +394 -361
- {nltk-3.2 → nltk-3.3}/nltk/ccg/combinator.py +343 -312
- {nltk-3.2 → nltk-3.3}/nltk/ccg/lexicon.py +328 -290
- nltk-3.3/nltk/ccg/logic.py +46 -0
- {nltk-3.2 → nltk-3.3}/nltk/chat/__init__.py +49 -49
- {nltk-3.2 → nltk-3.3}/nltk/chat/eliza.py +244 -244
- {nltk-3.2 → nltk-3.3}/nltk/chat/iesha.py +140 -140
- {nltk-3.2 → nltk-3.3}/nltk/chat/rude.py +92 -92
- {nltk-3.2 → nltk-3.3}/nltk/chat/suntsu.py +117 -117
- {nltk-3.2 → nltk-3.3}/nltk/chat/util.py +122 -120
- {nltk-3.2 → nltk-3.3}/nltk/chat/zen.py +282 -282
- {nltk-3.2 → nltk-3.3}/nltk/chunk/__init__.py +190 -190
- {nltk-3.2 → nltk-3.3}/nltk/chunk/api.py +51 -51
- {nltk-3.2 → nltk-3.3}/nltk/chunk/named_entity.py +331 -331
- {nltk-3.2 → nltk-3.3}/nltk/chunk/regexp.py +1390 -1384
- {nltk-3.2 → nltk-3.3}/nltk/chunk/util.py +600 -600
- {nltk-3.2 → nltk-3.3}/nltk/classify/__init__.py +98 -98
- {nltk-3.2 → nltk-3.3}/nltk/classify/api.py +193 -193
- {nltk-3.2 → nltk-3.3}/nltk/classify/decisiontree.py +295 -295
- {nltk-3.2 → nltk-3.3}/nltk/classify/maxent.py +1510 -1492
- {nltk-3.2 → nltk-3.3}/nltk/classify/megam.py +179 -178
- {nltk-3.2 → nltk-3.3}/nltk/classify/naivebayes.py +245 -242
- {nltk-3.2 → nltk-3.3}/nltk/classify/positivenaivebayes.py +170 -170
- {nltk-3.2 → nltk-3.3}/nltk/classify/rte_classify.py +160 -191
- {nltk-3.2 → nltk-3.3}/nltk/classify/scikitlearn.py +153 -151
- {nltk-3.2 → nltk-3.3}/nltk/classify/senna.py +183 -181
- {nltk-3.2 → nltk-3.3}/nltk/classify/svm.py +15 -15
- {nltk-3.2 → nltk-3.3}/nltk/classify/tadm.py +113 -112
- {nltk-3.2 → nltk-3.3}/nltk/classify/textcat.py +193 -193
- {nltk-3.2 → nltk-3.3}/nltk/classify/util.py +324 -312
- {nltk-3.2 → nltk-3.3}/nltk/classify/weka.py +346 -345
- {nltk-3.2 → nltk-3.3}/nltk/cluster/__init__.py +86 -86
- {nltk-3.2 → nltk-3.3}/nltk/cluster/api.py +74 -70
- {nltk-3.2 → nltk-3.3}/nltk/cluster/em.py +247 -247
- {nltk-3.2 → nltk-3.3}/nltk/cluster/gaac.py +168 -168
- {nltk-3.2 → nltk-3.3}/nltk/cluster/kmeans.py +221 -221
- {nltk-3.2 → nltk-3.3}/nltk/cluster/util.py +304 -289
- nltk-3.3/nltk/collections.py +640 -0
- {nltk-3.2 → nltk-3.3}/nltk/collocations.py +388 -388
- nltk-3.3/nltk/compat.py +365 -0
- {nltk-3.2 → nltk-3.3}/nltk/corpus/__init__.py +341 -320
- {nltk-3.2 → nltk-3.3}/nltk/corpus/europarl_raw.py +44 -44
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/__init__.py +147 -145
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/aligned.py +115 -114
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/api.py +448 -446
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/bnc.py +252 -252
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/bracket_parse.py +227 -227
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/categorized_sents.py +179 -178
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/chasen.py +140 -139
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/childes.py +522 -494
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/chunked.py +212 -212
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/cmudict.py +97 -95
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/comparative_sents.py +280 -278
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/conll.py +528 -521
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/crubadan.py +116 -116
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/dependency.py +101 -101
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/framenet.py +3059 -2093
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ieer.py +112 -111
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/indian.py +85 -86
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ipipan.py +332 -331
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/knbc.py +187 -187
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/lin.py +156 -156
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/mte.py +311 -310
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/nkjp.py +429 -428
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/nombank.py +421 -421
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/nps_chat.py +73 -73
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/opinion_lexicon.py +115 -115
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/panlex_lite.py +165 -165
- nltk-3.3/nltk/corpus/reader/pl196x.py +292 -0
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/plaintext.py +232 -233
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ppattach.py +96 -95
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/propbank.py +479 -481
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/pros_cons.py +128 -126
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/reviews.py +330 -328
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/rte.py +144 -146
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/semcor.py +256 -256
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/senseval.py +203 -201
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/sentiwordnet.py +137 -137
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/sinica_treebank.py +75 -75
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/string_category.py +60 -59
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/switchboard.py +119 -119
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/tagged.py +295 -294
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/timit.py +452 -450
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/toolbox.py +68 -68
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/twitter.py +155 -157
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/udhr.py +80 -80
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/util.py +804 -800
- nltk-3.3/nltk/corpus/reader/verbnet.py +595 -0
- nltk-3.3/nltk/corpus/reader/wordlist.py +134 -0
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/wordnet.py +2108 -2036
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/xmldocs.py +392 -387
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ycoe.py +243 -242
- {nltk-3.2 → nltk-3.3}/nltk/corpus/util.py +144 -127
- {nltk-3.2 → nltk-3.3}/nltk/data.py +1487 -1451
- {nltk-3.2 → nltk-3.3}/nltk/decorators.py +219 -215
- {nltk-3.2 → nltk-3.3}/nltk/downloader.py +2278 -2274
- {nltk-3.2 → nltk-3.3}/nltk/draw/__init__.py +27 -27
- {nltk-3.2 → nltk-3.3}/nltk/draw/cfg.py +774 -777
- {nltk-3.2 → nltk-3.3}/nltk/draw/dispersion.py +58 -58
- {nltk-3.2 → nltk-3.3}/nltk/draw/table.py +1098 -1098
- {nltk-3.2 → nltk-3.3}/nltk/draw/tree.py +963 -966
- {nltk-3.2 → nltk-3.3}/nltk/draw/util.py +2356 -2356
- {nltk-3.2 → nltk-3.3}/nltk/featstruct.py +2617 -2500
- {nltk-3.2 → nltk-3.3}/nltk/grammar.py +1559 -1516
- {nltk-3.2 → nltk-3.3}/nltk/help.py +56 -56
- {nltk-3.2 → nltk-3.3}/nltk/inference/__init__.py +20 -20
- {nltk-3.2 → nltk-3.3}/nltk/inference/api.py +597 -589
- {nltk-3.2 → nltk-3.3}/nltk/inference/discourse.py +612 -608
- {nltk-3.2 → nltk-3.3}/nltk/inference/mace.py +311 -311
- {nltk-3.2 → nltk-3.3}/nltk/inference/nonmonotonic.py +509 -509
- {nltk-3.2 → nltk-3.3}/nltk/inference/prover9.py +431 -431
- {nltk-3.2 → nltk-3.3}/nltk/inference/resolution.py +687 -687
- {nltk-3.2 → nltk-3.3}/nltk/inference/tableau.py +607 -607
- {nltk-3.2 → nltk-3.3}/nltk/internals.py +982 -982
- {nltk-3.2 → nltk-3.3}/nltk/jsontags.py +63 -63
- {nltk-3.2 → nltk-3.3}/nltk/lazyimport.py +142 -142
- {nltk-3.2 → nltk-3.3}/nltk/metrics/__init__.py +30 -29
- {nltk-3.2 → nltk-3.3}/nltk/metrics/agreement.py +421 -401
- nltk-3.3/nltk/metrics/aline.py +607 -0
- {nltk-3.2 → nltk-3.3}/nltk/metrics/association.py +414 -410
- {nltk-3.2 → nltk-3.3}/nltk/metrics/confusionmatrix.py +206 -206
- {nltk-3.2 → nltk-3.3}/nltk/metrics/distance.py +207 -202
- {nltk-3.2 → nltk-3.3}/nltk/metrics/paice.py +381 -381
- {nltk-3.2 → nltk-3.3}/nltk/metrics/scores.py +228 -228
- {nltk-3.2 → nltk-3.3}/nltk/metrics/segmentation.py +231 -232
- {nltk-3.2 → nltk-3.3}/nltk/metrics/spearman.py +69 -69
- {nltk-3.2 → nltk-3.3}/nltk/misc/__init__.py +11 -11
- {nltk-3.2 → nltk-3.3}/nltk/misc/babelfish.py +10 -10
- {nltk-3.2 → nltk-3.3}/nltk/misc/chomsky.py +135 -133
- {nltk-3.2 → nltk-3.3}/nltk/misc/minimalset.py +83 -83
- {nltk-3.2 → nltk-3.3}/nltk/misc/sort.py +157 -157
- {nltk-3.2 → nltk-3.3}/nltk/misc/wordfinder.py +129 -129
- {nltk-3.2 → nltk-3.3}/nltk/parse/__init__.py +81 -80
- {nltk-3.2 → nltk-3.3}/nltk/parse/api.py +66 -66
- {nltk-3.2 → nltk-3.3}/nltk/parse/bllip.py +282 -282
- {nltk-3.2 → nltk-3.3}/nltk/parse/chart.py +1682 -1681
- nltk-3.3/nltk/parse/corenlp.py +780 -0
- {nltk-3.2 → nltk-3.3}/nltk/parse/dependencygraph.py +760 -758
- {nltk-3.2 → nltk-3.3}/nltk/parse/earleychart.py +452 -451
- {nltk-3.2 → nltk-3.3}/nltk/parse/evaluate.py +129 -129
- {nltk-3.2 → nltk-3.3}/nltk/parse/featurechart.py +580 -578
- {nltk-3.2 → nltk-3.3}/nltk/parse/generate.py +87 -76
- {nltk-3.2 → nltk-3.3}/nltk/parse/malt.py +356 -356
- {nltk-3.2 → nltk-3.3}/nltk/parse/nonprojectivedependencyparser.py +783 -783
- {nltk-3.2 → nltk-3.3}/nltk/parse/pchart.py +522 -522
- {nltk-3.2 → nltk-3.3}/nltk/parse/projectivedependencyparser.py +579 -578
- {nltk-3.2 → nltk-3.3}/nltk/parse/recursivedescent.py +655 -656
- {nltk-3.2 → nltk-3.3}/nltk/parse/shiftreduce.py +458 -459
- {nltk-3.2 → nltk-3.3}/nltk/parse/stanford.py +431 -406
- {nltk-3.2 → nltk-3.3}/nltk/parse/transitionparser.py +774 -772
- {nltk-3.2 → nltk-3.3}/nltk/parse/util.py +231 -231
- {nltk-3.2 → nltk-3.3}/nltk/parse/viterbi.py +401 -401
- {nltk-3.2 → nltk-3.3}/nltk/probability.py +2410 -2355
- {nltk-3.2 → nltk-3.3}/nltk/sem/__init__.py +61 -61
- {nltk-3.2 → nltk-3.3}/nltk/sem/boxer.py +1261 -1261
- {nltk-3.2 → nltk-3.3}/nltk/sem/chat80.py +782 -782
- {nltk-3.2 → nltk-3.3}/nltk/sem/cooper_storage.py +118 -118
- {nltk-3.2 → nltk-3.3}/nltk/sem/drt.py +1258 -1257
- {nltk-3.2 → nltk-3.3}/nltk/sem/drt_glue_demo.py +485 -489
- {nltk-3.2 → nltk-3.3}/nltk/sem/evaluate.py +790 -784
- {nltk-3.2 → nltk-3.3}/nltk/sem/glue.py +696 -694
- {nltk-3.2 → nltk-3.3}/nltk/sem/hole.py +386 -384
- {nltk-3.2 → nltk-3.3}/nltk/sem/lfg.py +210 -208
- {nltk-3.2 → nltk-3.3}/nltk/sem/linearlogic.py +451 -449
- {nltk-3.2 → nltk-3.3}/nltk/sem/logic.py +1910 -1910
- {nltk-3.2 → nltk-3.3}/nltk/sem/relextract.py +476 -483
- {nltk-3.2 → nltk-3.3}/nltk/sem/skolemize.py +101 -101
- {nltk-3.2 → nltk-3.3}/nltk/sem/util.py +249 -249
- {nltk-3.2 → nltk-3.3}/nltk/sentiment/__init__.py +14 -14
- {nltk-3.2 → nltk-3.3}/nltk/sentiment/sentiment_analyzer.py +229 -229
- {nltk-3.2 → nltk-3.3}/nltk/sentiment/util.py +762 -752
- {nltk-3.2 → nltk-3.3}/nltk/sentiment/vader.py +443 -445
- {nltk-3.2 → nltk-3.3}/nltk/stem/__init__.py +31 -33
- {nltk-3.2 → nltk-3.3}/nltk/stem/api.py +28 -25
- nltk-3.3/nltk/stem/arlstem.py +369 -0
- {nltk-3.2 → nltk-3.3}/nltk/stem/isri.py +345 -345
- {nltk-3.2 → nltk-3.3}/nltk/stem/lancaster.py +337 -310
- nltk-3.3/nltk/stem/porter.py +710 -0
- {nltk-3.2 → nltk-3.3}/nltk/stem/regexp.py +61 -61
- {nltk-3.2 → nltk-3.3}/nltk/stem/rslp.py +142 -142
- {nltk-3.2 → nltk-3.3}/nltk/stem/snowball.py +4236 -3712
- {nltk-3.2 → nltk-3.3}/nltk/stem/util.py +22 -12
- {nltk-3.2 → nltk-3.3}/nltk/stem/wordnet.py +51 -51
- {nltk-3.2 → nltk-3.3}/nltk/tag/__init__.py +152 -127
- {nltk-3.2 → nltk-3.3}/nltk/tag/api.py +86 -81
- {nltk-3.2 → nltk-3.3}/nltk/tag/brill.py +424 -426
- {nltk-3.2 → nltk-3.3}/nltk/tag/brill_trainer.py +608 -608
- {nltk-3.2 → nltk-3.3}/nltk/tag/crf.py +204 -200
- {nltk-3.2 → nltk-3.3}/nltk/tag/hmm.py +1277 -1277
- {nltk-3.2 → nltk-3.3}/nltk/tag/hunpos.py +132 -132
- {nltk-3.2 → nltk-3.3}/nltk/tag/mapping.py +101 -101
- {nltk-3.2 → nltk-3.3}/nltk/tag/perceptron.py +329 -314
- {nltk-3.2 → nltk-3.3}/nltk/tag/senna.py +147 -145
- {nltk-3.2 → nltk-3.3}/nltk/tag/sequential.py +740 -736
- {nltk-3.2 → nltk-3.3}/nltk/tag/stanford.py +219 -196
- {nltk-3.2 → nltk-3.3}/nltk/tag/tnt.py +595 -604
- {nltk-3.2 → nltk-3.3}/nltk/tag/util.py +72 -72
- {nltk-3.2 → nltk-3.3}/nltk/tbl/__init__.py +28 -28
- {nltk-3.2 → nltk-3.3}/nltk/tbl/api.py +1 -1
- {nltk-3.2 → nltk-3.3}/nltk/tbl/demo.py +366 -366
- {nltk-3.2 → nltk-3.3}/nltk/tbl/erroranalysis.py +39 -39
- {nltk-3.2 → nltk-3.3}/nltk/tbl/feature.py +259 -261
- {nltk-3.2 → nltk-3.3}/nltk/tbl/rule.py +321 -321
- {nltk-3.2 → nltk-3.3}/nltk/tbl/template.py +303 -308
- {nltk-3.2 → nltk-3.3}/nltk/test/__init__.py +18 -18
- {nltk-3.2 → nltk-3.3}/nltk/test/all.py +24 -23
- {nltk-3.2 → nltk-3.3}/nltk/test/bleu.doctest +14 -14
- {nltk-3.2 → nltk-3.3}/nltk/test/bnc.doctest +60 -60
- {nltk-3.2 → nltk-3.3}/nltk/test/ccg.doctest +376 -395
- nltk-3.3/nltk/test/ccg_semantics.doctest +553 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/chat80.doctest +234 -234
- {nltk-3.2 → nltk-3.3}/nltk/test/childes.doctest +184 -184
- {nltk-3.2 → nltk-3.3}/nltk/test/childes_fixt.py +14 -13
- {nltk-3.2 → nltk-3.3}/nltk/test/chunk.doctest +373 -373
- {nltk-3.2 → nltk-3.3}/nltk/test/classify.doctest +185 -185
- {nltk-3.2 → nltk-3.3}/nltk/test/classify_fixt.py +11 -10
- nltk-3.3/nltk/test/collections.doctest +20 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/collocations.doctest +276 -276
- {nltk-3.2 → nltk-3.3}/nltk/test/compat.doctest +134 -138
- {nltk-3.2 → nltk-3.3}/nltk/test/compat_fixt.py +9 -8
- nltk-3.3/nltk/test/concordance.doctest +55 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/corpus.doctest +2200 -2212
- {nltk-3.2 → nltk-3.3}/nltk/test/corpus_fixt.py +3 -3
- {nltk-3.2 → nltk-3.3}/nltk/test/crubadan.doctest +65 -65
- {nltk-3.2 → nltk-3.3}/nltk/test/data.doctest +379 -379
- {nltk-3.2 → nltk-3.3}/nltk/test/dependency.doctest +241 -241
- {nltk-3.2 → nltk-3.3}/nltk/test/discourse.doctest +546 -546
- {nltk-3.2 → nltk-3.3}/nltk/test/discourse_fixt.py +14 -13
- {nltk-3.2 → nltk-3.3}/nltk/test/doctest_nose_plugin.py +159 -156
- {nltk-3.2 → nltk-3.3}/nltk/test/drt.doctest +517 -517
- {nltk-3.2 → nltk-3.3}/nltk/test/featgram.doctest +607 -607
- {nltk-3.2 → nltk-3.3}/nltk/test/featstruct.doctest +1229 -1230
- {nltk-3.2 → nltk-3.3}/nltk/test/framenet.doctest +288 -239
- {nltk-3.2 → nltk-3.3}/nltk/test/generate.doctest +67 -67
- {nltk-3.2 → nltk-3.3}/nltk/test/gensim.doctest +140 -141
- {nltk-3.2 → nltk-3.3}/nltk/test/gensim_fixt.py +10 -9
- {nltk-3.2 → nltk-3.3}/nltk/test/gluesemantics.doctest +384 -386
- {nltk-3.2 → nltk-3.3}/nltk/test/gluesemantics_malt.doctest +68 -68
- {nltk-3.2 → nltk-3.3}/nltk/test/gluesemantics_malt_fixt.py +12 -11
- {nltk-3.2 → nltk-3.3}/nltk/test/grammar.doctest +48 -48
- {nltk-3.2 → nltk-3.3}/nltk/test/grammartestsuites.doctest +109 -109
- {nltk-3.2 → nltk-3.3}/nltk/test/index.doctest +100 -100
- {nltk-3.2 → nltk-3.3}/nltk/test/inference.doctest +534 -534
- {nltk-3.2 → nltk-3.3}/nltk/test/inference_fixt.py +12 -11
- {nltk-3.2 → nltk-3.3}/nltk/test/internals.doctest +140 -140
- {nltk-3.2 → nltk-3.3}/nltk/test/japanese.doctest +48 -48
- {nltk-3.2 → nltk-3.3}/nltk/test/logic.doctest +1098 -1098
- {nltk-3.2 → nltk-3.3}/nltk/test/metrics.doctest +270 -270
- {nltk-3.2 → nltk-3.3}/nltk/test/misc.doctest +118 -118
- {nltk-3.2 → nltk-3.3}/nltk/test/nonmonotonic.doctest +286 -286
- {nltk-3.2 → nltk-3.3}/nltk/test/nonmonotonic_fixt.py +12 -11
- {nltk-3.2 → nltk-3.3}/nltk/test/paice.doctest +35 -35
- {nltk-3.2 → nltk-3.3}/nltk/test/parse.doctest +884 -884
- {nltk-3.2 → nltk-3.3}/nltk/test/portuguese_en.doctest +565 -565
- {nltk-3.2 → nltk-3.3}/nltk/test/portuguese_en_fixt.py +16 -16
- {nltk-3.2 → nltk-3.3}/nltk/test/probability.doctest +304 -304
- {nltk-3.2 → nltk-3.3}/nltk/test/probability_fixt.py +13 -12
- {nltk-3.2 → nltk-3.3}/nltk/test/propbank.doctest +176 -176
- {nltk-3.2 → nltk-3.3}/nltk/test/relextract.doctest +263 -263
- {nltk-3.2 → nltk-3.3}/nltk/test/resolution.doctest +221 -221
- {nltk-3.2 → nltk-3.3}/nltk/test/runtests.py +86 -64
- {nltk-3.2 → nltk-3.3}/nltk/test/segmentation_fixt.py +10 -10
- {nltk-3.2 → nltk-3.3}/nltk/test/semantics.doctest +665 -665
- {nltk-3.2 → nltk-3.3}/nltk/test/semantics_fixt.py +7 -7
- {nltk-3.2 → nltk-3.3}/nltk/test/sentiment.doctest +233 -233
- {nltk-3.2 → nltk-3.3}/nltk/test/sentiwordnet.doctest +41 -41
- {nltk-3.2 → nltk-3.3}/nltk/test/simple.doctest +85 -85
- {nltk-3.2 → nltk-3.3}/nltk/test/stem.doctest +78 -78
- {nltk-3.2 → nltk-3.3}/nltk/test/tag.doctest +33 -33
- {nltk-3.2 → nltk-3.3}/nltk/test/tokenize.doctest +273 -166
- {nltk-3.2 → nltk-3.3}/nltk/test/toolbox.doctest +307 -307
- {nltk-3.2 → nltk-3.3}/nltk/test/translate.doctest +242 -242
- {nltk-3.2 → nltk-3.3}/nltk/test/translate_fixt.py +3 -3
- {nltk-3.2 → nltk-3.3}/nltk/test/tree.doctest +1101 -1101
- {nltk-3.2 → nltk-3.3}/nltk/test/treeprettyprinter.doctest +127 -127
- {nltk-3.2 → nltk-3.3}/nltk/test/treetransforms.doctest +156 -156
- nltk-3.3/nltk/test/unit/test_2x_compat.py +63 -0
- nltk-3.3/nltk/test/unit/test_aline.py +52 -0
- nltk-3.3/nltk/test/unit/test_chunk.py +56 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_classify.py +55 -53
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_collocations.py +88 -79
- nltk-3.3/nltk/test/unit/test_concordance.py +98 -0
- nltk-3.3/nltk/test/unit/test_corenlp.py +1000 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_corpora.py +200 -183
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_corpus_views.py +45 -45
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_hmm.py +88 -88
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_json2csv_corpus.py +188 -187
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_naivebayes.py +26 -26
- nltk-3.3/nltk/test/unit/test_rte_classify.py +86 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_seekable_unicode_stream_reader.py +124 -124
- nltk-3.3/nltk/test/unit/test_senna.py +74 -0
- nltk-3.3/nltk/test/unit/test_stem.py +121 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_tag.py +20 -20
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_tgrep.py +628 -626
- nltk-3.3/nltk/test/unit/test_tokenize.py +168 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_twitter_auth.py +185 -185
- nltk-3.3/nltk/test/unit/test_wordnet.py +134 -0
- nltk-3.3/nltk/test/unit/translate/test_bleu.py +244 -0
- nltk-3.3/nltk/test/unit/translate/test_gdfa.py +57 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm1.py +75 -75
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm2.py +87 -87
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm3.py +105 -105
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm4.py +126 -126
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm5.py +166 -166
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm_model.py +270 -270
- nltk-3.3/nltk/test/unit/translate/test_nist.py +37 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_stack_decoder.py +300 -300
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/utils.py +41 -41
- {nltk-3.2 → nltk-3.3}/nltk/test/util.doctest +48 -48
- {nltk-3.2 → nltk-3.3}/nltk/test/wordnet.doctest +595 -581
- {nltk-3.2 → nltk-3.3}/nltk/test/wordnet_fixt.py +7 -6
- {nltk-3.2 → nltk-3.3}/nltk/test/wordnet_lch.doctest +53 -53
- {nltk-3.2 → nltk-3.3}/nltk/test/wsd.doctest +68 -68
- {nltk-3.2 → nltk-3.3}/nltk/text.py +663 -610
- {nltk-3.2 → nltk-3.3}/nltk/tgrep.py +938 -937
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/__init__.py +130 -107
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/api.py +77 -78
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/casual.py +343 -342
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/mwe.py +120 -120
- nltk-3.3/nltk/tokenize/nist.py +173 -0
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/punkt.py +1620 -1601
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/regexp.py +208 -202
- nltk-3.3/nltk/tokenize/repp.py +151 -0
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/sexpr.py +140 -142
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/simple.py +138 -136
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/stanford.py +117 -113
- nltk-3.3/nltk/tokenize/stanford_segmenter.py +246 -0
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/texttiling.py +458 -460
- nltk-3.3/nltk/tokenize/toktok.py +155 -0
- nltk-3.3/nltk/tokenize/treebank.py +363 -0
- nltk-3.3/nltk/tokenize/util.py +261 -0
- {nltk-3.2 → nltk-3.3}/nltk/toolbox.py +499 -497
- {nltk-3.2 → nltk-3.3}/nltk/translate/__init__.py +24 -24
- {nltk-3.2 → nltk-3.3}/nltk/translate/api.py +331 -321
- {nltk-3.2 → nltk-3.3}/nltk/translate/bleu_score.py +615 -556
- nltk-3.3/nltk/translate/chrf_score.py +137 -0
- {nltk-3.2 → nltk-3.3}/nltk/translate/gale_church.py +248 -235
- {nltk-3.2 → nltk-3.3}/nltk/translate/gdfa.py +135 -131
- nltk-3.3/nltk/translate/gleu_score.py +193 -0
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm1.py +250 -241
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm2.py +308 -308
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm3.py +337 -337
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm4.py +467 -467
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm5.py +639 -639
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm_model.py +536 -533
- {nltk-3.2 → nltk-3.3}/nltk/translate/metrics.py +40 -40
- nltk-3.3/nltk/translate/nist_score.py +178 -0
- {nltk-3.2 → nltk-3.3}/nltk/translate/phrase_based.py +175 -175
- {nltk-3.2 → nltk-3.3}/nltk/translate/ribes_score.py +325 -325
- {nltk-3.2 → nltk-3.3}/nltk/translate/stack_decoder.py +499 -495
- {nltk-3.2 → nltk-3.3}/nltk/tree.py +1605 -1603
- {nltk-3.2 → nltk-3.3}/nltk/treeprettyprinter.py +564 -564
- {nltk-3.2 → nltk-3.3}/nltk/treetransforms.py +309 -309
- {nltk-3.2 → nltk-3.3}/nltk/twitter/__init__.py +28 -28
- {nltk-3.2 → nltk-3.3}/nltk/twitter/api.py +141 -136
- {nltk-3.2 → nltk-3.3}/nltk/twitter/common.py +264 -258
- {nltk-3.2 → nltk-3.3}/nltk/twitter/twitter_demo.py +288 -288
- {nltk-3.2 → nltk-3.3}/nltk/twitter/twitterclient.py +530 -530
- {nltk-3.2 → nltk-3.3}/nltk/twitter/util.py +147 -146
- nltk-3.3/nltk/util.py +751 -0
- {nltk-3.2 → nltk-3.3}/nltk/wsd.py +53 -53
- {nltk-3.2 → nltk-3.3/nltk.egg-info}/PKG-INFO +13 -5
- {nltk-3.2 → nltk-3.3}/nltk.egg-info/SOURCES.txt +26 -3
- nltk-3.3/nltk.egg-info/requires.txt +31 -0
- {nltk-3.2 → nltk-3.3}/setup.cfg +0 -1
- {nltk-3.2 → nltk-3.3}/setup.py +102 -75
- nltk-3.2/nltk/VERSION +0 -1
- nltk-3.2/nltk/compat.py +0 -680
- nltk-3.2/nltk/corpus/reader/pl196x.py +0 -278
- nltk-3.2/nltk/corpus/reader/verbnet.py +0 -408
- nltk-3.2/nltk/corpus/reader/wordlist.py +0 -37
- nltk-3.2/nltk/sentiment/vader_lexicon.txt +0 -7517
- nltk-3.2/nltk/six.py +0 -868
- nltk-3.2/nltk/stem/porter.py +0 -692
- nltk-3.2/nltk/test/unit/test_2x_compat.py +0 -26
- nltk-3.2/nltk/test/unit/test_stem.py +0 -36
- nltk-3.2/nltk/test/unit/test_tokenize.py +0 -23
- nltk-3.2/nltk/test/unit/translate/test_bleu.py +0 -84
- nltk-3.2/nltk/tokenize/treebank.py +0 -124
- nltk-3.2/nltk/tokenize/util.py +0 -90
- nltk-3.2/nltk/util.py +0 -1364
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/__init__.py +0 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/__init__.py +0 -0
- {nltk-3.2 → nltk-3.3}/nltk.egg-info/dependency_links.txt +0 -0
- {nltk-3.2 → nltk-3.3}/nltk.egg-info/not-zip-safe +0 -0
- {nltk-3.2 → nltk-3.3}/nltk.egg-info/top_level.txt +0 -0
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
from __future__ import absolute_import
|
|
3
|
-
from nltk.compat import PY3
|
|
4
|
-
|
|
5
|
-
from nltk.corpus import teardown_module
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
def setup_module(module):
|
|
9
|
-
from nose import SkipTest
|
|
10
|
-
|
|
11
|
-
raise SkipTest("portuguese_en.doctest imports nltk.examples.pt which doesn't exist!")
|
|
12
|
-
|
|
13
|
-
if not PY3:
|
|
14
|
-
raise SkipTest(
|
|
15
|
-
"portuguese_en.doctest was skipped because non-ascii doctests are not supported under Python 2.x"
|
|
16
|
-
)
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
from __future__ import absolute_import
|
|
3
|
+
from nltk.compat import PY3
|
|
4
|
+
|
|
5
|
+
from nltk.corpus import teardown_module
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def setup_module(module):
|
|
9
|
+
from nose import SkipTest
|
|
10
|
+
|
|
11
|
+
raise SkipTest("portuguese_en.doctest imports nltk.examples.pt which doesn't exist!")
|
|
12
|
+
|
|
13
|
+
if not PY3:
|
|
14
|
+
raise SkipTest(
|
|
15
|
+
"portuguese_en.doctest was skipped because non-ascii doctests are not supported under Python 2.x"
|
|
16
|
+
)
|
|
@@ -1,304 +1,304 @@
|
|
|
1
|
-
.. Copyright (C) 2001-
|
|
2
|
-
.. For license information, see LICENSE.TXT
|
|
3
|
-
|
|
4
|
-
===========
|
|
5
|
-
Probability
|
|
6
|
-
===========
|
|
7
|
-
|
|
8
|
-
>>> import nltk
|
|
9
|
-
>>> from nltk.probability import *
|
|
10
|
-
|
|
11
|
-
FreqDist
|
|
12
|
-
--------
|
|
13
|
-
|
|
14
|
-
>>> text1 = ['no', 'good', 'fish', 'goes', 'anywhere', 'without', 'a', 'porpoise', '!']
|
|
15
|
-
>>> text2 = ['no', 'good', 'porpoise', 'likes', 'to', 'fish', 'fish', 'anywhere', '.']
|
|
16
|
-
|
|
17
|
-
>>> fd1 = nltk.FreqDist(text1)
|
|
18
|
-
>>> fd1 == nltk.FreqDist(text1)
|
|
19
|
-
True
|
|
20
|
-
|
|
21
|
-
Note that items are sorted in order of decreasing frequency; two items of the same frequency appear in indeterminate order.
|
|
22
|
-
|
|
23
|
-
>>> import itertools
|
|
24
|
-
>>> both = nltk.FreqDist(text1 + text2)
|
|
25
|
-
>>> both_most_common = both.most_common()
|
|
26
|
-
>>> list(itertools.chain(*(sorted(ys) for k, ys in itertools.groupby(both_most_common, key=lambda t: t[1]))))
|
|
27
|
-
[('fish', 3), ('anywhere', 2), ('good', 2), ('no', 2), ('porpoise', 2), ('!', 1), ('.', 1), ('a', 1), ('goes', 1), ('likes', 1), ('to', 1), ('without', 1)]
|
|
28
|
-
|
|
29
|
-
>>> both == fd1 + nltk.FreqDist(text2)
|
|
30
|
-
True
|
|
31
|
-
>>> fd1 == nltk.FreqDist(text1) # But fd1 is unchanged
|
|
32
|
-
True
|
|
33
|
-
|
|
34
|
-
>>> fd2 = nltk.FreqDist(text2)
|
|
35
|
-
>>> fd1.update(fd2)
|
|
36
|
-
>>> fd1 == both
|
|
37
|
-
True
|
|
38
|
-
|
|
39
|
-
>>> fd1 = nltk.FreqDist(text1)
|
|
40
|
-
>>> fd1.update(text2)
|
|
41
|
-
>>> fd1 == both
|
|
42
|
-
True
|
|
43
|
-
|
|
44
|
-
>>> fd1 = nltk.FreqDist(text1)
|
|
45
|
-
>>> fd2 = nltk.FreqDist(fd1)
|
|
46
|
-
>>> fd2 == fd1
|
|
47
|
-
True
|
|
48
|
-
|
|
49
|
-
``nltk.FreqDist`` can be pickled:
|
|
50
|
-
|
|
51
|
-
>>> import pickle
|
|
52
|
-
>>> fd1 = nltk.FreqDist(text1)
|
|
53
|
-
>>> pickled = pickle.dumps(fd1)
|
|
54
|
-
>>> fd1 == pickle.loads(pickled)
|
|
55
|
-
True
|
|
56
|
-
|
|
57
|
-
Mathematical operations:
|
|
58
|
-
|
|
59
|
-
>>> FreqDist('abbb') + FreqDist('bcc')
|
|
60
|
-
FreqDist({'b': 4, 'c': 2, 'a': 1})
|
|
61
|
-
>>> FreqDist('abbbc') - FreqDist('bccd')
|
|
62
|
-
FreqDist({'b': 2, 'a': 1})
|
|
63
|
-
>>> FreqDist('abbb') | FreqDist('bcc')
|
|
64
|
-
FreqDist({'b': 3, 'c': 2, 'a': 1})
|
|
65
|
-
>>> FreqDist('abbb') & FreqDist('bcc')
|
|
66
|
-
FreqDist({'b': 1})
|
|
67
|
-
|
|
68
|
-
ConditionalFreqDist
|
|
69
|
-
-------------------
|
|
70
|
-
|
|
71
|
-
>>> cfd1 = ConditionalFreqDist()
|
|
72
|
-
>>> cfd1[1] = FreqDist('abbbb')
|
|
73
|
-
>>> cfd1[2] = FreqDist('xxxxyy')
|
|
74
|
-
>>> cfd1
|
|
75
|
-
<ConditionalFreqDist with 2 conditions>
|
|
76
|
-
|
|
77
|
-
>>> cfd2 = ConditionalFreqDist()
|
|
78
|
-
>>> cfd2[1] = FreqDist('bbccc')
|
|
79
|
-
>>> cfd2[2] = FreqDist('xxxyyyzz')
|
|
80
|
-
>>> cfd2[3] = FreqDist('m')
|
|
81
|
-
>>> cfd2
|
|
82
|
-
<ConditionalFreqDist with 3 conditions>
|
|
83
|
-
|
|
84
|
-
>>> r = cfd1 + cfd2
|
|
85
|
-
>>> [(i,r[i]) for i in r.conditions()]
|
|
86
|
-
[(1, FreqDist({'b': 6, 'c': 3, 'a': 1})), (2, FreqDist({'x': 7, 'y': 5, 'z': 2})), (3, FreqDist({'m': 1}))]
|
|
87
|
-
|
|
88
|
-
>>> r = cfd1 - cfd2
|
|
89
|
-
>>> [(i,r[i]) for i in r.conditions()]
|
|
90
|
-
[(1, FreqDist({'b': 2, 'a': 1})), (2, FreqDist({'x': 1}))]
|
|
91
|
-
|
|
92
|
-
>>> r = cfd1 | cfd2
|
|
93
|
-
>>> [(i,r[i]) for i in r.conditions()]
|
|
94
|
-
[(1, FreqDist({'b': 4, 'c': 3, 'a': 1})), (2, FreqDist({'x': 4, 'y': 3, 'z': 2})), (3, FreqDist({'m': 1}))]
|
|
95
|
-
|
|
96
|
-
>>> r = cfd1 & cfd2
|
|
97
|
-
>>> [(i,r[i]) for i in r.conditions()]
|
|
98
|
-
[(1, FreqDist({'b': 2})), (2, FreqDist({'x': 3, 'y': 2}))]
|
|
99
|
-
|
|
100
|
-
Testing some HMM estimators
|
|
101
|
-
---------------------------
|
|
102
|
-
|
|
103
|
-
We extract a small part (500 sentences) of the Brown corpus
|
|
104
|
-
|
|
105
|
-
>>> corpus = nltk.corpus.brown.tagged_sents(categories='adventure')[:500]
|
|
106
|
-
>>> print(len(corpus))
|
|
107
|
-
500
|
|
108
|
-
|
|
109
|
-
We create a HMM trainer - note that we need the tags and symbols
|
|
110
|
-
from the whole corpus, not just the training corpus
|
|
111
|
-
|
|
112
|
-
>>> from nltk.util import unique_list
|
|
113
|
-
>>> tag_set = unique_list(tag for sent in corpus for (word,tag) in sent)
|
|
114
|
-
>>> print(len(tag_set))
|
|
115
|
-
92
|
|
116
|
-
>>> symbols = unique_list(word for sent in corpus for (word,tag) in sent)
|
|
117
|
-
>>> print(len(symbols))
|
|
118
|
-
1464
|
|
119
|
-
>>> trainer = nltk.tag.HiddenMarkovModelTrainer(tag_set, symbols)
|
|
120
|
-
|
|
121
|
-
We divide the corpus into 90% training and 10% testing
|
|
122
|
-
|
|
123
|
-
>>> train_corpus = []
|
|
124
|
-
>>> test_corpus = []
|
|
125
|
-
>>> for i in range(len(corpus)):
|
|
126
|
-
... if i % 10:
|
|
127
|
-
... train_corpus += [corpus[i]]
|
|
128
|
-
... else:
|
|
129
|
-
... test_corpus += [corpus[i]]
|
|
130
|
-
>>> print(len(train_corpus))
|
|
131
|
-
450
|
|
132
|
-
>>> print(len(test_corpus))
|
|
133
|
-
50
|
|
134
|
-
|
|
135
|
-
And now we can test the estimators
|
|
136
|
-
|
|
137
|
-
>>> def train_and_test(est):
|
|
138
|
-
... hmm = trainer.train_supervised(train_corpus, estimator=est)
|
|
139
|
-
... print('%.2f%%' % (100 * hmm.evaluate(test_corpus)))
|
|
140
|
-
|
|
141
|
-
Maximum Likelihood Estimation
|
|
142
|
-
-----------------------------
|
|
143
|
-
- this resulted in an initialization error before r7209
|
|
144
|
-
|
|
145
|
-
>>> mle = lambda fd, bins: MLEProbDist(fd)
|
|
146
|
-
>>> train_and_test(mle)
|
|
147
|
-
22.75%
|
|
148
|
-
|
|
149
|
-
Laplace (= Lidstone with gamma==1)
|
|
150
|
-
|
|
151
|
-
>>> train_and_test(LaplaceProbDist)
|
|
152
|
-
66.04%
|
|
153
|
-
|
|
154
|
-
Expected Likelihood Estimation (= Lidstone with gamma==0.5)
|
|
155
|
-
|
|
156
|
-
>>> train_and_test(ELEProbDist)
|
|
157
|
-
73.01%
|
|
158
|
-
|
|
159
|
-
Lidstone Estimation, for gamma==0.1, 0.5 and 1
|
|
160
|
-
(the later two should be exactly equal to MLE and ELE above)
|
|
161
|
-
|
|
162
|
-
>>> def lidstone(gamma):
|
|
163
|
-
... return lambda fd, bins: LidstoneProbDist(fd, gamma, bins)
|
|
164
|
-
>>> train_and_test(lidstone(0.1))
|
|
165
|
-
82.51%
|
|
166
|
-
>>> train_and_test(lidstone(0.5))
|
|
167
|
-
73.01%
|
|
168
|
-
>>> train_and_test(lidstone(1.0))
|
|
169
|
-
66.04%
|
|
170
|
-
|
|
171
|
-
Witten Bell Estimation
|
|
172
|
-
----------------------
|
|
173
|
-
- This resulted in ZeroDivisionError before r7209
|
|
174
|
-
|
|
175
|
-
>>> train_and_test(WittenBellProbDist)
|
|
176
|
-
88.12%
|
|
177
|
-
|
|
178
|
-
Good Turing Estimation
|
|
179
|
-
|
|
180
|
-
>>> gt = lambda fd, bins: SimpleGoodTuringProbDist(fd, bins=1e5)
|
|
181
|
-
>>> train_and_test(gt)
|
|
182
|
-
86.93%
|
|
183
|
-
|
|
184
|
-
Kneser Ney Estimation
|
|
185
|
-
---------------------
|
|
186
|
-
Since the Kneser-Ney distribution is best suited for trigrams, we must adjust
|
|
187
|
-
our testing accordingly.
|
|
188
|
-
|
|
189
|
-
>>> corpus = [[((x[0],y[0],z[0]),(x[1],y[1],z[1]))
|
|
190
|
-
... for x, y, z in nltk.trigrams(sent)]
|
|
191
|
-
... for sent in corpus[:100]]
|
|
192
|
-
|
|
193
|
-
We will then need to redefine the rest of the training/testing variables
|
|
194
|
-
>>> tag_set = unique_list(tag for sent in corpus for (word,tag) in sent)
|
|
195
|
-
>>> len(tag_set)
|
|
196
|
-
906
|
|
197
|
-
|
|
198
|
-
>>> symbols = unique_list(word for sent in corpus for (word,tag) in sent)
|
|
199
|
-
>>> len(symbols)
|
|
200
|
-
1341
|
|
201
|
-
|
|
202
|
-
>>> trainer = nltk.tag.HiddenMarkovModelTrainer(tag_set, symbols)
|
|
203
|
-
>>> train_corpus = []
|
|
204
|
-
>>> test_corpus = []
|
|
205
|
-
|
|
206
|
-
>>> for i in range(len(corpus)):
|
|
207
|
-
... if i % 10:
|
|
208
|
-
... train_corpus += [corpus[i]]
|
|
209
|
-
... else:
|
|
210
|
-
... test_corpus += [corpus[i]]
|
|
211
|
-
|
|
212
|
-
>>> len(train_corpus)
|
|
213
|
-
90
|
|
214
|
-
>>> len(test_corpus)
|
|
215
|
-
10
|
|
216
|
-
|
|
217
|
-
>>> kn = lambda fd, bins: KneserNeyProbDist(fd)
|
|
218
|
-
>>> train_and_test(kn)
|
|
219
|
-
0.86%
|
|
220
|
-
|
|
221
|
-
Remains to be added:
|
|
222
|
-
- Tests for HeldoutProbDist, CrossValidationProbDist and MutableProbDist
|
|
223
|
-
|
|
224
|
-
Squashed bugs
|
|
225
|
-
-------------
|
|
226
|
-
|
|
227
|
-
Issue 511: override pop and popitem to invalidate the cache
|
|
228
|
-
|
|
229
|
-
>>> fd = nltk.FreqDist('a')
|
|
230
|
-
>>> list(fd.keys())
|
|
231
|
-
['a']
|
|
232
|
-
>>> fd.pop('a')
|
|
233
|
-
1
|
|
234
|
-
>>> list(fd.keys())
|
|
235
|
-
[]
|
|
236
|
-
|
|
237
|
-
Issue 533: access cumulative frequencies with no arguments
|
|
238
|
-
|
|
239
|
-
>>> fd = nltk.FreqDist('aab')
|
|
240
|
-
>>> list(fd._cumulative_frequencies(['a']))
|
|
241
|
-
[2.0]
|
|
242
|
-
>>> list(fd._cumulative_frequencies(['a', 'b']))
|
|
243
|
-
[2.0, 3.0]
|
|
244
|
-
|
|
245
|
-
Issue 579: override clear to reset some variables
|
|
246
|
-
|
|
247
|
-
>>> fd = FreqDist('aab')
|
|
248
|
-
>>> fd.clear()
|
|
249
|
-
>>> fd.N()
|
|
250
|
-
0
|
|
251
|
-
|
|
252
|
-
Issue 351: fix fileids method of CategorizedCorpusReader to inadvertently
|
|
253
|
-
add errant categories
|
|
254
|
-
|
|
255
|
-
>>> from nltk.corpus import brown
|
|
256
|
-
>>> brown.fileids('blah')
|
|
257
|
-
Traceback (most recent call last):
|
|
258
|
-
...
|
|
259
|
-
ValueError: Category blah not found
|
|
260
|
-
>>> brown.categories()
|
|
261
|
-
['adventure', 'belles_lettres', 'editorial', 'fiction', 'government', 'hobbies', 'humor', 'learned', 'lore', 'mystery', 'news', 'religion', 'reviews', 'romance', 'science_fiction']
|
|
262
|
-
|
|
263
|
-
Issue 175: add the unseen bin to SimpleGoodTuringProbDist by default
|
|
264
|
-
otherwise any unseen events get a probability of zero, i.e.,
|
|
265
|
-
they don't get smoothed
|
|
266
|
-
|
|
267
|
-
>>> from nltk import SimpleGoodTuringProbDist, FreqDist
|
|
268
|
-
>>> fd = FreqDist({'a':1, 'b':1, 'c': 2, 'd': 3, 'e': 4, 'f': 4, 'g': 4, 'h': 5, 'i': 5, 'j': 6, 'k': 6, 'l': 6, 'm': 7, 'n': 7, 'o': 8, 'p': 9, 'q': 10})
|
|
269
|
-
>>> p = SimpleGoodTuringProbDist(fd)
|
|
270
|
-
>>> p.prob('a')
|
|
271
|
-
0.017649766667026317...
|
|
272
|
-
>>> p.prob('o')
|
|
273
|
-
0.08433050215340411...
|
|
274
|
-
>>> p.prob('z')
|
|
275
|
-
0.022727272727272728...
|
|
276
|
-
>>> p.prob('foobar')
|
|
277
|
-
0.022727272727272728...
|
|
278
|
-
|
|
279
|
-
``MLEProbDist``, ``ConditionalProbDist'', ``DictionaryConditionalProbDist`` and
|
|
280
|
-
``ConditionalFreqDist`` can be pickled:
|
|
281
|
-
|
|
282
|
-
>>> import pickle
|
|
283
|
-
>>> pd = MLEProbDist(fd)
|
|
284
|
-
>>> sorted(pd.samples()) == sorted(pickle.loads(pickle.dumps(pd)).samples())
|
|
285
|
-
True
|
|
286
|
-
>>> dpd = DictionaryConditionalProbDist({'x': pd})
|
|
287
|
-
>>> unpickled = pickle.loads(pickle.dumps(dpd))
|
|
288
|
-
>>> dpd['x'].prob('a')
|
|
289
|
-
0.011363636...
|
|
290
|
-
>>> dpd['x'].prob('a') == unpickled['x'].prob('a')
|
|
291
|
-
True
|
|
292
|
-
>>> cfd = nltk.probability.ConditionalFreqDist()
|
|
293
|
-
>>> cfd['foo']['hello'] += 1
|
|
294
|
-
>>> cfd['foo']['hello'] += 1
|
|
295
|
-
>>> cfd['bar']['hello'] += 1
|
|
296
|
-
>>> cfd2 = pickle.loads(pickle.dumps(cfd))
|
|
297
|
-
>>> cfd2 == cfd
|
|
298
|
-
True
|
|
299
|
-
>>> cpd = ConditionalProbDist(cfd, SimpleGoodTuringProbDist)
|
|
300
|
-
>>> cpd2 = pickle.loads(pickle.dumps(cpd))
|
|
301
|
-
>>> cpd['foo'].prob('hello') == cpd2['foo'].prob('hello')
|
|
302
|
-
True
|
|
303
|
-
|
|
304
|
-
|
|
1
|
+
.. Copyright (C) 2001-2018 NLTK Project
|
|
2
|
+
.. For license information, see LICENSE.TXT
|
|
3
|
+
|
|
4
|
+
===========
|
|
5
|
+
Probability
|
|
6
|
+
===========
|
|
7
|
+
|
|
8
|
+
>>> import nltk
|
|
9
|
+
>>> from nltk.probability import *
|
|
10
|
+
|
|
11
|
+
FreqDist
|
|
12
|
+
--------
|
|
13
|
+
|
|
14
|
+
>>> text1 = ['no', 'good', 'fish', 'goes', 'anywhere', 'without', 'a', 'porpoise', '!']
|
|
15
|
+
>>> text2 = ['no', 'good', 'porpoise', 'likes', 'to', 'fish', 'fish', 'anywhere', '.']
|
|
16
|
+
|
|
17
|
+
>>> fd1 = nltk.FreqDist(text1)
|
|
18
|
+
>>> fd1 == nltk.FreqDist(text1)
|
|
19
|
+
True
|
|
20
|
+
|
|
21
|
+
Note that items are sorted in order of decreasing frequency; two items of the same frequency appear in indeterminate order.
|
|
22
|
+
|
|
23
|
+
>>> import itertools
|
|
24
|
+
>>> both = nltk.FreqDist(text1 + text2)
|
|
25
|
+
>>> both_most_common = both.most_common()
|
|
26
|
+
>>> list(itertools.chain(*(sorted(ys) for k, ys in itertools.groupby(both_most_common, key=lambda t: t[1]))))
|
|
27
|
+
[('fish', 3), ('anywhere', 2), ('good', 2), ('no', 2), ('porpoise', 2), ('!', 1), ('.', 1), ('a', 1), ('goes', 1), ('likes', 1), ('to', 1), ('without', 1)]
|
|
28
|
+
|
|
29
|
+
>>> both == fd1 + nltk.FreqDist(text2)
|
|
30
|
+
True
|
|
31
|
+
>>> fd1 == nltk.FreqDist(text1) # But fd1 is unchanged
|
|
32
|
+
True
|
|
33
|
+
|
|
34
|
+
>>> fd2 = nltk.FreqDist(text2)
|
|
35
|
+
>>> fd1.update(fd2)
|
|
36
|
+
>>> fd1 == both
|
|
37
|
+
True
|
|
38
|
+
|
|
39
|
+
>>> fd1 = nltk.FreqDist(text1)
|
|
40
|
+
>>> fd1.update(text2)
|
|
41
|
+
>>> fd1 == both
|
|
42
|
+
True
|
|
43
|
+
|
|
44
|
+
>>> fd1 = nltk.FreqDist(text1)
|
|
45
|
+
>>> fd2 = nltk.FreqDist(fd1)
|
|
46
|
+
>>> fd2 == fd1
|
|
47
|
+
True
|
|
48
|
+
|
|
49
|
+
``nltk.FreqDist`` can be pickled:
|
|
50
|
+
|
|
51
|
+
>>> import pickle
|
|
52
|
+
>>> fd1 = nltk.FreqDist(text1)
|
|
53
|
+
>>> pickled = pickle.dumps(fd1)
|
|
54
|
+
>>> fd1 == pickle.loads(pickled)
|
|
55
|
+
True
|
|
56
|
+
|
|
57
|
+
Mathematical operations:
|
|
58
|
+
|
|
59
|
+
>>> FreqDist('abbb') + FreqDist('bcc')
|
|
60
|
+
FreqDist({'b': 4, 'c': 2, 'a': 1})
|
|
61
|
+
>>> FreqDist('abbbc') - FreqDist('bccd')
|
|
62
|
+
FreqDist({'b': 2, 'a': 1})
|
|
63
|
+
>>> FreqDist('abbb') | FreqDist('bcc')
|
|
64
|
+
FreqDist({'b': 3, 'c': 2, 'a': 1})
|
|
65
|
+
>>> FreqDist('abbb') & FreqDist('bcc')
|
|
66
|
+
FreqDist({'b': 1})
|
|
67
|
+
|
|
68
|
+
ConditionalFreqDist
|
|
69
|
+
-------------------
|
|
70
|
+
|
|
71
|
+
>>> cfd1 = ConditionalFreqDist()
|
|
72
|
+
>>> cfd1[1] = FreqDist('abbbb')
|
|
73
|
+
>>> cfd1[2] = FreqDist('xxxxyy')
|
|
74
|
+
>>> cfd1
|
|
75
|
+
<ConditionalFreqDist with 2 conditions>
|
|
76
|
+
|
|
77
|
+
>>> cfd2 = ConditionalFreqDist()
|
|
78
|
+
>>> cfd2[1] = FreqDist('bbccc')
|
|
79
|
+
>>> cfd2[2] = FreqDist('xxxyyyzz')
|
|
80
|
+
>>> cfd2[3] = FreqDist('m')
|
|
81
|
+
>>> cfd2
|
|
82
|
+
<ConditionalFreqDist with 3 conditions>
|
|
83
|
+
|
|
84
|
+
>>> r = cfd1 + cfd2
|
|
85
|
+
>>> [(i,r[i]) for i in r.conditions()]
|
|
86
|
+
[(1, FreqDist({'b': 6, 'c': 3, 'a': 1})), (2, FreqDist({'x': 7, 'y': 5, 'z': 2})), (3, FreqDist({'m': 1}))]
|
|
87
|
+
|
|
88
|
+
>>> r = cfd1 - cfd2
|
|
89
|
+
>>> [(i,r[i]) for i in r.conditions()]
|
|
90
|
+
[(1, FreqDist({'b': 2, 'a': 1})), (2, FreqDist({'x': 1}))]
|
|
91
|
+
|
|
92
|
+
>>> r = cfd1 | cfd2
|
|
93
|
+
>>> [(i,r[i]) for i in r.conditions()]
|
|
94
|
+
[(1, FreqDist({'b': 4, 'c': 3, 'a': 1})), (2, FreqDist({'x': 4, 'y': 3, 'z': 2})), (3, FreqDist({'m': 1}))]
|
|
95
|
+
|
|
96
|
+
>>> r = cfd1 & cfd2
|
|
97
|
+
>>> [(i,r[i]) for i in r.conditions()]
|
|
98
|
+
[(1, FreqDist({'b': 2})), (2, FreqDist({'x': 3, 'y': 2}))]
|
|
99
|
+
|
|
100
|
+
Testing some HMM estimators
|
|
101
|
+
---------------------------
|
|
102
|
+
|
|
103
|
+
We extract a small part (500 sentences) of the Brown corpus
|
|
104
|
+
|
|
105
|
+
>>> corpus = nltk.corpus.brown.tagged_sents(categories='adventure')[:500]
|
|
106
|
+
>>> print(len(corpus))
|
|
107
|
+
500
|
|
108
|
+
|
|
109
|
+
We create a HMM trainer - note that we need the tags and symbols
|
|
110
|
+
from the whole corpus, not just the training corpus
|
|
111
|
+
|
|
112
|
+
>>> from nltk.util import unique_list
|
|
113
|
+
>>> tag_set = unique_list(tag for sent in corpus for (word,tag) in sent)
|
|
114
|
+
>>> print(len(tag_set))
|
|
115
|
+
92
|
|
116
|
+
>>> symbols = unique_list(word for sent in corpus for (word,tag) in sent)
|
|
117
|
+
>>> print(len(symbols))
|
|
118
|
+
1464
|
|
119
|
+
>>> trainer = nltk.tag.HiddenMarkovModelTrainer(tag_set, symbols)
|
|
120
|
+
|
|
121
|
+
We divide the corpus into 90% training and 10% testing
|
|
122
|
+
|
|
123
|
+
>>> train_corpus = []
|
|
124
|
+
>>> test_corpus = []
|
|
125
|
+
>>> for i in range(len(corpus)):
|
|
126
|
+
... if i % 10:
|
|
127
|
+
... train_corpus += [corpus[i]]
|
|
128
|
+
... else:
|
|
129
|
+
... test_corpus += [corpus[i]]
|
|
130
|
+
>>> print(len(train_corpus))
|
|
131
|
+
450
|
|
132
|
+
>>> print(len(test_corpus))
|
|
133
|
+
50
|
|
134
|
+
|
|
135
|
+
And now we can test the estimators
|
|
136
|
+
|
|
137
|
+
>>> def train_and_test(est):
|
|
138
|
+
... hmm = trainer.train_supervised(train_corpus, estimator=est)
|
|
139
|
+
... print('%.2f%%' % (100 * hmm.evaluate(test_corpus)))
|
|
140
|
+
|
|
141
|
+
Maximum Likelihood Estimation
|
|
142
|
+
-----------------------------
|
|
143
|
+
- this resulted in an initialization error before r7209
|
|
144
|
+
|
|
145
|
+
>>> mle = lambda fd, bins: MLEProbDist(fd)
|
|
146
|
+
>>> train_and_test(mle)
|
|
147
|
+
22.75%
|
|
148
|
+
|
|
149
|
+
Laplace (= Lidstone with gamma==1)
|
|
150
|
+
|
|
151
|
+
>>> train_and_test(LaplaceProbDist)
|
|
152
|
+
66.04%
|
|
153
|
+
|
|
154
|
+
Expected Likelihood Estimation (= Lidstone with gamma==0.5)
|
|
155
|
+
|
|
156
|
+
>>> train_and_test(ELEProbDist)
|
|
157
|
+
73.01%
|
|
158
|
+
|
|
159
|
+
Lidstone Estimation, for gamma==0.1, 0.5 and 1
|
|
160
|
+
(the later two should be exactly equal to MLE and ELE above)
|
|
161
|
+
|
|
162
|
+
>>> def lidstone(gamma):
|
|
163
|
+
... return lambda fd, bins: LidstoneProbDist(fd, gamma, bins)
|
|
164
|
+
>>> train_and_test(lidstone(0.1))
|
|
165
|
+
82.51%
|
|
166
|
+
>>> train_and_test(lidstone(0.5))
|
|
167
|
+
73.01%
|
|
168
|
+
>>> train_and_test(lidstone(1.0))
|
|
169
|
+
66.04%
|
|
170
|
+
|
|
171
|
+
Witten Bell Estimation
|
|
172
|
+
----------------------
|
|
173
|
+
- This resulted in ZeroDivisionError before r7209
|
|
174
|
+
|
|
175
|
+
>>> train_and_test(WittenBellProbDist)
|
|
176
|
+
88.12%
|
|
177
|
+
|
|
178
|
+
Good Turing Estimation
|
|
179
|
+
|
|
180
|
+
>>> gt = lambda fd, bins: SimpleGoodTuringProbDist(fd, bins=1e5)
|
|
181
|
+
>>> train_and_test(gt)
|
|
182
|
+
86.93%
|
|
183
|
+
|
|
184
|
+
Kneser Ney Estimation
|
|
185
|
+
---------------------
|
|
186
|
+
Since the Kneser-Ney distribution is best suited for trigrams, we must adjust
|
|
187
|
+
our testing accordingly.
|
|
188
|
+
|
|
189
|
+
>>> corpus = [[((x[0],y[0],z[0]),(x[1],y[1],z[1]))
|
|
190
|
+
... for x, y, z in nltk.trigrams(sent)]
|
|
191
|
+
... for sent in corpus[:100]]
|
|
192
|
+
|
|
193
|
+
We will then need to redefine the rest of the training/testing variables
|
|
194
|
+
>>> tag_set = unique_list(tag for sent in corpus for (word,tag) in sent)
|
|
195
|
+
>>> len(tag_set)
|
|
196
|
+
906
|
|
197
|
+
|
|
198
|
+
>>> symbols = unique_list(word for sent in corpus for (word,tag) in sent)
|
|
199
|
+
>>> len(symbols)
|
|
200
|
+
1341
|
|
201
|
+
|
|
202
|
+
>>> trainer = nltk.tag.HiddenMarkovModelTrainer(tag_set, symbols)
|
|
203
|
+
>>> train_corpus = []
|
|
204
|
+
>>> test_corpus = []
|
|
205
|
+
|
|
206
|
+
>>> for i in range(len(corpus)):
|
|
207
|
+
... if i % 10:
|
|
208
|
+
... train_corpus += [corpus[i]]
|
|
209
|
+
... else:
|
|
210
|
+
... test_corpus += [corpus[i]]
|
|
211
|
+
|
|
212
|
+
>>> len(train_corpus)
|
|
213
|
+
90
|
|
214
|
+
>>> len(test_corpus)
|
|
215
|
+
10
|
|
216
|
+
|
|
217
|
+
>>> kn = lambda fd, bins: KneserNeyProbDist(fd)
|
|
218
|
+
>>> train_and_test(kn)
|
|
219
|
+
0.86%
|
|
220
|
+
|
|
221
|
+
Remains to be added:
|
|
222
|
+
- Tests for HeldoutProbDist, CrossValidationProbDist and MutableProbDist
|
|
223
|
+
|
|
224
|
+
Squashed bugs
|
|
225
|
+
-------------
|
|
226
|
+
|
|
227
|
+
Issue 511: override pop and popitem to invalidate the cache
|
|
228
|
+
|
|
229
|
+
>>> fd = nltk.FreqDist('a')
|
|
230
|
+
>>> list(fd.keys())
|
|
231
|
+
['a']
|
|
232
|
+
>>> fd.pop('a')
|
|
233
|
+
1
|
|
234
|
+
>>> list(fd.keys())
|
|
235
|
+
[]
|
|
236
|
+
|
|
237
|
+
Issue 533: access cumulative frequencies with no arguments
|
|
238
|
+
|
|
239
|
+
>>> fd = nltk.FreqDist('aab')
|
|
240
|
+
>>> list(fd._cumulative_frequencies(['a']))
|
|
241
|
+
[2.0]
|
|
242
|
+
>>> list(fd._cumulative_frequencies(['a', 'b']))
|
|
243
|
+
[2.0, 3.0]
|
|
244
|
+
|
|
245
|
+
Issue 579: override clear to reset some variables
|
|
246
|
+
|
|
247
|
+
>>> fd = FreqDist('aab')
|
|
248
|
+
>>> fd.clear()
|
|
249
|
+
>>> fd.N()
|
|
250
|
+
0
|
|
251
|
+
|
|
252
|
+
Issue 351: fix fileids method of CategorizedCorpusReader to inadvertently
|
|
253
|
+
add errant categories
|
|
254
|
+
|
|
255
|
+
>>> from nltk.corpus import brown
|
|
256
|
+
>>> brown.fileids('blah')
|
|
257
|
+
Traceback (most recent call last):
|
|
258
|
+
...
|
|
259
|
+
ValueError: Category blah not found
|
|
260
|
+
>>> brown.categories()
|
|
261
|
+
['adventure', 'belles_lettres', 'editorial', 'fiction', 'government', 'hobbies', 'humor', 'learned', 'lore', 'mystery', 'news', 'religion', 'reviews', 'romance', 'science_fiction']
|
|
262
|
+
|
|
263
|
+
Issue 175: add the unseen bin to SimpleGoodTuringProbDist by default
|
|
264
|
+
otherwise any unseen events get a probability of zero, i.e.,
|
|
265
|
+
they don't get smoothed
|
|
266
|
+
|
|
267
|
+
>>> from nltk import SimpleGoodTuringProbDist, FreqDist
|
|
268
|
+
>>> fd = FreqDist({'a':1, 'b':1, 'c': 2, 'd': 3, 'e': 4, 'f': 4, 'g': 4, 'h': 5, 'i': 5, 'j': 6, 'k': 6, 'l': 6, 'm': 7, 'n': 7, 'o': 8, 'p': 9, 'q': 10})
|
|
269
|
+
>>> p = SimpleGoodTuringProbDist(fd)
|
|
270
|
+
>>> p.prob('a')
|
|
271
|
+
0.017649766667026317...
|
|
272
|
+
>>> p.prob('o')
|
|
273
|
+
0.08433050215340411...
|
|
274
|
+
>>> p.prob('z')
|
|
275
|
+
0.022727272727272728...
|
|
276
|
+
>>> p.prob('foobar')
|
|
277
|
+
0.022727272727272728...
|
|
278
|
+
|
|
279
|
+
``MLEProbDist``, ``ConditionalProbDist'', ``DictionaryConditionalProbDist`` and
|
|
280
|
+
``ConditionalFreqDist`` can be pickled:
|
|
281
|
+
|
|
282
|
+
>>> import pickle
|
|
283
|
+
>>> pd = MLEProbDist(fd)
|
|
284
|
+
>>> sorted(pd.samples()) == sorted(pickle.loads(pickle.dumps(pd)).samples())
|
|
285
|
+
True
|
|
286
|
+
>>> dpd = DictionaryConditionalProbDist({'x': pd})
|
|
287
|
+
>>> unpickled = pickle.loads(pickle.dumps(dpd))
|
|
288
|
+
>>> dpd['x'].prob('a')
|
|
289
|
+
0.011363636...
|
|
290
|
+
>>> dpd['x'].prob('a') == unpickled['x'].prob('a')
|
|
291
|
+
True
|
|
292
|
+
>>> cfd = nltk.probability.ConditionalFreqDist()
|
|
293
|
+
>>> cfd['foo']['hello'] += 1
|
|
294
|
+
>>> cfd['foo']['hello'] += 1
|
|
295
|
+
>>> cfd['bar']['hello'] += 1
|
|
296
|
+
>>> cfd2 = pickle.loads(pickle.dumps(cfd))
|
|
297
|
+
>>> cfd2 == cfd
|
|
298
|
+
True
|
|
299
|
+
>>> cpd = ConditionalProbDist(cfd, SimpleGoodTuringProbDist)
|
|
300
|
+
>>> cpd2 = pickle.loads(pickle.dumps(cpd))
|
|
301
|
+
>>> cpd['foo'].prob('hello') == cpd2['foo'].prob('hello')
|
|
302
|
+
True
|
|
303
|
+
|
|
304
|
+
|
|
@@ -1,12 +1,13 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
from __future__ import absolute_import
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
#
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
from __future__ import absolute_import
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
# probability.doctest uses HMM which requires numpy;
|
|
6
|
+
# skip probability.doctest if numpy is not available
|
|
7
|
+
|
|
8
|
+
def setup_module(module):
|
|
9
|
+
from nose import SkipTest
|
|
10
|
+
try:
|
|
11
|
+
import numpy
|
|
12
|
+
except ImportError:
|
|
13
|
+
raise SkipTest("probability.doctest requires numpy")
|