nltk 3.2__zip → 3.3__zip
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nltk-3.2 → nltk-3.3}/INSTALL.txt +6 -6
- {nltk-3.2 → nltk-3.3}/LICENSE.txt +13 -13
- {nltk-3.2 → nltk-3.3}/MANIFEST.in +6 -6
- {nltk-3.2/nltk.egg-info → nltk-3.3}/PKG-INFO +13 -5
- nltk-3.2/README.txt → nltk-3.3/README.md +49 -48
- nltk-3.3/nltk/VERSION +1 -0
- {nltk-3.2 → nltk-3.3}/nltk/__init__.py +185 -185
- {nltk-3.2 → nltk-3.3}/nltk/app/__init__.py +52 -53
- {nltk-3.2 → nltk-3.3}/nltk/app/chartparser_app.py +2276 -2276
- {nltk-3.2 → nltk-3.3}/nltk/app/chunkparser_app.py +1262 -1264
- {nltk-3.2 → nltk-3.3}/nltk/app/collocations_app.py +347 -350
- {nltk-3.2 → nltk-3.3}/nltk/app/concordance_app.py +568 -570
- {nltk-3.2 → nltk-3.3}/nltk/app/nemo_app.py +156 -155
- {nltk-3.2 → nltk-3.3}/nltk/app/rdparser_app.py +892 -894
- {nltk-3.2 → nltk-3.3}/nltk/app/srparser_app.py +809 -809
- {nltk-3.2 → nltk-3.3}/nltk/app/wordfreq_app.py +32 -32
- {nltk-3.2 → nltk-3.3}/nltk/app/wordnet_app.py +968 -968
- {nltk-3.2 → nltk-3.3}/nltk/book.py +91 -90
- {nltk-3.2 → nltk-3.3}/nltk/ccg/__init__.py +22 -22
- {nltk-3.2 → nltk-3.3}/nltk/ccg/api.py +359 -335
- {nltk-3.2 → nltk-3.3}/nltk/ccg/chart.py +394 -361
- {nltk-3.2 → nltk-3.3}/nltk/ccg/combinator.py +343 -312
- {nltk-3.2 → nltk-3.3}/nltk/ccg/lexicon.py +328 -290
- nltk-3.3/nltk/ccg/logic.py +46 -0
- {nltk-3.2 → nltk-3.3}/nltk/chat/__init__.py +49 -49
- {nltk-3.2 → nltk-3.3}/nltk/chat/eliza.py +244 -244
- {nltk-3.2 → nltk-3.3}/nltk/chat/iesha.py +140 -140
- {nltk-3.2 → nltk-3.3}/nltk/chat/rude.py +92 -92
- {nltk-3.2 → nltk-3.3}/nltk/chat/suntsu.py +117 -117
- {nltk-3.2 → nltk-3.3}/nltk/chat/util.py +122 -120
- {nltk-3.2 → nltk-3.3}/nltk/chat/zen.py +282 -282
- {nltk-3.2 → nltk-3.3}/nltk/chunk/__init__.py +190 -190
- {nltk-3.2 → nltk-3.3}/nltk/chunk/api.py +51 -51
- {nltk-3.2 → nltk-3.3}/nltk/chunk/named_entity.py +331 -331
- {nltk-3.2 → nltk-3.3}/nltk/chunk/regexp.py +1390 -1384
- {nltk-3.2 → nltk-3.3}/nltk/chunk/util.py +600 -600
- {nltk-3.2 → nltk-3.3}/nltk/classify/__init__.py +98 -98
- {nltk-3.2 → nltk-3.3}/nltk/classify/api.py +193 -193
- {nltk-3.2 → nltk-3.3}/nltk/classify/decisiontree.py +295 -295
- {nltk-3.2 → nltk-3.3}/nltk/classify/maxent.py +1510 -1492
- {nltk-3.2 → nltk-3.3}/nltk/classify/megam.py +179 -178
- {nltk-3.2 → nltk-3.3}/nltk/classify/naivebayes.py +245 -242
- {nltk-3.2 → nltk-3.3}/nltk/classify/positivenaivebayes.py +170 -170
- {nltk-3.2 → nltk-3.3}/nltk/classify/rte_classify.py +160 -191
- {nltk-3.2 → nltk-3.3}/nltk/classify/scikitlearn.py +153 -151
- {nltk-3.2 → nltk-3.3}/nltk/classify/senna.py +183 -181
- {nltk-3.2 → nltk-3.3}/nltk/classify/svm.py +15 -15
- {nltk-3.2 → nltk-3.3}/nltk/classify/tadm.py +113 -112
- {nltk-3.2 → nltk-3.3}/nltk/classify/textcat.py +193 -193
- {nltk-3.2 → nltk-3.3}/nltk/classify/util.py +324 -312
- {nltk-3.2 → nltk-3.3}/nltk/classify/weka.py +346 -345
- {nltk-3.2 → nltk-3.3}/nltk/cluster/__init__.py +86 -86
- {nltk-3.2 → nltk-3.3}/nltk/cluster/api.py +74 -70
- {nltk-3.2 → nltk-3.3}/nltk/cluster/em.py +247 -247
- {nltk-3.2 → nltk-3.3}/nltk/cluster/gaac.py +168 -168
- {nltk-3.2 → nltk-3.3}/nltk/cluster/kmeans.py +221 -221
- {nltk-3.2 → nltk-3.3}/nltk/cluster/util.py +304 -289
- nltk-3.3/nltk/collections.py +640 -0
- {nltk-3.2 → nltk-3.3}/nltk/collocations.py +388 -388
- nltk-3.3/nltk/compat.py +365 -0
- {nltk-3.2 → nltk-3.3}/nltk/corpus/__init__.py +341 -320
- {nltk-3.2 → nltk-3.3}/nltk/corpus/europarl_raw.py +44 -44
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/__init__.py +147 -145
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/aligned.py +115 -114
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/api.py +448 -446
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/bnc.py +252 -252
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/bracket_parse.py +227 -227
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/categorized_sents.py +179 -178
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/chasen.py +140 -139
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/childes.py +522 -494
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/chunked.py +212 -212
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/cmudict.py +97 -95
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/comparative_sents.py +280 -278
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/conll.py +528 -521
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/crubadan.py +116 -116
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/dependency.py +101 -101
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/framenet.py +3059 -2093
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ieer.py +112 -111
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/indian.py +85 -86
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ipipan.py +332 -331
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/knbc.py +187 -187
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/lin.py +156 -156
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/mte.py +311 -310
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/nkjp.py +429 -428
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/nombank.py +421 -421
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/nps_chat.py +73 -73
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/opinion_lexicon.py +115 -115
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/panlex_lite.py +165 -165
- nltk-3.3/nltk/corpus/reader/pl196x.py +292 -0
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/plaintext.py +232 -233
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ppattach.py +96 -95
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/propbank.py +479 -481
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/pros_cons.py +128 -126
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/reviews.py +330 -328
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/rte.py +144 -146
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/semcor.py +256 -256
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/senseval.py +203 -201
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/sentiwordnet.py +137 -137
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/sinica_treebank.py +75 -75
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/string_category.py +60 -59
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/switchboard.py +119 -119
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/tagged.py +295 -294
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/timit.py +452 -450
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/toolbox.py +68 -68
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/twitter.py +155 -157
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/udhr.py +80 -80
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/util.py +804 -800
- nltk-3.3/nltk/corpus/reader/verbnet.py +595 -0
- nltk-3.3/nltk/corpus/reader/wordlist.py +134 -0
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/wordnet.py +2108 -2036
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/xmldocs.py +392 -387
- {nltk-3.2 → nltk-3.3}/nltk/corpus/reader/ycoe.py +243 -242
- {nltk-3.2 → nltk-3.3}/nltk/corpus/util.py +144 -127
- {nltk-3.2 → nltk-3.3}/nltk/data.py +1487 -1451
- {nltk-3.2 → nltk-3.3}/nltk/decorators.py +219 -215
- {nltk-3.2 → nltk-3.3}/nltk/downloader.py +2278 -2274
- {nltk-3.2 → nltk-3.3}/nltk/draw/__init__.py +27 -27
- {nltk-3.2 → nltk-3.3}/nltk/draw/cfg.py +774 -777
- {nltk-3.2 → nltk-3.3}/nltk/draw/dispersion.py +58 -58
- {nltk-3.2 → nltk-3.3}/nltk/draw/table.py +1098 -1098
- {nltk-3.2 → nltk-3.3}/nltk/draw/tree.py +963 -966
- {nltk-3.2 → nltk-3.3}/nltk/draw/util.py +2356 -2356
- {nltk-3.2 → nltk-3.3}/nltk/featstruct.py +2617 -2500
- {nltk-3.2 → nltk-3.3}/nltk/grammar.py +1559 -1516
- {nltk-3.2 → nltk-3.3}/nltk/help.py +56 -56
- {nltk-3.2 → nltk-3.3}/nltk/inference/__init__.py +20 -20
- {nltk-3.2 → nltk-3.3}/nltk/inference/api.py +597 -589
- {nltk-3.2 → nltk-3.3}/nltk/inference/discourse.py +612 -608
- {nltk-3.2 → nltk-3.3}/nltk/inference/mace.py +311 -311
- {nltk-3.2 → nltk-3.3}/nltk/inference/nonmonotonic.py +509 -509
- {nltk-3.2 → nltk-3.3}/nltk/inference/prover9.py +431 -431
- {nltk-3.2 → nltk-3.3}/nltk/inference/resolution.py +687 -687
- {nltk-3.2 → nltk-3.3}/nltk/inference/tableau.py +607 -607
- {nltk-3.2 → nltk-3.3}/nltk/internals.py +982 -982
- {nltk-3.2 → nltk-3.3}/nltk/jsontags.py +63 -63
- {nltk-3.2 → nltk-3.3}/nltk/lazyimport.py +142 -142
- {nltk-3.2 → nltk-3.3}/nltk/metrics/__init__.py +30 -29
- {nltk-3.2 → nltk-3.3}/nltk/metrics/agreement.py +421 -401
- nltk-3.3/nltk/metrics/aline.py +607 -0
- {nltk-3.2 → nltk-3.3}/nltk/metrics/association.py +414 -410
- {nltk-3.2 → nltk-3.3}/nltk/metrics/confusionmatrix.py +206 -206
- {nltk-3.2 → nltk-3.3}/nltk/metrics/distance.py +207 -202
- {nltk-3.2 → nltk-3.3}/nltk/metrics/paice.py +381 -381
- {nltk-3.2 → nltk-3.3}/nltk/metrics/scores.py +228 -228
- {nltk-3.2 → nltk-3.3}/nltk/metrics/segmentation.py +231 -232
- {nltk-3.2 → nltk-3.3}/nltk/metrics/spearman.py +69 -69
- {nltk-3.2 → nltk-3.3}/nltk/misc/__init__.py +11 -11
- {nltk-3.2 → nltk-3.3}/nltk/misc/babelfish.py +10 -10
- {nltk-3.2 → nltk-3.3}/nltk/misc/chomsky.py +135 -133
- {nltk-3.2 → nltk-3.3}/nltk/misc/minimalset.py +83 -83
- {nltk-3.2 → nltk-3.3}/nltk/misc/sort.py +157 -157
- {nltk-3.2 → nltk-3.3}/nltk/misc/wordfinder.py +129 -129
- {nltk-3.2 → nltk-3.3}/nltk/parse/__init__.py +81 -80
- {nltk-3.2 → nltk-3.3}/nltk/parse/api.py +66 -66
- {nltk-3.2 → nltk-3.3}/nltk/parse/bllip.py +282 -282
- {nltk-3.2 → nltk-3.3}/nltk/parse/chart.py +1682 -1681
- nltk-3.3/nltk/parse/corenlp.py +780 -0
- {nltk-3.2 → nltk-3.3}/nltk/parse/dependencygraph.py +760 -758
- {nltk-3.2 → nltk-3.3}/nltk/parse/earleychart.py +452 -451
- {nltk-3.2 → nltk-3.3}/nltk/parse/evaluate.py +129 -129
- {nltk-3.2 → nltk-3.3}/nltk/parse/featurechart.py +580 -578
- {nltk-3.2 → nltk-3.3}/nltk/parse/generate.py +87 -76
- {nltk-3.2 → nltk-3.3}/nltk/parse/malt.py +356 -356
- {nltk-3.2 → nltk-3.3}/nltk/parse/nonprojectivedependencyparser.py +783 -783
- {nltk-3.2 → nltk-3.3}/nltk/parse/pchart.py +522 -522
- {nltk-3.2 → nltk-3.3}/nltk/parse/projectivedependencyparser.py +579 -578
- {nltk-3.2 → nltk-3.3}/nltk/parse/recursivedescent.py +655 -656
- {nltk-3.2 → nltk-3.3}/nltk/parse/shiftreduce.py +458 -459
- {nltk-3.2 → nltk-3.3}/nltk/parse/stanford.py +431 -406
- {nltk-3.2 → nltk-3.3}/nltk/parse/transitionparser.py +774 -772
- {nltk-3.2 → nltk-3.3}/nltk/parse/util.py +231 -231
- {nltk-3.2 → nltk-3.3}/nltk/parse/viterbi.py +401 -401
- {nltk-3.2 → nltk-3.3}/nltk/probability.py +2410 -2355
- {nltk-3.2 → nltk-3.3}/nltk/sem/__init__.py +61 -61
- {nltk-3.2 → nltk-3.3}/nltk/sem/boxer.py +1261 -1261
- {nltk-3.2 → nltk-3.3}/nltk/sem/chat80.py +782 -782
- {nltk-3.2 → nltk-3.3}/nltk/sem/cooper_storage.py +118 -118
- {nltk-3.2 → nltk-3.3}/nltk/sem/drt.py +1258 -1257
- {nltk-3.2 → nltk-3.3}/nltk/sem/drt_glue_demo.py +485 -489
- {nltk-3.2 → nltk-3.3}/nltk/sem/evaluate.py +790 -784
- {nltk-3.2 → nltk-3.3}/nltk/sem/glue.py +696 -694
- {nltk-3.2 → nltk-3.3}/nltk/sem/hole.py +386 -384
- {nltk-3.2 → nltk-3.3}/nltk/sem/lfg.py +210 -208
- {nltk-3.2 → nltk-3.3}/nltk/sem/linearlogic.py +451 -449
- {nltk-3.2 → nltk-3.3}/nltk/sem/logic.py +1910 -1910
- {nltk-3.2 → nltk-3.3}/nltk/sem/relextract.py +476 -483
- {nltk-3.2 → nltk-3.3}/nltk/sem/skolemize.py +101 -101
- {nltk-3.2 → nltk-3.3}/nltk/sem/util.py +249 -249
- {nltk-3.2 → nltk-3.3}/nltk/sentiment/__init__.py +14 -14
- {nltk-3.2 → nltk-3.3}/nltk/sentiment/sentiment_analyzer.py +229 -229
- {nltk-3.2 → nltk-3.3}/nltk/sentiment/util.py +762 -752
- {nltk-3.2 → nltk-3.3}/nltk/sentiment/vader.py +443 -445
- {nltk-3.2 → nltk-3.3}/nltk/stem/__init__.py +31 -33
- {nltk-3.2 → nltk-3.3}/nltk/stem/api.py +28 -25
- nltk-3.3/nltk/stem/arlstem.py +369 -0
- {nltk-3.2 → nltk-3.3}/nltk/stem/isri.py +345 -345
- {nltk-3.2 → nltk-3.3}/nltk/stem/lancaster.py +337 -310
- nltk-3.3/nltk/stem/porter.py +710 -0
- {nltk-3.2 → nltk-3.3}/nltk/stem/regexp.py +61 -61
- {nltk-3.2 → nltk-3.3}/nltk/stem/rslp.py +142 -142
- {nltk-3.2 → nltk-3.3}/nltk/stem/snowball.py +4236 -3712
- {nltk-3.2 → nltk-3.3}/nltk/stem/util.py +22 -12
- {nltk-3.2 → nltk-3.3}/nltk/stem/wordnet.py +51 -51
- {nltk-3.2 → nltk-3.3}/nltk/tag/__init__.py +152 -127
- {nltk-3.2 → nltk-3.3}/nltk/tag/api.py +86 -81
- {nltk-3.2 → nltk-3.3}/nltk/tag/brill.py +424 -426
- {nltk-3.2 → nltk-3.3}/nltk/tag/brill_trainer.py +608 -608
- {nltk-3.2 → nltk-3.3}/nltk/tag/crf.py +204 -200
- {nltk-3.2 → nltk-3.3}/nltk/tag/hmm.py +1277 -1277
- {nltk-3.2 → nltk-3.3}/nltk/tag/hunpos.py +132 -132
- {nltk-3.2 → nltk-3.3}/nltk/tag/mapping.py +101 -101
- {nltk-3.2 → nltk-3.3}/nltk/tag/perceptron.py +329 -314
- {nltk-3.2 → nltk-3.3}/nltk/tag/senna.py +147 -145
- {nltk-3.2 → nltk-3.3}/nltk/tag/sequential.py +740 -736
- {nltk-3.2 → nltk-3.3}/nltk/tag/stanford.py +219 -196
- {nltk-3.2 → nltk-3.3}/nltk/tag/tnt.py +595 -604
- {nltk-3.2 → nltk-3.3}/nltk/tag/util.py +72 -72
- {nltk-3.2 → nltk-3.3}/nltk/tbl/__init__.py +28 -28
- {nltk-3.2 → nltk-3.3}/nltk/tbl/api.py +1 -1
- {nltk-3.2 → nltk-3.3}/nltk/tbl/demo.py +366 -366
- {nltk-3.2 → nltk-3.3}/nltk/tbl/erroranalysis.py +39 -39
- {nltk-3.2 → nltk-3.3}/nltk/tbl/feature.py +259 -261
- {nltk-3.2 → nltk-3.3}/nltk/tbl/rule.py +321 -321
- {nltk-3.2 → nltk-3.3}/nltk/tbl/template.py +303 -308
- {nltk-3.2 → nltk-3.3}/nltk/test/__init__.py +18 -18
- {nltk-3.2 → nltk-3.3}/nltk/test/all.py +24 -23
- {nltk-3.2 → nltk-3.3}/nltk/test/bleu.doctest +14 -14
- {nltk-3.2 → nltk-3.3}/nltk/test/bnc.doctest +60 -60
- {nltk-3.2 → nltk-3.3}/nltk/test/ccg.doctest +376 -395
- nltk-3.3/nltk/test/ccg_semantics.doctest +553 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/chat80.doctest +234 -234
- {nltk-3.2 → nltk-3.3}/nltk/test/childes.doctest +184 -184
- {nltk-3.2 → nltk-3.3}/nltk/test/childes_fixt.py +14 -13
- {nltk-3.2 → nltk-3.3}/nltk/test/chunk.doctest +373 -373
- {nltk-3.2 → nltk-3.3}/nltk/test/classify.doctest +185 -185
- {nltk-3.2 → nltk-3.3}/nltk/test/classify_fixt.py +11 -10
- nltk-3.3/nltk/test/collections.doctest +20 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/collocations.doctest +276 -276
- {nltk-3.2 → nltk-3.3}/nltk/test/compat.doctest +134 -138
- {nltk-3.2 → nltk-3.3}/nltk/test/compat_fixt.py +9 -8
- nltk-3.3/nltk/test/concordance.doctest +55 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/corpus.doctest +2200 -2212
- {nltk-3.2 → nltk-3.3}/nltk/test/corpus_fixt.py +3 -3
- {nltk-3.2 → nltk-3.3}/nltk/test/crubadan.doctest +65 -65
- {nltk-3.2 → nltk-3.3}/nltk/test/data.doctest +379 -379
- {nltk-3.2 → nltk-3.3}/nltk/test/dependency.doctest +241 -241
- {nltk-3.2 → nltk-3.3}/nltk/test/discourse.doctest +546 -546
- {nltk-3.2 → nltk-3.3}/nltk/test/discourse_fixt.py +14 -13
- {nltk-3.2 → nltk-3.3}/nltk/test/doctest_nose_plugin.py +159 -156
- {nltk-3.2 → nltk-3.3}/nltk/test/drt.doctest +517 -517
- {nltk-3.2 → nltk-3.3}/nltk/test/featgram.doctest +607 -607
- {nltk-3.2 → nltk-3.3}/nltk/test/featstruct.doctest +1229 -1230
- {nltk-3.2 → nltk-3.3}/nltk/test/framenet.doctest +288 -239
- {nltk-3.2 → nltk-3.3}/nltk/test/generate.doctest +67 -67
- {nltk-3.2 → nltk-3.3}/nltk/test/gensim.doctest +140 -141
- {nltk-3.2 → nltk-3.3}/nltk/test/gensim_fixt.py +10 -9
- {nltk-3.2 → nltk-3.3}/nltk/test/gluesemantics.doctest +384 -386
- {nltk-3.2 → nltk-3.3}/nltk/test/gluesemantics_malt.doctest +68 -68
- {nltk-3.2 → nltk-3.3}/nltk/test/gluesemantics_malt_fixt.py +12 -11
- {nltk-3.2 → nltk-3.3}/nltk/test/grammar.doctest +48 -48
- {nltk-3.2 → nltk-3.3}/nltk/test/grammartestsuites.doctest +109 -109
- {nltk-3.2 → nltk-3.3}/nltk/test/index.doctest +100 -100
- {nltk-3.2 → nltk-3.3}/nltk/test/inference.doctest +534 -534
- {nltk-3.2 → nltk-3.3}/nltk/test/inference_fixt.py +12 -11
- {nltk-3.2 → nltk-3.3}/nltk/test/internals.doctest +140 -140
- {nltk-3.2 → nltk-3.3}/nltk/test/japanese.doctest +48 -48
- {nltk-3.2 → nltk-3.3}/nltk/test/logic.doctest +1098 -1098
- {nltk-3.2 → nltk-3.3}/nltk/test/metrics.doctest +270 -270
- {nltk-3.2 → nltk-3.3}/nltk/test/misc.doctest +118 -118
- {nltk-3.2 → nltk-3.3}/nltk/test/nonmonotonic.doctest +286 -286
- {nltk-3.2 → nltk-3.3}/nltk/test/nonmonotonic_fixt.py +12 -11
- {nltk-3.2 → nltk-3.3}/nltk/test/paice.doctest +35 -35
- {nltk-3.2 → nltk-3.3}/nltk/test/parse.doctest +884 -884
- {nltk-3.2 → nltk-3.3}/nltk/test/portuguese_en.doctest +565 -565
- {nltk-3.2 → nltk-3.3}/nltk/test/portuguese_en_fixt.py +16 -16
- {nltk-3.2 → nltk-3.3}/nltk/test/probability.doctest +304 -304
- {nltk-3.2 → nltk-3.3}/nltk/test/probability_fixt.py +13 -12
- {nltk-3.2 → nltk-3.3}/nltk/test/propbank.doctest +176 -176
- {nltk-3.2 → nltk-3.3}/nltk/test/relextract.doctest +263 -263
- {nltk-3.2 → nltk-3.3}/nltk/test/resolution.doctest +221 -221
- {nltk-3.2 → nltk-3.3}/nltk/test/runtests.py +86 -64
- {nltk-3.2 → nltk-3.3}/nltk/test/segmentation_fixt.py +10 -10
- {nltk-3.2 → nltk-3.3}/nltk/test/semantics.doctest +665 -665
- {nltk-3.2 → nltk-3.3}/nltk/test/semantics_fixt.py +7 -7
- {nltk-3.2 → nltk-3.3}/nltk/test/sentiment.doctest +233 -233
- {nltk-3.2 → nltk-3.3}/nltk/test/sentiwordnet.doctest +41 -41
- {nltk-3.2 → nltk-3.3}/nltk/test/simple.doctest +85 -85
- {nltk-3.2 → nltk-3.3}/nltk/test/stem.doctest +78 -78
- {nltk-3.2 → nltk-3.3}/nltk/test/tag.doctest +33 -33
- {nltk-3.2 → nltk-3.3}/nltk/test/tokenize.doctest +273 -166
- {nltk-3.2 → nltk-3.3}/nltk/test/toolbox.doctest +307 -307
- {nltk-3.2 → nltk-3.3}/nltk/test/translate.doctest +242 -242
- {nltk-3.2 → nltk-3.3}/nltk/test/translate_fixt.py +3 -3
- {nltk-3.2 → nltk-3.3}/nltk/test/tree.doctest +1101 -1101
- {nltk-3.2 → nltk-3.3}/nltk/test/treeprettyprinter.doctest +127 -127
- {nltk-3.2 → nltk-3.3}/nltk/test/treetransforms.doctest +156 -156
- nltk-3.3/nltk/test/unit/test_2x_compat.py +63 -0
- nltk-3.3/nltk/test/unit/test_aline.py +52 -0
- nltk-3.3/nltk/test/unit/test_chunk.py +56 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_classify.py +55 -53
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_collocations.py +88 -79
- nltk-3.3/nltk/test/unit/test_concordance.py +98 -0
- nltk-3.3/nltk/test/unit/test_corenlp.py +1000 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_corpora.py +200 -183
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_corpus_views.py +45 -45
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_hmm.py +88 -88
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_json2csv_corpus.py +188 -187
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_naivebayes.py +26 -26
- nltk-3.3/nltk/test/unit/test_rte_classify.py +86 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_seekable_unicode_stream_reader.py +124 -124
- nltk-3.3/nltk/test/unit/test_senna.py +74 -0
- nltk-3.3/nltk/test/unit/test_stem.py +121 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_tag.py +20 -20
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_tgrep.py +628 -626
- nltk-3.3/nltk/test/unit/test_tokenize.py +168 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/test_twitter_auth.py +185 -185
- nltk-3.3/nltk/test/unit/test_wordnet.py +134 -0
- nltk-3.3/nltk/test/unit/translate/test_bleu.py +244 -0
- nltk-3.3/nltk/test/unit/translate/test_gdfa.py +57 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm1.py +75 -75
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm2.py +87 -87
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm3.py +105 -105
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm4.py +126 -126
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm5.py +166 -166
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_ibm_model.py +270 -270
- nltk-3.3/nltk/test/unit/translate/test_nist.py +37 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/test_stack_decoder.py +300 -300
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/utils.py +41 -41
- {nltk-3.2 → nltk-3.3}/nltk/test/util.doctest +48 -48
- {nltk-3.2 → nltk-3.3}/nltk/test/wordnet.doctest +595 -581
- {nltk-3.2 → nltk-3.3}/nltk/test/wordnet_fixt.py +7 -6
- {nltk-3.2 → nltk-3.3}/nltk/test/wordnet_lch.doctest +53 -53
- {nltk-3.2 → nltk-3.3}/nltk/test/wsd.doctest +68 -68
- {nltk-3.2 → nltk-3.3}/nltk/text.py +663 -610
- {nltk-3.2 → nltk-3.3}/nltk/tgrep.py +938 -937
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/__init__.py +130 -107
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/api.py +77 -78
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/casual.py +343 -342
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/mwe.py +120 -120
- nltk-3.3/nltk/tokenize/nist.py +173 -0
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/punkt.py +1620 -1601
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/regexp.py +208 -202
- nltk-3.3/nltk/tokenize/repp.py +151 -0
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/sexpr.py +140 -142
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/simple.py +138 -136
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/stanford.py +117 -113
- nltk-3.3/nltk/tokenize/stanford_segmenter.py +246 -0
- {nltk-3.2 → nltk-3.3}/nltk/tokenize/texttiling.py +458 -460
- nltk-3.3/nltk/tokenize/toktok.py +155 -0
- nltk-3.3/nltk/tokenize/treebank.py +363 -0
- nltk-3.3/nltk/tokenize/util.py +261 -0
- {nltk-3.2 → nltk-3.3}/nltk/toolbox.py +499 -497
- {nltk-3.2 → nltk-3.3}/nltk/translate/__init__.py +24 -24
- {nltk-3.2 → nltk-3.3}/nltk/translate/api.py +331 -321
- {nltk-3.2 → nltk-3.3}/nltk/translate/bleu_score.py +615 -556
- nltk-3.3/nltk/translate/chrf_score.py +137 -0
- {nltk-3.2 → nltk-3.3}/nltk/translate/gale_church.py +248 -235
- {nltk-3.2 → nltk-3.3}/nltk/translate/gdfa.py +135 -131
- nltk-3.3/nltk/translate/gleu_score.py +193 -0
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm1.py +250 -241
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm2.py +308 -308
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm3.py +337 -337
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm4.py +467 -467
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm5.py +639 -639
- {nltk-3.2 → nltk-3.3}/nltk/translate/ibm_model.py +536 -533
- {nltk-3.2 → nltk-3.3}/nltk/translate/metrics.py +40 -40
- nltk-3.3/nltk/translate/nist_score.py +178 -0
- {nltk-3.2 → nltk-3.3}/nltk/translate/phrase_based.py +175 -175
- {nltk-3.2 → nltk-3.3}/nltk/translate/ribes_score.py +325 -325
- {nltk-3.2 → nltk-3.3}/nltk/translate/stack_decoder.py +499 -495
- {nltk-3.2 → nltk-3.3}/nltk/tree.py +1605 -1603
- {nltk-3.2 → nltk-3.3}/nltk/treeprettyprinter.py +564 -564
- {nltk-3.2 → nltk-3.3}/nltk/treetransforms.py +309 -309
- {nltk-3.2 → nltk-3.3}/nltk/twitter/__init__.py +28 -28
- {nltk-3.2 → nltk-3.3}/nltk/twitter/api.py +141 -136
- {nltk-3.2 → nltk-3.3}/nltk/twitter/common.py +264 -258
- {nltk-3.2 → nltk-3.3}/nltk/twitter/twitter_demo.py +288 -288
- {nltk-3.2 → nltk-3.3}/nltk/twitter/twitterclient.py +530 -530
- {nltk-3.2 → nltk-3.3}/nltk/twitter/util.py +147 -146
- nltk-3.3/nltk/util.py +751 -0
- {nltk-3.2 → nltk-3.3}/nltk/wsd.py +53 -53
- {nltk-3.2 → nltk-3.3/nltk.egg-info}/PKG-INFO +13 -5
- {nltk-3.2 → nltk-3.3}/nltk.egg-info/SOURCES.txt +26 -3
- nltk-3.3/nltk.egg-info/requires.txt +31 -0
- {nltk-3.2 → nltk-3.3}/setup.cfg +0 -1
- {nltk-3.2 → nltk-3.3}/setup.py +102 -75
- nltk-3.2/nltk/VERSION +0 -1
- nltk-3.2/nltk/compat.py +0 -680
- nltk-3.2/nltk/corpus/reader/pl196x.py +0 -278
- nltk-3.2/nltk/corpus/reader/verbnet.py +0 -408
- nltk-3.2/nltk/corpus/reader/wordlist.py +0 -37
- nltk-3.2/nltk/sentiment/vader_lexicon.txt +0 -7517
- nltk-3.2/nltk/six.py +0 -868
- nltk-3.2/nltk/stem/porter.py +0 -692
- nltk-3.2/nltk/test/unit/test_2x_compat.py +0 -26
- nltk-3.2/nltk/test/unit/test_stem.py +0 -36
- nltk-3.2/nltk/test/unit/test_tokenize.py +0 -23
- nltk-3.2/nltk/test/unit/translate/test_bleu.py +0 -84
- nltk-3.2/nltk/tokenize/treebank.py +0 -124
- nltk-3.2/nltk/tokenize/util.py +0 -90
- nltk-3.2/nltk/util.py +0 -1364
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/__init__.py +0 -0
- {nltk-3.2 → nltk-3.3}/nltk/test/unit/translate/__init__.py +0 -0
- {nltk-3.2 → nltk-3.3}/nltk.egg-info/dependency_links.txt +0 -0
- {nltk-3.2 → nltk-3.3}/nltk.egg-info/not-zip-safe +0 -0
- {nltk-3.2 → nltk-3.3}/nltk.egg-info/top_level.txt +0 -0
|
@@ -1,607 +1,607 @@
|
|
|
1
|
-
.. Copyright (C) 2001-
|
|
2
|
-
.. For license information, see LICENSE.TXT
|
|
3
|
-
|
|
4
|
-
=========================
|
|
5
|
-
Feature Grammar Parsing
|
|
6
|
-
=========================
|
|
7
|
-
|
|
8
|
-
.. include:: ../../../nltk_book/definitions.rst
|
|
9
|
-
|
|
10
|
-
Grammars can be parsed from strings.
|
|
11
|
-
|
|
12
|
-
>>> from __future__ import print_function
|
|
13
|
-
>>> import nltk
|
|
14
|
-
>>> from nltk import grammar, parse
|
|
15
|
-
>>> g = """
|
|
16
|
-
... % start DP
|
|
17
|
-
... DP[AGR=?a] -> D[AGR=?a] N[AGR=?a]
|
|
18
|
-
... D[AGR=[NUM='sg', PERS=3]] -> 'this' | 'that'
|
|
19
|
-
... D[AGR=[NUM='pl', PERS=3]] -> 'these' | 'those'
|
|
20
|
-
... D[AGR=[NUM='pl', PERS=1]] -> 'we'
|
|
21
|
-
... D[AGR=[PERS=2]] -> 'you'
|
|
22
|
-
... N[AGR=[NUM='sg', GND='m']] -> 'boy'
|
|
23
|
-
... N[AGR=[NUM='pl', GND='m']] -> 'boys'
|
|
24
|
-
... N[AGR=[NUM='sg', GND='f']] -> 'girl'
|
|
25
|
-
... N[AGR=[NUM='pl', GND='f']] -> 'girls'
|
|
26
|
-
... N[AGR=[NUM='sg']] -> 'student'
|
|
27
|
-
... N[AGR=[NUM='pl']] -> 'students'
|
|
28
|
-
... """
|
|
29
|
-
>>> grammar = grammar.FeatureGrammar.fromstring(g)
|
|
30
|
-
>>> tokens = 'these girls'.split()
|
|
31
|
-
>>> parser = parse.FeatureEarleyChartParser(grammar)
|
|
32
|
-
>>> trees = parser.parse(tokens)
|
|
33
|
-
>>> for tree in trees: print(tree)
|
|
34
|
-
(DP[AGR=[GND='f', NUM='pl', PERS=3]]
|
|
35
|
-
(D[AGR=[NUM='pl', PERS=3]] these)
|
|
36
|
-
(N[AGR=[GND='f', NUM='pl']] girls))
|
|
37
|
-
|
|
38
|
-
In general, when we are trying to develop even a very small grammar,
|
|
39
|
-
it is convenient to put the rules in a file where they can be edited,
|
|
40
|
-
tested and revised. Let's assume that we have saved feat0cfg_ as a file named
|
|
41
|
-
``'feat0.fcfg'`` and placed it in the NLTK ``data`` directory. We can
|
|
42
|
-
inspect it as follows:
|
|
43
|
-
|
|
44
|
-
.. _feat0cfg: http://nltk.svn.sourceforge.net/svnroot/nltk/trunk/nltk/data/grammars/feat0.fcfg
|
|
45
|
-
|
|
46
|
-
>>> nltk.data.show_cfg('grammars/book_grammars/feat0.fcfg')
|
|
47
|
-
% start S
|
|
48
|
-
# ###################
|
|
49
|
-
# Grammar Productions
|
|
50
|
-
# ###################
|
|
51
|
-
# S expansion productions
|
|
52
|
-
S -> NP[NUM=?n] VP[NUM=?n]
|
|
53
|
-
# NP expansion productions
|
|
54
|
-
NP[NUM=?n] -> N[NUM=?n]
|
|
55
|
-
NP[NUM=?n] -> PropN[NUM=?n]
|
|
56
|
-
NP[NUM=?n] -> Det[NUM=?n] N[NUM=?n]
|
|
57
|
-
NP[NUM=pl] -> N[NUM=pl]
|
|
58
|
-
# VP expansion productions
|
|
59
|
-
VP[TENSE=?t, NUM=?n] -> IV[TENSE=?t, NUM=?n]
|
|
60
|
-
VP[TENSE=?t, NUM=?n] -> TV[TENSE=?t, NUM=?n] NP
|
|
61
|
-
# ###################
|
|
62
|
-
# Lexical Productions
|
|
63
|
-
# ###################
|
|
64
|
-
Det[NUM=sg] -> 'this' | 'every'
|
|
65
|
-
Det[NUM=pl] -> 'these' | 'all'
|
|
66
|
-
Det -> 'the' | 'some' | 'several'
|
|
67
|
-
PropN[NUM=sg]-> 'Kim' | 'Jody'
|
|
68
|
-
N[NUM=sg] -> 'dog' | 'girl' | 'car' | 'child'
|
|
69
|
-
N[NUM=pl] -> 'dogs' | 'girls' | 'cars' | 'children'
|
|
70
|
-
IV[TENSE=pres, NUM=sg] -> 'disappears' | 'walks'
|
|
71
|
-
TV[TENSE=pres, NUM=sg] -> 'sees' | 'likes'
|
|
72
|
-
IV[TENSE=pres, NUM=pl] -> 'disappear' | 'walk'
|
|
73
|
-
TV[TENSE=pres, NUM=pl] -> 'see' | 'like'
|
|
74
|
-
IV[TENSE=past] -> 'disappeared' | 'walked'
|
|
75
|
-
TV[TENSE=past] -> 'saw' | 'liked'
|
|
76
|
-
|
|
77
|
-
Assuming we have saved feat0cfg_ as a file named
|
|
78
|
-
``'feat0.fcfg'``, the function ``parse.load_parser`` allows us to
|
|
79
|
-
read the grammar into NLTK, ready for use in parsing.
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
>>> cp = parse.load_parser('grammars/book_grammars/feat0.fcfg', trace=1)
|
|
83
|
-
>>> sent = 'Kim likes children'
|
|
84
|
-
>>> tokens = sent.split()
|
|
85
|
-
>>> tokens
|
|
86
|
-
['Kim', 'likes', 'children']
|
|
87
|
-
>>> trees = cp.parse(tokens)
|
|
88
|
-
|.Kim .like.chil.|
|
|
89
|
-
|[----] . .| [0:1] 'Kim'
|
|
90
|
-
|. [----] .| [1:2] 'likes'
|
|
91
|
-
|. . [----]| [2:3] 'children'
|
|
92
|
-
|[----] . .| [0:1] PropN[NUM='sg'] -> 'Kim' *
|
|
93
|
-
|[----] . .| [0:1] NP[NUM='sg'] -> PropN[NUM='sg'] *
|
|
94
|
-
|[----> . .| [0:1] S[] -> NP[NUM=?n] * VP[NUM=?n] {?n: 'sg'}
|
|
95
|
-
|. [----] .| [1:2] TV[NUM='sg', TENSE='pres'] -> 'likes' *
|
|
96
|
-
|. [----> .| [1:2] VP[NUM=?n, TENSE=?t] -> TV[NUM=?n, TENSE=?t] * NP[] {?n: 'sg', ?t: 'pres'}
|
|
97
|
-
|. . [----]| [2:3] N[NUM='pl'] -> 'children' *
|
|
98
|
-
|. . [----]| [2:3] NP[NUM='pl'] -> N[NUM='pl'] *
|
|
99
|
-
|. . [---->| [2:3] S[] -> NP[NUM=?n] * VP[NUM=?n] {?n: 'pl'}
|
|
100
|
-
|. [---------]| [1:3] VP[NUM='sg', TENSE='pres'] -> TV[NUM='sg', TENSE='pres'] NP[] *
|
|
101
|
-
|[==============]| [0:3] S[] -> NP[NUM='sg'] VP[NUM='sg'] *
|
|
102
|
-
>>> for tree in trees: print(tree)
|
|
103
|
-
(S[]
|
|
104
|
-
(NP[NUM='sg'] (PropN[NUM='sg'] Kim))
|
|
105
|
-
(VP[NUM='sg', TENSE='pres']
|
|
106
|
-
(TV[NUM='sg', TENSE='pres'] likes)
|
|
107
|
-
(NP[NUM='pl'] (N[NUM='pl'] children))))
|
|
108
|
-
|
|
109
|
-
The parser works directly with
|
|
110
|
-
the underspecified productions given by the grammar. That is, the
|
|
111
|
-
Predictor rule does not attempt to compile out all admissible feature
|
|
112
|
-
combinations before trying to expand the non-terminals on the left hand
|
|
113
|
-
side of a production. However, when the Scanner matches an input word
|
|
114
|
-
against a lexical production that has been predicted, the new edge will
|
|
115
|
-
typically contain fully specified features; e.g., the edge
|
|
116
|
-
[PropN[`num`:feat: = `sg`:fval:] |rarr| 'Kim', (0, 1)]. Recall from
|
|
117
|
-
Chapter 8 that the Fundamental (or Completer) Rule in
|
|
118
|
-
standard CFGs is used to combine an incomplete edge that's expecting a
|
|
119
|
-
nonterminal *B* with a following, complete edge whose left hand side
|
|
120
|
-
matches *B*. In our current setting, rather than checking for a
|
|
121
|
-
complete match, we test whether the expected category *B* will
|
|
122
|
-
`unify`:dt: with the left hand side *B'* of a following complete
|
|
123
|
-
edge. We will explain in more detail in Section 9.2 how
|
|
124
|
-
unification works; for the moment, it is enough to know that as a
|
|
125
|
-
result of unification, any variable values of features in *B* will be
|
|
126
|
-
instantiated by constant values in the corresponding feature structure
|
|
127
|
-
in *B'*, and these instantiated values will be used in the new edge
|
|
128
|
-
added by the Completer. This instantiation can be seen, for example,
|
|
129
|
-
in the edge
|
|
130
|
-
[NP [`num`:feat:\ =\ `sg`:fval:] |rarr| PropN[`num`:feat:\ =\ `sg`:fval:] |dot|, (0, 1)]
|
|
131
|
-
in Example 9.2, where the feature `num`:feat: has been assigned the value `sg`:fval:.
|
|
132
|
-
|
|
133
|
-
Feature structures in NLTK are ... Atomic feature values can be strings or
|
|
134
|
-
integers.
|
|
135
|
-
|
|
136
|
-
>>> fs1 = nltk.FeatStruct(TENSE='past', NUM='sg')
|
|
137
|
-
>>> print(fs1)
|
|
138
|
-
[ NUM = 'sg' ]
|
|
139
|
-
[ TENSE = 'past' ]
|
|
140
|
-
|
|
141
|
-
We can think of a feature structure as being like a Python dictionary,
|
|
142
|
-
and access its values by indexing in the usual way.
|
|
143
|
-
|
|
144
|
-
>>> fs1 = nltk.FeatStruct(PER=3, NUM='pl', GND='fem')
|
|
145
|
-
>>> print(fs1['GND'])
|
|
146
|
-
fem
|
|
147
|
-
|
|
148
|
-
We can also define feature structures which have complex values, as
|
|
149
|
-
discussed earlier.
|
|
150
|
-
|
|
151
|
-
>>> fs2 = nltk.FeatStruct(POS='N', AGR=fs1)
|
|
152
|
-
>>> print(fs2)
|
|
153
|
-
[ [ GND = 'fem' ] ]
|
|
154
|
-
[ AGR = [ NUM = 'pl' ] ]
|
|
155
|
-
[ [ PER = 3 ] ]
|
|
156
|
-
[ ]
|
|
157
|
-
[ POS = 'N' ]
|
|
158
|
-
>>> print(fs2['AGR'])
|
|
159
|
-
[ GND = 'fem' ]
|
|
160
|
-
[ NUM = 'pl' ]
|
|
161
|
-
[ PER = 3 ]
|
|
162
|
-
>>> print(fs2['AGR']['PER'])
|
|
163
|
-
3
|
|
164
|
-
|
|
165
|
-
Feature structures can also be constructed using the ``parse()``
|
|
166
|
-
method of the ``nltk.FeatStruct`` class. Note that in this case, atomic
|
|
167
|
-
feature values do not need to be enclosed in quotes.
|
|
168
|
-
|
|
169
|
-
>>> f1 = nltk.FeatStruct("[NUMBER = sg]")
|
|
170
|
-
>>> f2 = nltk.FeatStruct("[PERSON = 3]")
|
|
171
|
-
>>> print(nltk.unify(f1, f2))
|
|
172
|
-
[ NUMBER = 'sg' ]
|
|
173
|
-
[ PERSON = 3 ]
|
|
174
|
-
|
|
175
|
-
>>> f1 = nltk.FeatStruct("[A = [B = b, D = d]]")
|
|
176
|
-
>>> f2 = nltk.FeatStruct("[A = [C = c, D = d]]")
|
|
177
|
-
>>> print(nltk.unify(f1, f2))
|
|
178
|
-
[ [ B = 'b' ] ]
|
|
179
|
-
[ A = [ C = 'c' ] ]
|
|
180
|
-
[ [ D = 'd' ] ]
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
Feature Structures as Graphs
|
|
184
|
-
----------------------------
|
|
185
|
-
|
|
186
|
-
Feature structures are not inherently tied to linguistic objects; they are
|
|
187
|
-
general purpose structures for representing knowledge. For example, we
|
|
188
|
-
could encode information about a person in a feature structure:
|
|
189
|
-
|
|
190
|
-
>>> person01 = nltk.FeatStruct("[NAME=Lee, TELNO='01 27 86 42 96',AGE=33]")
|
|
191
|
-
>>> print(person01)
|
|
192
|
-
[ AGE = 33 ]
|
|
193
|
-
[ NAME = 'Lee' ]
|
|
194
|
-
[ TELNO = '01 27 86 42 96' ]
|
|
195
|
-
|
|
196
|
-
There are a number of notations for representing reentrancy in
|
|
197
|
-
matrix-style representations of feature structures. In NLTK, we adopt
|
|
198
|
-
the following convention: the first occurrence of a shared feature structure
|
|
199
|
-
is prefixed with an integer in parentheses, such as ``(1)``, and any
|
|
200
|
-
subsequent reference to that structure uses the notation
|
|
201
|
-
``->(1)``, as shown below.
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
>>> fs = nltk.FeatStruct("""[NAME=Lee, ADDRESS=(1)[NUMBER=74, STREET='rue Pascal'],
|
|
205
|
-
... SPOUSE=[NAME=Kim, ADDRESS->(1)]]""")
|
|
206
|
-
>>> print(fs)
|
|
207
|
-
[ ADDRESS = (1) [ NUMBER = 74 ] ]
|
|
208
|
-
[ [ STREET = 'rue Pascal' ] ]
|
|
209
|
-
[ ]
|
|
210
|
-
[ NAME = 'Lee' ]
|
|
211
|
-
[ ]
|
|
212
|
-
[ SPOUSE = [ ADDRESS -> (1) ] ]
|
|
213
|
-
[ [ NAME = 'Kim' ] ]
|
|
214
|
-
|
|
215
|
-
There can be any number of tags within a single feature structure.
|
|
216
|
-
|
|
217
|
-
>>> fs3 = nltk.FeatStruct("[A=(1)[B=b], C=(2)[], D->(1), E->(2)]")
|
|
218
|
-
>>> print(fs3)
|
|
219
|
-
[ A = (1) [ B = 'b' ] ]
|
|
220
|
-
[ ]
|
|
221
|
-
[ C = (2) [] ]
|
|
222
|
-
[ ]
|
|
223
|
-
[ D -> (1) ]
|
|
224
|
-
[ E -> (2) ]
|
|
225
|
-
>>> fs1 = nltk.FeatStruct(NUMBER=74, STREET='rue Pascal')
|
|
226
|
-
>>> fs2 = nltk.FeatStruct(CITY='Paris')
|
|
227
|
-
>>> print(nltk.unify(fs1, fs2))
|
|
228
|
-
[ CITY = 'Paris' ]
|
|
229
|
-
[ NUMBER = 74 ]
|
|
230
|
-
[ STREET = 'rue Pascal' ]
|
|
231
|
-
|
|
232
|
-
Unification is symmetric:
|
|
233
|
-
|
|
234
|
-
>>> nltk.unify(fs1, fs2) == nltk.unify(fs2, fs1)
|
|
235
|
-
True
|
|
236
|
-
|
|
237
|
-
Unification is commutative:
|
|
238
|
-
|
|
239
|
-
>>> fs3 = nltk.FeatStruct(TELNO='01 27 86 42 96')
|
|
240
|
-
>>> nltk.unify(nltk.unify(fs1, fs2), fs3) == nltk.unify(fs1, nltk.unify(fs2, fs3))
|
|
241
|
-
True
|
|
242
|
-
|
|
243
|
-
Unification between `FS`:math:\ :subscript:`0` and `FS`:math:\
|
|
244
|
-
:subscript:`1` will fail if the two feature structures share a path |pi|,
|
|
245
|
-
but the value of |pi| in `FS`:math:\ :subscript:`0` is a distinct
|
|
246
|
-
atom from the value of |pi| in `FS`:math:\ :subscript:`1`. In NLTK,
|
|
247
|
-
this is implemented by setting the result of unification to be
|
|
248
|
-
``None``.
|
|
249
|
-
|
|
250
|
-
>>> fs0 = nltk.FeatStruct(A='a')
|
|
251
|
-
>>> fs1 = nltk.FeatStruct(A='b')
|
|
252
|
-
>>> print(nltk.unify(fs0, fs1))
|
|
253
|
-
None
|
|
254
|
-
|
|
255
|
-
Now, if we look at how unification interacts with structure-sharing,
|
|
256
|
-
things become really interesting.
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
>>> fs0 = nltk.FeatStruct("""[NAME=Lee,
|
|
261
|
-
... ADDRESS=[NUMBER=74,
|
|
262
|
-
... STREET='rue Pascal'],
|
|
263
|
-
... SPOUSE= [NAME=Kim,
|
|
264
|
-
... ADDRESS=[NUMBER=74,
|
|
265
|
-
... STREET='rue Pascal']]]""")
|
|
266
|
-
>>> print(fs0)
|
|
267
|
-
[ ADDRESS = [ NUMBER = 74 ] ]
|
|
268
|
-
[ [ STREET = 'rue Pascal' ] ]
|
|
269
|
-
[ ]
|
|
270
|
-
[ NAME = 'Lee' ]
|
|
271
|
-
[ ]
|
|
272
|
-
[ [ ADDRESS = [ NUMBER = 74 ] ] ]
|
|
273
|
-
[ SPOUSE = [ [ STREET = 'rue Pascal' ] ] ]
|
|
274
|
-
[ [ ] ]
|
|
275
|
-
[ [ NAME = 'Kim' ] ]
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
>>> fs1 = nltk.FeatStruct("[SPOUSE=[ADDRESS=[CITY=Paris]]]")
|
|
279
|
-
>>> print(nltk.unify(fs0, fs1))
|
|
280
|
-
[ ADDRESS = [ NUMBER = 74 ] ]
|
|
281
|
-
[ [ STREET = 'rue Pascal' ] ]
|
|
282
|
-
[ ]
|
|
283
|
-
[ NAME = 'Lee' ]
|
|
284
|
-
[ ]
|
|
285
|
-
[ [ [ CITY = 'Paris' ] ] ]
|
|
286
|
-
[ [ ADDRESS = [ NUMBER = 74 ] ] ]
|
|
287
|
-
[ SPOUSE = [ [ STREET = 'rue Pascal' ] ] ]
|
|
288
|
-
[ [ ] ]
|
|
289
|
-
[ [ NAME = 'Kim' ] ]
|
|
290
|
-
|
|
291
|
-
>>> fs2 = nltk.FeatStruct("""[NAME=Lee, ADDRESS=(1)[NUMBER=74, STREET='rue Pascal'],
|
|
292
|
-
... SPOUSE=[NAME=Kim, ADDRESS->(1)]]""")
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
>>> print(fs2)
|
|
296
|
-
[ ADDRESS = (1) [ NUMBER = 74 ] ]
|
|
297
|
-
[ [ STREET = 'rue Pascal' ] ]
|
|
298
|
-
[ ]
|
|
299
|
-
[ NAME = 'Lee' ]
|
|
300
|
-
[ ]
|
|
301
|
-
[ SPOUSE = [ ADDRESS -> (1) ] ]
|
|
302
|
-
[ [ NAME = 'Kim' ] ]
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
>>> print(nltk.unify(fs2, fs1))
|
|
306
|
-
[ [ CITY = 'Paris' ] ]
|
|
307
|
-
[ ADDRESS = (1) [ NUMBER = 74 ] ]
|
|
308
|
-
[ [ STREET = 'rue Pascal' ] ]
|
|
309
|
-
[ ]
|
|
310
|
-
[ NAME = 'Lee' ]
|
|
311
|
-
[ ]
|
|
312
|
-
[ SPOUSE = [ ADDRESS -> (1) ] ]
|
|
313
|
-
[ [ NAME = 'Kim' ] ]
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
>>> fs1 = nltk.FeatStruct("[ADDRESS1=[NUMBER=74, STREET='rue Pascal']]")
|
|
317
|
-
>>> fs2 = nltk.FeatStruct("[ADDRESS1=?x, ADDRESS2=?x]")
|
|
318
|
-
>>> print(fs2)
|
|
319
|
-
[ ADDRESS1 = ?x ]
|
|
320
|
-
[ ADDRESS2 = ?x ]
|
|
321
|
-
>>> print(nltk.unify(fs1, fs2))
|
|
322
|
-
[ ADDRESS1 = (1) [ NUMBER = 74 ] ]
|
|
323
|
-
[ [ STREET = 'rue Pascal' ] ]
|
|
324
|
-
[ ]
|
|
325
|
-
[ ADDRESS2 -> (1) ]
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
>>> sent = 'who do you claim that you like'
|
|
331
|
-
>>> tokens = sent.split()
|
|
332
|
-
>>> cp = parse.load_parser('grammars/book_grammars/feat1.fcfg', trace=1)
|
|
333
|
-
>>> trees = cp.parse(tokens)
|
|
334
|
-
|.w.d.y.c.t.y.l.|
|
|
335
|
-
|[-] . . . . . .| [0:1] 'who'
|
|
336
|
-
|. [-] . . . . .| [1:2] 'do'
|
|
337
|
-
|. . [-] . . . .| [2:3] 'you'
|
|
338
|
-
|. . . [-] . . .| [3:4] 'claim'
|
|
339
|
-
|. . . . [-] . .| [4:5] 'that'
|
|
340
|
-
|. . . . . [-] .| [5:6] 'you'
|
|
341
|
-
|. . . . . . [-]| [6:7] 'like'
|
|
342
|
-
|# . . . . . . .| [0:0] NP[]/NP[] -> *
|
|
343
|
-
|. # . . . . . .| [1:1] NP[]/NP[] -> *
|
|
344
|
-
|. . # . . . . .| [2:2] NP[]/NP[] -> *
|
|
345
|
-
|. . . # . . . .| [3:3] NP[]/NP[] -> *
|
|
346
|
-
|. . . . # . . .| [4:4] NP[]/NP[] -> *
|
|
347
|
-
|. . . . . # . .| [5:5] NP[]/NP[] -> *
|
|
348
|
-
|. . . . . . # .| [6:6] NP[]/NP[] -> *
|
|
349
|
-
|. . . . . . . #| [7:7] NP[]/NP[] -> *
|
|
350
|
-
|[-] . . . . . .| [0:1] NP[+WH] -> 'who' *
|
|
351
|
-
|[-> . . . . . .| [0:1] S[-INV] -> NP[] * VP[] {}
|
|
352
|
-
|[-> . . . . . .| [0:1] S[-INV]/?x[] -> NP[] * VP[]/?x[] {}
|
|
353
|
-
|[-> . . . . . .| [0:1] S[-INV] -> NP[] * S[]/NP[] {}
|
|
354
|
-
|. [-] . . . . .| [1:2] V[+AUX] -> 'do' *
|
|
355
|
-
|. [-> . . . . .| [1:2] S[+INV] -> V[+AUX] * NP[] VP[] {}
|
|
356
|
-
|. [-> . . . . .| [1:2] S[+INV]/?x[] -> V[+AUX] * NP[] VP[]/?x[] {}
|
|
357
|
-
|. [-> . . . . .| [1:2] VP[] -> V[+AUX] * VP[] {}
|
|
358
|
-
|. [-> . . . . .| [1:2] VP[]/?x[] -> V[+AUX] * VP[]/?x[] {}
|
|
359
|
-
|. . [-] . . . .| [2:3] NP[-WH] -> 'you' *
|
|
360
|
-
|. . [-> . . . .| [2:3] S[-INV] -> NP[] * VP[] {}
|
|
361
|
-
|. . [-> . . . .| [2:3] S[-INV]/?x[] -> NP[] * VP[]/?x[] {}
|
|
362
|
-
|. . [-> . . . .| [2:3] S[-INV] -> NP[] * S[]/NP[] {}
|
|
363
|
-
|. [---> . . . .| [1:3] S[+INV] -> V[+AUX] NP[] * VP[] {}
|
|
364
|
-
|. [---> . . . .| [1:3] S[+INV]/?x[] -> V[+AUX] NP[] * VP[]/?x[] {}
|
|
365
|
-
|. . . [-] . . .| [3:4] V[-AUX, SUBCAT='clause'] -> 'claim' *
|
|
366
|
-
|. . . [-> . . .| [3:4] VP[] -> V[-AUX, SUBCAT='clause'] * SBar[] {}
|
|
367
|
-
|. . . [-> . . .| [3:4] VP[]/?x[] -> V[-AUX, SUBCAT='clause'] * SBar[]/?x[] {}
|
|
368
|
-
|. . . . [-] . .| [4:5] Comp[] -> 'that' *
|
|
369
|
-
|. . . . [-> . .| [4:5] SBar[] -> Comp[] * S[-INV] {}
|
|
370
|
-
|. . . . [-> . .| [4:5] SBar[]/?x[] -> Comp[] * S[-INV]/?x[] {}
|
|
371
|
-
|. . . . . [-] .| [5:6] NP[-WH] -> 'you' *
|
|
372
|
-
|. . . . . [-> .| [5:6] S[-INV] -> NP[] * VP[] {}
|
|
373
|
-
|. . . . . [-> .| [5:6] S[-INV]/?x[] -> NP[] * VP[]/?x[] {}
|
|
374
|
-
|. . . . . [-> .| [5:6] S[-INV] -> NP[] * S[]/NP[] {}
|
|
375
|
-
|. . . . . . [-]| [6:7] V[-AUX, SUBCAT='trans'] -> 'like' *
|
|
376
|
-
|. . . . . . [->| [6:7] VP[] -> V[-AUX, SUBCAT='trans'] * NP[] {}
|
|
377
|
-
|. . . . . . [->| [6:7] VP[]/?x[] -> V[-AUX, SUBCAT='trans'] * NP[]/?x[] {}
|
|
378
|
-
|. . . . . . [-]| [6:7] VP[]/NP[] -> V[-AUX, SUBCAT='trans'] NP[]/NP[] *
|
|
379
|
-
|. . . . . [---]| [5:7] S[-INV]/NP[] -> NP[] VP[]/NP[] *
|
|
380
|
-
|. . . . [-----]| [4:7] SBar[]/NP[] -> Comp[] S[-INV]/NP[] *
|
|
381
|
-
|. . . [-------]| [3:7] VP[]/NP[] -> V[-AUX, SUBCAT='clause'] SBar[]/NP[] *
|
|
382
|
-
|. . [---------]| [2:7] S[-INV]/NP[] -> NP[] VP[]/NP[] *
|
|
383
|
-
|. [-----------]| [1:7] S[+INV]/NP[] -> V[+AUX] NP[] VP[]/NP[] *
|
|
384
|
-
|[=============]| [0:7] S[-INV] -> NP[] S[]/NP[] *
|
|
385
|
-
|
|
386
|
-
>>> trees = list(trees)
|
|
387
|
-
>>> for tree in trees: print(tree)
|
|
388
|
-
(S[-INV]
|
|
389
|
-
(NP[+WH] who)
|
|
390
|
-
(S[+INV]/NP[]
|
|
391
|
-
(V[+AUX] do)
|
|
392
|
-
(NP[-WH] you)
|
|
393
|
-
(VP[]/NP[]
|
|
394
|
-
(V[-AUX, SUBCAT='clause'] claim)
|
|
395
|
-
(SBar[]/NP[]
|
|
396
|
-
(Comp[] that)
|
|
397
|
-
(S[-INV]/NP[]
|
|
398
|
-
(NP[-WH] you)
|
|
399
|
-
(VP[]/NP[] (V[-AUX, SUBCAT='trans'] like) (NP[]/NP[] )))))))
|
|
400
|
-
|
|
401
|
-
A different parser should give the same parse trees, but perhaps in a different order:
|
|
402
|
-
|
|
403
|
-
>>> cp2 = parse.load_parser('grammars/book_grammars/feat1.fcfg', trace=1,
|
|
404
|
-
... parser=parse.FeatureEarleyChartParser)
|
|
405
|
-
>>> trees2 = cp2.parse(tokens)
|
|
406
|
-
|.w.d.y.c.t.y.l.|
|
|
407
|
-
|[-] . . . . . .| [0:1] 'who'
|
|
408
|
-
|. [-] . . . . .| [1:2] 'do'
|
|
409
|
-
|. . [-] . . . .| [2:3] 'you'
|
|
410
|
-
|. . . [-] . . .| [3:4] 'claim'
|
|
411
|
-
|. . . . [-] . .| [4:5] 'that'
|
|
412
|
-
|. . . . . [-] .| [5:6] 'you'
|
|
413
|
-
|. . . . . . [-]| [6:7] 'like'
|
|
414
|
-
|> . . . . . . .| [0:0] S[-INV] -> * NP[] VP[] {}
|
|
415
|
-
|> . . . . . . .| [0:0] S[-INV]/?x[] -> * NP[] VP[]/?x[] {}
|
|
416
|
-
|> . . . . . . .| [0:0] S[-INV] -> * NP[] S[]/NP[] {}
|
|
417
|
-
|> . . . . . . .| [0:0] S[-INV] -> * Adv[+NEG] S[+INV] {}
|
|
418
|
-
|> . . . . . . .| [0:0] S[+INV] -> * V[+AUX] NP[] VP[] {}
|
|
419
|
-
|> . . . . . . .| [0:0] S[+INV]/?x[] -> * V[+AUX] NP[] VP[]/?x[] {}
|
|
420
|
-
|> . . . . . . .| [0:0] NP[+WH] -> * 'who' {}
|
|
421
|
-
|[-] . . . . . .| [0:1] NP[+WH] -> 'who' *
|
|
422
|
-
|[-> . . . . . .| [0:1] S[-INV] -> NP[] * VP[] {}
|
|
423
|
-
|[-> . . . . . .| [0:1] S[-INV]/?x[] -> NP[] * VP[]/?x[] {}
|
|
424
|
-
|[-> . . . . . .| [0:1] S[-INV] -> NP[] * S[]/NP[] {}
|
|
425
|
-
|. > . . . . . .| [1:1] S[-INV]/?x[] -> * NP[] VP[]/?x[] {}
|
|
426
|
-
|. > . . . . . .| [1:1] S[+INV]/?x[] -> * V[+AUX] NP[] VP[]/?x[] {}
|
|
427
|
-
|. > . . . . . .| [1:1] V[+AUX] -> * 'do' {}
|
|
428
|
-
|. > . . . . . .| [1:1] VP[]/?x[] -> * V[-AUX, SUBCAT='trans'] NP[]/?x[] {}
|
|
429
|
-
|. > . . . . . .| [1:1] VP[]/?x[] -> * V[-AUX, SUBCAT='clause'] SBar[]/?x[] {}
|
|
430
|
-
|. > . . . . . .| [1:1] VP[]/?x[] -> * V[+AUX] VP[]/?x[] {}
|
|
431
|
-
|. > . . . . . .| [1:1] VP[] -> * V[-AUX, SUBCAT='intrans'] {}
|
|
432
|
-
|. > . . . . . .| [1:1] VP[] -> * V[-AUX, SUBCAT='trans'] NP[] {}
|
|
433
|
-
|. > . . . . . .| [1:1] VP[] -> * V[-AUX, SUBCAT='clause'] SBar[] {}
|
|
434
|
-
|. > . . . . . .| [1:1] VP[] -> * V[+AUX] VP[] {}
|
|
435
|
-
|. [-] . . . . .| [1:2] V[+AUX] -> 'do' *
|
|
436
|
-
|. [-> . . . . .| [1:2] S[+INV]/?x[] -> V[+AUX] * NP[] VP[]/?x[] {}
|
|
437
|
-
|. [-> . . . . .| [1:2] VP[]/?x[] -> V[+AUX] * VP[]/?x[] {}
|
|
438
|
-
|. [-> . . . . .| [1:2] VP[] -> V[+AUX] * VP[] {}
|
|
439
|
-
|. . > . . . . .| [2:2] VP[] -> * V[-AUX, SUBCAT='intrans'] {}
|
|
440
|
-
|. . > . . . . .| [2:2] VP[] -> * V[-AUX, SUBCAT='trans'] NP[] {}
|
|
441
|
-
|. . > . . . . .| [2:2] VP[] -> * V[-AUX, SUBCAT='clause'] SBar[] {}
|
|
442
|
-
|. . > . . . . .| [2:2] VP[] -> * V[+AUX] VP[] {}
|
|
443
|
-
|. . > . . . . .| [2:2] VP[]/?x[] -> * V[-AUX, SUBCAT='trans'] NP[]/?x[] {}
|
|
444
|
-
|. . > . . . . .| [2:2] VP[]/?x[] -> * V[-AUX, SUBCAT='clause'] SBar[]/?x[] {}
|
|
445
|
-
|. . > . . . . .| [2:2] VP[]/?x[] -> * V[+AUX] VP[]/?x[] {}
|
|
446
|
-
|. . > . . . . .| [2:2] NP[-WH] -> * 'you' {}
|
|
447
|
-
|. . [-] . . . .| [2:3] NP[-WH] -> 'you' *
|
|
448
|
-
|. [---> . . . .| [1:3] S[+INV]/?x[] -> V[+AUX] NP[] * VP[]/?x[] {}
|
|
449
|
-
|. . . > . . . .| [3:3] VP[]/?x[] -> * V[-AUX, SUBCAT='trans'] NP[]/?x[] {}
|
|
450
|
-
|. . . > . . . .| [3:3] VP[]/?x[] -> * V[-AUX, SUBCAT='clause'] SBar[]/?x[] {}
|
|
451
|
-
|. . . > . . . .| [3:3] VP[]/?x[] -> * V[+AUX] VP[]/?x[] {}
|
|
452
|
-
|. . . > . . . .| [3:3] V[-AUX, SUBCAT='clause'] -> * 'claim' {}
|
|
453
|
-
|. . . [-] . . .| [3:4] V[-AUX, SUBCAT='clause'] -> 'claim' *
|
|
454
|
-
|. . . [-> . . .| [3:4] VP[]/?x[] -> V[-AUX, SUBCAT='clause'] * SBar[]/?x[] {}
|
|
455
|
-
|. . . . > . . .| [4:4] SBar[]/?x[] -> * Comp[] S[-INV]/?x[] {}
|
|
456
|
-
|. . . . > . . .| [4:4] Comp[] -> * 'that' {}
|
|
457
|
-
|. . . . [-] . .| [4:5] Comp[] -> 'that' *
|
|
458
|
-
|. . . . [-> . .| [4:5] SBar[]/?x[] -> Comp[] * S[-INV]/?x[] {}
|
|
459
|
-
|. . . . . > . .| [5:5] S[-INV]/?x[] -> * NP[] VP[]/?x[] {}
|
|
460
|
-
|. . . . . > . .| [5:5] NP[-WH] -> * 'you' {}
|
|
461
|
-
|. . . . . [-] .| [5:6] NP[-WH] -> 'you' *
|
|
462
|
-
|. . . . . [-> .| [5:6] S[-INV]/?x[] -> NP[] * VP[]/?x[] {}
|
|
463
|
-
|. . . . . . > .| [6:6] VP[]/?x[] -> * V[-AUX, SUBCAT='trans'] NP[]/?x[] {}
|
|
464
|
-
|. . . . . . > .| [6:6] VP[]/?x[] -> * V[-AUX, SUBCAT='clause'] SBar[]/?x[] {}
|
|
465
|
-
|. . . . . . > .| [6:6] VP[]/?x[] -> * V[+AUX] VP[]/?x[] {}
|
|
466
|
-
|. . . . . . > .| [6:6] V[-AUX, SUBCAT='trans'] -> * 'like' {}
|
|
467
|
-
|. . . . . . [-]| [6:7] V[-AUX, SUBCAT='trans'] -> 'like' *
|
|
468
|
-
|. . . . . . [->| [6:7] VP[]/?x[] -> V[-AUX, SUBCAT='trans'] * NP[]/?x[] {}
|
|
469
|
-
|. . . . . . . #| [7:7] NP[]/NP[] -> *
|
|
470
|
-
|. . . . . . [-]| [6:7] VP[]/NP[] -> V[-AUX, SUBCAT='trans'] NP[]/NP[] *
|
|
471
|
-
|. . . . . [---]| [5:7] S[-INV]/NP[] -> NP[] VP[]/NP[] *
|
|
472
|
-
|. . . . [-----]| [4:7] SBar[]/NP[] -> Comp[] S[-INV]/NP[] *
|
|
473
|
-
|. . . [-------]| [3:7] VP[]/NP[] -> V[-AUX, SUBCAT='clause'] SBar[]/NP[] *
|
|
474
|
-
|. [-----------]| [1:7] S[+INV]/NP[] -> V[+AUX] NP[] VP[]/NP[] *
|
|
475
|
-
|[=============]| [0:7] S[-INV] -> NP[] S[]/NP[] *
|
|
476
|
-
|
|
477
|
-
>>> sorted(trees) == sorted(trees2)
|
|
478
|
-
True
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
Let's load a German grammar:
|
|
482
|
-
|
|
483
|
-
>>> cp = parse.load_parser('grammars/book_grammars/german.fcfg', trace=0)
|
|
484
|
-
>>> sent = 'die Katze sieht den Hund'
|
|
485
|
-
>>> tokens = sent.split()
|
|
486
|
-
>>> trees = cp.parse(tokens)
|
|
487
|
-
>>> for tree in trees: print(tree)
|
|
488
|
-
(S[]
|
|
489
|
-
(NP[AGR=[GND='fem', NUM='sg', PER=3], CASE='nom']
|
|
490
|
-
(Det[AGR=[GND='fem', NUM='sg', PER=3], CASE='nom'] die)
|
|
491
|
-
(N[AGR=[GND='fem', NUM='sg', PER=3]] Katze))
|
|
492
|
-
(VP[AGR=[NUM='sg', PER=3]]
|
|
493
|
-
(TV[AGR=[NUM='sg', PER=3], OBJCASE='acc'] sieht)
|
|
494
|
-
(NP[AGR=[GND='masc', NUM='sg', PER=3], CASE='acc']
|
|
495
|
-
(Det[AGR=[GND='masc', NUM='sg', PER=3], CASE='acc'] den)
|
|
496
|
-
(N[AGR=[GND='masc', NUM='sg', PER=3]] Hund))))
|
|
497
|
-
|
|
498
|
-
Grammar with Binding Operators
|
|
499
|
-
------------------------------
|
|
500
|
-
The `bindop.fcfg`_ grammar is a semantic grammar that uses lambda
|
|
501
|
-
calculus. Each element has a core semantics, which is a single lambda
|
|
502
|
-
calculus expression; and a set of binding operators, which bind
|
|
503
|
-
variables.
|
|
504
|
-
|
|
505
|
-
.. _bindop.fcfg: http://nltk.svn.sourceforge.net/svnroot/nltk/trunk/nltk/data/grammars/bindop.fcfg
|
|
506
|
-
|
|
507
|
-
In order to make the binding operators work right, they need to
|
|
508
|
-
instantiate their bound variable every time they are added to the
|
|
509
|
-
chart. To do this, we use a special subclass of `Chart`, called
|
|
510
|
-
`InstantiateVarsChart`.
|
|
511
|
-
|
|
512
|
-
>>> from nltk.parse.featurechart import InstantiateVarsChart
|
|
513
|
-
>>> cp = parse.load_parser('grammars/sample_grammars/bindop.fcfg', trace=1,
|
|
514
|
-
... chart_class=InstantiateVarsChart)
|
|
515
|
-
>>> print(cp.grammar())
|
|
516
|
-
Grammar with 15 productions (start state = S[])
|
|
517
|
-
S[SEM=[BO={?b1+?b2}, CORE=<?vp(?subj)>]] -> NP[SEM=[BO=?b1, CORE=?subj]] VP[SEM=[BO=?b2, CORE=?vp]]
|
|
518
|
-
VP[SEM=[BO={?b1+?b2}, CORE=<?v(?obj)>]] -> TV[SEM=[BO=?b1, CORE=?v]] NP[SEM=[BO=?b2, CORE=?obj]]
|
|
519
|
-
VP[SEM=?s] -> IV[SEM=?s]
|
|
520
|
-
NP[SEM=[BO={?b1+?b2+{bo(?det(?n),@x)}}, CORE=<@x>]] -> Det[SEM=[BO=?b1, CORE=?det]] N[SEM=[BO=?b2, CORE=?n]]
|
|
521
|
-
Det[SEM=[BO={/}, CORE=<\Q P.exists x.(Q(x) & P(x))>]] -> 'a'
|
|
522
|
-
N[SEM=[BO={/}, CORE=<dog>]] -> 'dog'
|
|
523
|
-
N[SEM=[BO={/}, CORE=<dog>]] -> 'cat'
|
|
524
|
-
N[SEM=[BO={/}, CORE=<dog>]] -> 'mouse'
|
|
525
|
-
IV[SEM=[BO={/}, CORE=<\x.bark(x)>]] -> 'barks'
|
|
526
|
-
IV[SEM=[BO={/}, CORE=<\x.bark(x)>]] -> 'eats'
|
|
527
|
-
IV[SEM=[BO={/}, CORE=<\x.bark(x)>]] -> 'walks'
|
|
528
|
-
TV[SEM=[BO={/}, CORE=<\x y.feed(y,x)>]] -> 'feeds'
|
|
529
|
-
TV[SEM=[BO={/}, CORE=<\x y.feed(y,x)>]] -> 'walks'
|
|
530
|
-
NP[SEM=[BO={bo(\P.P(John),@x)}, CORE=<@x>]] -> 'john'
|
|
531
|
-
NP[SEM=[BO={bo(\P.P(John),@x)}, CORE=<@x>]] -> 'alex'
|
|
532
|
-
|
|
533
|
-
A simple intransitive sentence:
|
|
534
|
-
|
|
535
|
-
>>> from nltk.sem import logic
|
|
536
|
-
>>> logic._counter._value = 100
|
|
537
|
-
|
|
538
|
-
>>> trees = cp.parse('john barks'.split())
|
|
539
|
-
|. john.barks.|
|
|
540
|
-
|[-----] .| [0:1] 'john'
|
|
541
|
-
|. [-----]| [1:2] 'barks'
|
|
542
|
-
|[-----] .| [0:1] NP[SEM=[BO={bo(\P.P(John),z101)}, CORE=<z101>]] -> 'john' *
|
|
543
|
-
|[-----> .| [0:1] S[SEM=[BO={?b1+?b2}, CORE=<?vp(?subj)>]] -> NP[SEM=[BO=?b1, CORE=?subj]] * VP[SEM=[BO=?b2, CORE=?vp]] {?b1: {bo(\P.P(John),z2)}, ?subj: <IndividualVariableExpression z2>}
|
|
544
|
-
|. [-----]| [1:2] IV[SEM=[BO={/}, CORE=<\x.bark(x)>]] -> 'barks' *
|
|
545
|
-
|. [-----]| [1:2] VP[SEM=[BO={/}, CORE=<\x.bark(x)>]] -> IV[SEM=[BO={/}, CORE=<\x.bark(x)>]] *
|
|
546
|
-
|[===========]| [0:2] S[SEM=[BO={bo(\P.P(John),z2)}, CORE=<bark(z2)>]] -> NP[SEM=[BO={bo(\P.P(John),z2)}, CORE=<z2>]] VP[SEM=[BO={/}, CORE=<\x.bark(x)>]] *
|
|
547
|
-
>>> for tree in trees: print(tree)
|
|
548
|
-
(S[SEM=[BO={bo(\P.P(John),z2)}, CORE=<bark(z2)>]]
|
|
549
|
-
(NP[SEM=[BO={bo(\P.P(John),z101)}, CORE=<z101>]] john)
|
|
550
|
-
(VP[SEM=[BO={/}, CORE=<\x.bark(x)>]]
|
|
551
|
-
(IV[SEM=[BO={/}, CORE=<\x.bark(x)>]] barks)))
|
|
552
|
-
|
|
553
|
-
A transitive sentence:
|
|
554
|
-
|
|
555
|
-
>>> trees = cp.parse('john feeds a dog'.split())
|
|
556
|
-
|.joh.fee. a .dog.|
|
|
557
|
-
|[---] . . .| [0:1] 'john'
|
|
558
|
-
|. [---] . .| [1:2] 'feeds'
|
|
559
|
-
|. . [---] .| [2:3] 'a'
|
|
560
|
-
|. . . [---]| [3:4] 'dog'
|
|
561
|
-
|[---] . . .| [0:1] NP[SEM=[BO={bo(\P.P(John),z102)}, CORE=<z102>]] -> 'john' *
|
|
562
|
-
|[---> . . .| [0:1] S[SEM=[BO={?b1+?b2}, CORE=<?vp(?subj)>]] -> NP[SEM=[BO=?b1, CORE=?subj]] * VP[SEM=[BO=?b2, CORE=?vp]] {?b1: {bo(\P.P(John),z2)}, ?subj: <IndividualVariableExpression z2>}
|
|
563
|
-
|. [---] . .| [1:2] TV[SEM=[BO={/}, CORE=<\x y.feed(y,x)>]] -> 'feeds' *
|
|
564
|
-
|. [---> . .| [1:2] VP[SEM=[BO={?b1+?b2}, CORE=<?v(?obj)>]] -> TV[SEM=[BO=?b1, CORE=?v]] * NP[SEM=[BO=?b2, CORE=?obj]] {?b1: {/}, ?v: <LambdaExpression \x y.feed(y,x)>}
|
|
565
|
-
|. . [---] .| [2:3] Det[SEM=[BO={/}, CORE=<\Q P.exists x.(Q(x) & P(x))>]] -> 'a' *
|
|
566
|
-
|. . [---> .| [2:3] NP[SEM=[BO={?b1+?b2+{bo(?det(?n),@x)}}, CORE=<@x>]] -> Det[SEM=[BO=?b1, CORE=?det]] * N[SEM=[BO=?b2, CORE=?n]] {?b1: {/}, ?det: <LambdaExpression \Q P.exists x.(Q(x) & P(x))>}
|
|
567
|
-
|. . . [---]| [3:4] N[SEM=[BO={/}, CORE=<dog>]] -> 'dog' *
|
|
568
|
-
|. . [-------]| [2:4] NP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z103)}, CORE=<z103>]] -> Det[SEM=[BO={/}, CORE=<\Q P.exists x.(Q(x) & P(x))>]] N[SEM=[BO={/}, CORE=<dog>]] *
|
|
569
|
-
|. . [------->| [2:4] S[SEM=[BO={?b1+?b2}, CORE=<?vp(?subj)>]] -> NP[SEM=[BO=?b1, CORE=?subj]] * VP[SEM=[BO=?b2, CORE=?vp]] {?b1: {bo(\P.exists x.(dog(x) & P(x)),z2)}, ?subj: <IndividualVariableExpression z2>}
|
|
570
|
-
|. [-----------]| [1:4] VP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z2)}, CORE=<\y.feed(y,z2)>]] -> TV[SEM=[BO={/}, CORE=<\x y.feed(y,x)>]] NP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z2)}, CORE=<z2>]] *
|
|
571
|
-
|[===============]| [0:4] S[SEM=[BO={bo(\P.P(John),z2), bo(\P.exists x.(dog(x) & P(x)),z3)}, CORE=<feed(z2,z3)>]] -> NP[SEM=[BO={bo(\P.P(John),z2)}, CORE=<z2>]] VP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z3)}, CORE=<\y.feed(y,z3)>]] *
|
|
572
|
-
|
|
573
|
-
>>> for tree in trees: print(tree)
|
|
574
|
-
(S[SEM=[BO={bo(\P.P(John),z2), bo(\P.exists x.(dog(x) & P(x)),z3)}, CORE=<feed(z2,z3)>]]
|
|
575
|
-
(NP[SEM=[BO={bo(\P.P(John),z102)}, CORE=<z102>]] john)
|
|
576
|
-
(VP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z2)}, CORE=<\y.feed(y,z2)>]]
|
|
577
|
-
(TV[SEM=[BO={/}, CORE=<\x y.feed(y,x)>]] feeds)
|
|
578
|
-
(NP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z103)}, CORE=<z103>]]
|
|
579
|
-
(Det[SEM=[BO={/}, CORE=<\Q P.exists x.(Q(x) & P(x))>]] a)
|
|
580
|
-
(N[SEM=[BO={/}, CORE=<dog>]] dog))))
|
|
581
|
-
|
|
582
|
-
Turn down the verbosity:
|
|
583
|
-
|
|
584
|
-
>>> cp = parse.load_parser('grammars/sample_grammars/bindop.fcfg', trace=0,
|
|
585
|
-
... chart_class=InstantiateVarsChart)
|
|
586
|
-
|
|
587
|
-
Reuse the same lexical item twice:
|
|
588
|
-
|
|
589
|
-
>>> trees = cp.parse('john feeds john'.split())
|
|
590
|
-
>>> for tree in trees: print(tree)
|
|
591
|
-
(S[SEM=[BO={bo(\P.P(John),z2), bo(\P.P(John),z3)}, CORE=<feed(z2,z3)>]]
|
|
592
|
-
(NP[SEM=[BO={bo(\P.P(John),z104)}, CORE=<z104>]] john)
|
|
593
|
-
(VP[SEM=[BO={bo(\P.P(John),z2)}, CORE=<\y.feed(y,z2)>]]
|
|
594
|
-
(TV[SEM=[BO={/}, CORE=<\x y.feed(y,x)>]] feeds)
|
|
595
|
-
(NP[SEM=[BO={bo(\P.P(John),z105)}, CORE=<z105>]] john)))
|
|
596
|
-
|
|
597
|
-
>>> trees = cp.parse('a dog feeds a dog'.split())
|
|
598
|
-
>>> for tree in trees: print(tree)
|
|
599
|
-
(S[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z2), bo(\P.exists x.(dog(x) & P(x)),z3)}, CORE=<feed(z2,z3)>]]
|
|
600
|
-
(NP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z106)}, CORE=<z106>]]
|
|
601
|
-
(Det[SEM=[BO={/}, CORE=<\Q P.exists x.(Q(x) & P(x))>]] a)
|
|
602
|
-
(N[SEM=[BO={/}, CORE=<dog>]] dog))
|
|
603
|
-
(VP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z2)}, CORE=<\y.feed(y,z2)>]]
|
|
604
|
-
(TV[SEM=[BO={/}, CORE=<\x y.feed(y,x)>]] feeds)
|
|
605
|
-
(NP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z107)}, CORE=<z107>]]
|
|
606
|
-
(Det[SEM=[BO={/}, CORE=<\Q P.exists x.(Q(x) & P(x))>]] a)
|
|
607
|
-
(N[SEM=[BO={/}, CORE=<dog>]] dog))))
|
|
1
|
+
.. Copyright (C) 2001-2018 NLTK Project
|
|
2
|
+
.. For license information, see LICENSE.TXT
|
|
3
|
+
|
|
4
|
+
=========================
|
|
5
|
+
Feature Grammar Parsing
|
|
6
|
+
=========================
|
|
7
|
+
|
|
8
|
+
.. include:: ../../../nltk_book/definitions.rst
|
|
9
|
+
|
|
10
|
+
Grammars can be parsed from strings.
|
|
11
|
+
|
|
12
|
+
>>> from __future__ import print_function
|
|
13
|
+
>>> import nltk
|
|
14
|
+
>>> from nltk import grammar, parse
|
|
15
|
+
>>> g = """
|
|
16
|
+
... % start DP
|
|
17
|
+
... DP[AGR=?a] -> D[AGR=?a] N[AGR=?a]
|
|
18
|
+
... D[AGR=[NUM='sg', PERS=3]] -> 'this' | 'that'
|
|
19
|
+
... D[AGR=[NUM='pl', PERS=3]] -> 'these' | 'those'
|
|
20
|
+
... D[AGR=[NUM='pl', PERS=1]] -> 'we'
|
|
21
|
+
... D[AGR=[PERS=2]] -> 'you'
|
|
22
|
+
... N[AGR=[NUM='sg', GND='m']] -> 'boy'
|
|
23
|
+
... N[AGR=[NUM='pl', GND='m']] -> 'boys'
|
|
24
|
+
... N[AGR=[NUM='sg', GND='f']] -> 'girl'
|
|
25
|
+
... N[AGR=[NUM='pl', GND='f']] -> 'girls'
|
|
26
|
+
... N[AGR=[NUM='sg']] -> 'student'
|
|
27
|
+
... N[AGR=[NUM='pl']] -> 'students'
|
|
28
|
+
... """
|
|
29
|
+
>>> grammar = grammar.FeatureGrammar.fromstring(g)
|
|
30
|
+
>>> tokens = 'these girls'.split()
|
|
31
|
+
>>> parser = parse.FeatureEarleyChartParser(grammar)
|
|
32
|
+
>>> trees = parser.parse(tokens)
|
|
33
|
+
>>> for tree in trees: print(tree)
|
|
34
|
+
(DP[AGR=[GND='f', NUM='pl', PERS=3]]
|
|
35
|
+
(D[AGR=[NUM='pl', PERS=3]] these)
|
|
36
|
+
(N[AGR=[GND='f', NUM='pl']] girls))
|
|
37
|
+
|
|
38
|
+
In general, when we are trying to develop even a very small grammar,
|
|
39
|
+
it is convenient to put the rules in a file where they can be edited,
|
|
40
|
+
tested and revised. Let's assume that we have saved feat0cfg_ as a file named
|
|
41
|
+
``'feat0.fcfg'`` and placed it in the NLTK ``data`` directory. We can
|
|
42
|
+
inspect it as follows:
|
|
43
|
+
|
|
44
|
+
.. _feat0cfg: http://nltk.svn.sourceforge.net/svnroot/nltk/trunk/nltk/data/grammars/feat0.fcfg
|
|
45
|
+
|
|
46
|
+
>>> nltk.data.show_cfg('grammars/book_grammars/feat0.fcfg')
|
|
47
|
+
% start S
|
|
48
|
+
# ###################
|
|
49
|
+
# Grammar Productions
|
|
50
|
+
# ###################
|
|
51
|
+
# S expansion productions
|
|
52
|
+
S -> NP[NUM=?n] VP[NUM=?n]
|
|
53
|
+
# NP expansion productions
|
|
54
|
+
NP[NUM=?n] -> N[NUM=?n]
|
|
55
|
+
NP[NUM=?n] -> PropN[NUM=?n]
|
|
56
|
+
NP[NUM=?n] -> Det[NUM=?n] N[NUM=?n]
|
|
57
|
+
NP[NUM=pl] -> N[NUM=pl]
|
|
58
|
+
# VP expansion productions
|
|
59
|
+
VP[TENSE=?t, NUM=?n] -> IV[TENSE=?t, NUM=?n]
|
|
60
|
+
VP[TENSE=?t, NUM=?n] -> TV[TENSE=?t, NUM=?n] NP
|
|
61
|
+
# ###################
|
|
62
|
+
# Lexical Productions
|
|
63
|
+
# ###################
|
|
64
|
+
Det[NUM=sg] -> 'this' | 'every'
|
|
65
|
+
Det[NUM=pl] -> 'these' | 'all'
|
|
66
|
+
Det -> 'the' | 'some' | 'several'
|
|
67
|
+
PropN[NUM=sg]-> 'Kim' | 'Jody'
|
|
68
|
+
N[NUM=sg] -> 'dog' | 'girl' | 'car' | 'child'
|
|
69
|
+
N[NUM=pl] -> 'dogs' | 'girls' | 'cars' | 'children'
|
|
70
|
+
IV[TENSE=pres, NUM=sg] -> 'disappears' | 'walks'
|
|
71
|
+
TV[TENSE=pres, NUM=sg] -> 'sees' | 'likes'
|
|
72
|
+
IV[TENSE=pres, NUM=pl] -> 'disappear' | 'walk'
|
|
73
|
+
TV[TENSE=pres, NUM=pl] -> 'see' | 'like'
|
|
74
|
+
IV[TENSE=past] -> 'disappeared' | 'walked'
|
|
75
|
+
TV[TENSE=past] -> 'saw' | 'liked'
|
|
76
|
+
|
|
77
|
+
Assuming we have saved feat0cfg_ as a file named
|
|
78
|
+
``'feat0.fcfg'``, the function ``parse.load_parser`` allows us to
|
|
79
|
+
read the grammar into NLTK, ready for use in parsing.
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
>>> cp = parse.load_parser('grammars/book_grammars/feat0.fcfg', trace=1)
|
|
83
|
+
>>> sent = 'Kim likes children'
|
|
84
|
+
>>> tokens = sent.split()
|
|
85
|
+
>>> tokens
|
|
86
|
+
['Kim', 'likes', 'children']
|
|
87
|
+
>>> trees = cp.parse(tokens)
|
|
88
|
+
|.Kim .like.chil.|
|
|
89
|
+
|[----] . .| [0:1] 'Kim'
|
|
90
|
+
|. [----] .| [1:2] 'likes'
|
|
91
|
+
|. . [----]| [2:3] 'children'
|
|
92
|
+
|[----] . .| [0:1] PropN[NUM='sg'] -> 'Kim' *
|
|
93
|
+
|[----] . .| [0:1] NP[NUM='sg'] -> PropN[NUM='sg'] *
|
|
94
|
+
|[----> . .| [0:1] S[] -> NP[NUM=?n] * VP[NUM=?n] {?n: 'sg'}
|
|
95
|
+
|. [----] .| [1:2] TV[NUM='sg', TENSE='pres'] -> 'likes' *
|
|
96
|
+
|. [----> .| [1:2] VP[NUM=?n, TENSE=?t] -> TV[NUM=?n, TENSE=?t] * NP[] {?n: 'sg', ?t: 'pres'}
|
|
97
|
+
|. . [----]| [2:3] N[NUM='pl'] -> 'children' *
|
|
98
|
+
|. . [----]| [2:3] NP[NUM='pl'] -> N[NUM='pl'] *
|
|
99
|
+
|. . [---->| [2:3] S[] -> NP[NUM=?n] * VP[NUM=?n] {?n: 'pl'}
|
|
100
|
+
|. [---------]| [1:3] VP[NUM='sg', TENSE='pres'] -> TV[NUM='sg', TENSE='pres'] NP[] *
|
|
101
|
+
|[==============]| [0:3] S[] -> NP[NUM='sg'] VP[NUM='sg'] *
|
|
102
|
+
>>> for tree in trees: print(tree)
|
|
103
|
+
(S[]
|
|
104
|
+
(NP[NUM='sg'] (PropN[NUM='sg'] Kim))
|
|
105
|
+
(VP[NUM='sg', TENSE='pres']
|
|
106
|
+
(TV[NUM='sg', TENSE='pres'] likes)
|
|
107
|
+
(NP[NUM='pl'] (N[NUM='pl'] children))))
|
|
108
|
+
|
|
109
|
+
The parser works directly with
|
|
110
|
+
the underspecified productions given by the grammar. That is, the
|
|
111
|
+
Predictor rule does not attempt to compile out all admissible feature
|
|
112
|
+
combinations before trying to expand the non-terminals on the left hand
|
|
113
|
+
side of a production. However, when the Scanner matches an input word
|
|
114
|
+
against a lexical production that has been predicted, the new edge will
|
|
115
|
+
typically contain fully specified features; e.g., the edge
|
|
116
|
+
[PropN[`num`:feat: = `sg`:fval:] |rarr| 'Kim', (0, 1)]. Recall from
|
|
117
|
+
Chapter 8 that the Fundamental (or Completer) Rule in
|
|
118
|
+
standard CFGs is used to combine an incomplete edge that's expecting a
|
|
119
|
+
nonterminal *B* with a following, complete edge whose left hand side
|
|
120
|
+
matches *B*. In our current setting, rather than checking for a
|
|
121
|
+
complete match, we test whether the expected category *B* will
|
|
122
|
+
`unify`:dt: with the left hand side *B'* of a following complete
|
|
123
|
+
edge. We will explain in more detail in Section 9.2 how
|
|
124
|
+
unification works; for the moment, it is enough to know that as a
|
|
125
|
+
result of unification, any variable values of features in *B* will be
|
|
126
|
+
instantiated by constant values in the corresponding feature structure
|
|
127
|
+
in *B'*, and these instantiated values will be used in the new edge
|
|
128
|
+
added by the Completer. This instantiation can be seen, for example,
|
|
129
|
+
in the edge
|
|
130
|
+
[NP [`num`:feat:\ =\ `sg`:fval:] |rarr| PropN[`num`:feat:\ =\ `sg`:fval:] |dot|, (0, 1)]
|
|
131
|
+
in Example 9.2, where the feature `num`:feat: has been assigned the value `sg`:fval:.
|
|
132
|
+
|
|
133
|
+
Feature structures in NLTK are ... Atomic feature values can be strings or
|
|
134
|
+
integers.
|
|
135
|
+
|
|
136
|
+
>>> fs1 = nltk.FeatStruct(TENSE='past', NUM='sg')
|
|
137
|
+
>>> print(fs1)
|
|
138
|
+
[ NUM = 'sg' ]
|
|
139
|
+
[ TENSE = 'past' ]
|
|
140
|
+
|
|
141
|
+
We can think of a feature structure as being like a Python dictionary,
|
|
142
|
+
and access its values by indexing in the usual way.
|
|
143
|
+
|
|
144
|
+
>>> fs1 = nltk.FeatStruct(PER=3, NUM='pl', GND='fem')
|
|
145
|
+
>>> print(fs1['GND'])
|
|
146
|
+
fem
|
|
147
|
+
|
|
148
|
+
We can also define feature structures which have complex values, as
|
|
149
|
+
discussed earlier.
|
|
150
|
+
|
|
151
|
+
>>> fs2 = nltk.FeatStruct(POS='N', AGR=fs1)
|
|
152
|
+
>>> print(fs2)
|
|
153
|
+
[ [ GND = 'fem' ] ]
|
|
154
|
+
[ AGR = [ NUM = 'pl' ] ]
|
|
155
|
+
[ [ PER = 3 ] ]
|
|
156
|
+
[ ]
|
|
157
|
+
[ POS = 'N' ]
|
|
158
|
+
>>> print(fs2['AGR'])
|
|
159
|
+
[ GND = 'fem' ]
|
|
160
|
+
[ NUM = 'pl' ]
|
|
161
|
+
[ PER = 3 ]
|
|
162
|
+
>>> print(fs2['AGR']['PER'])
|
|
163
|
+
3
|
|
164
|
+
|
|
165
|
+
Feature structures can also be constructed using the ``parse()``
|
|
166
|
+
method of the ``nltk.FeatStruct`` class. Note that in this case, atomic
|
|
167
|
+
feature values do not need to be enclosed in quotes.
|
|
168
|
+
|
|
169
|
+
>>> f1 = nltk.FeatStruct("[NUMBER = sg]")
|
|
170
|
+
>>> f2 = nltk.FeatStruct("[PERSON = 3]")
|
|
171
|
+
>>> print(nltk.unify(f1, f2))
|
|
172
|
+
[ NUMBER = 'sg' ]
|
|
173
|
+
[ PERSON = 3 ]
|
|
174
|
+
|
|
175
|
+
>>> f1 = nltk.FeatStruct("[A = [B = b, D = d]]")
|
|
176
|
+
>>> f2 = nltk.FeatStruct("[A = [C = c, D = d]]")
|
|
177
|
+
>>> print(nltk.unify(f1, f2))
|
|
178
|
+
[ [ B = 'b' ] ]
|
|
179
|
+
[ A = [ C = 'c' ] ]
|
|
180
|
+
[ [ D = 'd' ] ]
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
Feature Structures as Graphs
|
|
184
|
+
----------------------------
|
|
185
|
+
|
|
186
|
+
Feature structures are not inherently tied to linguistic objects; they are
|
|
187
|
+
general purpose structures for representing knowledge. For example, we
|
|
188
|
+
could encode information about a person in a feature structure:
|
|
189
|
+
|
|
190
|
+
>>> person01 = nltk.FeatStruct("[NAME=Lee, TELNO='01 27 86 42 96',AGE=33]")
|
|
191
|
+
>>> print(person01)
|
|
192
|
+
[ AGE = 33 ]
|
|
193
|
+
[ NAME = 'Lee' ]
|
|
194
|
+
[ TELNO = '01 27 86 42 96' ]
|
|
195
|
+
|
|
196
|
+
There are a number of notations for representing reentrancy in
|
|
197
|
+
matrix-style representations of feature structures. In NLTK, we adopt
|
|
198
|
+
the following convention: the first occurrence of a shared feature structure
|
|
199
|
+
is prefixed with an integer in parentheses, such as ``(1)``, and any
|
|
200
|
+
subsequent reference to that structure uses the notation
|
|
201
|
+
``->(1)``, as shown below.
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
>>> fs = nltk.FeatStruct("""[NAME=Lee, ADDRESS=(1)[NUMBER=74, STREET='rue Pascal'],
|
|
205
|
+
... SPOUSE=[NAME=Kim, ADDRESS->(1)]]""")
|
|
206
|
+
>>> print(fs)
|
|
207
|
+
[ ADDRESS = (1) [ NUMBER = 74 ] ]
|
|
208
|
+
[ [ STREET = 'rue Pascal' ] ]
|
|
209
|
+
[ ]
|
|
210
|
+
[ NAME = 'Lee' ]
|
|
211
|
+
[ ]
|
|
212
|
+
[ SPOUSE = [ ADDRESS -> (1) ] ]
|
|
213
|
+
[ [ NAME = 'Kim' ] ]
|
|
214
|
+
|
|
215
|
+
There can be any number of tags within a single feature structure.
|
|
216
|
+
|
|
217
|
+
>>> fs3 = nltk.FeatStruct("[A=(1)[B=b], C=(2)[], D->(1), E->(2)]")
|
|
218
|
+
>>> print(fs3)
|
|
219
|
+
[ A = (1) [ B = 'b' ] ]
|
|
220
|
+
[ ]
|
|
221
|
+
[ C = (2) [] ]
|
|
222
|
+
[ ]
|
|
223
|
+
[ D -> (1) ]
|
|
224
|
+
[ E -> (2) ]
|
|
225
|
+
>>> fs1 = nltk.FeatStruct(NUMBER=74, STREET='rue Pascal')
|
|
226
|
+
>>> fs2 = nltk.FeatStruct(CITY='Paris')
|
|
227
|
+
>>> print(nltk.unify(fs1, fs2))
|
|
228
|
+
[ CITY = 'Paris' ]
|
|
229
|
+
[ NUMBER = 74 ]
|
|
230
|
+
[ STREET = 'rue Pascal' ]
|
|
231
|
+
|
|
232
|
+
Unification is symmetric:
|
|
233
|
+
|
|
234
|
+
>>> nltk.unify(fs1, fs2) == nltk.unify(fs2, fs1)
|
|
235
|
+
True
|
|
236
|
+
|
|
237
|
+
Unification is commutative:
|
|
238
|
+
|
|
239
|
+
>>> fs3 = nltk.FeatStruct(TELNO='01 27 86 42 96')
|
|
240
|
+
>>> nltk.unify(nltk.unify(fs1, fs2), fs3) == nltk.unify(fs1, nltk.unify(fs2, fs3))
|
|
241
|
+
True
|
|
242
|
+
|
|
243
|
+
Unification between `FS`:math:\ :subscript:`0` and `FS`:math:\
|
|
244
|
+
:subscript:`1` will fail if the two feature structures share a path |pi|,
|
|
245
|
+
but the value of |pi| in `FS`:math:\ :subscript:`0` is a distinct
|
|
246
|
+
atom from the value of |pi| in `FS`:math:\ :subscript:`1`. In NLTK,
|
|
247
|
+
this is implemented by setting the result of unification to be
|
|
248
|
+
``None``.
|
|
249
|
+
|
|
250
|
+
>>> fs0 = nltk.FeatStruct(A='a')
|
|
251
|
+
>>> fs1 = nltk.FeatStruct(A='b')
|
|
252
|
+
>>> print(nltk.unify(fs0, fs1))
|
|
253
|
+
None
|
|
254
|
+
|
|
255
|
+
Now, if we look at how unification interacts with structure-sharing,
|
|
256
|
+
things become really interesting.
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
>>> fs0 = nltk.FeatStruct("""[NAME=Lee,
|
|
261
|
+
... ADDRESS=[NUMBER=74,
|
|
262
|
+
... STREET='rue Pascal'],
|
|
263
|
+
... SPOUSE= [NAME=Kim,
|
|
264
|
+
... ADDRESS=[NUMBER=74,
|
|
265
|
+
... STREET='rue Pascal']]]""")
|
|
266
|
+
>>> print(fs0)
|
|
267
|
+
[ ADDRESS = [ NUMBER = 74 ] ]
|
|
268
|
+
[ [ STREET = 'rue Pascal' ] ]
|
|
269
|
+
[ ]
|
|
270
|
+
[ NAME = 'Lee' ]
|
|
271
|
+
[ ]
|
|
272
|
+
[ [ ADDRESS = [ NUMBER = 74 ] ] ]
|
|
273
|
+
[ SPOUSE = [ [ STREET = 'rue Pascal' ] ] ]
|
|
274
|
+
[ [ ] ]
|
|
275
|
+
[ [ NAME = 'Kim' ] ]
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
>>> fs1 = nltk.FeatStruct("[SPOUSE=[ADDRESS=[CITY=Paris]]]")
|
|
279
|
+
>>> print(nltk.unify(fs0, fs1))
|
|
280
|
+
[ ADDRESS = [ NUMBER = 74 ] ]
|
|
281
|
+
[ [ STREET = 'rue Pascal' ] ]
|
|
282
|
+
[ ]
|
|
283
|
+
[ NAME = 'Lee' ]
|
|
284
|
+
[ ]
|
|
285
|
+
[ [ [ CITY = 'Paris' ] ] ]
|
|
286
|
+
[ [ ADDRESS = [ NUMBER = 74 ] ] ]
|
|
287
|
+
[ SPOUSE = [ [ STREET = 'rue Pascal' ] ] ]
|
|
288
|
+
[ [ ] ]
|
|
289
|
+
[ [ NAME = 'Kim' ] ]
|
|
290
|
+
|
|
291
|
+
>>> fs2 = nltk.FeatStruct("""[NAME=Lee, ADDRESS=(1)[NUMBER=74, STREET='rue Pascal'],
|
|
292
|
+
... SPOUSE=[NAME=Kim, ADDRESS->(1)]]""")
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
>>> print(fs2)
|
|
296
|
+
[ ADDRESS = (1) [ NUMBER = 74 ] ]
|
|
297
|
+
[ [ STREET = 'rue Pascal' ] ]
|
|
298
|
+
[ ]
|
|
299
|
+
[ NAME = 'Lee' ]
|
|
300
|
+
[ ]
|
|
301
|
+
[ SPOUSE = [ ADDRESS -> (1) ] ]
|
|
302
|
+
[ [ NAME = 'Kim' ] ]
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
>>> print(nltk.unify(fs2, fs1))
|
|
306
|
+
[ [ CITY = 'Paris' ] ]
|
|
307
|
+
[ ADDRESS = (1) [ NUMBER = 74 ] ]
|
|
308
|
+
[ [ STREET = 'rue Pascal' ] ]
|
|
309
|
+
[ ]
|
|
310
|
+
[ NAME = 'Lee' ]
|
|
311
|
+
[ ]
|
|
312
|
+
[ SPOUSE = [ ADDRESS -> (1) ] ]
|
|
313
|
+
[ [ NAME = 'Kim' ] ]
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
>>> fs1 = nltk.FeatStruct("[ADDRESS1=[NUMBER=74, STREET='rue Pascal']]")
|
|
317
|
+
>>> fs2 = nltk.FeatStruct("[ADDRESS1=?x, ADDRESS2=?x]")
|
|
318
|
+
>>> print(fs2)
|
|
319
|
+
[ ADDRESS1 = ?x ]
|
|
320
|
+
[ ADDRESS2 = ?x ]
|
|
321
|
+
>>> print(nltk.unify(fs1, fs2))
|
|
322
|
+
[ ADDRESS1 = (1) [ NUMBER = 74 ] ]
|
|
323
|
+
[ [ STREET = 'rue Pascal' ] ]
|
|
324
|
+
[ ]
|
|
325
|
+
[ ADDRESS2 -> (1) ]
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
>>> sent = 'who do you claim that you like'
|
|
331
|
+
>>> tokens = sent.split()
|
|
332
|
+
>>> cp = parse.load_parser('grammars/book_grammars/feat1.fcfg', trace=1)
|
|
333
|
+
>>> trees = cp.parse(tokens)
|
|
334
|
+
|.w.d.y.c.t.y.l.|
|
|
335
|
+
|[-] . . . . . .| [0:1] 'who'
|
|
336
|
+
|. [-] . . . . .| [1:2] 'do'
|
|
337
|
+
|. . [-] . . . .| [2:3] 'you'
|
|
338
|
+
|. . . [-] . . .| [3:4] 'claim'
|
|
339
|
+
|. . . . [-] . .| [4:5] 'that'
|
|
340
|
+
|. . . . . [-] .| [5:6] 'you'
|
|
341
|
+
|. . . . . . [-]| [6:7] 'like'
|
|
342
|
+
|# . . . . . . .| [0:0] NP[]/NP[] -> *
|
|
343
|
+
|. # . . . . . .| [1:1] NP[]/NP[] -> *
|
|
344
|
+
|. . # . . . . .| [2:2] NP[]/NP[] -> *
|
|
345
|
+
|. . . # . . . .| [3:3] NP[]/NP[] -> *
|
|
346
|
+
|. . . . # . . .| [4:4] NP[]/NP[] -> *
|
|
347
|
+
|. . . . . # . .| [5:5] NP[]/NP[] -> *
|
|
348
|
+
|. . . . . . # .| [6:6] NP[]/NP[] -> *
|
|
349
|
+
|. . . . . . . #| [7:7] NP[]/NP[] -> *
|
|
350
|
+
|[-] . . . . . .| [0:1] NP[+WH] -> 'who' *
|
|
351
|
+
|[-> . . . . . .| [0:1] S[-INV] -> NP[] * VP[] {}
|
|
352
|
+
|[-> . . . . . .| [0:1] S[-INV]/?x[] -> NP[] * VP[]/?x[] {}
|
|
353
|
+
|[-> . . . . . .| [0:1] S[-INV] -> NP[] * S[]/NP[] {}
|
|
354
|
+
|. [-] . . . . .| [1:2] V[+AUX] -> 'do' *
|
|
355
|
+
|. [-> . . . . .| [1:2] S[+INV] -> V[+AUX] * NP[] VP[] {}
|
|
356
|
+
|. [-> . . . . .| [1:2] S[+INV]/?x[] -> V[+AUX] * NP[] VP[]/?x[] {}
|
|
357
|
+
|. [-> . . . . .| [1:2] VP[] -> V[+AUX] * VP[] {}
|
|
358
|
+
|. [-> . . . . .| [1:2] VP[]/?x[] -> V[+AUX] * VP[]/?x[] {}
|
|
359
|
+
|. . [-] . . . .| [2:3] NP[-WH] -> 'you' *
|
|
360
|
+
|. . [-> . . . .| [2:3] S[-INV] -> NP[] * VP[] {}
|
|
361
|
+
|. . [-> . . . .| [2:3] S[-INV]/?x[] -> NP[] * VP[]/?x[] {}
|
|
362
|
+
|. . [-> . . . .| [2:3] S[-INV] -> NP[] * S[]/NP[] {}
|
|
363
|
+
|. [---> . . . .| [1:3] S[+INV] -> V[+AUX] NP[] * VP[] {}
|
|
364
|
+
|. [---> . . . .| [1:3] S[+INV]/?x[] -> V[+AUX] NP[] * VP[]/?x[] {}
|
|
365
|
+
|. . . [-] . . .| [3:4] V[-AUX, SUBCAT='clause'] -> 'claim' *
|
|
366
|
+
|. . . [-> . . .| [3:4] VP[] -> V[-AUX, SUBCAT='clause'] * SBar[] {}
|
|
367
|
+
|. . . [-> . . .| [3:4] VP[]/?x[] -> V[-AUX, SUBCAT='clause'] * SBar[]/?x[] {}
|
|
368
|
+
|. . . . [-] . .| [4:5] Comp[] -> 'that' *
|
|
369
|
+
|. . . . [-> . .| [4:5] SBar[] -> Comp[] * S[-INV] {}
|
|
370
|
+
|. . . . [-> . .| [4:5] SBar[]/?x[] -> Comp[] * S[-INV]/?x[] {}
|
|
371
|
+
|. . . . . [-] .| [5:6] NP[-WH] -> 'you' *
|
|
372
|
+
|. . . . . [-> .| [5:6] S[-INV] -> NP[] * VP[] {}
|
|
373
|
+
|. . . . . [-> .| [5:6] S[-INV]/?x[] -> NP[] * VP[]/?x[] {}
|
|
374
|
+
|. . . . . [-> .| [5:6] S[-INV] -> NP[] * S[]/NP[] {}
|
|
375
|
+
|. . . . . . [-]| [6:7] V[-AUX, SUBCAT='trans'] -> 'like' *
|
|
376
|
+
|. . . . . . [->| [6:7] VP[] -> V[-AUX, SUBCAT='trans'] * NP[] {}
|
|
377
|
+
|. . . . . . [->| [6:7] VP[]/?x[] -> V[-AUX, SUBCAT='trans'] * NP[]/?x[] {}
|
|
378
|
+
|. . . . . . [-]| [6:7] VP[]/NP[] -> V[-AUX, SUBCAT='trans'] NP[]/NP[] *
|
|
379
|
+
|. . . . . [---]| [5:7] S[-INV]/NP[] -> NP[] VP[]/NP[] *
|
|
380
|
+
|. . . . [-----]| [4:7] SBar[]/NP[] -> Comp[] S[-INV]/NP[] *
|
|
381
|
+
|. . . [-------]| [3:7] VP[]/NP[] -> V[-AUX, SUBCAT='clause'] SBar[]/NP[] *
|
|
382
|
+
|. . [---------]| [2:7] S[-INV]/NP[] -> NP[] VP[]/NP[] *
|
|
383
|
+
|. [-----------]| [1:7] S[+INV]/NP[] -> V[+AUX] NP[] VP[]/NP[] *
|
|
384
|
+
|[=============]| [0:7] S[-INV] -> NP[] S[]/NP[] *
|
|
385
|
+
|
|
386
|
+
>>> trees = list(trees)
|
|
387
|
+
>>> for tree in trees: print(tree)
|
|
388
|
+
(S[-INV]
|
|
389
|
+
(NP[+WH] who)
|
|
390
|
+
(S[+INV]/NP[]
|
|
391
|
+
(V[+AUX] do)
|
|
392
|
+
(NP[-WH] you)
|
|
393
|
+
(VP[]/NP[]
|
|
394
|
+
(V[-AUX, SUBCAT='clause'] claim)
|
|
395
|
+
(SBar[]/NP[]
|
|
396
|
+
(Comp[] that)
|
|
397
|
+
(S[-INV]/NP[]
|
|
398
|
+
(NP[-WH] you)
|
|
399
|
+
(VP[]/NP[] (V[-AUX, SUBCAT='trans'] like) (NP[]/NP[] )))))))
|
|
400
|
+
|
|
401
|
+
A different parser should give the same parse trees, but perhaps in a different order:
|
|
402
|
+
|
|
403
|
+
>>> cp2 = parse.load_parser('grammars/book_grammars/feat1.fcfg', trace=1,
|
|
404
|
+
... parser=parse.FeatureEarleyChartParser)
|
|
405
|
+
>>> trees2 = cp2.parse(tokens)
|
|
406
|
+
|.w.d.y.c.t.y.l.|
|
|
407
|
+
|[-] . . . . . .| [0:1] 'who'
|
|
408
|
+
|. [-] . . . . .| [1:2] 'do'
|
|
409
|
+
|. . [-] . . . .| [2:3] 'you'
|
|
410
|
+
|. . . [-] . . .| [3:4] 'claim'
|
|
411
|
+
|. . . . [-] . .| [4:5] 'that'
|
|
412
|
+
|. . . . . [-] .| [5:6] 'you'
|
|
413
|
+
|. . . . . . [-]| [6:7] 'like'
|
|
414
|
+
|> . . . . . . .| [0:0] S[-INV] -> * NP[] VP[] {}
|
|
415
|
+
|> . . . . . . .| [0:0] S[-INV]/?x[] -> * NP[] VP[]/?x[] {}
|
|
416
|
+
|> . . . . . . .| [0:0] S[-INV] -> * NP[] S[]/NP[] {}
|
|
417
|
+
|> . . . . . . .| [0:0] S[-INV] -> * Adv[+NEG] S[+INV] {}
|
|
418
|
+
|> . . . . . . .| [0:0] S[+INV] -> * V[+AUX] NP[] VP[] {}
|
|
419
|
+
|> . . . . . . .| [0:0] S[+INV]/?x[] -> * V[+AUX] NP[] VP[]/?x[] {}
|
|
420
|
+
|> . . . . . . .| [0:0] NP[+WH] -> * 'who' {}
|
|
421
|
+
|[-] . . . . . .| [0:1] NP[+WH] -> 'who' *
|
|
422
|
+
|[-> . . . . . .| [0:1] S[-INV] -> NP[] * VP[] {}
|
|
423
|
+
|[-> . . . . . .| [0:1] S[-INV]/?x[] -> NP[] * VP[]/?x[] {}
|
|
424
|
+
|[-> . . . . . .| [0:1] S[-INV] -> NP[] * S[]/NP[] {}
|
|
425
|
+
|. > . . . . . .| [1:1] S[-INV]/?x[] -> * NP[] VP[]/?x[] {}
|
|
426
|
+
|. > . . . . . .| [1:1] S[+INV]/?x[] -> * V[+AUX] NP[] VP[]/?x[] {}
|
|
427
|
+
|. > . . . . . .| [1:1] V[+AUX] -> * 'do' {}
|
|
428
|
+
|. > . . . . . .| [1:1] VP[]/?x[] -> * V[-AUX, SUBCAT='trans'] NP[]/?x[] {}
|
|
429
|
+
|. > . . . . . .| [1:1] VP[]/?x[] -> * V[-AUX, SUBCAT='clause'] SBar[]/?x[] {}
|
|
430
|
+
|. > . . . . . .| [1:1] VP[]/?x[] -> * V[+AUX] VP[]/?x[] {}
|
|
431
|
+
|. > . . . . . .| [1:1] VP[] -> * V[-AUX, SUBCAT='intrans'] {}
|
|
432
|
+
|. > . . . . . .| [1:1] VP[] -> * V[-AUX, SUBCAT='trans'] NP[] {}
|
|
433
|
+
|. > . . . . . .| [1:1] VP[] -> * V[-AUX, SUBCAT='clause'] SBar[] {}
|
|
434
|
+
|. > . . . . . .| [1:1] VP[] -> * V[+AUX] VP[] {}
|
|
435
|
+
|. [-] . . . . .| [1:2] V[+AUX] -> 'do' *
|
|
436
|
+
|. [-> . . . . .| [1:2] S[+INV]/?x[] -> V[+AUX] * NP[] VP[]/?x[] {}
|
|
437
|
+
|. [-> . . . . .| [1:2] VP[]/?x[] -> V[+AUX] * VP[]/?x[] {}
|
|
438
|
+
|. [-> . . . . .| [1:2] VP[] -> V[+AUX] * VP[] {}
|
|
439
|
+
|. . > . . . . .| [2:2] VP[] -> * V[-AUX, SUBCAT='intrans'] {}
|
|
440
|
+
|. . > . . . . .| [2:2] VP[] -> * V[-AUX, SUBCAT='trans'] NP[] {}
|
|
441
|
+
|. . > . . . . .| [2:2] VP[] -> * V[-AUX, SUBCAT='clause'] SBar[] {}
|
|
442
|
+
|. . > . . . . .| [2:2] VP[] -> * V[+AUX] VP[] {}
|
|
443
|
+
|. . > . . . . .| [2:2] VP[]/?x[] -> * V[-AUX, SUBCAT='trans'] NP[]/?x[] {}
|
|
444
|
+
|. . > . . . . .| [2:2] VP[]/?x[] -> * V[-AUX, SUBCAT='clause'] SBar[]/?x[] {}
|
|
445
|
+
|. . > . . . . .| [2:2] VP[]/?x[] -> * V[+AUX] VP[]/?x[] {}
|
|
446
|
+
|. . > . . . . .| [2:2] NP[-WH] -> * 'you' {}
|
|
447
|
+
|. . [-] . . . .| [2:3] NP[-WH] -> 'you' *
|
|
448
|
+
|. [---> . . . .| [1:3] S[+INV]/?x[] -> V[+AUX] NP[] * VP[]/?x[] {}
|
|
449
|
+
|. . . > . . . .| [3:3] VP[]/?x[] -> * V[-AUX, SUBCAT='trans'] NP[]/?x[] {}
|
|
450
|
+
|. . . > . . . .| [3:3] VP[]/?x[] -> * V[-AUX, SUBCAT='clause'] SBar[]/?x[] {}
|
|
451
|
+
|. . . > . . . .| [3:3] VP[]/?x[] -> * V[+AUX] VP[]/?x[] {}
|
|
452
|
+
|. . . > . . . .| [3:3] V[-AUX, SUBCAT='clause'] -> * 'claim' {}
|
|
453
|
+
|. . . [-] . . .| [3:4] V[-AUX, SUBCAT='clause'] -> 'claim' *
|
|
454
|
+
|. . . [-> . . .| [3:4] VP[]/?x[] -> V[-AUX, SUBCAT='clause'] * SBar[]/?x[] {}
|
|
455
|
+
|. . . . > . . .| [4:4] SBar[]/?x[] -> * Comp[] S[-INV]/?x[] {}
|
|
456
|
+
|. . . . > . . .| [4:4] Comp[] -> * 'that' {}
|
|
457
|
+
|. . . . [-] . .| [4:5] Comp[] -> 'that' *
|
|
458
|
+
|. . . . [-> . .| [4:5] SBar[]/?x[] -> Comp[] * S[-INV]/?x[] {}
|
|
459
|
+
|. . . . . > . .| [5:5] S[-INV]/?x[] -> * NP[] VP[]/?x[] {}
|
|
460
|
+
|. . . . . > . .| [5:5] NP[-WH] -> * 'you' {}
|
|
461
|
+
|. . . . . [-] .| [5:6] NP[-WH] -> 'you' *
|
|
462
|
+
|. . . . . [-> .| [5:6] S[-INV]/?x[] -> NP[] * VP[]/?x[] {}
|
|
463
|
+
|. . . . . . > .| [6:6] VP[]/?x[] -> * V[-AUX, SUBCAT='trans'] NP[]/?x[] {}
|
|
464
|
+
|. . . . . . > .| [6:6] VP[]/?x[] -> * V[-AUX, SUBCAT='clause'] SBar[]/?x[] {}
|
|
465
|
+
|. . . . . . > .| [6:6] VP[]/?x[] -> * V[+AUX] VP[]/?x[] {}
|
|
466
|
+
|. . . . . . > .| [6:6] V[-AUX, SUBCAT='trans'] -> * 'like' {}
|
|
467
|
+
|. . . . . . [-]| [6:7] V[-AUX, SUBCAT='trans'] -> 'like' *
|
|
468
|
+
|. . . . . . [->| [6:7] VP[]/?x[] -> V[-AUX, SUBCAT='trans'] * NP[]/?x[] {}
|
|
469
|
+
|. . . . . . . #| [7:7] NP[]/NP[] -> *
|
|
470
|
+
|. . . . . . [-]| [6:7] VP[]/NP[] -> V[-AUX, SUBCAT='trans'] NP[]/NP[] *
|
|
471
|
+
|. . . . . [---]| [5:7] S[-INV]/NP[] -> NP[] VP[]/NP[] *
|
|
472
|
+
|. . . . [-----]| [4:7] SBar[]/NP[] -> Comp[] S[-INV]/NP[] *
|
|
473
|
+
|. . . [-------]| [3:7] VP[]/NP[] -> V[-AUX, SUBCAT='clause'] SBar[]/NP[] *
|
|
474
|
+
|. [-----------]| [1:7] S[+INV]/NP[] -> V[+AUX] NP[] VP[]/NP[] *
|
|
475
|
+
|[=============]| [0:7] S[-INV] -> NP[] S[]/NP[] *
|
|
476
|
+
|
|
477
|
+
>>> sorted(trees) == sorted(trees2)
|
|
478
|
+
True
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
Let's load a German grammar:
|
|
482
|
+
|
|
483
|
+
>>> cp = parse.load_parser('grammars/book_grammars/german.fcfg', trace=0)
|
|
484
|
+
>>> sent = 'die Katze sieht den Hund'
|
|
485
|
+
>>> tokens = sent.split()
|
|
486
|
+
>>> trees = cp.parse(tokens)
|
|
487
|
+
>>> for tree in trees: print(tree)
|
|
488
|
+
(S[]
|
|
489
|
+
(NP[AGR=[GND='fem', NUM='sg', PER=3], CASE='nom']
|
|
490
|
+
(Det[AGR=[GND='fem', NUM='sg', PER=3], CASE='nom'] die)
|
|
491
|
+
(N[AGR=[GND='fem', NUM='sg', PER=3]] Katze))
|
|
492
|
+
(VP[AGR=[NUM='sg', PER=3]]
|
|
493
|
+
(TV[AGR=[NUM='sg', PER=3], OBJCASE='acc'] sieht)
|
|
494
|
+
(NP[AGR=[GND='masc', NUM='sg', PER=3], CASE='acc']
|
|
495
|
+
(Det[AGR=[GND='masc', NUM='sg', PER=3], CASE='acc'] den)
|
|
496
|
+
(N[AGR=[GND='masc', NUM='sg', PER=3]] Hund))))
|
|
497
|
+
|
|
498
|
+
Grammar with Binding Operators
|
|
499
|
+
------------------------------
|
|
500
|
+
The `bindop.fcfg`_ grammar is a semantic grammar that uses lambda
|
|
501
|
+
calculus. Each element has a core semantics, which is a single lambda
|
|
502
|
+
calculus expression; and a set of binding operators, which bind
|
|
503
|
+
variables.
|
|
504
|
+
|
|
505
|
+
.. _bindop.fcfg: http://nltk.svn.sourceforge.net/svnroot/nltk/trunk/nltk/data/grammars/bindop.fcfg
|
|
506
|
+
|
|
507
|
+
In order to make the binding operators work right, they need to
|
|
508
|
+
instantiate their bound variable every time they are added to the
|
|
509
|
+
chart. To do this, we use a special subclass of `Chart`, called
|
|
510
|
+
`InstantiateVarsChart`.
|
|
511
|
+
|
|
512
|
+
>>> from nltk.parse.featurechart import InstantiateVarsChart
|
|
513
|
+
>>> cp = parse.load_parser('grammars/sample_grammars/bindop.fcfg', trace=1,
|
|
514
|
+
... chart_class=InstantiateVarsChart)
|
|
515
|
+
>>> print(cp.grammar())
|
|
516
|
+
Grammar with 15 productions (start state = S[])
|
|
517
|
+
S[SEM=[BO={?b1+?b2}, CORE=<?vp(?subj)>]] -> NP[SEM=[BO=?b1, CORE=?subj]] VP[SEM=[BO=?b2, CORE=?vp]]
|
|
518
|
+
VP[SEM=[BO={?b1+?b2}, CORE=<?v(?obj)>]] -> TV[SEM=[BO=?b1, CORE=?v]] NP[SEM=[BO=?b2, CORE=?obj]]
|
|
519
|
+
VP[SEM=?s] -> IV[SEM=?s]
|
|
520
|
+
NP[SEM=[BO={?b1+?b2+{bo(?det(?n),@x)}}, CORE=<@x>]] -> Det[SEM=[BO=?b1, CORE=?det]] N[SEM=[BO=?b2, CORE=?n]]
|
|
521
|
+
Det[SEM=[BO={/}, CORE=<\Q P.exists x.(Q(x) & P(x))>]] -> 'a'
|
|
522
|
+
N[SEM=[BO={/}, CORE=<dog>]] -> 'dog'
|
|
523
|
+
N[SEM=[BO={/}, CORE=<dog>]] -> 'cat'
|
|
524
|
+
N[SEM=[BO={/}, CORE=<dog>]] -> 'mouse'
|
|
525
|
+
IV[SEM=[BO={/}, CORE=<\x.bark(x)>]] -> 'barks'
|
|
526
|
+
IV[SEM=[BO={/}, CORE=<\x.bark(x)>]] -> 'eats'
|
|
527
|
+
IV[SEM=[BO={/}, CORE=<\x.bark(x)>]] -> 'walks'
|
|
528
|
+
TV[SEM=[BO={/}, CORE=<\x y.feed(y,x)>]] -> 'feeds'
|
|
529
|
+
TV[SEM=[BO={/}, CORE=<\x y.feed(y,x)>]] -> 'walks'
|
|
530
|
+
NP[SEM=[BO={bo(\P.P(John),@x)}, CORE=<@x>]] -> 'john'
|
|
531
|
+
NP[SEM=[BO={bo(\P.P(John),@x)}, CORE=<@x>]] -> 'alex'
|
|
532
|
+
|
|
533
|
+
A simple intransitive sentence:
|
|
534
|
+
|
|
535
|
+
>>> from nltk.sem import logic
|
|
536
|
+
>>> logic._counter._value = 100
|
|
537
|
+
|
|
538
|
+
>>> trees = cp.parse('john barks'.split())
|
|
539
|
+
|. john.barks.|
|
|
540
|
+
|[-----] .| [0:1] 'john'
|
|
541
|
+
|. [-----]| [1:2] 'barks'
|
|
542
|
+
|[-----] .| [0:1] NP[SEM=[BO={bo(\P.P(John),z101)}, CORE=<z101>]] -> 'john' *
|
|
543
|
+
|[-----> .| [0:1] S[SEM=[BO={?b1+?b2}, CORE=<?vp(?subj)>]] -> NP[SEM=[BO=?b1, CORE=?subj]] * VP[SEM=[BO=?b2, CORE=?vp]] {?b1: {bo(\P.P(John),z2)}, ?subj: <IndividualVariableExpression z2>}
|
|
544
|
+
|. [-----]| [1:2] IV[SEM=[BO={/}, CORE=<\x.bark(x)>]] -> 'barks' *
|
|
545
|
+
|. [-----]| [1:2] VP[SEM=[BO={/}, CORE=<\x.bark(x)>]] -> IV[SEM=[BO={/}, CORE=<\x.bark(x)>]] *
|
|
546
|
+
|[===========]| [0:2] S[SEM=[BO={bo(\P.P(John),z2)}, CORE=<bark(z2)>]] -> NP[SEM=[BO={bo(\P.P(John),z2)}, CORE=<z2>]] VP[SEM=[BO={/}, CORE=<\x.bark(x)>]] *
|
|
547
|
+
>>> for tree in trees: print(tree)
|
|
548
|
+
(S[SEM=[BO={bo(\P.P(John),z2)}, CORE=<bark(z2)>]]
|
|
549
|
+
(NP[SEM=[BO={bo(\P.P(John),z101)}, CORE=<z101>]] john)
|
|
550
|
+
(VP[SEM=[BO={/}, CORE=<\x.bark(x)>]]
|
|
551
|
+
(IV[SEM=[BO={/}, CORE=<\x.bark(x)>]] barks)))
|
|
552
|
+
|
|
553
|
+
A transitive sentence:
|
|
554
|
+
|
|
555
|
+
>>> trees = cp.parse('john feeds a dog'.split())
|
|
556
|
+
|.joh.fee. a .dog.|
|
|
557
|
+
|[---] . . .| [0:1] 'john'
|
|
558
|
+
|. [---] . .| [1:2] 'feeds'
|
|
559
|
+
|. . [---] .| [2:3] 'a'
|
|
560
|
+
|. . . [---]| [3:4] 'dog'
|
|
561
|
+
|[---] . . .| [0:1] NP[SEM=[BO={bo(\P.P(John),z102)}, CORE=<z102>]] -> 'john' *
|
|
562
|
+
|[---> . . .| [0:1] S[SEM=[BO={?b1+?b2}, CORE=<?vp(?subj)>]] -> NP[SEM=[BO=?b1, CORE=?subj]] * VP[SEM=[BO=?b2, CORE=?vp]] {?b1: {bo(\P.P(John),z2)}, ?subj: <IndividualVariableExpression z2>}
|
|
563
|
+
|. [---] . .| [1:2] TV[SEM=[BO={/}, CORE=<\x y.feed(y,x)>]] -> 'feeds' *
|
|
564
|
+
|. [---> . .| [1:2] VP[SEM=[BO={?b1+?b2}, CORE=<?v(?obj)>]] -> TV[SEM=[BO=?b1, CORE=?v]] * NP[SEM=[BO=?b2, CORE=?obj]] {?b1: {/}, ?v: <LambdaExpression \x y.feed(y,x)>}
|
|
565
|
+
|. . [---] .| [2:3] Det[SEM=[BO={/}, CORE=<\Q P.exists x.(Q(x) & P(x))>]] -> 'a' *
|
|
566
|
+
|. . [---> .| [2:3] NP[SEM=[BO={?b1+?b2+{bo(?det(?n),@x)}}, CORE=<@x>]] -> Det[SEM=[BO=?b1, CORE=?det]] * N[SEM=[BO=?b2, CORE=?n]] {?b1: {/}, ?det: <LambdaExpression \Q P.exists x.(Q(x) & P(x))>}
|
|
567
|
+
|. . . [---]| [3:4] N[SEM=[BO={/}, CORE=<dog>]] -> 'dog' *
|
|
568
|
+
|. . [-------]| [2:4] NP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z103)}, CORE=<z103>]] -> Det[SEM=[BO={/}, CORE=<\Q P.exists x.(Q(x) & P(x))>]] N[SEM=[BO={/}, CORE=<dog>]] *
|
|
569
|
+
|. . [------->| [2:4] S[SEM=[BO={?b1+?b2}, CORE=<?vp(?subj)>]] -> NP[SEM=[BO=?b1, CORE=?subj]] * VP[SEM=[BO=?b2, CORE=?vp]] {?b1: {bo(\P.exists x.(dog(x) & P(x)),z2)}, ?subj: <IndividualVariableExpression z2>}
|
|
570
|
+
|. [-----------]| [1:4] VP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z2)}, CORE=<\y.feed(y,z2)>]] -> TV[SEM=[BO={/}, CORE=<\x y.feed(y,x)>]] NP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z2)}, CORE=<z2>]] *
|
|
571
|
+
|[===============]| [0:4] S[SEM=[BO={bo(\P.P(John),z2), bo(\P.exists x.(dog(x) & P(x)),z3)}, CORE=<feed(z2,z3)>]] -> NP[SEM=[BO={bo(\P.P(John),z2)}, CORE=<z2>]] VP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z3)}, CORE=<\y.feed(y,z3)>]] *
|
|
572
|
+
|
|
573
|
+
>>> for tree in trees: print(tree)
|
|
574
|
+
(S[SEM=[BO={bo(\P.P(John),z2), bo(\P.exists x.(dog(x) & P(x)),z3)}, CORE=<feed(z2,z3)>]]
|
|
575
|
+
(NP[SEM=[BO={bo(\P.P(John),z102)}, CORE=<z102>]] john)
|
|
576
|
+
(VP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z2)}, CORE=<\y.feed(y,z2)>]]
|
|
577
|
+
(TV[SEM=[BO={/}, CORE=<\x y.feed(y,x)>]] feeds)
|
|
578
|
+
(NP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z103)}, CORE=<z103>]]
|
|
579
|
+
(Det[SEM=[BO={/}, CORE=<\Q P.exists x.(Q(x) & P(x))>]] a)
|
|
580
|
+
(N[SEM=[BO={/}, CORE=<dog>]] dog))))
|
|
581
|
+
|
|
582
|
+
Turn down the verbosity:
|
|
583
|
+
|
|
584
|
+
>>> cp = parse.load_parser('grammars/sample_grammars/bindop.fcfg', trace=0,
|
|
585
|
+
... chart_class=InstantiateVarsChart)
|
|
586
|
+
|
|
587
|
+
Reuse the same lexical item twice:
|
|
588
|
+
|
|
589
|
+
>>> trees = cp.parse('john feeds john'.split())
|
|
590
|
+
>>> for tree in trees: print(tree)
|
|
591
|
+
(S[SEM=[BO={bo(\P.P(John),z2), bo(\P.P(John),z3)}, CORE=<feed(z2,z3)>]]
|
|
592
|
+
(NP[SEM=[BO={bo(\P.P(John),z104)}, CORE=<z104>]] john)
|
|
593
|
+
(VP[SEM=[BO={bo(\P.P(John),z2)}, CORE=<\y.feed(y,z2)>]]
|
|
594
|
+
(TV[SEM=[BO={/}, CORE=<\x y.feed(y,x)>]] feeds)
|
|
595
|
+
(NP[SEM=[BO={bo(\P.P(John),z105)}, CORE=<z105>]] john)))
|
|
596
|
+
|
|
597
|
+
>>> trees = cp.parse('a dog feeds a dog'.split())
|
|
598
|
+
>>> for tree in trees: print(tree)
|
|
599
|
+
(S[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z2), bo(\P.exists x.(dog(x) & P(x)),z3)}, CORE=<feed(z2,z3)>]]
|
|
600
|
+
(NP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z106)}, CORE=<z106>]]
|
|
601
|
+
(Det[SEM=[BO={/}, CORE=<\Q P.exists x.(Q(x) & P(x))>]] a)
|
|
602
|
+
(N[SEM=[BO={/}, CORE=<dog>]] dog))
|
|
603
|
+
(VP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z2)}, CORE=<\y.feed(y,z2)>]]
|
|
604
|
+
(TV[SEM=[BO={/}, CORE=<\x y.feed(y,x)>]] feeds)
|
|
605
|
+
(NP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z107)}, CORE=<z107>]]
|
|
606
|
+
(Det[SEM=[BO={/}, CORE=<\Q P.exists x.(Q(x) & P(x))>]] a)
|
|
607
|
+
(N[SEM=[BO={/}, CORE=<dog>]] dog))))
|