Document Classification

Natural Language Processing with Python

Chapter 6.1

由于nltk.FreqDist的排序问题，获取电影文本特征词的代码有些微改动。

 import nltk
 from nltk.corpus import movie_reviews as mr   

 def document_features(document,words_features):
     document_words=set(document)
     features={}
     for word in words_features:
         features['has(%s)' %word] = (word in document_words)
     return features   

 def test_doc_classification():
     documents=[(list(mr.words(fileid)),category)
                 for category in mr.categories()
                 for fileid in mr.fileids(categories=category)]
     all_words_dist=nltk.FreqDist(w.lower() for w in mr.words())
     words_freq =sorted(all_words_dist.items(), key=lambda x: (-1*x[1], x[0]))[:2000]
     words_features=[word[0] for word in words_freq]

     featuresets=[(document_features(doc,words_features),c) for (doc,c) in
                     documents]

     train_set, test_set= featuresets[100:],featuresets[:100]
     classifier=nltk.NaiveBayesClassifier.train(train_set)

     print nltk.classify.accuracy(classifier,test_set)

     classifier.show_most_informative_features(5)

结果如下，accuracy为0.86：

0.86
Most Informative Features
has(outstanding) = True pos : neg = 10.4 : 1.0
has(seagal) = True neg : pos = 8.7 : 1.0
has(mulan) = True pos : neg = 8.1 : 1.0
has(wonderfully) = True pos : neg = 6.3 : 1.0
has(damon) = True pos : neg = 5.7 : 1.0