python应用：主题分类（gensim lda）

安装第三方包：gensim

首先，执行去停词操作（去除与主题无关的词）

 #-*-coding:utf8-*-

 import jieba

 def stopwordslist(filepath):
     stopwords = [line.strip() for line in open(filepath, 'r').readlines()]
     return stopwords

 def seg_sentence(sentence):
     sentence_seged = jieba.cut(sentence.strip())
     stopwords = stopwordslist('stopWords/stopwords.txt')
     outstr = ''
     for word in sentence_seged:
         word = word.lower()
         if word not in stopwords:
             if word != '\t':
                 outstr += word
                 outstr += " "
     return outstr

 inputs = open('input/copurs.txt', 'r')   

 outputs = open('input/copurs_out.txt', 'w')
 for line in inputs:
     line_seg = seg_sentence(line)
     outputs.write(line_seg + '\n')
 outputs.close()
 inputs.close()

然后，执行主题分类操作

 import codecs
 from gensim import corpora
 from gensim.models import LdaModel
 from gensim import models
 from gensim.corpora import Dictionary

 te = []
 fp = codecs.open('input/copurs_out.txt','r')
 for line in fp:
     line = line.split()
     te.append([ w for w in line  ])
 print len(te)
 dictionary = corpora.Dictionary(te)
 corpus = [ dictionary.doc2bow(text) for text in te ]

 #tfidf = models.TfidfModel(corpus)
 #corpus_tfidf = tfidf[corpus]

 #########Run the LDA model for XX topics ###############################
 lda = LdaModel(corpus=corpus, id2word=dictionary, num_topics=50,passes=2000)
 doc_topic = [a for a in lda[corpus]]

 ####### write the topics in file topics_result.txt ####################
 topics_r = lda.print_topics(num_topics = 50, num_words = 10)
 topic_name = codecs.open('output/topics_result.txt','w')
 for v in topics_r:
     topic_name.write(str(v)+'\n')

 ######################  write the class results to file  #########################
 ###################### each document belongs to which topic ######################

 fp2 = codecs.open('output/documents_result.txt','w')
 for t in doc_topic:
     c = []
     c.append([a[1] for a in t])
     m = max(c[0])

     for i in range(0, len(t)):
         if m in t[i]:
             #print(t[i])
             fp2.write(str(t[i][0]) + '  ' + str(t[i][1]) + '\n')
             break
 ################################ OVER ############################################

注意：上述主题分类，仅使用lda模型（根据频数计算）

也可混合使用tf-idf模型XX-topic下代码改为如下即可：

方式一
#########Run the LDA model for XX topics ###############################
lda = LdaModel(corpus=corpus_tfidf, id2word=dictionary, num_topics=50,passes=2000)
doc_topic = [a for a in lda[corpus_tfidf]]

或

方式二
#########Run the LDA model for XX topics ###############################
lda = LdaModel(corpus=corpus, id2word=dictionary, num_topics=50,passes=2000)
doc_topic = [a for a in lda[corpus_tfidf]]

常用方式为方式一，作者暂时为弄清楚这两种方式的区别，后期将会继续完善