import jieba
from jieba import analyse
import numpy
import gensim
import codecs
import pandas as pd
import jieba.posseg as pog
from gensim.models import Word2Vec
from gensim.models.word2vec import LineSentence
#获取训练语料
def data_handle(data):
n = data.shape[0]
data_str = ''
for i in numpy.arange(n):
data_str += str(data.ix[i, 'comment'])
return data_str
def fenci(data_str,stop_property,stopfile):
# 停用词
stop_word = [word.strip() for word in open(stopfile, encoding='utf-8').readlines()]
# 分词
word_cut = pog.cut(data_str) with open('weibo.txt','w',encoding='utf-8') as f:
for word, flag in word_cut:
if flag not in stop_property:
if word not in stop_word:
f.write(word+'\n') # 原始的训练语料转化成一个sentence的迭代器,每一次迭代返回的sentence是一个word(utf8格式)的列表
def vctor_word():
wiki_news = open('weibo.txt', 'r',encoding='utf-8')
sentences=LineSentence(wiki_news)
model=Word2Vec(sentences,sg=0,size=100,window=5,min_count=5,workers=9)
model.save('weibo.word2vec') # 实现给出任意字符串,获取字符串中某字符的位置以及出现的总次数
def get_char_pos(string, char):
chPos = []
try:
chPos = list(((pos, char) for pos, val in enumerate(string) if (val == char)))
except:
pass
return chPos # 利用训练好的词向量获取关键词的词向量 def cut_data(data,stopfile):
data.fillna(0,inplace=True)
stop_word = [word.strip() for word in open(stopfile, encoding='utf-8').readlines()]
charater=['a', 'nr', 'ns', 'nt', 'ng', 'vn', 'vi', 'l', 'n', 'v']
m=data.shape[0]
with open('seg_word.txt', 'w', encoding='utf-8') as f: for i in range(m):
str_cut = ''
str=data.ix[i,'comment']
if str!=0:
segs=jieba.posseg.cut(str)
for word,flag in segs:
if flag in charater:
if word not in stop_word:
str_cut+=word+'/'
f.write(str_cut )
else:
str_cut=''
f.write('\n ') def get_vector(data,model):#str
wordvec_size = 100
word_vec_all = numpy.zeros(wordvec_size)
space_pos = get_char_pos(data, '/')
first_word = data[0:space_pos[0][0]]
print('first_word', first_word)
if first_word in model:
print('yes')
word_vec_all = word_vec_all + model[first_word] for i in range(len(space_pos) - 2):
word = data[space_pos[i][0]:space_pos[i + 1][0]]
print('word',word)
if word in model:
print('yes')
word_vec_all = word_vec_all + model[first_word] print('word_vec_all',word_vec_all)
return word_vec_all def word2vec(file_name, model,str): DataFile = codecs.open(file_name, "r", encoding='utf-8')
DataSet = DataFile.readlines()[:-1] score_list=[] str_vector=get_vector(str,model)
for data in DataSet: #
if data.strip()!='':
word_vec_all=get_vector(data,model)
score=simlarityCalu(word_vec_all, str_vector)
else:
score=0
score_list.append(score)
print('score_list',score_list)
return score_list # 词向量相似度计算代码:余弦
def simlarityCalu(vector1, vector2):
vector1Mod = numpy.sqrt(vector1.dot(vector1))
vector2Mod = numpy.sqrt(vector2.dot(vector2))
if vector2Mod != 0 and vector1Mod != 0:
simlarity = (vector1.dot(vector2)) / (vector1Mod * vector2Mod)
else:
simlarity = 0
return simlarity if __name__ == '__main__': stop_property = ['b', 'c', 'd', 'e', 'f', 'm', 'o', 'p', 'q', 'r', 't', 'u', 'x', 'y', 'z', 'uj', 'nrt', 'eng',
'zg', 'ul']
stop_file='stop.txt' # 读取数据
data = pd.read_excel('C:/E/weibo.xlsx')
data.rename(columns={'粉丝ID': 'fans_id', '粉丝': 'fans_name', '微博账户id': 'weibo_user_id', '微博名': 'weibo_name',
'微博id': 'weibo_id', '评论id': 'comment_id', '评论': 'comment'}, inplace=True) # 获取评论字符串
comment_str=data_handle(data) #获取语料
fenci(comment_str, stop_property, stop_file)
#训练模型
vctor_word()
#获取关键词
cut_data(data, stop_file) p1_keywords = 'seg_word.txt'
str1 = '农农/陈利农/宝贝'
# model = gensim.models.Word2Vec.load('weibo.word2vec')
model = gensim.models.Word2Vec.load('zhiwiki_news.word2vec')
p1_vec = word2vec(p1_keywords, model,str1) str2='舒蔻 尤妮佳 买'

Wordvec_句子相似度的更多相关文章

  1. NLP入门(一)词袋模型及句子相似度

      本文作为笔者NLP入门系列文章第一篇,以后我们就要步入NLP时代.   本文将会介绍NLP中常见的词袋模型(Bag of Words)以及如何利用词袋模型来计算句子间的相似度(余弦相似度,cosi ...

  2. [LeetCode] 737. Sentence Similarity II 句子相似度 II

    Given two sentences words1, words2 (each represented as an array of strings), and a list of similar ...

  3. [LeetCode] 734. Sentence Similarity 句子相似度

    Given two sentences words1, words2 (each represented as an array of strings), and a list of similar ...

  4. 使用 TF-IDF 加权的空间向量模型实现句子相似度计算

    使用 TF-IDF 加权的空间向量模型实现句子相似度计算 字符匹配层次计算句子相似度 计算两个句子相似度的算法有很多种,但是对于从未了解过这方面算法的人来说,可能最容易想到的就是使用字符串匹配相关的算 ...

  5. LSTM 句子相似度分析

    使用句子中出现单词的Vector加权平均进行文本相似度分析虽然简单,但也有比较明显的缺点:没有考虑词序且词向量区别不明确.如下面两个句子: "北京的首都是中国"与"中国的 ...

  6. [LeetCode] Sentence Similarity 句子相似度

    Given two sentences words1, words2 (each represented as an array of strings), and a list of similar ...

  7. [LeetCode] Sentence Similarity II 句子相似度之二

    Given two sentences words1, words2 (each represented as an array of strings), and a list of similar ...

  8. 句子相似度_tf/idf

    import mathfrom math import isnanimport pandas as pd#结巴分词,切开之后,有分隔符def jieba_function(sent): import ...

  9. [LeetCode] 737. Sentence Similarity II 句子相似度之二

    Given two sentences words1, words2 (each represented as an array of strings), and a list of similar ...

随机推荐

  1. 1.4、CDH 搭建Hadoop在安装之前(推荐的群集主机和角色分配)

    推荐的群集主机和角色分配 要点:本主题描述了Cloudera Manager管理的CDH群集的建议角色分配.您为部署选择的实际分配可能会有所不同,具体取决于工作负载的类型和数量,群集中部署的服务,硬件 ...

  2. http4e eclipse plugin 插件介绍

    感谢作者的分享: http://blog.csdn.net/wiker_yong/article/details/10066905 以及作者的破解jar.目前看网站留言说已经git了. 官网链接地址: ...

  3. 【C++】Mandelbrot集绘制(生成ppm文件)

    曼德勃罗特集是人类有史以来做出的最奇异,最瑰丽的几何图形.曾被称为"上帝的指纹". 这个点集均出自公式:Zn+1=(Zn)^2+C.(此处Z.C均为复数)所有使得该公式无限迭代后的 ...

  4. 网页请求get方式

    方法都是博客中的大神写的,谢谢各路大神. 方法一:(亲测有效) //Get请求方式 private string RequestGet(string Url) { string PageStr = s ...

  5. oracle数据库导入导出问题

    场景描述: 1.做一个从UAT到PRD的Schema迁移,UAT环境有sys用户,PRD环境没有sys用户,由于权限限制,没办法使用expdp/impdp,只好选择exp/imp命令: 2.UAT和P ...

  6. 转移动APP测试实践

    http://blog.csdn.net/hgstclyh/article/details/53115325

  7. ACM-ICPC 2018 南京赛区网络预赛 G. Lpl and Energy-saving Lamps(二分+线段树区间最小)

    During tea-drinking, princess, amongst other things, asked why has such a good-natured and cute Drag ...

  8. f5版本升级

    1)上传系统IOS及Hotfix 点击import按钮,选择要上传的文件.上传成功的话就会显示上传成功的10.2.4的iso文件 2)通过CLI命令行方式上传补丁 通过SSH工具将ISO以及Hotfi ...

  9. layer使用

    1引入js <script src="${pageContext.request.contextPath }/js/jquery-1.9.1.min.js" type=&qu ...

  10. Java基本语法之动手动脑

    1.枚举类型 运行EnumTest.java 运行结果:false,false,true,SMALL,MEDIUM,LARGE 结论:枚举类型是引用类型,枚举不属于原始数据类型,它的每个具体值都引用一 ...