【笔记3】用pandas实现矩阵数据格式的推荐算法 (基于用户的协同)

原书作者使用字典dict实现推荐算法，并且惊叹于18行代码实现了向量的余弦夹角公式。

我用pandas实现相同的公式只要3行。

特别说明：本篇笔记是针对矩阵数据，下篇笔记是针对条目数据。

'''
基于用户的协同推荐

矩阵数据
'''

import pandas as pd
from io import StringIO
import json

#数据类型一：csv矩阵（用户-商品）（适用于小数据量）
csv_txt = '''"user","Blues Traveler","Broken Bells","Deadmau5","Norah Jones","Phoenix","Slightly Stoopid","The Strokes","Vampire Weekend"
"Angelica",3.5,2.0,,4.5,5.0,1.5,2.5,2.0
"Bill",2.0,3.5,4.0,,2.0,3.5,,3.0
"Chan",5.0,1.0,1.0,3.0,5,1.0,,
"Dan",3.0,4.0,4.5,,3.0,4.5,4.0,2.0
"Hailey",,4.0,1.0,4.0,,,4.0,1.0
"Jordyn",,4.5,4.0,5.0,5.0,4.5,4.0,4.0
"Sam",5.0,2.0,,3.0,5.0,4.0,5.0,
"Veronica",3.0,,,5.0,4.0,2.5,3.0,'''

#数据类型二：json数据（用户、商品、打分）
json_txt = '''{"Angelica": {"Blues Traveler": 3.5, "Broken Bells": 2.0,
                      "Norah Jones": 4.5, "Phoenix": 5.0,
                      "Slightly Stoopid": 1.5,
                      "The Strokes": 2.5, "Vampire Weekend": 2.0},

         "Bill":{"Blues Traveler": 2.0, "Broken Bells": 3.5,
                 "Deadmau5": 4.0, "Phoenix": 2.0,
                 "Slightly Stoopid": 3.5, "Vampire Weekend": 3.0},

         "Chan": {"Blues Traveler": 5.0, "Broken Bells": 1.0,
                  "Deadmau5": 1.0, "Norah Jones": 3.0, "Phoenix": 5,
                  "Slightly Stoopid": 1.0},

         "Dan": {"Blues Traveler": 3.0, "Broken Bells": 4.0,
                 "Deadmau5": 4.5, "Phoenix": 3.0,
                 "Slightly Stoopid": 4.5, "The Strokes": 4.0,
                 "Vampire Weekend": 2.0},

         "Hailey": {"Broken Bells": 4.0, "Deadmau5": 1.0,
                    "Norah Jones": 4.0, "The Strokes": 4.0,
                    "Vampire Weekend": 1.0},

         "Jordyn":  {"Broken Bells": 4.5, "Deadmau5": 4.0,
                     "Norah Jones": 5.0, "Phoenix": 5.0,
                     "Slightly Stoopid": 4.5, "The Strokes": 4.0,
                     "Vampire Weekend": 4.0},

         "Sam": {"Blues Traveler": 5.0, "Broken Bells": 2.0,
                 "Norah Jones": 3.0, "Phoenix": 5.0,
                 "Slightly Stoopid": 4.0, "The Strokes": 5.0},

         "Veronica": {"Blues Traveler": 3.0, "Norah Jones": 5.0,
                      "Phoenix": 4.0, "Slightly Stoopid": 2.5,
                      "The Strokes": 3.0}
}'''

df = None

#方式一：加载csv数据
def load_csv_txt():
    global df
    df = pd.read_csv(StringIO(csv_txt), header=0, index_col="user")

#方式二：加载json数据（把json读成矩阵）
def load_json_txt():
    global df
    df = pd.read_json(json_txt, orient='index')

#测试：读取数据
load_csv_txt()
#load_json_txt()

def build_xy(user_name1, user_name2):
    #df2 = df.ix[[user_name1, user_name2]].dropna(axis=1)
    #return df2.ix[user_name1], df2.ix[user_name2]

    bool_array = df.ix[user_name1].notnull() & df.ix[user_name2].notnull()
    return df.ix[user_name1, bool_array], df.ix[user_name2, bool_array]

#曼哈顿距离
def manhattan(user_name1, user_name2):
    x, y = build_xy(user_name1, user_name2)
    return sum(abs(x - y))

#欧几里德距离
def euclidean(user_name1, user_name2):
    x, y = build_xy(user_name1, user_name2)
    return sum((x - y)**2)**0.5

#闵可夫斯基距离
def minkowski(user_name1, user_name2, r):
    x, y = build_xy(user_name1, user_name2)
    return sum(abs(x - y)**r)**(1/r)

#皮尔逊相关系数
def pearson(user_name1, user_name2):
    x, y = build_xy(user_name1, user_name2)
    mean1, mean2 = x.mean(), y.mean()
    #分母
    denominator = (sum((x-mean1)**2)*sum((y-mean2)**2))**0.5
    return [sum((x-mean1)*(y-mean2))/denominator, 0][denominator == 0]

#余弦相似度（数据的稀疏性问题，在文本挖掘中应用得较多）
def cosine(user_name1, user_name2):
    x, y = build_xy(user_name1, user_name2)
    #分母
    denominator = (sum(x*x)*sum(y*y))**0.5
    return [sum(x*y)/denominator, 0][denominator == 0]

metric_funcs = {
    'manhattan': manhattan,
    'euclidean': euclidean,
    'minkowski': minkowski,
    'pearson': pearson,
    'cosine': cosine
}

#df.ix[["Angelica","Bill"]].dropna(axis=1)
print(manhattan("Angelica","Bill"))

#计算最近的邻居
def computeNearestNeighbor(user_name, metric='pearson', k=3, r=2):
    '''
    metric: 度量函数
    k:      返回k个邻居
    r:      闵可夫斯基距离专用

    返回：pd.Series，其中index是邻居名称，values是距离
    '''
    if metric in ['manhattan', 'euclidean']:
        return df.drop(user_name).index.to_series().apply(metric_funcs[metric], args=(user_name,)).nsmallest(k)
    elif metric in ['minkowski']:
        return df.drop(user_name).index.to_series().apply(metric_funcs[metric], args=(user_name, r,)).nsmallest(k)
    elif metric in ['pearson', 'cosine']:
        return df.drop(user_name).index.to_series().apply(metric_funcs[metric], args=(user_name,)).nlargest(k)

print(computeNearestNeighbor('Hailey', metric='pearson'))

#向给定用户推荐（返回：pd.Series）
def recommend(user_name):
    # 找到距离最近的用户名
    nearest_username = computeNearestNeighbor(user_name).index[0]

    # 找出邻居评价过、但自己未曾评价的乐队（或商品）
    # 结果：index是商品名称，values是评分
    return df.ix[nearest_username, df.ix[user_name].isnull() & df.ix[nearest_username].notnull()].sort_values()

#为Hailey做推荐
print(recommend('Hailey'))

#向给定用户推荐
def recommend2(user_name, metric='pearson', k=3, n=5, r=2):
    '''
    metric: 度量函数
    k:      根据k个最近邻居，协同推荐
    r:      闵可夫斯基距离专用
    n:      推荐的商品数目

    返回：pd.Series，其中index是商品名称，values是加权评分
    '''
    # 找到距离最近的k个邻居
    nearest_neighbors = computeNearestNeighbor(user_name, metric='pearson', k=k, r=r)

    # 计算权值
    if metric in ['manhattan', 'euclidean', 'minkowski']: # 距离越小，越类似
        nearest_neighbors = 1 / nearest_neighbors # 所以，取倒数（或者别的减函数，如：y=2**-x）
    elif metric in ['pearson', 'cosine']:                 # 距离越大，越类似
        pass

    nearest_neighbors = nearest_neighbors / nearest_neighbors.sum() #已经变为权值（pd.Series）

    # 逐个邻居找出其评价过、但自己未曾评价的乐队（或商品）的评分，并乘以权值
    neighbors_rate_with_weight = []
    for neighbor_name in nearest_neighbors.index:
        # 每个结果：pd.Series，其中index是商品名称，values是评分（已乘权值）
        neighbors_rate_with_weight.append(df.ix[neighbor_name, df.ix[user_name].isnull() & df.ix[neighbor_name].notnull()] * nearest_neighbors[neighbor_name])

    # 把邻居们的加权评分拼接成pd.DataFrame，按列累加，取最大的前n个商品的评分
    return pd.concat(neighbors_rate_with_weight, axis=1).sum(axis=1, skipna=True).nlargest(n)

#为Hailey做推荐
print(recommend2('Hailey', metric='manhattan', k=3, n=5))

#为Hailey做推荐
print(recommend2('Hailey', metric='euclidean', k=3, n=5, r=2))

#为Hailey做推荐
print(recommend2('Hailey', metric='pearson', k=1, n=5))