python数据分析Adult-Salary预测
具体文档戳下方网站
代码如下：
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import scipy.stats as stats # adult_data = pd.read_csv("adult.data")
adult_test = pd.read_csv("adult.test", header=None, na_values=' ?') # 不要表头
adult_train = pd.read_csv("adult.data", header=None, na_values=' ?')
Names = ['age', 'workclass', 'fnlwgt', 'education', 'education-num', 'marital-status', 'occupation',
'relationship', 'race', 'sex', 'capital-gain', 'capital-loss', 'hours-per-week',
'native-country', 'income']
adult_test.columns = Names
adult_train.columns = Names
adult_train.info()
# print(adult_train.head())
# -------------------------------------------数据查看-------------------------------------------
#
# # ------------------------------对分类型数据的处理------------------------------
#
# # --------------对workclass的处理----------------
# work_income_less = adult_train['workclass'][adult_train['income'] == ' <=50K'].value_counts()
# work_income_more = adult_train['workclass'][adult_train['income'] == ' >50K'].value_counts()
# work_income = pd.concat([work_income_less, work_income_more], axis=1, sort=True)
# work_income.columns = ['less than 50K', 'more than 50K']
# work_income.fillna(0, inplace=True)
# work_income.plot(kind='bar', figsize=(10, 6))
# plt.xticks(rotation=45, size=6)
# plt.title('Salery about Workclass', size=20)
# # plt.savefig("Salery_about_workclass.png")
# # 独立性检验
# from scipy.stats import chi2_contingency
# g, p, dof, expected = chi2_contingency(work_income)
# '''
# g: z值
# p: 概率值
# dof: 自由度
# expected: 期望值
# '''
# # 绘制自由度为dof的卡方分布表
# from scipy.stats import chi2
# import numpy as np
# x = np.linspace(chi2.ppf(0.01, dof), chi2.ppf(0.99, dof), 100)
# plt.figure()
# plt.plot(x,chi2.pdf(x,dof))
#
# # 以95%置信区间，查看小概率事件区间
# plt.axvline(chi2.ppf(0.975, dof), color='r')
# plt.axvline(chi2.ppf(0.025, dof), color='r')
# plt.title('chi2 distribution'+' whose dof is '+str(dof),size=20)
# plt.text(chi2.ppf(0.975, dof), 0.02, 'q=0.95,z='+str(round(chi2.ppf(0.975, dof), 2)), ha='right', va='top', color='g', alpha=0.8, size=15)
# plt.text(chi2.ppf(0.025, dof), 0.02, 'q=0.05,z='+str(round(chi2.ppf(0.025, dof), 2)), ha='left', va='top', color='g', alpha=0.8, size=15)
# plt.xlabel('g='+str(round(g,2)), color='orange',size=15)
# # plt.savefig("W_and_Income_chi2_distribution.png")
#
# # --------------对education的处理----------------
# education_income_less = adult_train['education'][adult_train['income'] == ' <=50K'].value_counts()
# education_income_more = adult_train['education'][adult_train['income'] == ' >50K'].value_counts()
# education_income = pd.concat([education_income_less, education_income_more], axis=1, sort=True)
# education_income.columns = ['less than 50K', 'more than 50K']
# education_income.fillna(0, inplace=True)
# education_income.plot(kind='bar', figsize=(10, 6))
# plt.xticks(rotation=45, size=6)
# plt.title('Salery about education', size=20)
# # plt.savefig("Salery_about_education.png")
#
# # 独立性检验
# from scipy.stats import chi2_contingency
# g, p, dof, expected = chi2_contingency(education_income)
# '''
# g: z值
# p: 概率值
# dof: 自由度
# expected: 期望值
# '''
# from scipy.stats import chi2
# import numpy as np
# x = np.linspace(chi2.ppf(0.01, dof), chi2.ppf(0.99, dof), 100)
# plt.figure()
# plt.plot(x,chi2.pdf(x,dof))
#
# # 以95%置信区间，查看小概率事件区间
# plt.axvline(chi2.ppf(0.975, dof), color='r')
# plt.axvline(chi2.ppf(0.025, dof), color='r')
# plt.title('chi2 distribution'+' whose dof is '+str(dof),size=20)
# plt.text(chi2.ppf(0.975, dof), 0.02, 'q=0.975,z='+str(round(chi2.ppf(0.975, dof), 2)), ha='right', va='top', color='g', alpha=0.8, size=15)
# plt.text(chi2.ppf(0.025, dof), 0.02, 'q=0.025,z='+str(round(chi2.ppf(0.025, dof), 2)), ha='left', va='top', color='g', alpha=0.8, size=15)
# plt.xlabel('g='+str(round(g,2))+', '+'p='+str(round(p,2)), color='orange',size=15)
# # plt.savefig("E_and_Income_chi2_distribution.png")
#
#
#
# # --------------对marital_status的处理----------------
# marital_status_income_less = adult_train['marital-status'][adult_train['income'] == ' <=50K'].value_counts()
# marital_status_income_more = adult_train['marital-status'][adult_train['income'] == ' >50K'].value_counts()
# marital_status_income = pd.concat([marital_status_income_less, marital_status_income_more], axis=1, sort=True)
# marital_status_income.columns = ['less than 50K', 'more than 50K']
# marital_status_income.fillna(0, inplace=True)
# marital_status_income.plot(kind='bar', figsize=(10, 6))
# plt.xticks(rotation=45, size=6)
# plt.title('Salery about Marital_status', size=20)
# # plt.savefig("Salery_about_Marital_status.png")
#
# # 独立性检验
# from scipy.stats import chi2_contingency
# g, p, dof, expected = chi2_contingency(marital_status_income)
# '''
# g: z值
# p: 概率值
# dof: 自由度
# expected: 期望值
# '''
# from scipy.stats import chi2
# import numpy as np
# x = np.linspace(chi2.ppf(0.01, dof), chi2.ppf(0.99, dof), 100)
# plt.figure()
# plt.plot(x,chi2.pdf(x,dof))
#
# # 以95%置信区间，查看小概率事件区间
# plt.axvline(chi2.ppf(0.975, dof), color='r')
# plt.axvline(chi2.ppf(0.025, dof), color='r')
# plt.title('chi2 distribution'+' whose dof is '+str(dof),size=20)
# plt.text(chi2.ppf(0.975, dof), 0.02, 'q=0.975,z='+str(round(chi2.ppf(0.975, dof), 2)), ha='right', va='top', color='g', alpha=0.8, size=15)
# plt.text(chi2.ppf(0.025, dof), 0.02, 'q=0.025,z='+str(round(chi2.ppf(0.025, dof), 2)), ha='left', va='top', color='g', alpha=0.8, size=15)
# plt.xlabel('g='+str(round(g,2))+', '+'p='+str(round(p,2)), color='orange',size=15)
# plt.savefig("M_and_Income_chi2_distribution.png")
#
# # --------------对occupation的处理----------------
# occupation_income_less = adult_train['occupation'][adult_train['income'] == ' <=50K'].value_counts()
# occupation_income_more = adult_train['occupation'][adult_train['income'] == ' >50K'].value_counts()
# occupation_income = pd.concat([occupation_income_less, occupation_income_more], axis=1, sort=True)
# occupation_income.columns = ['less than 50K', 'more than 50K']
# occupation_income.fillna(0, inplace=True)
# occupation_income.plot(kind='bar', figsize=(10, 6))
# plt.xticks(rotation=45, size=6)
# plt.title('Salery about Occupation', size=20)
# plt.savefig("Salery_about_Occupation.png")
#
# # 独立性检验
# from scipy.stats import chi2_contingency
# g, p, dof, expected = chi2_contingency(occupation_income)
# '''
# g: z值
# p: 概率值
# dof: 自由度
# expected: 期望值
# '''
# from scipy.stats import chi2
# import numpy as np
# x = np.linspace(chi2.ppf(0.01, dof), chi2.ppf(0.99, dof), 100)
# plt.figure()
# plt.plot(x,chi2.pdf(x,dof))
#
# # 以95%置信区间，查看小概率事件区间
# plt.axvline(chi2.ppf(0.975, dof), color='r')
# plt.axvline(chi2.ppf(0.025, dof), color='r')
# plt.title('chi2 distribution'+' whose dof is '+str(dof),size=20)
# plt.text(chi2.ppf(0.975, dof), 0.02, 'q=0.975,z='+str(round(chi2.ppf(0.975, dof), 2)), ha='right', va='top', color='g', alpha=0.8, size=15)
# plt.text(chi2.ppf(0.025, dof), 0.02, 'q=0.025,z='+str(round(chi2.ppf(0.025, dof), 2)), ha='left', va='top', color='g', alpha=0.8, size=15)
# plt.xlabel('g='+str(round(g,2))+', '+'p='+str(round(p,2)), color='orange',size=15)
# plt.savefig("O_and_Income_chi2_distribution.png")
#
# # --------------对relationship的处理----------------
# relationship_income_less = adult_train['relationship'][adult_train['income'] == ' <=50K'].value_counts()
# relationship_income_more = adult_train['relationship'][adult_train['income'] == ' >50K'].value_counts()
# relationship_income = pd.concat([relationship_income_less, relationship_income_more], axis=1, sort=True)
# relationship_income.columns = ['less than 50K', 'more than 50K']
# relationship_income.fillna(0, inplace=True)
# relationship_income.plot(kind='bar', figsize=(10, 6)) # 设置图片大小
# plt.xticks(rotation=45, size=6)
# plt.title('Salery about Relationship', size=20)
# plt.savefig("Salery_about_Relationship.png")
#
# # 独立性检验
# from scipy.stats import chi2_contingency
# g, p, dof, expected = chi2_contingency(relationship_income)
# '''
# g: z值
# p: 概率值
# dof: 自由度
# expected: 期望值
# '''
# from scipy.stats import chi2
# import numpy as np
# x = np.linspace(chi2.ppf(0.01, dof), chi2.ppf(0.99, dof), 100)
# plt.figure()
# plt.plot(x,chi2.pdf(x,dof))
#
# # 以95%置信区间，查看小概率事件区间
# plt.axvline(chi2.ppf(0.975, dof), color='r')
# plt.axvline(chi2.ppf(0.025, dof), color='r')
# plt.title('chi2 distribution'+' whose dof is '+str(dof),size=20)
# plt.text(chi2.ppf(0.975, dof), 0.02, 'q=0.975,z='+str(round(chi2.ppf(0.975, dof), 2)), ha='right', va='top', color='g', alpha=0.8, size=15)
# plt.text(chi2.ppf(0.025, dof), 0.02, 'q=0.025,z='+str(round(chi2.ppf(0.025, dof), 2)), ha='left', va='top', color='g', alpha=0.8, size=15)
# plt.xlabel('g='+str(round(g,2))+', '+'p='+str(round(p,2)), color='orange',size=15)
# plt.savefig("R_and_Income_chi2_distribution.png")
#
# # --------------对race的处理----------------
# race_income_less = adult_train['race'][adult_train['income'] == ' <=50K'].value_counts()
# race_income_more = adult_train['race'][adult_train['income'] == ' >50K'].value_counts()
# race_income = pd.concat([race_income_less, race_income_more], axis=1, sort=True)
# race_income.columns = ['less than 50K', 'more than 50K']
# race_income.fillna(0, inplace=True)
# race_income.plot(kind='bar', figsize=(10, 6))
# plt.xticks(rotation=45, size=6)
# plt.title('Salery about race', size=20)
# # plt.savefig("Salery_about_Race.png")
#
# # 独立性检验
# from scipy.stats import chi2_contingency
# g, p, dof, expected = chi2_contingency(race_income)
# '''
# g: z值
# p: 概率值
# dof: 自由度
# expected: 期望值
# '''
# from scipy.stats import chi2
# import numpy as np
# x = np.linspace(chi2.ppf(0.01, dof), chi2.ppf(0.99, dof), 100)
# plt.figure()
# plt.plot(x,chi2.pdf(x,dof))
#
# # 以95%置信区间，查看小概率事件区间
# plt.axvline(chi2.ppf(0.975, dof), color='r')
# plt.axvline(chi2.ppf(0.025, dof), color='r')
# plt.title('chi2 distribution'+' whose dof is '+str(dof),size=20)
# plt.text(chi2.ppf(0.975, dof), 0.02, 'q=0.975,z='+str(round(chi2.ppf(0.975, dof), 2)), ha='right', va='top', color='g', alpha=0.8, size=15)
# plt.text(chi2.ppf(0.025, dof), 0.02, 'q=0.025,z='+str(round(chi2.ppf(0.025, dof), 2)), ha='left', va='top', color='g', alpha=0.8, size=15)
# plt.xlabel('g='+str(round(g,2))+', '+'p='+str(round(p,2)), color='orange',size=15)
# plt.savefig("Ra_and_Income_chi2_distribution.png")
#
# # --------------对sex的处理----------------
# sex_income_less = adult_train['sex'][adult_train['income'] == ' <=50K'].value_counts()
# sex_income_more = adult_train['sex'][adult_train['income'] == ' >50K'].value_counts()
# sex_income = pd.concat([sex_income_less, sex_income_more], axis=1, sort=True)
# sex_income.columns = ['less than 50K', 'more than 50K']
# sex_income.fillna(0, inplace=True)
# sex_income.plot(kind='bar', figsize=(10, 6))
# plt.xticks(rotation=45, size=6)
# plt.title('Salery about sex', size=20)
# plt.savefig("Salery_about_Sex.png")
#
# # 独立性检验
# from scipy.stats import chi2_contingency
# g, p, dof, expected = chi2_contingency(sex_income)
# '''
# g: z值
# p: 概率值
# dof: 自由度
# expected: 期望值
# '''
# from scipy.stats import chi2
# import numpy as np
# x = np.linspace(chi2.ppf(0.01, dof), chi2.ppf(0.99, dof), 100)
# plt.figure()
# plt.plot(x,chi2.pdf(x,dof))
#
# # 以95%置信区间，查看小概率事件区间
# plt.axvline(chi2.ppf(0.975, dof), color='r')
# plt.axvline(chi2.ppf(0.025, dof), color='r')
# plt.title('chi2 distribution'+' whose dof is '+str(dof),size=20)
# plt.text(chi2.ppf(0.975, dof), 0.02, 'q=0.975,z='+str(round(chi2.ppf(0.975, dof), 2)), ha='right', va='top', color='g', alpha=0.8, size=15)
# plt.text(chi2.ppf(0.025, dof), 0.02, 'q=0.025,z='+str(round(chi2.ppf(0.025, dof), 2)), ha='left', va='top', color='g', alpha=0.8, size=15)
# plt.xlabel('g='+str(round(g,2))+', '+'p='+str(round(p,2)), color='orange',size=15)
# plt.savefig("S_and_Income_chi2_distribution.png")
#
#
# # 对native_country进行查看数据，看看国家与工资的关系
# native_country_income_less = adult_train['native-country'][adult_train['income'] == ' <=50K'].value_counts()
# native_country_income_more = adult_train['native-country'][adult_train['income'] == ' >50K'].value_counts()
# native_country_income = pd.concat([native_country_income_less, native_country_income_more], axis=1, sort=True)
# native_country_income.columns = ['less than 50K', 'more than 50K']
# native_country_income.fillna(0,inplace=True)
# print(native_country_income)
# # 独立性检验
# from scipy.stats import chi2_contingency
# g, p, dof, expected = chi2_contingency(native_country_income)
# '''
# g: z值
# p: 概率值
# dof: 自由度
# expected: 期望值
# '''
# from scipy.stats import chi2
# import numpy as np
# x = np.linspace(chi2.ppf(0.01, dof), chi2.ppf(0.99, dof), 100)
# plt.figure()
# plt.plot(x,chi2.pdf(x,dof))
#
# # 以95%置信区间，查看小概率事件区间
# plt.axvline(chi2.ppf(0.975, dof), color='r')
# plt.axvline(chi2.ppf(0.025, dof), color='r')
# plt.title('chi2 distribution'+' whose dof is '+str(dof),size=20)
# plt.text(chi2.ppf(0.975, dof), 0.02, 'q=0.975,z='+str(round(chi2.ppf(0.975, dof), 2)), ha='right', va='top', color='g', alpha=0.8, size=15)
# plt.text(chi2.ppf(0.025, dof), 0.02, 'q=0.025,z='+str(round(chi2.ppf(0.025, dof), 2)), ha='left', va='top', color='g', alpha=0.8, size=15)
# plt.xlabel('g='+str(round(g,2))+', '+'p='+str(round(p,2)), color='orange',size=15)
# plt.savefig("NC_and_Income_chi2_distribution.png")
#
# # --------------对Age的处理，数据进行离散化----------------
adult_train['age_set'] = pd.cut(adult_train['age'], bins=4, labels=['Teenager', 'young man', 'elder', 'old man'])
# # 离散数据绘图
# age_set_income_less = adult_train['age_set'][adult_train['income'] == ' <=50K'].value_counts()
# age_set_income_more = adult_train['age_set'][adult_train['income'] == ' >50K'].value_counts()
# age_set_income = pd.concat([age_set_income_less, age_set_income_more], axis=1, sort=True)
# age_set_income.columns = ['less than 50K', 'more than 50K']
# age_set_income.fillna(0, inplace=True)
# age_set_income.plot(kind='bar', figsize=(10, 6))
# plt.xticks(rotation=45)
# plt.title('Salery about Age Set', size=20)
# plt.savefig("Salery_about_Age_Set.png")
#
# # 绘制线性图
# plt.figure()
# adult_train['age'][adult_train['income'] == ' <=50K'].plot(kind='kde', figsize=(10, 6)) # 分布图
# adult_train['age'][adult_train['income'] == ' >50K'].plot(kind='kde')
# adult_train['age'].plot(kind='kde')
# plt.xlabel("Age")
# plt.ylabel("Density")
# plt.title("Age about higher_salery")
# plt.legend(('<=50K', '>50K','all age'), loc='best')
# plt.savefig("Age_kernel_density_estimation.png")
#
# # 绘制正态拟合曲线
# M_S = stats.norm.fit(adult_train['age']) # 正态拟合的平均值与标准差
# plt.figure()
# adult_train['age'].plot(kind='kde') # 原本的概率密度分布图
#
# normalDistribution = stats.norm(M_S[0], M_S[1]) # 绘制拟合的正态分布图
# x = np.linspace(normalDistribution.ppf(0.01), normalDistribution.ppf(0.99), 100)
# plt.plot(x, normalDistribution.pdf(x), c='orange')
# plt.xlabel('Age about Adult '+'mean='+str(round(M_S[0], 5))+',std='+str(round(M_S[1], 5)))
# plt.title('Age_NormalDistibution_Origin', size=20)
# plt.legend(['Origin', 'NormDistribution'])
# plt.savefig("Age_NormalDistibution_Origin.png")
#
#
# # --------------对education_num的处理，数据进行离散化----------------
# enum_income_less = adult_train['education-num'][adult_train['income'] == ' <=50K'].value_counts()
# enum_income_more = adult_train['education-num'][adult_train['income'] == ' >50K'].value_counts()
# enum_income = pd.concat([enum_income_less, enum_income_more], axis=1, sort=True)
# enum_income.columns = ['less than 50K', 'more than 50K']
# enum_income.fillna(0, inplace=True)
# enum_income.plot(kind='bar', figsize=(10, 6))
# plt.xticks(rotation=0)
# plt.title('Salery about e_num', size=20)
# # plt.savefig("Salery_about_e_num.png")
#
# # 独立性检验
# from scipy.stats import chi2_contingency
# g, p, dof, expected = chi2_contingency(enum_income)
# '''
# g: z值
# p: 概率值
# dof: 自由度
# expected: 期望值
# '''
# from scipy.stats import chi2
# import numpy as np
# x = np.linspace(chi2.ppf(0.01, dof), chi2.ppf(0.99, dof), 100)
# plt.figure()
# plt.plot(x,chi2.pdf(x,dof))
#
# # 以95%置信区间，查看小概率事件区间
# plt.axvline(chi2.ppf(0.975, dof), color='r')
# plt.axvline(chi2.ppf(0.025, dof), color='r')
# plt.title('chi2 distribution'+' whose dof is '+str(dof),size=20)
# plt.text(chi2.ppf(0.975, dof), 0.02, 'q=0.975,z='+str(round(chi2.ppf(0.975, dof), 2)), ha='right', va='top', color='g', alpha=0.8, size=15)
# plt.text(chi2.ppf(0.025, dof), 0.02, 'q=0.025,z='+str(round(chi2.ppf(0.025, dof), 2)), ha='left', va='top', color='g', alpha=0.8, size=15)
# plt.xlabel('g='+str(round(g,2))+', '+'p='+str(round(p,2)), color='orange',size=15)
# plt.savefig("EN_and_Income_chi2_distribution.png")
#
# # --------------对hours-per-week的处理，数据进行离散化----------------
# adult_train['hours_set'] = pd.cut(adult_train['hours-per-week'], bins=4, labels=['too free', 'few', 'common', 'too busy'])
# # 离散数据绘图
# hours_income_less = adult_train['hours_set'][adult_train['income'] == ' <=50K'].value_counts()
# hours_income_more = adult_train['hours_set'][adult_train['income'] == ' >50K'].value_counts()
# hours_income = pd.concat([hours_income_less, hours_income_more], axis=1, sort=True)
# hours_income.columns = ['less than 50K', 'more than 50K']
# print(hours_income)
# hours_income.fillna(0, inplace=True)
# hours_income.plot(kind='bar', figsize=(10, 6))
# plt.xticks(rotation=0)
# plt.title('Salery about hours set', size=20)
# # plt.savefig("Salery_about_hours_set.png")
#
# # 绘制线性图
# plt.figure()
# adult_train['hours-per-week'][adult_train['income'] == ' <=50K'].plot(kind='kde', figsize=(10, 6)) # 分布图
# adult_train['hours-per-week'][adult_train['income'] == ' >50K'].plot(kind='kde')
# adult_train['hours-per-week'].plot(kind='kde')
# plt.xlabel("hours-per-week")
# plt.ylabel("Density")
# plt.title("hours-per-week about higher_salery")
# plt.legend(('<=50K', '>50K','all '), loc='best')
# plt.savefig("hours_per_week_kernel_density_estimation.png")
#
#
# # --------------对capital-gain的处理----------------
# # 绘制线性图
# plt.figure()
# adult_train['capital-gain'][adult_train['income'] == ' <=50K'].plot(kind='kde', figsize=(10, 6)) # 分布图
# adult_train['capital-gain'][adult_train['income'] == ' >50K'].plot(kind='kde')
# adult_train['capital-gain'].plot(kind='kde')
# plt.xlabel("capital-gain")
# plt.ylabel("Density")
# plt.title("capital-gain about higher_salery")
# plt.legend(('<=50K', '>50K','all '), loc='best')
# plt.savefig("capital_gain_kernel_density_estimation.png")
#
# # --------------对capital-loss的处理----------------
# # 绘制线性图
# plt.figure()
# adult_train['capital-loss'][adult_train['income'] == ' <=50K'].plot(kind='kde', figsize=(10, 6)) # 分布图
# adult_train['capital-loss'][adult_train['income'] == ' >50K'].plot(kind='kde')
# adult_train['capital-loss'].plot(kind='kde')
# plt.xlabel("capital-loss")
# plt.ylabel("Density")
# plt.title("capital-loss about higher_salery")
# plt.legend(('<=50K', '>50K','all '), loc='best')
# plt.savefig("capital_loss_kernel_density_estimation.png")
#
# # 查看capital-gain与capital-loss数据
# # print(adult_train[['capital-gain','capital-loss']].head(10))
# gain_loss = adult_train.groupby(['capital-gain','capital-loss'])
# print(gain_loss.size().head())
#
# # --------------对fnlwgt的处理----------------
# # 绘制线性图
# plt.figure()
# adult_train['fnlwgt'][adult_train['income'] == ' <=50K'].plot(kind='kde', figsize=(10, 6)) # 分布图
# adult_train['fnlwgt'][adult_train['income'] == ' >50K'].plot(kind='kde')
# adult_train['fnlwgt'].plot(kind='kde')
# plt.xlabel("fnlwgt")
# plt.ylabel("Density")
# plt.title("fnlwgt about higher_salery")
# plt.legend(('<=50K', '>50K','all '), loc='best')
# plt.savefig("fnlwgt_kernel_density_estimation.png") # -------------------------------------------数据处理-------------------------------------------
# 删除不符合条件的列
adult_train_good = adult_train.drop(['fnlwgt','hours-per-week','capital-gain','capital-loss'],axis=1)
print(adult_train_good.columns) # 查看每一列的缺失率
print(adult_train_good.isnull().sum()/adult_train_good.shape[0])
#
# # -------------异常值处理-------------
# # 3σ原则检测异常值
# # 定义3σ法则识别异常值函数
# def outRange(Ser1):
# '''
# Ser1：表示传入DataFrame的某一列。
# '''
# boolInd = (Ser1.mean()-3*Ser1.std() > Ser1) | (Ser1.mean()+3*Ser1.std() < Ser1)
# index = np.arange(Ser1.shape[0])[boolInd] # range to array
# outrange = Ser1.iloc[index]
# return outrange
#
# print(outRange(adult_train_good['age'])) # # 提取出训练集
X_train = adult_train_good.drop(['income','age'], axis=1)
Y_train = adult_train_good['income'] X_train = pd.get_dummies(X_train, prefix_sep=':', columns=['workclass', 'education', 'education-num',
'marital-status', 'occupation','race',
'relationship', 'sex', 'native-country'
,'age_set'], dummy_na=True) print(X_train.head())
Y_train = pd.get_dummies(Y_train, drop_first=True).iloc[:,0] # 取一列
print(Y_train)
# 哑变量处理
# 把空白值也当作变量处理 # 数据集划分
from sklearn.model_selection import train_test_split X = X_train
y = Y_train X_train, X_test, y_train, y_test = train_test_split(X_train,Y_train, test_size=0.2, random_state=12345) # 标准化
from sklearn.preprocessing import StandardScaler
Standard = StandardScaler().fit(X_train) # 训练产生标准化的规则,因为数据集分为训练与测试，测试相当于后来的。 Xtrain = Standard.transform(X_train) # 将规则应用于训练集
Xtest = Standard.transform(X_test) # 将规则应用于测试集 # 进行分类算法
from sklearn.neighbors import KNeighborsClassifier
clf = KNeighborsClassifier().fit(X_train, y_train)
y_pred = clf.predict(Xtest) # 判定分类算法
from sklearn.metrics import classification_report, auc
print(classification_report(y_test, y_pred)) # 绘制roc曲线
from sklearn.metrics import roc_curve
import matplotlib.pyplot as plt
plt.rcParams['font.sans-serif'] = 'SimHei' # 改字体
# 求出ROC曲线的x轴和Y轴
fpr, tpr, thresholds = roc_curve(y_test, y_pred)
print(auc(fpr, tpr))
plt.figure(figsize=(10, 6))
plt.xlim(0, 1) # 设定x轴的范围
plt.ylim(0.0, 1.1) # 设定y轴的范围
plt.xlabel('假正率')
plt.ylabel('真正率')
plt.plot(fpr, tpr, linewidth=2, linestyle="-", color='red')
plt.title('Line Roc of X_train by KNeighborsClassifier()', size=20)
plt.show() print(type(Standard.transform(X)))
print(type(y))
print(Standard.transform(X).shape)
print(y.shape) #
# #交叉验证
# from sklearn.cross_validation import cross_val_score
# k_score=[]
X = Standard.transform(X)
# for i in range(1,3):
# knn = KNeighborsClassifier(n_neighbors=i)
# scores = cross_val_score(knn, X, y, scoring='accuracy',cv=5)
# k_score.append(scores.mean())
# print(k_score) # 查看学习曲线
from sklearn.learning_curve import learning_curve
train_sizes, train_loss, test_loss = learning_curve(clf,X,y,cv=10,scoring='mean_squared_error',train_sizes=[0.1,0.25,0.5,0.75,1])
train_loss_mean = -np.mean(train_loss,axis=1)
test_loss_mean = -np.mean(test_loss,axis=1)
plt.plot(train_sizes,train_loss_mean,'o-',color='r', label='Training')
plt.plot(train_sizes,test_loss_mean,'o-',color='g', label='Cross_validation')
plt.legend(loc='best')
plt.show()
python数据分析Adult-Salary预测的更多相关文章
- 【读书笔记与思考】《python数据分析与挖掘实战》-张良均
[读书笔记与思考]<python数据分析与挖掘实战>-张良均 最近看一些机器学习相关书籍,主要是为了拓宽视野.在阅读这本书前最吸引我的地方是实战篇,我通读全书后给我印象最深的还是实战篇.基 ...
- 《Python数据分析与挖掘实战》读书笔记
大致扫了一遍,具体的代码基本都没看了,毕竟我还不懂python,并且在手机端的排版,这些代码没法看. 有收获,至少了解到以下几点: 一. Python的语法挺有意思的 有一些类似于JavaSc ...
- 小象学院Python数据分析第二期【升级版】
点击了解更多Python课程>>> 小象学院Python数据分析第二期[升级版] 主讲老师: 梁斌 资深算法工程师 查尔斯特大学(Charles Sturt University)计 ...
- Python数据分析【炼数成金15周完整课程】
点击了解更多Python课程>>> Python数据分析[炼数成金15周完整课程] 课程简介: Python是一种面向对象.直译式计算机程序设计语言.也是一种功能强大而完善的通用型语 ...
- Python数据分析简介
1,Python作为一门编程语言开发效率快,运行效率被人诟病,但是Python核心部分使用c/c++等更高效的语言来编写的还有强大的numpy, padnas, matplotlib,scipy库等应 ...
- Python数据分析常用的库总结
Python之所以能够成为数据分析与挖掘领域的最佳语言,是有其独特的优势的.因为他有很多这个领域相关的库可以用,而且很好用,比如Numpy.SciPy.Matploglib.Pandas.Scikit ...
- 快速入门 Python 数据分析实用指南
Python 现如今已成为数据分析和数据科学使用上的标准语言和标准平台之一.那么作为一个新手小白,该如何快速入门 Python 数据分析呢? 下面根据数据分析的一般工作流程,梳理了相关知识技能以及学习 ...
- python数据分析与应用
python数据分析与应用笔记 使用sklearn构建模型 1.使用sklearn转换器处理数据 import numpy as np from sklearn.datasets import loa ...
- python数据分析中常用的库
Python是数据处理常用工具,可以处理数量级从几K至几T不等的数据,具有较高的开发效率和可维护性,还具有较强的通用性和跨平台性,这里就为大家分享几个不错的数据分析工具,需要的朋友可以参考下 Pyth ...
- 小白学 Python 数据分析（1）：数据分析基础
各位同学好,小编接下来为大家分享一些有关 Python 数据分析方面的内容,希望大家能够喜欢. 人工植入广告: PS:小编最近两天偷了点懒,好久没有发原创了,最近是在 CSDN 开通了一个付费专栏,用 ...
随机推荐
- JS中 map, filter, some, every, forEach, for in, for of 用法总结
本文转载自:http://blog.csdn.net/gis_swb/article/details/52297343 1.map 有返回值,返回一个新的数组,每个元素为调用func的结果. let ...
- HTML5API之获取地理位置详解
在使用地理位置API之前先来了解一下什么是经度和纬度以及地理位置获取的原理 首先经度指的是南北极的连接线,纬度指的是东西的连接线 地理位置的获取原理是通过IP地址(基于ISP记录,能够知道这个IP地址 ...
- php面试流程
- apache配置文件：http.conf配置详解
Apache的配置文件http.conf参数含义详解 Apache的配置由httpd.conf文件配置,因此下面的配置指令都是在httpd.conf文件中修改.主站点的配置(基本配置) (1) 基本配 ...
- 2.2.2 RelativeLayout(相对布局)
本节引言 在上一节中我们对LinearLayout进行了详细的解析,LinearLayout也是我们 用的比较多的一个布局,我们更多的时候更钟情于他的weight(权重)属性,等比例划分,对屏幕适配还 ...
- Linux下RPM包的安装
Linux下RPM包安装 二进制包(RPM包.系统默认包) RPM安装 rpm -ivh 包全名(查询依赖网址:http://www.rpmfind.net) -i(install):安装 -v(ve ...
- 解决：linux 固定ip 导致ping 外网unknown host
首先说下问题产生场景:最近搞jenkins搭建持续集成,搞完后发现服务器ip(ifconfig 红色)老是变化,一怒之下果断修改ip [root@bogon etc]# ifconfigeth0 Li ...
- laytpl js模板引擎
laytpl js模板引擎.laytpl是一款非常轻量的JavaScript模板引擎.地址:http://www.layui.com/laytpl/ 用法与handlebar.js类似,但是比较轻量级 ...
- FPGA按一下按键,对应端口输出单个脉冲
对于FPGA的verilog语言,,,规定一个变量不能在多个always中被赋值.但是可以在多个alway块中做判断--结合状态机思想 module state(key,led,clk); input ...
- qmlscene: could not find a Qt installation of ''
qt └── helloqt.qml 在qt目录下运行helloqt.qml文件时出现这个提示: $ qmlscene helloqt.qml qmlscene: could not find a Q ...