Pandas分组

GroupBy技术

分组运算的过程可以用下面的流程图表示出来

import pandas as pd
from pandas import Series
import numpy as np

df = pd.DataFrame({
    'key1':['a','a','b','b','a'],
    'key2':['one','two','one','two','one'],
    'data1':np.random.randn(5),
    'data2':np.random.randn(5)
})
df

   key1	key2	data1	    data2
0	a	one	  -0.687740	   1.536286
1	a	two	   0.177156	  -1.132658
2	b	one	   2.024931	   0.755856
3	b	two	  -0.304281	  -0.629296
4	a	one	   0.202405	   0.911700

# 访问key1调用groupby
grouped = df['data1'].groupby(df['key1'])
# grouped是一个GroupBy对象，它实际上还没有进行任何计算。但是它已经包含了执行分组运算的一切信息
grouped

<pandas.core.groupby.groupby.SeriesGroupBy object at 0x0000020473C51978>

# 例如，算平均值
grouped.mean()

      key1
a   -0.102726
b    0.860325
Name: data1, dtype: float64

means = df['data1'].groupby([df['key1'],df['key2']]).mean()
means

      key1      key2
a     one    -0.242668
      two     0.177156
b     one     2.024931
      two    -0.304281
Name: data1, dtype: float64

# 默认操作的是最内层的
means.unstack()

key2	one	      two
key1
a	 -0.242668	 0.177156
b	  2.024931	-0.304281

# 自定义分组键
states = np.array(['Ohio','California','California','Ohio','Ohio'])
years = np.array([2005,2005,2006,2005,2006])
df['data1'].groupby([states,years]).mean()

California  2005    0.177156
            2006    2.024931
Ohio        2005   -0.496010
            2006    0.202405
Name: data1, dtype: float64

# 将列名用作分组键

df.groupby('key1').mean()

         data1	   data2
key1
a	  -0.102726	  0.438443
b	   0.860325	  0.063280

df.groupby(['key1','key2']).mean()

                  data1	          data2
key1	key2
a	    one	    -0.242668	     1.223993
        two	     0.177156	    -1.132658
b	    one	     2.024931	     0.755856
        two	    -0.304281	    -0.629296

# groupby.size方法，返回一个含有分组大小的Series,统计分组的数量
df.groupby(['key1','key2']).size()

      key1  key2
a     one     2
      two     1
b     one     1
      two     1
dtype: int64

df.groupby('key1').size()
    key1
a    3
b    2
dtype: int64

对分组进行迭代

for name,group in df.groupby('key1'):
    print(name)
    print(group)

a
  key1 key2     data1     data2
0    a  one -0.687740  1.536286
1    a  two  0.177156 -1.132658
4    a  one  0.202405  0.911700
b
  key1 key2     data1     data2
2    b  one  2.024931  0.755856
3    b  two -0.304281 -0.629296

# 对于多重键的情况
for k1,group in df.groupby(['key1','key2']):
    print(k1)
    print(group)

('a', 'one')
  key1 key2     data1     data2
0    a  one -0.687740  1.536286
4    a  one  0.202405  0.911700
('a', 'two')
  key1 key2     data1     data2
1    a  two  0.177156 -1.132658
('b', 'one')
  key1 key2     data1     data2
2    b  one  2.024931  0.755856
('b', 'two')
  key1 key2     data1     data2
3    b  two -0.304281 -0.629296

pieces = dict(list(df.groupby('key1')))
pieces['a']

   key1	key2	data1	  data2
0	a	one	 -0.687740	 1.536286
1	a	two	  0.177156	-1.132658
4	a	one	  0.202405	 0.911700

# 按数据类型对列尽心分组
df.dtypes

key1      object
key2      object
data1    float64
data2    float64
dtype: object

grouped1 = df.groupby(df.dtypes, axis=1)
dict(list(grouped1))

{dtype('float64'):       data1     data2
 0 -0.687740  1.536286
 1  0.177156 -1.132658
 2  2.024931  0.755856
 3 -0.304281 -0.629296
 4  0.202405  0.911700, dtype('O'):   key1 key2
 0    a  one
 1    a  two
 2    b  one
 3    b  two
 4    a  one}

选取一个或一个组列

# 分组通常和运算是一起的，只分组，只能是出来一个分组对象
df['data1'].groupby(df['key1'])
<pandas.core.groupby.groupby.SeriesGroupBy object at 0x000002047407C0F0>

df.groupby(['key1','key2'])[['data2']].mean()

              data2
key1 key2
a	 one	 1.223993
     two	-1.132658
b	 one	 0.755856
     two	-0.629296

通过字典或Series进行分组

people = pd.DataFrame(np.random.randn(5,5),
                      columns = ['a','b','c','d','e'],
                      index = ['Joe','Steve','Wes','Jim','Travis']
                     )
people

           a	        b	        c	        d	        e
Joe	   -0.721168	-0.332402	-0.512183	 0.765475	-0.119330
Steve	0.166230	 2.095133	-0.298735	-2.167574	-0.273929
Wes	   -1.428962	-2.109915	-0.686155	-1.337891	 0.020153
Jim	    1.041952	 0.130567	 0.776067	 0.508666	 0.100419
Travis -0.215110	 0.611383	-0.011443	-0.552580	-0.274047

people.ix['Steve':'Wes',['b','c']] = np.nan
people

              a	       b	       c	       d	        e
Joe	    -0.721168	-0.332402	-0.512183	0.765475	-0.119330
Steve	0.166230	NaN	          NaN	    -2.167574	-0.273929
Wes	    -1.428962	NaN	          NaN	    -1.337891	 0.020153
Jim	    1.041952	0.130567	 0.776067	 0.508666	 0.100419
Travis	-0.215110	0.611383	-0.011443	-0.552580	-0.274047

# 根据分组计算列的总计
mapping = {'a':'red','b':'red','c':'blue',
          'd':'blue','e':'red','f':'orange'}
by_column = people.groupby(mapping,axis=1).sum()
by_column

         blue	       red
Joe	     0.253291	-1.172899
Steve	-2.167574	-0.107699
Wes	    -1.337891	-1.408809
Jim	     1.284733	 1.272937
Travis	-0.564023	 0.122226

# 检查映射的对应关系
map_series = Series(mapping)
map_series

a       red
b       red
c      blue
d      blue
e       red
f    orange
dtype: object

people.groupby(map_series,axis=1).count()

      blue	red
Joe	    2	3
Steve	1	2
Wes	    1	2
Jim	    2	3
Travis	2	3

通过函数进行分组

# 仅仅传入len，就可以实现按名字长度分组
people.groupby(len).sum()

       a	       b	      c	           d	      e
3	-1.108178	-0.201835	0.263884	-0.063751	0.001242
5	0.166230	0.000000	0.000000	-2.167574	-0.273929
6	-0.215110	0.611383	-0.011443	-0.552580	-0.274047

key_list = ['one','one','one','two','two']
people.groupby([len,key_list]).min()

           a	      b	          c	           d	       e
3	one	-1.428962	-0.332402	-0.512183	-1.337891	-0.119330
    two	 1.041952	0.130567	 0.776067	 0.508666	 0.100419
5	one	 0.166230	NaN	         NaN	    -2.167574	-0.273929
6	two	-0.215110	0.611383	-0.011443	-0.552580	-0.274047

根据索引级别分组

columns = pd.MultiIndex.from_arrays([['US','US','US','JP','JP'],[1,3,5,1,3]],names=['cty','tenor'])
hier_df = pd.DataFrame(np.random.randn(4,5),columns=columns)
hier_df

cty	     US	                                    JP
tenor	1	        3	         5	         1	        3
0	-0.637036	 0.230604	 0.425606	 0.579673	-0.408535
1	 0.987018	-1.313152	-0.322781	-1.393266	-0.393867
2	-2.343672	 0.385606	 0.992302	-0.791528	-0.774870
3	-0.809241	-0.654219	-0.860250	 0.222547	 0.049766

hier_df.groupby(level='cty',axis=1).count()

cty	JP	US
0	2	3
1	2	3
2	2	3
3	2	3