collections——高性能容器数据类型

　　由于最近对机器学习算法感兴趣，一直知道python有一个包collections封装了一些比dict,list之类高级点的类，所以抽空研究下，为接下来的工作准备。

　　主要参考是https://docs.python.org/2/library/collections.html#defaultdict-objects官方的文档，根据不高的英文水平翻译和理解总结出来的，如果有错误欢迎提醒，万一，您有兴趣转载的也请注明是@瓜棚

collections封装的结构主要有5个:

###########################################################################################################################################
Counter            *            字典(dict)的子类用来统计可哈希对象的值                                    *      new in py 2.7   *
deque              *            双端队列，两端都可以作为队列的结束，方便前端插入需求                         *      new in py 2.4   *
namedtuple         *            tuple的子类，可以用于按名字标识元组                                       *      new in py 2.6   *
OrderedDict        *            dict的子类，创建一个可哈希的有序字典                                      *      new in py 2.7   *
defaultdict        *            dict的子类，当某个key不存在时，一共一个默认值，而不是报KeyError              *      new in py 2.5   *

---

Counter类

example:

from collections import Counter
cnt = Counter()
for word in ['','','','','','']:
    cnt[word] +=1
cnt
#Counter({'1':3,'2':2,'3':1})
#####################################
#统计一段话里出现次数最多的10个词和各自的次数
text = ['a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'can','for', 'from', 'have', 'if', 'in', 'is', 'it', 'may','not', 'of', 'on', 'or', 'tbd', 'that', 'the', 'this','to', 'us', 'we', 'when', 'will', 'with', 'yet','you', 'your', '的', '了', '和','or', 'tbd', 'that', 'the', 'this','to', 'us', 'we', 'when', 'will','when']
Counter(text).most_common(10)
#[('when', 3), ('to', 2), ('we', 2), ('that', 2), ('tbd', 2), ('this', 2), ('us',2), ('will', 2), ('the', 2), ('or', 2)]

Counter类是dict的子类，接受参数可以是iterable或者mapping.Counter是一个无序的集合。

c = Counter()     #一个新的空的Counter
c = Counter('kwejrkhdskf')    #以可迭代对象'kwejrkhdskf'为基础创建Counter
c = Counter({'red': 4, 'blue': 2})    #以mapping{'red': 4, 'blue': 2}为基础创建Counter
c = Counter(cats=4, dogs=8)    #以keywords args为基础创建Counter

如果索引的key不存在，Counter类不会报一个KeyError,相应地，它扩展了dict类，如果索引的key不存在，则返回0，如果key存在，则返回对应的值。

>>> c = Counter(['eggs', 'ham','eggs'])
>>> c['bacon']
0    #在c.keys中不存在'bacon'，故返回0
>>> c['eggs']
2    #在c.keys中存在'bacon'，故返回对应的value

设置一个key的值为0，并不意味着把key从Counter中删除，相应的用del 关键词可以达到想要的效果。

>>> c['sausage'] = 0  #设置'sausage'的个数为0
>>> del c['sausage']    #从Counter中删除'sausage'

Counter的一些常用方法

 ##########
elements()
    Return an iterator over elements repeating each as many times as its count. Elements are returned in arbitrary order. If an element’s count is less than one, elements() will ignore it.
##########
most_common([n])
Return a list of the n most common elements and their counts from the most common to the least. If n is omitted or None, most_common() returns all elements in the counter. Elements with equal counts are ordered arbitrarily:
#########################
subtract([iterable-or-mapping])
Elements are subtracted from an iterable or from another mapping (or counter). Like dict.update() but subtracts counts instead of replacing them. Both inputs and outputs may be zero or negative.
##########################
sum(c.values())                 # 求和
c.clear()                       # 重置
list(c)                         # 转换为list
set(c)                          # 转换为set
dict(c)                         # 转换为dict
c.items()                       # 类似dict的items()
c += Counter()                  # 删除掉值为0和负数的count

Counter还提供加、减、与、或运算。

>>> c = Counter(a=3, b=1)
>>> d = Counter(a=1, b=2)
>>> c + d                       # 相加:  c[x] + d[x]
Counter({'a': 4, 'b': 3})
>>> c - d                       # 相减 (舍掉非整数值)
Counter({'a': 2})
>>> c & d                       # 取最小值:  min(c[x], d[x])
Counter({'a': 1, 'b': 1})
>>> c | d                       # 取最大值:  max(c[x], d[x])
Counter({'a': 3, 'b': 2})

---

 deque

example:

>>> from collections import deque
>>> d = deque('ghi')                 # 创建实例
>>> for elem in d:                   # 迭代d
...     print elem.upper()
G
H
I

>>> d.append('j')                    # 在最右边加
>>> d.appendleft('f')                # 在最左边加
>>> d                                # show
deque(['f', 'g', 'h', 'i', 'j'])

>>> d.pop()                          # 在右边pop
'j'
>>> d.popleft()                      #在左边pop
'f'
>>> list(d)                          # 转换为list
['g', 'h', 'i']
>>> d[0]                             #用下标获取元素
'g'
>>> d[-1]                            # 类比list语法
'i'

>>> list(reversed(d))
['i', 'h', 'g']
>>> 'h' in d
True
>>> d.extend('jkl')                  # 拼接一个可迭代对象
>>> d
deque(['g', 'h', 'i', 'j', 'k', 'l'])
>>> d.rotate(1)                      #顺时针旋转
>>> d
deque(['l', 'g', 'h', 'i', 'j', 'k'])
>>> d.rotate(-1)                     #逆时针旋转
>>> d
deque(['g', 'h', 'i', 'j', 'k', 'l'])

>>> deque(reversed(d))
deque(['l', 'k', 'j', 'i', 'h', 'g'])
>>> d.clear()                        # 清空
>>> d.pop()                          # 没有元素不可以pop
Traceback (most recent call last):
  File "<pyshell#6>", line 1, in -toplevel-
    d.pop()
IndexError: pop from an empty deque

>>> d.extendleft('abc')              # reverse input
>>> d
deque(['c', 'b', 'a'])

del d[n] 相当于：

def del_(d,n):
    d.rorate(-n)
    d.popleft()
    d.rorate(n)

一个有趣的例子是计算MA：

#####################
算法：
    例如：[40, 30, 50, 46, 39, 44] --> 40.0 42.0 45.0 43.0
计算公式： MA = (C1+C2+C3+C4+C5+....+Cn)/n C 为收盘价，n 为移动平均周期数例如，现货黄金的 5 日移动平均价格计算方法为： MA 5 = （前四天收盘价+前三天收盘价+前天收盘价+昨天收盘价+今天收盘价）/5
#####################
def moving_average(iterable, n=3):
    # http://en.wikipedia.org/wiki/Moving_average
    it = iter(iterable)
    d = deque(itertools.islice(it, n-1))
    d.appendleft(0)
    s = sum(d)
    for elem in it:
        s += elem - d.popleft()
        d.append(elem)
        yield s / float(n)

---

namedtuple()

example:

>>> Point = namedtuple('Point', ['x', 'y'], verbose=True)
class Point(tuple):
    'Point(x, y)'

    __slots__ = ()

    _fields = ('x', 'y')

    def __new__(_cls, x, y):
        'Create a new instance of Point(x, y)'
        return _tuple.__new__(_cls, (x, y))

    @classmethod
    def _make(cls, iterable, new=tuple.__new__, len=len):
        'Make a new Point object from a sequence or iterable'
        result = new(cls, iterable)
        if len(result) != 2:
            raise TypeError('Expected 2 arguments, got %d' % len(result))
        return result

    def __repr__(self):
        'Return a nicely formatted representation string'
        return 'Point(x=%r, y=%r)' % self

    def _asdict(self):
        'Return a new OrderedDict which maps field names to their values'
        return OrderedDict(zip(self._fields, self))

    def _replace(_self, **kwds):
        'Return a new Point object replacing specified fields with new values'
        result = _self._make(map(kwds.pop, ('x', 'y'), _self))
        if kwds:
            raise ValueError('Got unexpected field names: %r' % kwds.keys())
        return result

    def __getnewargs__(self):
        'Return self as a plain tuple.   Used by copy and pickle.'
        return tuple(self)

    __dict__ = _property(_asdict)

    def __getstate__(self):
        'Exclude the OrderedDict from pickling'
        pass

    x = _property(_itemgetter(0), doc='Alias for field number 0')

    y = _property(_itemgetter(1), doc='Alias for field number 1')

>>> p = Point(11, y=22)     # 实例化一个点对象
>>> p[0] + p[1]             # 索引方式相加
33
>>> x, y = p                # unpack like a regular tuple
>>> x, y
(11, 22)
>>> p.x + p.y               # 属性方式相加
33
>>> p                       # __repr__实例的值
Point(x=11, y=22)

namedtuple对于导入csv和sqlite3的数据十分方便。以下是官方的demo

EmployeeRecord = namedtuple('EmployeeRecord', 'name, age, title, department, paygrade')

import csv
for emp in map(EmployeeRecord._make, csv.reader(open("employees.csv", "rb"))):
    print emp.name, emp.title

import sqlite3
conn = sqlite3.connect('/companydata')
cursor = conn.cursor()
cursor.execute('SELECT name, age, title, department, paygrade FROM employees')
for emp in map(EmployeeRecord._make, cursor.fetchall()):
    print emp.name, emp.title

namedtuple的一些封装方法

>>> t = [11, 22]
>>> Point._make(t)    #通过_make(可迭代对象)对实例传值
Point(x=11, y=22)
>>> p._asdict()    #返回一个有序字典（py2.7更新的功能）
OrderedDict([('x', 11), ('y', 22)])
>>> p = Point(x=11, y=22)
>>> p._replace(x=33) #_replace方法替换值
Point(x=33, y=22)

>>> for partnum, record in inventory.items():
        inventory[partnum] = record._replace(price=newprices[partnum], timestamp=time.now())
>>> p._fields            # 查看 fields名字
('x', 'y')

>>> Color = namedtuple('Color', 'red green blue')
>>> Pixel = namedtuple('Pixel', Point._fields + Color._fields)
>>> Pixel(11, 22, 128, 255, 0)
Pixel(x=11, y=22, red=128, green=255, blue=0)
>>> getattr(p, 'x') #获取p实例的x的值
11
>>> d = {'x': 11, 'y': 22}
>>> Point(**d)    #用"**"表示传的参数是一个字典
Point(x=11, y=22)
>>> class Point(namedtuple('Point', 'x y')):
        __slots__ = ()
        @property
        def hypot(self):
            return (self.x ** 2 + self.y ** 2) ** 0.5
        def __str__(self):
            return 'Point: x=%6.3f  y=%6.3f  hypot=%6.3f' % (self.x, self.y, self.hypot)
>>> Point3D = namedtuple('Point3D', Point._fields + ('z',))
>>> Account = namedtuple('Account', 'owner balance transaction_count')
>>> default_account = Account('<owner name>', 0.0, 0)
>>> johns_account = default_account._replace(owner='John')
>>> Status = namedtuple('Status', 'open pending closed')._make(range(3)) #实例化时，也可以同时初始化对象
>>> Status.open, Status.pending, Status.closed
(0, 1, 2)
>>> class Status:
        open, pending, closed = range(3)

---

OrderedDict

普通字典是无序结构，不是可哈希的值，对于某些应用情况可能不方便，OrderedDict提供的就是无序字典结构有序化的方法。

example:

>>> # 普通的字典
>>> d = {'banana': 3, 'apple':4, 'pear': 1, 'orange': 2}

>>> # 以key排序
>>> OrderedDict(sorted(d.items(), key=lambda t: t[0]))
OrderedDict([('apple', 4), ('banana', 3), ('orange', 2), ('pear', 1)])

>>> # 以value排序
>>> OrderedDict(sorted(d.items(), key=lambda t: t[1]))
OrderedDict([('pear', 1), ('orange', 2), ('banana', 3), ('apple', 4)])

>>> # 以key的长度排序
>>> OrderedDict(sorted(d.items(), key=lambda t: len(t[0])))
OrderedDict([('pear', 1), ('apple', 4), ('orange', 2), ('banana', 3)])

---

defaultdict

dict的子类，当某个key不存在时，提供一个默认值，而不是报错"keyerror"。

example

>>> s = [('yellow', 1), ('blue', 2), ('yellow', 3), ('blue', 4), ('red', 1)]
>>> d = defaultdict(list) #以list格式储存字典values
>>> for k, v in s:
...     d[k].append(v)
...
>>> d.items()
[('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])]

>>> d = {}
>>> for k, v in s:
...     d.setdefault(k, []).append(v) #另一种方式
...
>>> d.items()
[('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])]

>>> s = 'mississippi'
>>> d = defaultdict(int) #这种风格比较像counter
>>> for k in s:
...     d[k] += 1
...
>>> d.items()
[('i', 4), ('p', 2), ('s', 4), ('m', 1)]

>>> def constant_factory(value):
...     return itertools.repeat(value).next
>>> d = defaultdict(constant_factory('<missing>'))
>>> d.update(name='John', action='ran')
>>> '%(name)s %(action)s to %(object)s' % d #key=object是缺失的值，采用默认值
'John ran to <missing>'

>>> s = [('red', 1), ('blue', 2), ('red', 3), ('blue', 4), ('red', 1), ('blue', 4)]
>>> d = defaultdict(set) #以集合形式存储字典的values
>>> for k, v in s:
...     d[k].add(v)
...
>>> d.items()
[('blue', set([2, 4])), ('red', set([1, 3]))]