redis 5.0.7 源码阅读——整数集合intset

redis中整数集合intset相关的文件为：intset.h与intset.c

intset的所有操作与操作一个排序整形数组 int a[N]类似，只是根据类型做了内存上的优化。

一、数据结构

 typedef struct intset {
     uint32_t encoding;
     uint32_t length;
     int8_t contents[];
 } intset;

intset的数据结构比较简单，使用了一个变长结构体，成员length记录当前成员数量，成员encoding记录当前的int类型，共有以下三种：

 #define INTSET_ENC_INT16 (sizeof(int16_t))
 #define INTSET_ENC_INT32 (sizeof(int32_t))
 #define INTSET_ENC_INT64 (sizeof(int64_t))

并使用以下方法进行判断类型：

 static uint8_t _intsetValueEncoding(int64_t v) {
     if (v < INT32_MIN || v > INT32_MAX)
         return INTSET_ENC_INT64;
     else if (v < INT16_MIN || v > INT16_MAX)
         return INTSET_ENC_INT32;
     else
         return INTSET_ENC_INT16;
 }

intset是已排序好的整数集合，其大致结构如下：

 /*
 +--------+--------+--------...--------------+
 |encoding|length  |contents(encoding*length)|
 +--------+--------+--------...--------------+
 */

intset严格按照小端字节序进行存储，不论机器的字节序类型。如果是大端机器，需要进行转换，才进行存储。endianconv.h中有如下定义：

 #if (BYTE_ORDER == LITTLE_ENDIAN)
 #define memrev16ifbe(p) ((void)(0))
 #define memrev32ifbe(p) ((void)(0))
 #define memrev64ifbe(p) ((void)(0))
 #define intrev16ifbe(v) (v)
 #define intrev32ifbe(v) (v)
 #define intrev64ifbe(v) (v)
 #else
 #define memrev16ifbe(p) memrev16(p)
 #define memrev32ifbe(p) memrev32(p)
 #define memrev64ifbe(p) memrev64(p)
 #define intrev16ifbe(v) intrev16(v)
 #define intrev32ifbe(v) intrev32(v)
 #define intrev64ifbe(v) intrev64(v)
 #endif

具体实现在endianconv.c中，此处略过。

二、创建

 intset *intsetNew(void) {
     intset *is = zmalloc(sizeof(intset));
     is->encoding = intrev32ifbe(INTSET_ENC_INT16);
     is->length = ;
     return is;
 }

刚创建好的intset是空的，默认使用最小的类型。其结构为：

 /*此处用一根“-”表示一字节，后同
 +----+----+
 |  16|   0|
 +----+----+
 */

三、 操作

若有以下intset：

 /*
 +----+----+--+--+--+--+--+--+--+
 |  16|   7| 1| 2| 3| 4| 5| 7| 8|
 +----+----+--+--+--+--+--+--+--+
           |contents
 
 */

现在插入一个数字6，需要调用以下方法：

 /* Insert an integer in the intset */
 intset *intsetAdd(intset *is, int64_t value, uint8_t *success) {
     uint8_t valenc = _intsetValueEncoding(value);
     uint32_t pos;
     if (success) *success = ;
 
     /* Upgrade encoding if necessary. If we need to upgrade, we know that
      * this value should be either appended (if > 0) or prepended (if < 0),
      * because it lies outside the range of existing values. */
     if (valenc > intrev32ifbe(is->encoding)) {
         /* This always succeeds, so we don't need to curry *success. */
         return intsetUpgradeAndAdd(is,value);
     } else {
         /* Abort if the value is already present in the set.
          * This call will populate "pos" with the right position to insert
          * the value when it cannot be found. */
         if (intsetSearch(is,value,&pos)) {
             if (success) *success = ;
             return is;
         }
 
         is = intsetResize(is,intrev32ifbe(is->length)+);
         if (pos < intrev32ifbe(is->length)) intsetMoveTail(is,pos,pos+);
     }
 
     _intsetSet(is,pos,value);
     is->length = intrev32ifbe(intrev32ifbe(is->length)+);
     return is;
 }

因int16_t足以存储数字“6”，所以新插入数字的int类型与intset一致，然后需要查找插入的pos：

 static uint8_t intsetSearch(intset *is, int64_t value, uint32_t *pos) {
     int min = , max = intrev32ifbe(is->length)-, mid = -;
     int64_t cur = -;
 
     /* The value can never be found when the set is empty */
     if (intrev32ifbe(is->length) == ) {
         if (pos) *pos = ;
         return ;
     } else {
         /* Check for the case where we know we cannot find the value,
          * but do know the insert position. */
         if (value > _intsetGet(is,max)) {
             if (pos) *pos = intrev32ifbe(is->length);
             return ;
         } else if (value < _intsetGet(is,)) {
             if (pos) *pos = ;
             return ;
         }
     }
 
     while(max >= min) {
         mid = ((unsigned int)min + (unsigned int)max) >> ;
         cur = _intsetGet(is,mid);
         if (value > cur) {
             min = mid+;
         } else if (value < cur) {
             max = mid-;
         } else {
             break;
         }
     }
 
     if (value == cur) {
         if (pos) *pos = mid;
         return ;
     } else {
         if (pos) *pos = min;
         return ;
     }
 }

因intset是已排序好的，所以使用了二分查找。过程如下

 /*
 find 6
         +----+----+--+--+--+--+--+--+--+
         |  16|   7| 1| 2| 3| 4| 5| 7| 8|
         +----+----+--+--+--+--+--+--+--+
 pos               | 0| 1| 2| 3| 4| 5| 6|
 step1             |min=0
                                     |max=6
                            |mid=(0+6)>>1=3
                            |mid_val=4
 
 pos               | 0| 1| 2| 3| 4| 5| 6|
 step2                         |min=4
                                     |max=6
                                  |mid=(4+6)>>1=5
                                  |mid_val=7
 
 pos               | 0| 1| 2| 3| 4| 5| 6|
 step3                         |min=4
                               |max=4
                               |mid=(4+4)>>1=5
                               |mid_val=5
 
 pos               | 0| 1| 2| 3| 4| 5| 6|
 step4                            |min=5
                               |max=4
 min>max  break
 */

6在intset中不存在，查找到需要插入到pos=5的位置，此时首先要扩展intset的content：

 static intset *intsetResize(intset *is, uint32_t len) {
     uint32_t size = len*intrev32ifbe(is->encoding);
     is = zrealloc(is,sizeof(intset)+size);
     return is;
 }

扩展后：

 /*
 +----+----+--+--+--+--+--+--+--+--+
 |  16|   7| 1| 2| 3| 4| 5| 7| 8|  |
 +----+----+--+--+--+--+--+--+--+--+
 pos       | 0| 1| 2| 3| 4| 5| 6| 7|
 */

然后把原来在pos=5及之后的所有的元素向后移一格：

 static void intsetMoveTail(intset *is, uint32_t from, uint32_t to) {
     void *src, *dst;
     uint32_t bytes = intrev32ifbe(is->length)-from;
     uint32_t encoding = intrev32ifbe(is->encoding);
 
     if (encoding == INTSET_ENC_INT64) {
         src = (int64_t*)is->contents+from;
         dst = (int64_t*)is->contents+to;
         bytes *= sizeof(int64_t);
     } else if (encoding == INTSET_ENC_INT32) {
         src = (int32_t*)is->contents+from;
         dst = (int32_t*)is->contents+to;
         bytes *= sizeof(int32_t);
     } else {
         src = (int16_t*)is->contents+from;
         dst = (int16_t*)is->contents+to;
         bytes *= sizeof(int16_t);
     }
     memmove(dst,src,bytes);
 }

移动后：

 /*
 +----+----+--+--+--+--+--+--+--+--+
 |  16|   7| 1| 2| 3| 4| 5| 7| 7| 8|
 +----+----+--+--+--+--+--+--+--+--+
 pos       | 0| 1| 2| 3| 4| 5| 6| 7|
 */

其使用memmove，并不全修改未覆盖到的内存，所以此时pos=5的值 还是7

最后修改pos=5的值：

 static void _intsetSet(intset *is, int pos, int64_t value) {
     uint32_t encoding = intrev32ifbe(is->encoding);
 
     if (encoding == INTSET_ENC_INT64) {
         ((int64_t*)is->contents)[pos] = value;
         memrev64ifbe(((int64_t*)is->contents)+pos);
     } else if (encoding == INTSET_ENC_INT32) {
         ((int32_t*)is->contents)[pos] = value;
         memrev32ifbe(((int32_t*)is->contents)+pos);
     } else {
         ((int16_t*)is->contents)[pos] = value;
         memrev16ifbe(((int16_t*)is->contents)+pos);
     }
 }

修改后并增加了length：

 /*
 +----+----+--+--+--+--+--+--+--+--+
 |  16|   8| 1| 2| 3| 4| 5| 6| 7| 8|
 +----+----+--+--+--+--+--+--+--+--+
 pos       | 0| 1| 2| 3| 4| 5| 6| 7|
 */

如果此时要插入的数字是65536，超出了int16_t所能表示的范围，要先进行扩展int类型操作：

 static intset *intsetUpgradeAndAdd(intset *is, int64_t value) {
     uint8_t curenc = intrev32ifbe(is->encoding);
     uint8_t newenc = _intsetValueEncoding(value);
     int length = intrev32ifbe(is->length);
     int prepend = value <  ?  : ;
 
     /* First set new encoding and resize */
     is->encoding = intrev32ifbe(newenc);
     is = intsetResize(is,intrev32ifbe(is->length)+);
 
     /* Upgrade back-to-front so we don't overwrite values.
      * Note that the "prepend" variable is used to make sure we have an empty
      * space at either the beginning or the end of the intset. */
     while(length--)
         _intsetSet(is,length+prepend,_intsetGetEncoded(is,length,curenc));
 
     /* Set the value at the beginning or the end. */
     if (prepend)
         _intsetSet(is,,value);
     else
         _intsetSet(is,intrev32ifbe(is->length),value);
     is->length = intrev32ifbe(intrev32ifbe(is->length)+);
     return is;
 }

因其超出原来的int类型所能表示的范围，若为正数，一定是最大的，则应该插入在intset最后，否则应该在最前面。扩展完之后，从后往前将原来的数字，以新的int类型，放置在新的位置上，保证不会有未处理的数字被覆盖，处理完整。

删除操作：

 intset *intsetRemove(intset *is, int64_t value, int *success) {
     uint8_t valenc = _intsetValueEncoding(value);
     uint32_t pos;
     if (success) *success = ;
 
     if (valenc <= intrev32ifbe(is->encoding) && intsetSearch(is,value,&pos)) {
         uint32_t len = intrev32ifbe(is->length);
 
         /* We know we can delete */
         if (success) *success = ;
 
         /* Overwrite value with tail and update length */
         if (pos < (len-)) intsetMoveTail(is,pos+,pos);
         is = intsetResize(is,len-);
         is->length = intrev32ifbe(len-);
     }
     return is;
 }

找到指定元素之后，直接把后面的内存移至前面，然后resize。

redis 5.0.7 下载链接

http://download.redis.io/releases/redis-5.0.7.tar.gz

源码阅读顺序参考：

https://github.com/huangz1990/blog/blob/master/diary/2014/how-to-read-redis-source-code.rst