c#扩展出MapReduce方法

MapReduce方法主体：

 public static IDictionary<TKey, TResult> MapReduce<TInput, TKey, TValue, TResult>(this IList<TInput> inputList,
             Func<MapReduceData<TInput>, KeyValueClass<TKey, TValue>> map, Func<TKey, IList<TValue>, TResult> reduce)
         {
             object locker = new object();
             ConcurrentDictionary<TKey, TResult> result = new ConcurrentDictionary<TKey, TResult>();
             //保存map出来的结果
             ConcurrentDictionary<TKey, IList<TValue>> mapDic = new ConcurrentDictionary<TKey, IList<TValue>>();
             var parallelOptions = new ParallelOptions();
             parallelOptions.MaxDegreeOfParallelism = Environment.ProcessorCount;
             //并行map
             Parallel.For(, inputList.Count(), parallelOptions, t =>
             {
                 MapReduceData<TInput> data = new MapReduceData<TInput>
                 {
                     Data = inputList[t],
                     Index = t,
                     List = inputList,
                 };
                 var pair = map(data);
                 if (pair != null && pair.Valid)
                 {
                     //锁住防止并发操作list造成数据缺失
                     lock (locker)
                     {
                         //将匹配出来的结果加入结果集放入字典
                         IList<TValue> list = null;
                         if (mapDic.ContainsKey(pair.Key))
                         {
                             list = mapDic[pair.Key];
                         }
                         else
                         {
                             list = new List<TValue>();
                             mapDic[pair.Key] = list;
                         }
                         list.Add(pair.Value);
                     }
                 }
             });
 
             //并行reduce
             Parallel.For(, mapDic.Keys.Count, parallelOptions, t =>
             {
                 KeyValuePair<TKey, IList<TValue>> pair = mapDic.ElementAt(t);
                 result[pair.Key] = reduce(pair.Key, pair.Value);
             });
             return result;
         }

KeyValueClass定义：

 public class KeyValueClass<K, V>
     {
         public KeyValueClass(K key, V value)
         {
             Key = key;
             Value = value;
         }
 
         public KeyValueClass()
         {
 
         }
 
         public K Key { get; set; }
 
         public V Value { get; set; }
     }

Console测试：

 List<TestClass> listTestClass = new List<TestClass>();
             listTestClass.Add(new TestClass { a = "a", g =  });
             listTestClass.Add(new TestClass { a = "b", g =  });
             listTestClass.Add(new TestClass { a = "c", g =  });
             listTestClass.Add(new TestClass { a = "d", g =  });
             listTestClass.Add(new TestClass { a = "e", g =  });
             listTestClass.Add(new TestClass { a = "f", g =  });
             listTestClass.Add(new TestClass { a = "g", g =  });
             listTestClass.Add(new TestClass { a = "h", g =  });
             IDictionary<int, string> dic = listTestClass.MapReduce(t =>
             {
                 if (t.g < )
                 {
                     return new KeyValueClass<int, string>(t.g, t.a);
                 }
                 return null;
             }, (key, values) =>
            {
                return string.Join(",", values);
            });

TestClass定义：

 public class TestClass
     {
         public string a { get; set; }
         public string b { get; set; }
 
         public string d { get; set; }
 
         //public DateTime f { get; set; }
 
         public int g { get; set; }
 
         public List<TestClass> test { get; set; }
 
         public Dictionary<string, string> dic { get; set; }
     }

结果：

1：a,e

2：d,f

3：b

4：c

词频性能测试