skynet源码阅读<6>--线程调度

相比于上节我们提到的协程调度，skynet的线程调度从逻辑流程上来看要简单很多。下面我们就来具体做一分析。首先自然是以skynet_start.c为入口：

 static void
 start(int thread) {
     pthread_t pid[thread+];

     struct monitor *m = skynet_malloc(sizeof(*m));
     memset(m, , sizeof(*m));
     m->count = thread;
     m->sleep = ;

     m->m = skynet_malloc(thread * sizeof(struct skynet_monitor *));
     int i;
     for (i=;i<thread;i++) {
         m->m[i] = skynet_monitor_new();
     }
     if (pthread_mutex_init(&m->mutex, NULL)) {
         fprintf(stderr, "Init mutex error");
         exit();
     }
     if (pthread_cond_init(&m->cond, NULL)) {
         fprintf(stderr, "Init cond error");
         exit();
     }

     create_thread(&pid[], thread_monitor, m);
     create_thread(&pid[], thread_timer, m);
     create_thread(&pid[], thread_socket, m);

     static int weight[] = {
         -, -, -, -, , , , ,
         , , , , , , , ,
         , , , , , , , ,
         , , , , , , , , };
     struct worker_parm wp[thread];
     for (i=;i<thread;i++) {
         wp[i].m = m;
         wp[i].id = i;
         if (i < sizeof(weight)/sizeof(weight[])) {
             wp[i].weight= weight[i];
         } else {
             wp[i].weight = ;
         }
         create_thread(&pid[i+], thread_worker, &wp[i]);
     }

     for (i=;i<thread+;i++) {
         pthread_join(pid[i], NULL);
     }

     free_monitor(m);
 }

先是创建monitor、timer、socket三个线程分别执行监控、计时器、网络处理的工作，接着创建thread个用户线程，设置权重weight并开始执行。看下thread_worker里面做了什么：

 static void *
 thread_worker(void *p) {
     struct worker_parm *wp = p;
     int id = wp->id;
     int weight = wp->weight;
     struct monitor *m = wp->m;
     struct skynet_monitor *sm = m->m[id];
     skynet_initthread(THREAD_WORKER);
     struct message_queue * q = NULL;
     while (!m->quit) {
         q = skynet_context_message_dispatch(sm, q, weight);
         if (q == NULL) {
             if (pthread_mutex_lock(&m->mutex) == ) {
                 ++ m->sleep;
                 // "spurious wakeup" is harmless,
                 // because skynet_context_message_dispatch() can be call at any time.
                 if (!m->quit)
                     pthread_cond_wait(&m->cond, &m->mutex);
                 -- m->sleep;
                 if (pthread_mutex_unlock(&m->mutex)) {
                     fprintf(stderr, "unlock mutex error");
                     exit();
                 }
             }
         }
     }
     return NULL;
 }

每一个工作线程，先是分发消息，并获得下一次要分发的消息队列。如果没有的话就休息一会儿，等待合适的时机被唤醒。每次socket线程中有新的消息到来，或是timer-update时就唤醒所有休息在m->cond上的线程，继续消息分发。看一下具体的分发过程：

 struct message_queue *
 skynet_context_message_dispatch(struct skynet_monitor *sm, struct message_queue *q, int weight) {
     if (q == NULL) {
         q = skynet_globalmq_pop();
         if (q==NULL)
             return NULL;
     }

     uint32_t handle = skynet_mq_handle(q);

     struct skynet_context * ctx = skynet_handle_grab(handle);
     if (ctx == NULL) {
         struct drop_t d = { handle };
         skynet_mq_release(q, drop_message, &d);
         return skynet_globalmq_pop();
     }

     int i,n=;
     struct skynet_message msg;

     for (i=;i<n;i++) {
         if (skynet_mq_pop(q,&msg)) {
             skynet_context_release(ctx);
             return skynet_globalmq_pop();
         } else if (i== && weight >= ) {
             n = skynet_mq_length(q);
             n >>= weight;
         }
         int overload = skynet_mq_overload(q);
         if (overload) {
             skynet_error(ctx, "May overload, message queue length = %d", overload);
         }

         skynet_monitor_trigger(sm, msg.source , handle);

         if (ctx->cb == NULL) {
             skynet_free(msg.data);
         } else {
             dispatch_message(ctx, &msg);
         }

         skynet_monitor_trigger(sm, ,);
     }

     assert(q == ctx->queue);
     struct message_queue *nq = skynet_globalmq_pop();
     if (nq) {
         // If global mq is not empty , push q back, and return next queue (nq)
         // Else (global mq is empty or block, don't push q back, and return q again (for next dispatch)
         skynet_globalmq_push(q);
         q = nq;
     }
     skynet_context_release(ctx);

     return q;
 }

拿到当前消息队列q对应的handle和context，接着便从q中连续取出n个消息通过dispatch_message分发出去。权重越大的线程，当前可分发的消息数n就越大。如果在n范围内q的消息分发完了，那么就从全局队列globalmq中取出下一个消息队列q，返回等待下一次执行；如果n范围内q依然有消息剩余，那么就把当前消息队列q扔回到global_mq中去，并拿到新的消息队列nq返回，这样每个消息队列都会相对公平地得到执行。不同的线程只是在从全局队列global_mq拿消息队列q时对global_mq加锁，或者在向q中推入消息（比如向skynet-context的q发送消息），以及这里取出消息执行时需要对q加锁。
    具体的q或global_mq是按照链表实现的，这里就不废话了。最后看下消息分发的处理：

 static void
 dispatch_message(struct skynet_context *ctx, struct skynet_message *msg) {
     assert(ctx->init);
     CHECKCALLING_BEGIN(ctx)
     pthread_setspecific(G_NODE.handle_key, (void *)(uintptr_t)(ctx->handle));
     int type = msg->sz >> MESSAGE_TYPE_SHIFT;
     size_t sz = msg->sz & MESSAGE_TYPE_MASK;
     if (ctx->logfile) {
         skynet_log_output(ctx->logfile, msg->source, type, msg->session, msg->data, sz);
     }
     ++ctx->message_count;
     int reserve_msg;
     if (ctx->profile) {
         ctx->cpu_start = skynet_thread_time();
         reserve_msg = ctx->cb(ctx, ctx->cb_ud, type, msg->session, msg->source, msg->data, sz);
         uint64_t cost_time = skynet_thread_time() - ctx->cpu_start;
         ctx->cpu_cost += cost_time;
     } else {
         reserve_msg = ctx->cb(ctx, ctx->cb_ud, type, msg->session, msg->source, msg->data, sz);
     }
     if (!reserve_msg) {
         skynet_free(msg->data);
     }
     CHECKCALLING_END(ctx)
 }

关键就在于ctx->cb这个函数指针的调用。你肯定会好奇，这个回调是如何进入LUA代码的呢？回想skynet.lua中skynet.start(f)函数，其调用了c.callback将回调f注册到skynet-os中。因此就让我们回到skynet-lua.c中，看一看lcallback函数的处理：

 static int
 lcallback(lua_State *L) {
     struct skynet_context * context = lua_touserdata(L, lua_upvalueindex());
     int forward = lua_toboolean(L, );
     luaL_checktype(L,,LUA_TFUNCTION);
     lua_settop(L,);
     lua_rawsetp(L, LUA_REGISTRYINDEX, _cb);

     lua_rawgeti(L, LUA_REGISTRYINDEX, LUA_RIDX_MAINTHREAD);
     lua_State *gL = lua_tothread(L,-);

     if (forward) {
         skynet_callback(context, gL, forward_cb);
     } else {
         skynet_callback(context, gL, _cb);
     }

     return ;
 }

这里重点看下lua_rawsetp(L, LUA_REGISTRYINDEX, _cb)。此时栈顶为LUA_TFUNCTION元素f，这句的作用是从索引LUA_REGISTERINDEX处取出t，在这里也就是全局注册表，然后设置t[_cb]=f并弹出栈顶元素（此函数不触发__index元方法）。这样就将_cb与f之间关联起来了。最后调用skynet_callback将_cb设置到context.cb元素，而虚拟机L也设置到context.ud元素。了解了lcallback的注册过程，我们回到_cb函数，看看具体调用时发生了什么事情：

 static int
 _cb(struct skynet_context * context, void * ud, int type, int session, uint32_t source, const void * msg, size_t sz) {
     lua_State *L = ud;
     int trace = ;
     int r;
     int top = lua_gettop(L);
     if (top == ) {
         lua_pushcfunction(L, traceback);
         lua_rawgetp(L, LUA_REGISTRYINDEX, _cb);
     } else {
         assert(top == );
     }
     lua_pushvalue(L,);

     lua_pushinteger(L, type);
     lua_pushlightuserdata(L, (void *)msg);
     lua_pushinteger(L,sz);
     lua_pushinteger(L, session);
     lua_pushinteger(L, source);

     r = lua_pcall(L, ,  , trace);

     if (r == LUA_OK) {
         return ;
     }
 }

以_cb（函数地址）为键，调用lua_rawgetp从全局注册表中取出上述注册进来的LUA_TFUNCTION函数f，压入参数调用执行。
    至此，skynet的线程调度流程已经清楚了。下一次我们来谈谈锁的问题。