homework做了些什么？

第一步:get_new_guid_uid_pairs_{$ymd}

参数是时间和100上的文件.

那么100上的文件是从哪里来的呢？

我们进入到100机器上,打开root权限下的cron,看到如下内容:

### add by kamilzhou for clickflow system
8 * * * * /data/clickflow/hourly_data/hourlyGetLogDataStore.sh >> /data1/clickflow/hourly_data/get.log &
20 * * * * /data/clickflow/filters/correct-page-id-filter/clickflowlog-filter/bin/hourly_refer_filter.sh

所以我猜测是不是100上的数据是从这两个脚本拉过来的呢？

查看这俩个脚本内容.

 #!/bin/sh

 #获取当前脚本所在路径
 currdir=$(cd "$(dirname "$0")"; pwd)

 #进入脚本所在目录
 cd $currdir

 log_file_time=`date +%Y%m%d%H -d"-1 hours"`
 log_file_dir=`date +%Y%m%d -d"-1 hours"`

 if  [ $# = 1 ]; then
     log_file_time=$1
 fi

 if  [ $# = 2 ]; then
     log_file_time=$1
     log_file_dir=$2
 fi

 if [ ! -d "/data1/clickflow/by-time/""$log_file_dir" ]; then
     mkdir "/data1/clickflow/by-time/""$log_file_dir"
 fi

 log_filename="row_data_""${log_file_time}""*"
 src_log_filename="/data/clickflow/data_today/""${log_filename}"

 /usr/bin/expect recv_file.exp 10.206.8.75 36000 $src_log_filename "/data1/clickflow/by-time/""$log_file_dir" kamilzhou Zhou2013@tencent

hourlyGetLogDataStore.sh

 #!/bin/sh

 orig_log_dir="/data1/clickflow/by-time/"
 refer_filter_dir="/data1/clickflow/refer-filtered/"

 refer_filter_bin="/data/clickflow/filters/correct-page-id-filter/clickflowlog-filter/bin/correct-page-id-filter.sh"

 log_file_time=`date +%Y%m%d%H -d"-1 hours"`
 log_file_dir=`date +%Y%m%d -d"-1 hours"`

 if  [ $# = 1 ]; then
     log_file_time=$1
 fi

 if  [ $# = 2 ]; then
     log_file_time=$1
     log_file_dir=$2
 fi

 src_dir="$orig_log_dir""$log_file_dir""/"
 dest_dir="$refer_filter_dir""$log_file_dir""/"

 if [ ! -d "$dest_dir" ]; then
     mkdir "$dest_dir"
 fi

 src_files="$src_dir""row_data_""$log_file_time""*"

 for file in `ls $src_files`; do
     filename=`basename $file`;

     echo $filename
     $refer_filter_bin $file "$dest_dir""$filename"
 done

hourly_refer_filter.sh

hourlyGetLogDataStore.sh这个脚本每小时的8分钟执行,负责把前一个小时的75上的原始数据拉过来.

hourly_refer_filter.sh 这个脚本每小时的20分钟执行,代码里调用了很多其他脚本.目的是负责过滤不合法的数据.

详细了解请参考:

100上的数据过滤

参数是有了,那么输出是什么呢？

我们来看下这个任务的handler.

 /**
  * get new guid uid pairs and paritition the raw clickflow data        获得新的 guid和uid 的配对   raw是未经加工的意思
  * useing multi-processes
  *
  * @param array $request request of a task
  * @param boolean &$continued wheater the process should be continued
  *
  * @return boolean true when success, otherwise false
  * @author baronyuan
  */
 function getNewGuidUidPairsNPartitionRawDataSMP($in, $out, &$continued)
 {
     {  //参数检查
     if (!is_array($in)
         || !isset($in['date'])
         || !isset($in['rawdata'])
         || !is_array($out)
         || !isset($out['upairs'])
         || !isset($out['rawdata'])
     ) {
         Logger::err(sprintf(
             'invalid request with in[%s] or out[%s]',
             json_encode($in), json_encode($out)
         ));

         return false;
     }
     }

     $date = trim($in['date']);   //
     $remote_dir = trim($in['rawdata']);   //    sftp://10.191.152.100:36000/data1/clickflow/refer-filtered/20140806/   未经处理的数据（来自100 ）
     $result_pairs = $out['upairs'];     //      sftp://10.206.8.75:36000/data/clickflow/tmp/guid_uid-map/new/20140806/data //这个应该是临时存放文件夹，因为75上暂时看不见数据
     $results_rawdata = pattern_range2array($out['rawdata']);  //   sftp://10.206.8.75:36000/data/clickflow/tmp/rawdata/20140806/by-guid/data_0   ~   sftp://10.206.8.75:36000/data/clickflow/tmp/rawdata/20140806/by-guid/data_99

     $unikey = get_unikey();   //

     $resource_prefix =
         HOMEWORK_CLICKFLOW_TMP_PATH . 'tasks.' . $unikey . '_input_';       //         /data/tmp/clickflow/homework/tasks.unikey_input_  这个地址应该在75和85 上都有
     $output_prefix =
         HOMEWORK_CLICKFLOW_TMP_PATH . 'tasks.' . $unikey . '_output_';

     if (RemoteResourceAccessExists($remote_dir . 'row_data_' . $date . '000')) {   //正常情况下 ，这个条件是 成立的  row_data_20140806000
         $remote_resource_names = getSplitedRawDataNames($date);      //这个地方20140806000~20140806235
         $tasks_input = array();

         foreach ($remote_resource_names as $key => $name) {
             $resource_local = $resource_prefix . $key;               // HOMEWORK_CLICKFLOW_TMP_PATH . 'tasks.' . $unikey . '_input_0' 对应 row_data_20140806000
             $remote = $remote_dir . 'row_data_' . $name;            //sftp://10.191.152.100:36000/data1/clickflow/refer-filtered/20140806/row_data_20140806235

             if (!RemoteResourceAccessExists($remote)) {                  // 查看100上原始文件  是否存在
                 Logger::err(sprintf('%s not found', $remote));
                 return false;
             }

             $get_result = RemoteResourceAccessGetUntil(        //把100上的文件拿过来，放在本地
                 $remote_dir . 'row_data_' . $name,
                 $resource_local,                                //存放在本地的临时文件
                 $continued
             );

             if (!$continued) {
                 Logger::err('stop!');
                 return false;
             }

             FileAutoCleaner::add($resource_local);
             $tasks_input[] = $resource_local;                           //把拿过来的文件 存到一个数组里面
         }
     } else {
         $remote_resource_name = $remote_dir . 'row_data_' . $date;

         if (!RemoteResourceAccessExists($remote_resource_name)) {
             Logger::err(sprintf('%s not found', $remote_resource_name));
             return false;
         }

         $resource_local = $resource_prefix . 'inall';
         $get_result = RemoteResourceAccessGetUntil(
             $remote_resource_name,
             $resource_local,
             $continued
         );

         if (!$continued) {
             Logger::err('stop!');
             return false;
         }

         FileAutoCleaner::add($resource_local);

         if (false === split_file($resource_local, 144, $resource_prefix)) {
             Logger::err(sprintf(
                 'failed to split the file[%s]', $resource_prefix
             ));

             return false;
         }

         for ($i = 0; $i < 144; ++$i) {
             $tasks_input[] = $resource_prefix . $i;
         }

         FileAutoCleaner::add($tasks_input);
     }

     $tasks_output = array(                            //输出数组？？？、、。。。
         $output_prefix . 'pairs.output',
         pattern_range2array(
             $output_prefix . 'rawdata_[0-99]'
         )
     );

     file_put_contents($tasks_output[0], '');

     foreach ($tasks_output[1] as $task_rawdata_output) {
         file_put_contents($task_rawdata_output, '');
     }

     FileAutoCleaner::add($tasks_output[0], $tasks_output[1]);

     $num_of_processes = 40;
     $num_of_tasks = count($tasks_input);    // 从100上过来多少个文件  就有多少子任务
     $counter = -1;

     $task_divider = new TaskDivider();

     while ($num_of_tasks > 0) {
         if ($num_of_tasks < $num_of_processes) {
             $num_of_processes = $num_of_tasks;
         }

         $task_divider->clear();

         for ($i = 0; $i < $num_of_processes; ++ $i) {
             ++ $counter;

             $task_divider->add(
                 'getNewGuidUidPairsNPartitionRawDataTask',
                 array(
                     $tasks_input[$counter],
                     $tasks_output[0],
                     $tasks_output[1]
                 )
             );
         }

         $run_result = $task_divider->run($continued);

         if (false === $run_result) {
             Logger::err('fail to run tasks');
             return false;
         } elseif ($run_result !== 0) {
             Logger::err(sprintf('fail to run %d tasks', $run_result));
             return false;
         } else {
             // nothing to do ...
         }

         $num_of_tasks -= $num_of_processes;
     }

     $uniq_result = uniq_file($tasks_output[0]);

     if (false === $uniq_result) {
         Logger::err(sprintf('fail to uniq the file[%s]', $tasks_output[0]));
         return false;
     }

     $put_result = RemoteResourceAccessPutUntil(
         $tasks_output[0],
         $result_pairs,
         $continued
     );

     if (!$continued) {
         Logger::err('stop!');
         return false;
     }

     foreach ($tasks_output[1] as $key => $task_rawdata_output) {
         $local_path = $task_rawdata_output;
         $remote_path = $results_rawdata[$key];

         $put_result = RemoteResourceAccessPutUntil(
             $local_path,
             $remote_path,
             $continued
         );

         if (!$continued) {
             Logger::err('stop!');
             return false;
         }
     }

     return true;
 }

getNewGuidUidPairsNPartitionRawDataSMP

 /**
  * get new guid uid pairs sub task for a single process to handle
  *
  * @param string $input_file file to handle
  * @param string $output_pairs_file output of pairs
  * @param array $output_rawdata_files output of rawdata partitioned
  *
  * @return boolean true when success, otherwise failed
  * @author baronyuan
  */
 function getNewGuidUidPairsNPartitionRawDataTask(
     $input_file,
     $output_pairs_file,
     $output_rawdata_files
 ) {
     global $logger;
     $logger->uninit();
     $logger = new Logger(HOMEWORK_WORKER_LOG_NAME);

     $fp_input = @fopen($input_file, 'r');     //这个地方打开的应该是 已经从100上拿来的 存在本地的临时文件

     if (false === $fp_input) {
         Logger::err(sprintf('failed to open file[%s]', $input_file));
         return false;
     }

     $upairs = array();
     $fps_rawdata_output = array();

     for ($i = 0; $i < 100; ++ $i) {
         $fp_rawdata_output =
             $fps_rawdata_output[] =                           //这个地方打开了100个文件，存在一个数组里，准备往里面写
                 @fopen($output_rawdata_files[$i], 'a');   // a 以附加的方式打开只写文件。若文件不存在，则会建立该文件，如果文件存在，写入的数据会被加到文件尾，即文件原先的内容会被保留。

         if (false === $fp_rawdata_output) {
             Logger::err(
                 sprintf('fail to open file[%s]', $output_rawdata_files[$i])
             );

             return false;
         }
     }

     $special_key = -1;

     while (($line = fgets($fp_input, 10240)) !== false) {
         $cells = explode("\t", $line);

         if (!isset($cells[LOG_FIELD_MAX_IDX])
             || $cells[LOG_FIELD_IDX_LOG_TYPE] != LOG_TYPE_PV
             || $cells[LOG_FIELD_IDX_PAGE_ID] < 0
         ) {
             continue;
         }

         // iframe
         $page_id = (int)$cells[LOG_FIELD_IDX_PAGE_ID];
         $wh_id = (int)$cells[LOG_FIELD_IDX_WH_ID];

         if ($wh_id != 1971) {
             if ((12376 == $page_id) || (1237017 == $page_id)
                 || (63786 == $page_id) || (6378017 == $page_id)
                 || (63796 == $page_id) || (6379017 == $page_id)
                 || (63806 == $page_id) || (6380017 == $page_id)
                 || (63816 == $page_id) || (6381017 == $page_id)
                 || (63826 == $page_id) || (6382017 == $page_id)
                 || (5020 == $page_id) || (11718320 == $page_id)
                 || (!(int)$cells[LOG_FIELD_IDX_PAGE_LEVEL] &&
                 !preg_match('#buy\.(?:51buy|yixun)\.com#', $cells[LOG_FIELD_IDX_PAGE_URL]))
             ) {
                 continue;
             }
         }

         $uid = (int)$cells[LOG_FIELD_IDX_USER_ID];
         $guid = trim($cells[LOG_FIELD_IDX_GUID]);

         $hash_key = null;

         // tmp hack
         if (!in_array($wh_id, array(1, 1001, 2001, 3001, 4001, 5001, 1969, 1971))) {
             continue;
         }

         if (1970 == $wh_id) {
             $hash_key = $special_key = (++ $special_key) % 100;
         } else {
             $hash_key = crc32($guid) % 100;

             if ($uid && $guid && $guid != 'null' && $guid != 'Unknown') {
                 $upairs[$guid] = $uid;
             }
         }

         if ($hash_key < 0 || $hash_key > 99) {
             Logger::err(sprintf('invalid hash key %d', $hash_key));
         }

         $write_result = @fwrite($fps_rawdata_output[$hash_key], $line);

         if (false === $write_result) {
             Logger::err('failed to write a line into a file');
             return false;
         }
     }

     if (!feof($fp_input)) {
         Logger::err(sprintf('failed to open file[%s]', $input_file));
         return false;
     }

     fclose($fp_input);

     foreach ($fps_rawdata_output as $fp_rawdata_output) {
         @fclose($fp_rawdata_output);
     }

     $fp_pairs_output = @fopen($output_pairs_file, 'a');

     if (false === $fp_pairs_output) {
         return false;
     }

     foreach ($upairs as $guid => $uid) {
         $write_result = fwrite($fp_pairs_output, "{$guid}\t{$uid}\n");

         if (false === $write_result) {
             Logger:err(
                 sprintf(
                     'failed to write a line into file[%s]',
                     $output_pairs_file
                 )
             );

             return false;
         }
     }

     fclose($fp_pairs_output);
     return true;
 }

getNewGuidUidPairsNPartitionRawDataTask