Linux服务器性能日志收集和分析脚本(转)

最近老大要求分析服务器的性能数据，找到服务器运行的性能瓶颈，结果花了两天时间，写了两个脚本可以生成日志并可以进行数据提取，最终生成数据可以放到excel生成报表。过程中也学到了不少shell编程技术。

收集性能数据系统日志，每3秒收集一次，将脚本放到后台运行就行。

1. #!/bin/sh
2. while :
3. do
4. iostat -x -t >> /var/log/jciostat.log
5. vmstat -t -S M >> /var/log/jcvmstat.log
6. free -g >> /var/log/jcfree_g.log
7. top -b -n 1 |head -5 >> /var/log/jctop.log
8. sar -P ALL 1 1 | grep : | grep all | cut -d: -f2 >> /var/log/jccpu.log
9. sar -n DEV 1 1 | grep : | cut -d: -f2 >> /var/log/jcnetwork.log
10. if [ -f "/var/log/jciostat.log" ];then
11. if [ $(stat -c "%s" /var/log/jciostat.log) -gt $((100*1024*1024)) ];then
12. # file size is greater more than 200MB,clean file data
13. cd /var/log/ >/dev/null 2>&1
14. tar czvf jc.log.tar.gz jciostat.log jcvmstat.log jcfree_g.log jctop.log > /dev/null 2>&1
15. echo "" > /var/log/jciostat.log
16. echo "" > /var/log/jcvmstat.log
17. echo "" > /var/log/jcfree_g.log
18. echo "" > /var/log/jctop.log
19. cd - > /dev/null 2>&1
20. fi
21. fi
22. sleep 1
23. done

日志文件分析脚本

1. #!/bin/sh
2. print_help()
3. {
4. echo "use age: analyz.sh  -day <day> -start <start time> -end <end time> -<option1> <colum1,colum2...> -<option2> <colum1,colum2...> -<option3> <colum1,colum2...>"
5. echo "day: YYYY-MM-DD"
6. echo "start time:HH:MM:SS"
7. echo "end time:HH:MM:SS"
8. echo "                   1    2        3       4       5       6        7        8         9         10     11     12     13  14  15  16  17"
9. echo "-vmstat:           r    b        swpd    free    buff    cache    si       so        bi        bo     in     cs     us  sy  id  wa  st"
10. echo "-sda:                   rrqm/s   wrqm/s  r/s     w/s     rsec/s   wsec/s   avgrq-sz  avgqu-sz  await  svctm  %util"
11. echo "-sdb:                   rrqm/s   wrqm/s  r/s     w/s     rsec/s   wsec/s   avgrq-sz  avgqu-sz  await  svctm  %util"
12. echo "-network                rxpck/s  txpck/s rxkB/s  txkB/s  rxcmp/s  txcmp/s  rxmcst/s"
13. echo "-cpu                    us       ni      sy      wa      st       id"
14. echo "-mem:                   total    used    free    shared  buffers  cached"
15. echo "-swap:                  total    used    free"
16. echo "-la(load average): 5min 10min    15min"
17. echo "-network <netdev:[cloudbr0/bond0/eth0...]> <colum1,colum2...>"
18. echo "example:$0 -sda 1,2,3 -sdb 10,11,12 -network cloudbr0 2,3,4 -swap 3,4 -day 2016-07-08 -start 07:00:00 -end 08:00:00"
19. }
20. cp /var/log/jc*.log ./
21. day=""
22. start=""
23. end=""
24. vmstat=""
25. sda=""
26. sdb=""
27. mem=""
28. swap=""
29. la=""
30. cpu=""
31. network=""
32. netdev=""
33. while [ -n "$1" ]
34. do
35. case "$1" in
36. "-vmstat")
37. vmstat=$2
38. shift
39. ;;
40. "-sda")
41. sda=$2
42. shift
43. ;;
44. "-sdb")
45. sdb=$2
46. shift
47. ;;
48. "-mem")
49. mem=$2
50. shift
51. ;;
52. "-swap")
53. swap=$2
54. shift
55. ;;
56. "-la")
57. la=$2
58. shift
59. ;;
60. "-day")
61. day=$2
62. shift
63. ;;
64. "-start")
65. start=$2
66. shift
67. ;;
68. "-end")
69. end=$2
70. shift
71. ;;
72. "-cpu")
73. cpu=$2
74. shift
75. ;;
76. "-network")
77. netdev=$2
78. network=$3
79. shift
80. shift
81. ;;
82. "--help")
83. print_help
84. exit 0
85. ;;
86. *)
87. echo "$1 is not an option"
88. ;;
89. esac
90. shift
91. done
92. # 第一步：生成独立的日志csv文件
93. if [ ! -z $vmstat ];then
94. colum_name=("CST" "vmstat_r" "vmstat_b" "vmstat_swpd" "vmstat_free" "vmstat_buff" "vmstat_cache" "vmstat_si" "vmstat_so" "vmstat_bi" "vmstat_bo" "vmstat_in" "vmstat_cs" "vmstat_us" "vmstat_sy" "vmstat_id" "vmstat_wa" "vmstat_st")
95. OLD_IFS="$IFS"
96. IFS=","
97. colums=($vmstat)
98. IFS="$OLD_IFS"
99. o_colum=""
100. o_colum_name=""
101. for c in ${colums[@]}
102. do
103. if [ -z "${colum_name[$c]}" ] || [ $c -ge ${#colum_name[@]} ];then
104. continue
105. fi
106. o_colum=${o_colum}\$$c\",\"
107. o_colum_name=${o_colum_name}${colum_name[$c]},
108. done
109. o_colum=${o_colum%\"}
110. o_colum=${o_colum%,}
111. o_colum=${o_colum%\"}
112. o_colum_name=${o_colum_name%,}
113. echo $o_colum_name > vmstat.csv1
114. # 因为gawk '{print $o_colum}'引用$o_colum做输出控制，但是无法使用，只能转到临时脚本中再执行
115. echo '#!/bin/sh' > vmstat.sh
116. echo "grep ${colum_name[0]} jcvmstat.log | gawk '{print $o_colum}' >> vmstat.csv1" >> vmstat.sh
117. chmod u+x vmstat.sh
118. ./vmstat.sh
119. rm -rf vmstat.sh
120. fi
121. if [ ! -z $sda ];then
122. colum_name=("sda" "" "sda_rrqm/s" "sda_wrqm/s" "sda_r/s" "sda_w/s" "sda_rsec/s" "sda_wsec/s" "sda_avgrq-sz" "sda_avgqu-sz" "sda_await" "sda_svctm" "sda_%util")
123. OLD_IFS="$IFS"
124. IFS=","
125. colums=($sda)
126. IFS="$OLD_IFS"
127. o_colum=""
128. o_colum_name=""
129. for c in ${colums[@]}
130. do
131. if [ -z "${colum_name[$c]}" ] || [ $c -ge ${#colum_name[@]} ];then
132. continue
133. fi
134. o_colum=${o_colum}\$$c\",\"
135. o_colum_name=${o_colum_name}${colum_name[$c]},
136. done
137. o_colum=${o_colum%\"}
138. o_colum=${o_colum%,}
139. o_colum=${o_colum%\"}
140. o_colum_name=${o_colum_name%,}
141. echo $o_colum_name > sda_io.csv1
142. # 因为gawk '{print $o_colum}'引用$o_colum做输出控制，但是无法使用，只能转到临时脚本中再执行
143. echo '#!/bin/sh' > sda.sh
144. echo "grep ${colum_name[0]} jciostat.log | gawk '{print $o_colum}' >> sda_io.csv1" >> sda.sh
145. chmod u+x sda.sh
146. ./sda.sh
147. rm -rf sda.sh
148. fi
149. if [ ! -z $sdb ];then
150. colum_name=("sdb" "" "sdb_rrqm/s" "sdb_wrqm/s" "sdb_r/s" "sdb_w/s" "sdb_rsec/s" "sdb_wsec/s" "sdb_avgrq-sz" "sdb_avgqu-sz" "sdb_await" "sdb_svctm" "sdb_%util")
151. OLD_IFS="$IFS"
152. IFS=","
153. colums=($sdb)
154. IFS="$OLD_IFS"
155. o_colum=""
156. o_colum_name=""
157. for c in ${colums[@]}
158. do
159. if [ -z "${colum_name[$c]}" ] || [ $c -ge ${#colum_name[@]} ];then
160. continue
161. fi
162. o_colum=${o_colum}\$$c\",\"
163. o_colum_name=${o_colum_name}${colum_name[$c]},
164. done
165. o_colum=${o_colum%\"}
166. o_colum=${o_colum%,}
167. o_colum=${o_colum%\"}
168. o_colum_name=${o_colum_name%,}
169. echo $o_colum_name > sdb_io.csv1
170. # 因为gawk '{print $o_colum}'引用$o_colum做输出控制，但是无法使用，只能转到临时脚本中再执行
171. echo '#!/bin/sh' > sdb.sh
172. echo "grep ${colum_name[0]} jciostat.log | gawk '{print $o_colum}' >> sdb_io.csv1" >> sdb.sh
173. chmod u+x sdb.sh
174. ./sdb.sh
175. rm -rf sdb.sh
176. fi
177. if [ ! -z $mem ];then
178. colum_name=("Mem" "" "mem_total" "mem_used" "mem_free" "shared" "buffers" "cached")
179. OLD_IFS="$IFS"
180. IFS=","
181. colums=($mem)
182. IFS="$OLD_IFS"
183. o_colum=""
184. o_colum_name=""
185. for c in ${colums[@]}
186. do
187. if [ -z "${colum_name[$c]}" ] || [ $c -ge ${#colum_name[@]} ];then
188. continue
189. fi
190. o_colum=${o_colum}\$$c\",\"
191. o_colum_name=${o_colum_name}${colum_name[$c]},
192. done
193. o_colum=${o_colum%\"}
194. o_colum=${o_colum%,}
195. o_colum=${o_colum%\"}
196. o_colum_name=${o_colum_name%,}
197. echo $o_colum_name > mem_used.csv1
198. echo '#!/bin/sh' > mem.sh
199. echo "grep ${colum_name[0]} jcfree_g.log | gawk '{print $o_colum}' >> mem_used.csv1" >> mem.sh
200. chmod u+x mem.sh
201. ./mem.sh
202. rm -rf mem.sh
203. fi
204. if [ ! -z $swap ];then
205. colum_name=("Swap" "" "swap_total" "swap_used" "swap_free")
206. OLD_IFS="$IFS"
207. IFS=","
208. colums=($swap)
209. IFS="$OLD_IFS"
210. o_colum=""
211. o_colum_name=""
212. for c in ${colums[@]}
213. do
214. if [ -z "${colum_name[$c]}" ] || [ $c -ge ${#colum_name[@]} ];then
215. continue
216. fi
217. o_colum=${o_colum}\$$c\",\"
218. o_colum_name=${o_colum_name}${colum_name[$c]},
219. done
220. o_colum=${o_colum%\"}
221. o_colum=${o_colum%,}
222. o_colum=${o_colum%\"}
223. o_colum_name=${o_colum_name%,}
224. echo $o_colum_name > swap_used.csv1
225. echo '#!/bin/sh' > swap.sh
226. echo "grep ${colum_name[0]} jcfree_g.log | gawk '{print $o_colum}' >> swap_used.csv1" >> swap.sh
227. chmod u+x swap.sh
228. ./swap.sh
229. rm -rf swap.sh
230. fi
231. if [ ! -z $la ];then
232. colum_name=("load average" "load_5min" "load_10min" "load_15min")
233. OLD_IFS="$IFS"
234. IFS=","
235. colums=($la)
236. IFS="$OLD_IFS"
237. o_colum=""
238. o_colum_name=""
239. for c in ${colums[@]}
240. do
241. if [ -z "${colum_name[$c]}" ] || [ $c -ge ${#colum_name[@]} ];then
242. continue
243. fi
244. o_colum=${o_colum}\$$c
245. o_colum_name=${o_colum_name}${colum_name[$c]},
246. done
247. o_colum=${o_colum%\"}
248. o_colum=${o_colum%,}
249. o_colum=${o_colum%\"}
250. o_colum_name=${o_colum_name%,}
251. echo $o_colum_name > load.csv1
252. echo '#!/bin/sh' > la.sh
253. echo "grep \"${colum_name[0]}\" jctop.log | cut -d, -f3,4,5 | cut -d: -f2 | gawk '{print $o_colum}'>> load.csv1" >> la.sh
254. chmod u+x la.sh
255. ./la.sh
256. rm -rf la.sh
257. fi
258. if [ ! -z $cpu ];then
259. colum_name=("all" "" "us" "ni" "sy" "wa" "st" "id")
260. OLD_IFS="$IFS"
261. IFS=","
262. colums=($cpu)
263. IFS="$OLD_IFS"
264. o_colum=""
265. o_colum_name=""
266. for c in ${colums[@]}
267. do
268. if [ -z "${colum_name[$c]}" ] || [ $c -ge ${#colum_name[@]} ];then
269. continue
270. fi
271. o_colum=${o_colum}\$$c\",\"
272. o_colum_name=${o_colum_name}${colum_name[$c]},
273. done
274. o_colum=${o_colum%\"}
275. o_colum=${o_colum%,}
276. o_colum=${o_colum%\"}
277. o_colum_name=${o_colum_name%,}
278. echo $o_colum_name > cpu.csv1
279. echo '#!/bin/sh' > cpu.sh
280. echo "grep \"${colum_name[0]}\" jccpu.log | gawk '{print $o_colum}'>> cpu.csv1" >> cpu.sh
281. chmod u+x cpu.sh
282. ./cpu.sh
283. rm -rf cpu.sh
284. fi
285. if [ ! -z $network ];then
286. colum_name=("" "" "rxpck/s" "txpck/s" "rxkB/s" "txkB/s" "rxcmp/s" "txcmp/s" "rxmcst/s")
287. OLD_IFS="$IFS"
288. IFS=","
289. colums=($network)
290. IFS="$OLD_IFS"
291. o_colum=""
292. o_colum_name=""
293. for c in ${colums[@]}
294. do
295. if [ -z "${colum_name[$c]}" ] || [ $c -ge ${#colum_name[@]} ];then
296. continue
297. fi
298. o_colum=${o_colum}\$$c\",\"
299. o_colum_name=${o_colum_name}${colum_name[$c]}"_"${netdev},
300. done
301. o_colum=${o_colum%\"}
302. o_colum=${o_colum%,}
303. o_colum=${o_colum%\"}
304. o_colum_name=${o_colum_name%,}
305. echo $o_colum_name > network.csv1
306. echo '#!/bin/sh' > network.sh
307. echo "grep \"$netdev\" jcnetwork.log | gawk '{print $o_colum}'>> network.csv1" >> network.sh
308. chmod u+x network.sh
309. ./network.sh
310. rm -rf network.sh
311. fi
312. #输出时间
313. echo time > time.csv1
314. grep "CST" jcvmstat.log | gawk {'print $18"/"$19'} >> time.csv1
315. # 第二步：整合csv文件
316. i=0 # next csv file
317. j=0 # prev csv file
318. csv_files=`ls *.csv1|grep -v "time.csv1"`
319. for f in $csv_files
320. do
321. # 可能在行尾有逗号，删除这个逗号
322. sed -i 's/,$//g' $f
323. if [ $i -eq 0 ];then  # first
324. gawk 'NR==FNR{a[FNR]=$0;next}{print a[FNR]","$0;next}' time.csv1 $f > tmp$j.csv2
325. i=$(($i+1))
326. else # not first
327. gawk 'NR==FNR{a[FNR]=$0;next}{print a[FNR]","$0;next}' tmp$j.csv2 $f > tmp$i.csv2
328. i=$(($i+1))
329. j=$(($j+1))
330. fi
331. done
332. i=$(($i-1))
333. mv tmp$i.csv2  result.csv
334. sed -i 's/time/    /g' result.csv
335. #gawk 'NR==FNR{a[FNR]=$0;next}{print a[FNR]","$0;next}' time.csv swap_used.csv > tmp1.csv
336. #gawk 'NR==FNR{a[FNR]=$0;next}{print a[FNR]","$0;next}' tmp1.csv sda_used.csv > tmp2.csv
337. #gawk 'NR==FNR{a[FNR]=$0;next}{print a[FNR]","$0;next}' tmp2.csv sdb_used.csv > tmp3.csv
338. #gawk 'NR==FNR{a[FNR]=$0;next}{print a[FNR]","$0;next}' tmp3.csv load.csv > result.csv
339. #sed -i 's/time/    /g' result.csv
340. if [ ! -z $day ];then
341. date_str=`echo $day | grep -E '^[0-9]{4}-[0-9]{2}-[0-9]{2}'`
342. if [ ! -z "$date_str" ];then
343. head -1 result.csv > $date_str.csv
344. grep $date_str result.csv >> $date_str.csv
345. sed -i 's/ //g' $date_str.csv
346. if [ ! -z $start ] && [ ! -z $end ];then
347. st=`echo $start | grep -E '^[0-9]{2}:[0-9]{2}:[0-9]{2}'`
348. et=`echo $end | grep -E '^[0-9]{2}:[0-9]{2}:[0-9]{2}'`
349. if [ ! -z $st ] && [ ! -z $et ];then
350. stn=`echo $st|sed 's/://g'`
351. etn=`echo $et|sed 's/://g'`
352. filename=${date_str}-${stn}-${etn}.csv
353. head -1 $date_str.csv > $filename
354. lines=`cat $date_str.csv`
355. for line in $lines
356. do
357. ctn=`echo $line | cut -d',' -f1|cut -d'/' -f2|sed 's/://g'`
358. if [ `expr $ctn + 0` -gt `expr $stn + 0` ] && [ `expr $ctn + 0` -lt `expr $etn + 0` ];then
359. echo $line >> $filename
360. fi
361. done
362. else
363. echo "Time foramt error.Please input HH-MM-SS"
364. fi
365. fi
366. else
367. echo "Date foramt error.Please input YYYY-MM-DD"
368. fi
369. fi
370. rm -rf *.csv1
371. rm -rf *.csv2
372. rm -rf jc*.log

要生成 2016年7月8日 早上7点到8点之间内存的used和cache，swap的used和free，sda磁盘的%util 可以使用如下命令：

./analyz.sh -swap 3,4 -sda 12  -mem 3,7 -day 2016-07-08 -start 07:00:00 -end 08:00:00

将生成的csv文件用excel打开，就可以使用图表功能生成出性能曲线。