python3实践-从网站获取数据(Carbon Market Data-BJ) （pandas，bs4）

自己边看边实践一些简单的实际应用，下面的程序是从某个网站上获取需要的数据。

在编写的过程中，通过学习陆续了解到一些方法，发现Python真的是很便捷。

尤其是用pandas获取网页中的表格数据，真的是太方便了！！！

程序写的可能并不好，但基本上实现了自己的需求。

希望有高手来指点下~~

Version 04 （Jan 12 2017）【对于获取表格信息，推荐使用该方法】

 # Code based on Python 3.x
 # _*_ coding: utf-8 _*_
 # __Author: "LEMON"

 import pandas as pd

 url2 = 'http://www.bjets.com.cn/article/jyxx/?'
 links = []
 for n in range(2, 40):
     # 页面总数为39页，需要自己先从网页判断，也可以从页面抓取，后续可以完善
     link = url2 + str(n)
     links.append(link)
 links.insert(0, url2)

 df2 = pd.DataFrame()  # creates a new dataframe that's empty
 for url in links:
     # 利用pandas获取数据，需要安装 html5lib模块
     dfs = pd.read_html(url, header=0)
     for df in dfs:
         df2= df2.append(df, ignore_index= True)

 # df2.to_excel('MktDataBJ.xlsx') # 将数据存储在excel文件里
 df2.to_csv('MktDataBJ-1.csv')  # 将数据存储在csv文件里

Version 03 （Jan 12 2017）

 # Code based on Python 3.x
 # _*_ coding: utf-8 _*_
 # __Author: "LEMON"

 from bs4 import BeautifulSoup
 import requests
 import csv

 url2 = 'http://www.bjets.com.cn/article/jyxx/?'
 links = []
 for n in range(2, 40):
     # 页面总数为39页，需要自己先从网页判断，也可以从页面抓取，后续可以完善
     link = url2 + str(n)
     links.append(link)
 links.insert(0, url2)

 for url in links:
     rep = requests.get(url)
     # content = rep.text.encode(rep.encoding).decode('utf-8')
     # # 直接用requests时，中文内容需要转码

     soup = BeautifulSoup(rep.content, 'html.parser')

     # table = soup.table
     table = soup.find('table')  # 两种方式都可以

     trs = table.find_all('tr')
     trs2 = trs[1:len(trs)]
     list1 = []
     for tr in trs2:
         td = tr.find_all('td')
         row = [i.text for i in td]
         list1.append(row)

     with open('MktDataBJ.csv', 'a', errors='ignore', newline='') as f:
         f_csv = csv.writer(f)
         f_csv.writerows(list1)

Version 02 （Jan 09 2017）

 # Code based on Python 3.x
 # _*_ coding: utf-8 _*_
 # __Author: "LEMON"

 from bs4 import BeautifulSoup
 import requests
 import csv

 url2 = 'http://www.bjets.com.cn/article/jyxx/?'
 links = []
 for n in range(2, 40):
     # 页面总数为39页，需要自己先从网页判断，也可以从页面抓取，后续可以完善
     link = url2 + str(n)
     links.append(link)
 links.insert(0, url2)
 # print(links)

 for url in links:
     rep = requests.get(url)
     # content = rep.text.encode(rep.encoding).decode('utf-8')
     # # 直接用requests时，中文内容需要转码

     soup = BeautifulSoup(rep.content, 'html.parser')
     body = soup.body
     data = body.find('div', {'class': 'list_right'})

     quotes = data.find_all('tr')
     quotes1 = quotes[1:len(quotes)]

     list1 = []
     for x in quotes1:
         list2 = []
         for y in x.find_all('td'):
             list2.append(y.text)  # 每日的数据做一个单独的list
         list1.append(list2)
     # print(list1)  # list1为每日数据的总列表
     with open('MktDataBJ.csv', 'a', errors='ignore', newline='') as f:
         f_csv = csv.writer(f)
         f_csv.writerows(list1)

Version 01 （Jan 08 2017）

 # Code based on Python 3.x
 # _*_ coding: utf-8 _*_
 # __Author: "LEMON"

 from bs4 import BeautifulSoup
 import requests
 import csv

 urllink = 'http://www.bjets.com.cn/article/jyxx/?'
 links = []
 for n in range(2, 40):
     #页面总数为39页，需要自己先从网页判断，也可以从页面抓取，后续可以完善
     link = urllink + str(n)
     links.append(link)
 links.insert(0, urllink)
 # print(links)

 for url in links:

     rep = requests.get(url)
     # content = rep.text.encode(rep.encoding).decode('utf-8')
     # # 直接用requests时，中文内容需要转码

     soup = BeautifulSoup(rep.content, 'html.parser')

     # print(soup.prettify())
     # # prettify()

     body = soup.body
     data = body.find('div', {'class': 'list_right'})

     # table title
     titles = data.find_all('th')

     title = []
     for x in titles:
         title.append(x.text)
     # print(title)

     quotes = data.find_all('tr')
     quotes1 = quotes[1:len(quotes)]
     # print(quotes1)

     list1 = []
     for x in quotes1:
         for y in x.find_all('td'):
             list1.append(y.text)
     # print(list1)  # list为每日数据的总列表

     date = []
     volumes = []
     meanprice = []
     totalmoney = []

     for i in range(0, len(list1)):
         if i % 4 == 0:
             date.append(list1[i])
         elif i % 4 == 1:
             volumes.append(list1[i])
         elif i % 4 == 2:
             meanprice.append(list1[i])
         else:
             totalmoney.append(list1[i])

     # print(date)
     # print(volumes)
     # print(meanprice)
     # print(totalmoney)

     final = []
     for i in range(0, len(date)):
         temp = [date[i], volumes[i], meanprice[i], totalmoney[i]]
         final.append(temp)
     # print(final)
     with open('bj_carbon.csv', 'a', errors='ignore', newline='') as f:
         f_csv = csv.writer(f)
         f_csv.writerows(final)