Python爬虫之Requests库的基本使用

 import requests
 response = requests.get('http://www.baidu.com/')
 print(type(response))
 print(response.status_code)
 print(type(response.text))
 print(response.text)
 print(response.cookies)

 # 各种请求方式
 import requests
 requests.post('http://httpbin.org/post')
 requests.put('http://httpbin.org/put')
 requests.delete('http://httpbin.org/delete')
 requests.head('http://httpbin.org/get')
 requests.options('http://httpbin.org/get')

 # 基本GET请求
 import requests
 response = requests.get('http://httpbin.org/get')
 print(response.text)

 # 带参数GET请求
 import requests
 response = requests.get('http://httpbin.org/get?name=germey&age=22')
 print(response.text)

 import requests
 data = {
     'name': 'germey',
     'age': 22
 }
 response = requests.get('http://httpbin.org/get', params = data)
 print(response.text)

 # 解析json
 import requests
 import json
 response = requests.get('http://httpbin.org/get')
 print(type(response.text))
 print(response.json())
 print(json.loads(response.text))
 print(type(response.json()))

 # 获取二进制数据
 import requests
 response = requests.get('http://github.com/favicon.ico')
 print(type(response.text), type(response.content))
 print(response.text)
 print(response.content)

 # 保存图片
 import requests
 response = requests.get('http://github.com/favicon.ico')
 with open('1.ico', 'wb') as f:
     f.write(response.content)
     f.close()

 # 添加headers 不添加的话会请求失败的
 import requests
 response = requests.get('http://www.zhihu.com/explore')
 print(response.text)

 import requests
 headers = {
     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36'
 }
 response = requests.get('http://zhihu.com/explore', headers = headers)
 print(response.text)

 # 基本的POST请求
 import requests
 data = {'name': 'germey', 'age': 22}
 response = requests.post('http://httpbin.org/post', data = data)
 print(response.text)

 import requests
 data = {'name':'germey', 'age':22}
 headers = {
     'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36'
 }
 response = requests.post('http://httpbin.org/post', data = data, headers = headers)
 print(response.json())

 # response属性
 import requests
 response = requests.get('http://www.jianshu.com')
 print(type(response.status_code), response.status_code)
 print(type(response.headers), response.headers)
 print(type(response.cookies), response.cookies)
 print(type(response.url), response.url)
 print(type(response.history), response.history)

 # 文件上传
 import requests
 files = {'file':open('1.ico', 'rb')}
 response = requests.post('http://httpbin.org/post', files = files)
 print(response.text)

 # 获取cookie
 import requests
 response = requests.get('http://www.baidu.com')
 print(response.cookies)
 for key, value in response.cookies.items():
     print(key + ' = ' + value)

 # 会话维持 模拟登陆(第一个例子，相当于在两个不同的浏览器请求页面，所以获取不到cookies,所以要用第二个session对象)
 import requests
 requests.get('http://httpbin.org/cookies/set/number/123456789')
 response = requests.get('http://httpbin.org/cookies')
 print(response.text)

 import requests
 s = requests.session()
 s.get('http://httpbin.org/cookies/set/number/123456789')
 response = s.get('http://httpbin.org/cookies')
 print(response.text)

 # 证书验证
 import requests
 response = requests.get('https://www.12306.cn')
 print(response.status_code)

 import requests
 from requests.exceptions import ConnectTimeout, HTTPError, ReadTimeout, RequestException
 from requests.auth import HTTPBasicAuth
 import urllib3

 # 证书验证
 # 消除警告（下面的HTTPS页面的请求如果不进行验证就会报警告）
 urllib3.disable_warnings()
 # verify=False 请求HTTPS页面的时候不进行证书验证，默认为True
 response = requests.get('https://www.12306.cn', verify=False)
 print(response.status_code)

 # 代理设置
 proxies = {
     "http": "http://127.0.0.1:9743",
     "https": "https://127.0.0.1:9743",
 }
 response = requests.get("https://www.taobao.com", proxies = proxies)
 print(response.status_code)

 # 代理有用户名和密码的情况
 proxies = {
     "http": "http://user:password@127.0.0.1:9743/",
 }
 response = requests.get("https://www.taobao.com", proxies = proxies)
 print(response.status_code)

 # socks代理
 proxies = {
     "http": "socks5://127.0.0.1:9742",
     "https": "socks5://127.0.0.1:9742",
 }
 response = requests.get("https://www.taobao.com", proxies = proxies)
 print(response.status_code)

 # 超时设置
 try:
     response = requests.get("http://httpbin.org/get", timeout=0.2)
     print(response.status_code)
 except ConnectTimeout:
     print("timeout!")

 # 认证设置 下面两种方法都可以
 response = requests.get("http://120.27.34.24:9001", auth=HTTPBasicAuth("user", ""))
 print(response.status_code)

 response = requests.get("http://120.27.34.24:9001", auth=("user", ""))
 print(response.status_code)

 # 异常处理
 try:
     response = requests.get("http://httpbin.org/get", timeout=0.2)
     print(response.status_code)
 except ConnectTimeout:
     print("timeout!")
 except RequestException:
     print("RequestException!")
 except HTTPError:
     print("HttpError!")
 except ReadTimeout:
     print("ReadTimeout")

 import requests
 response = requests.get("http://www.baidu.com")
 print(response.cookies)
 print("----------")
 # 把cookie对象转化为字典
 d = requests.utils.dict_from_cookiejar(response.cookies)
 print(d)
 print("----------")
 # 把字典转化为cookie对象
 print(requests.utils.cookiejar_from_dict(d))

 # url解码
 print(requests.utils.unquote("http://tieba.baidu.com/f?kw=%D2%D7%D3%EF%D1%D4&fr=ala0&tpl=5", encoding="gb18030"))
 # url编码
 print(requests.utils.quote("http://tieba.baidu.com/f?kw=%D2%D7%D3%EF%D1%D4&fr=ala0&tpl=5"))