urllib

urllib

Python 标准库 urllib2 的使用细节

python 2.x

3.x

from urllib import request
 
with request.urlopen('https://api.douban.com/v2/book/2129650') as f:
    data = f.read()
    print('Status:', f.status, f.reason)
    for k, v in f.getheaders():
        print('%s: %s' % (k, v))
    print('Data:', data.decode('utf-8'))

模拟浏览器

from urllib import request
 
req = request.Request('http://www.douban.com/')
req.add_header('User-Agent', 'Mozilla/6.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/8.0 Mobile/10A5376e Safari/8536.25')
with request.urlopen(req) as f:
    print('Status:', f.status, f.reason)
    for k, v in f.getheaders():
        print('%s: %s' % (k, v))
    print('Data:', f.read().decode('utf-8'))

兼容Python2.x 3.x

#!/usr/bin/python
#-*-coding:utf-8-*- 
 
import json
import os
import re
 
try:
    from urllib.parse import urlencode
    from urllib.request import Request, urlopen
except:
    from urllib import urlencode
    from urllib2 import Request, urlopen
 
def get_token():
    req = Request("%s/authorizations/user_token/%s" % (os.environ.get('JPY_HUB_API_URL'), os.environ.get('JPY_USER')))
    req.add_header('Authorization', "token %s" % os.environ.get('JPY_HUB_API_TOKEN'))
    req.add_header("Host", "192.168.176.100")
    req.add_header("Referer", "%s/authorizations/user_token" % os.environ.get('JPY_HUB_API_URL'))
 
    data_success = None
    data_error = None
 
    try:
        f = urlopen(req)
    except:
        print('fail to open')
        exit(-1)
 
    try:
        data_success = f.read()
    except:
        data_error = {"user_token": ""}
    finally:
        f.close()
 
    if not data_error is None:
        data = data_error
    else:
        data = json.loads(data_success.decode('utf-8'))
 
    return data["user_token"]
 
def validate_year(v):
    pattern = re.compile(r'^\d{4}$')
 
    match = pattern.match(str(v))
 
    if match:
        return True
    else:
        return False
 
def get_data(url):
    req = Request(url)
    req.add_header('Authorization', "Bearer %s" % get_token())
    data_success = None
    data_error = None
 
    try:
        f = urlopen(req)
    except:
        print('fail to open')
        exit(-1)
 
    try:
        data_success = f.read()
    except:
        data_error = {"errmsg": "请求发生错误"}
    finally:
        f.close()
 
    if not data_error is None:
        data = data_error
    else:
        data = json.loads(data_success.decode('utf-8'))
 
    return data
 
def get_market_list():
    url = 'https://xxx.xx.xxx/quote/v1/market/list'
    #url = 'http://127.0.0.1:8000/json/market_list.json'
 
    return get_data(url)
 
def get_market_holiday(finance_mic, date=None):
    mic = ('O', 'N', 'A', 'HK', 'DCE', 'SHFE', 'SZ', 'SS', 'CZCE')
    if not finance_mic in mic:
        return {"errmsg": "参数错误"}
 
    if not date is None and \
        not validate_year(date):
        return {"errmsg": "参数错误"}
 
    search = urlencode([
        ('finance_mic', finance_mic),
        ('date', date)
    ])
 
    url = 'https://xxxx.xx.xxx/quote/v1/market/holiday?%s' % search
 
    return get_data(url)