urllib库的使用

# coding=utf-8
import urllib2
import urllib # htpbin模拟的环境
URL_IP="http://10.11.0.215:8080"
URL_GET = "http://10.11.0.215:8080/get" def use_simple_urllib2():
response = urllib2.urlopen(URL_IP)
print '>>>> Response Headers:'
print response.info()
print '>>>>Response Body:'
print ''.join([line for line in response.readlines()]) def use_params_urllib2():
# 构建请求参数
params = urllib.urlencode({'param1': 'hello','param2': 'world'})
print 'Request Params:'
print params
# 发送请求
response = urllib2.urlopen('?'.join([URL_GET, '%s']) % params)
# 处理响应
print '>>>Response Headers:'
print response.info()
print '>>>Status code'
print response.getcode()
print '>>>Response Body'
print ''.join([line for line in response.readlines()])
# print response.readlines() if __name__ == '__main__':
# print '>>>Use simple urllib2'
# use_simple_urllib2()
print '>>>Use params urllib2'
use_params_urllib2()

requests库的简单使用

# coding=utf-8

import requests

URL_IP="http://10.11.0.215:8080/ip"
URL_GET="http://10.11.0.215:8080/get" def use_simple_requests():
response = requests.get(URL_IP)
print ">>>Response Headers:"
print response.headers
print ">>>Response Code:"
print response.status_code
print "Response Body:"
print response.text def use_params_requests():
response = requests.get(URL_GET)
print ">>>Response Headers:"
print response.headers
print ">>>Response Code:"
print response.status_code
print response.reason
print "Response Body:"
print response.json() if __name__ == "__main__":
# print "simple requests:"
# use_simple_requests()
print "params requests:"
use_params_requests()

requests和github api的互动

# coding=utf-8
import json
import requests
from requests import exceptions URL = "https://api.github.com" def build_uri(endpoint):
# 拼凑url为最终的api路径
return '/'.join([URL, endpoint]) def better_print(json_str):
# 格式化输出, indent=4是缩进为4个空格
return json.dumps(json.loads(json_str), indent = 4) def request_method():
# 获取用户信息
# response = requests.get(build_uri('users/reblue520'))
# response = requests.get(build_uri('user/emails'), auth=('reblue520', 'reblue520'))
response = requests.get(build_uri('user/public_emails'), auth=('reblue520', 'reblue520'))
print(better_print(response.text)) def params_request():
response = requests.get(build_uri('users'), params={'since':11})
print better_print(response.text)
print response.request.headers
print response.url def json_request():
# 更新用户信息,邮箱必须是已经验证过的邮箱
# response = requests.patch(build_uri('user'), auth=('reblue520','reblue520'),json={'name':'hellojack2019','email':'reblue520@163.com'})
response = requests.post(build_uri('user/emails'), auth=('reblue520','Reblue0225520'),json=['hellojack2019@163.com'])
print better_print(response.text)
print response.request.headers
print response.request.body
print response.status_code def timeout_request():
# api异常处理:超时
try:
response = requests.get(build_uri('user/emails'), timeout=10)
response.raise_for_status()
except exceptions.Timeout as e:
print e.message
except exceptions.HTTPError as e:
print e.message
else:
print response.status_code
print response.text def hard_requests():
# 自定义request
from requests import Request, Session
s = Session()
headers = {'User-Agent': 'fake1.3.4'}
req = Request('GET', build_uri('user/emails'), auth=('reblue520', 'Reblue0225520'), headers=headers)
prepped = req.prepare()
print prepped.body
print prepped.headers resp = s.send(prepped, timeout = 5)
print resp.status_code
print resp.request.headers
print resp.text if __name__ == '__main__':
# request_method()
# params_request()
# json_request()
# timeout_request()
hard_requests()

response响应的常用api

响应的基本API
In []: import requests In []: response = requests.get("https://api.github.com") In []: response.status_code
Out[]: In []: response.reason
Out[]: 'OK' In []: response.headers
Out[]: {'Date': 'Sat, 20 Jul 2019 03:48:51 GMT', 'Content-Type': 'application/json; charset=utf-8', 'Transfer-Encoding': 'chunked', 'Server': 'GitHub.com', 'Status': '200 OK', 'X-RateLimit-Limit': '', 'X-RateLimit-Remaining': '', 'X-RateLimit-Reset': '', 'Cache-Control': 'public, max-age=60, s-maxage=60', 'Vary': 'Accept, Accept-Encoding', 'ETag': 'W/"7dc470913f1fe9bb6c7355b50a0737bc"', 'X-GitHub-Media-Type': 'github.v3; format=json', 'Access-Control-Expose-Headers': 'ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type', 'Access-Control-Allow-Origin': '*', 'Strict-Transport-Security': 'max-age=31536000; includeSubdomains; preload', 'X-Frame-Options': 'deny', 'X-Content-Type-Options': 'nosniff', 'X-XSS-Protection': '1; mode=block', 'Referrer-Policy': 'origin-when-cross-origin, strict-origin-when-cross-origin', 'Content-Security-Policy': "default-src 'none'", 'Content-Encoding': 'gzip', 'X-GitHub-Request-Id': '33D9:591B:9D084B:CF860E:5D328F23'} In []: response.url
Out[]: 'https://api.github.com/' In []: response.history
Out[]: [] In []: response = requests.get("http://api.github.com") In []: response.history
Out[]: [<Response []>] In []: response = requests.get("https://api.github.com") In []: response.elapsed
Out[]: datetime.timedelta(microseconds=) In []: response.request
Out[]: <PreparedRequest [GET]> In []: response.request.headers
Out[]: {'User-Agent': 'python-requests/2.22.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'} In []: response.encoding
Out[]: 'utf-8' In []: response.raw.read()
Out[]: b'' In []: response.content
Out[]: b'{"current_user_url":"https://api.github.com/user","current_user_authorizations_html_url":"https://github.com/settings/connections/applications{/client_id}","authorizations_url":"https://api.github.com/authorizations","code_search_url":"https://api.github.com/search/code?q={query}{&page,per_page,sort,order}","commit_search_url":"https://api.github.com/search/commits?q={query}{&page,per_page,sort,order}","emails_url":"https://api.github.com/user/emails","emojis_url":"https://api.github.com/emojis","events_url":"https://api.github.com/events","feeds_url":"https://api.github.com/feeds","followers_url":"https://api.github.com/user/followers","following_url":"https://api.github.com/user/following{/target}","gists_url":"https://api.github.com/gists{/gist_id}","hub_url":"https://api.github.com/hub","issue_search_url":"https://api.github.com/search/issues?q={query}{&page,per_page,sort,order}","issues_url":"https://api.github.com/issues","keys_url":"https://api.github.com/user/keys","notifications_url":"https://api.github.com/notifications","organization_repositories_url":"https://api.github.com/orgs/{org}/repos{?type,page,per_page,sort}","organization_url":"https://api.github.com/orgs/{org}","public_gists_url":"https://api.github.com/gists/public","rate_limit_url":"https://api.github.com/rate_limit","repository_url":"https://api.github.com/repos/{owner}/{repo}","repository_search_url":"https://api.github.com/search/repositories?q={query}{&page,per_page,sort,order}","current_user_repositories_url":"https://api.github.com/user/repos{?type,page,per_page,sort}","starred_url":"https://api.github.com/user/starred{/owner}{/repo}","starred_gists_url":"https://api.github.com/gists/starred","team_url":"https://api.github.com/teams","user_url":"https://api.github.com/users/{user}","user_organizations_url":"https://api.github.com/user/orgs","user_repositories_url":"https://api.github.com/users/{user}/repos{?type,page,per_page,sort}","user_search_url":"https://api.github.com/search/users?q={query}{&page,per_page,sort,order}"}' In []: response.json()
Out[]:
{'current_user_url': 'https://api.github.com/user',
'current_user_authorizations_html_url': 'https://github.com/settings/connections/applications{/client_id}',
'authorizations_url': 'https://api.github.com/authorizations',
'code_search_url': 'https://api.github.com/search/code?q={query}{&page,per_page,sort,order}',
'commit_search_url': 'https://api.github.com/search/commits?q={query}{&page,per_page,sort,order}',
'emails_url': 'https://api.github.com/user/emails',
'emojis_url': 'https://api.github.com/emojis',
'events_url': 'https://api.github.com/events',
'feeds_url': 'https://api.github.com/feeds',
'followers_url': 'https://api.github.com/user/followers',
'following_url': 'https://api.github.com/user/following{/target}',
'gists_url': 'https://api.github.com/gists{/gist_id}',
'hub_url': 'https://api.github.com/hub',
'issue_search_url': 'https://api.github.com/search/issues?q={query}{&page,per_page,sort,order}',
'issues_url': 'https://api.github.com/issues',
'keys_url': 'https://api.github.com/user/keys',
'notifications_url': 'https://api.github.com/notifications',
'organization_repositories_url': 'https://api.github.com/orgs/{org}/repos{?type,page,per_page,sort}',
'organization_url': 'https://api.github.com/orgs/{org}',
'public_gists_url': 'https://api.github.com/gists/public',
'rate_limit_url': 'https://api.github.com/rate_limit',
'repository_url': 'https://api.github.com/repos/{owner}/{repo}',
'repository_search_url': 'https://api.github.com/search/repositories?q={query}{&page,per_page,sort,order}',
'current_user_repositories_url': 'https://api.github.com/user/repos{?type,page,per_page,sort}',
'starred_url': 'https://api.github.com/user/starred{/owner}{/repo}',
'starred_gists_url': 'https://api.github.com/gists/starred',
'team_url': 'https://api.github.com/teams',
'user_url': 'https://api.github.com/users/{user}',
'user_organizations_url': 'https://api.github.com/user/orgs',
'user_repositories_url': 'https://api.github.com/users/{user}/repos{?type,page,per_page,sort}',
'user_search_url': 'https://api.github.com/search/users?q={query}{&page,per_page,sort,order}'}

从0开始学爬虫10之urllib和requests库与github/api的交互的更多相关文章

  1. 从0开始学爬虫11之使用requests库下载图片

    从0开始学爬虫11之使用requests库下载图片 # coding=utf-8 import requests def download_imgage(): ''' demo: 下载图片 ''' h ...

  2. 从0开始学爬虫8使用requests/pymysql和beautifulsoup4爬取维基百科词条链接并存入数据库

    从0开始学爬虫8使用requests和beautifulsoup4爬取维基百科词条链接并存入数据库 Python使用requests和beautifulsoup4爬取维基百科词条链接并存入数据库 参考 ...

  3. 从0开始学爬虫12之使用requests库基本认证

    从0开始学爬虫12之使用requests库基本认证 此处我们使用github的token进行简单测试验证 # coding=utf-8 import requests BASE_URL = " ...

  4. 从0开始学爬虫9之requests库的学习之环境搭建

    从0开始学爬虫9之requests库的学习之环境搭建 Requests库的环境搭建 环境:python2.7.9版本 参考文档:http://2.python-requests.org/zh_CN/l ...

  5. 从0开始学爬虫4之requests基础知识

    从0开始学爬虫4之requests基础知识 安装requestspip install requests get请求:可以用浏览器直接访问请求可以携带参数,但是又长度限制请求参数直接放在URL后面 P ...

  6. 从0开始学爬虫3之xpath的介绍和使用

    从0开始学爬虫3之xpath的介绍和使用 Xpath:一种HTML和XML的查询语言,它能在XML和HTML的树状结构中寻找节点 安装xpath: pip install lxml HTML 超文本标 ...

  7. 从0开始学爬虫2之json的介绍和使用

    从0开始学爬虫2之json的介绍和使用 Json 一种轻量级的数据交换格式,通用,跨平台 键值对的集合,值的有序列表 类似于python中的dict Json中的键值如果是字符串一定要用双引号 jso ...

  8. Python使用urllib,urllib3,requests库+beautifulsoup爬取网页

    Python使用urllib/urllib3/requests库+beautifulsoup爬取网页 urllib urllib3 requests 笔者在爬取时遇到的问题 1.结果不全 2.'抓取失 ...

  9. urllib和requests库

    目录 1. Python3 使用urllib库请求网络 1.1 基于urllib库的GET请求 1.2 使用User-Agent伪装后请求网站 1.3 基于urllib库的POST请求,并用Cooki ...

随机推荐

  1. generator 和yeild

    参考 https://www.cnblogs.com/xybaby/p/6322376.html 使用yield那么该函数就变成了一个生成器方法. def zheng(n,m): if n<m: ...

  2. 转 C#关于DateTime得到的当前时间的格式和用法

    DateTime.Now.ToShortTimeString() DateTime dt = DateTime.Now; dt.ToString();//2005-11-5 13:21:25 dt.T ...

  3. P2261 [CQOI2007]余数求和[整除分块]

    题目大意 给出正整数 n 和 k 计算 \(G(n, k)=k\ \bmod\ 1 + k\ \bmod\ 2 + k\ \bmod\ 3 + \cdots + k\ \bmod\ n\) 的值 其中 ...

  4. linux 中截取字符串

    shell中截取字符串的方法有很多中,${expression}一共有9种使用方法.${parameter:-word}${parameter:=word}${parameter:?word}${pa ...

  5. 通过php安装Imagick扩展给动态gif图片打水印

    通过php安装Imagick扩展给动态gif图片打水印 一直以来php处理图片都是以gd为主流,直到近些年Imagick的使用才渐渐变多. gd通常用来缩放图片,给图片打水印等基本功能,对于复杂效果如 ...

  6. linux下Boost序列化问题解决

    由于项目需要,要使用boost,所以在网上找了一些例子帮助理解,其中遇到很多问题,再次总结记录一下.#include <boost/archive/text_oarchive.hpp> # ...

  7. 学习Spring-Data-Jpa(三)---关联关系常用注解

    1.@JoinColumn: 用于指定 连接实体关联或元素集合的列. 属性 name: 外键列的名称,它所在的表取决于上下文. 如果连接是使用外键映射策略的一对一或多对一映射,则外键列位于源实体或可嵌 ...

  8. java之比较器

    java中的比较器有两种: 1.实体类实现Comparable接口,并实现其中的compareTo方法 2.在外部定义实现Comparator接口的比较器类,并实现其中的compare方法 Compa ...

  9. hello world 程序的生成过程

    一个c / c ++文件需要经过预先(预处理),编译(编译),编译(汇编)和链接(链接)等四步,才能生成可执行程序. 在日常编译中,通常“编译”统称这四步: gcc -c xxx .s:汇编 gcc ...

  10. vue组件传值的三种方式,文字版解释

    父传子: 当子组件子父组件中当标签使用的时候,给子组件添加一个自定义属性,值为需要传递的值(如: <Child v-bind:parentToChild="parentMsg" ...