Xpath解析库的使用

### Xpath常用规则
## nodename     选取此节点的所有子节点
## /            从当前节点选取直接子节点
## //           从当前节点选取子孙节点
## .            选取当前节点
## ..           选取当前节点的父节点
## @            选取属性

### 测试文本
text = '''
<ul id="dmr" name="liebiao">
<li data-closeper="" aria-label="查看更多" role="menuitem" aria-haspopup="true" data-groupid="104" class="J_Cat a-all">
<a data-cid="1" data-dataid="222878" >家电</a>
<a data-cid="1" data-dataid="222908" >数码</a>
<a data-cid="1" data-dataid="222879" >手机

<i aria-hidden="true" class="tb-ifont service-arrow"></i>
</li>
</ul>
'''

1. etree示例引入

## etree示例引入
from lxml import etree

# 构造一个etree的HTML节点对象(可供Xpath解析)
html = etree.HTML(text)
# 读取text文本内容进行构造节点对象
html2 = etree.parse('./text', etree.HTMLParser())
# 用tostring方法可以修正html代码，如上面代码缺失的</a>标签
result = etree.tostring(html)
result2 = etree.tostring(html2)
print(html, html2)
print(type(html), type(html2))
'''
输出内容：
<Element html at 0x2b47848> <lxml.etree._ElementTree object at 0x0000000002B47788>
<class 'lxml.etree._Element'> <class 'lxml.etree._ElementTree'>
'''
# 输出修正后的html代码
print(result.decode('utf-8'))
print(result2.decode('utf-8'))

2. 提取页面下的所有节点

## 提取页面下的所有节点
from lxml import etree

html = etree.HTML(text)
result = html.xpath('//*')
print(len(result))
print(result)

'''
输出结果：
8
[<Element html at 0x2b539c8>, <Element body at 0x2b53948>, <Element ul at 0x2b53a08>, <Element li at 0x2b53a48>, <Element a at 0x2b53a88>, <Element a at 0x2b53b08>, <Element a at 0x2b53b48>, <Element i at 0x2b53b88>]
'''

3. 提取子节点

## 提取子节点
from lxml import etree

html = etree.parse('./text', etree.HTMLParser())
# 通过/寻找li标签下的直接a子节点
result = html.xpath('//li/a')
# 通过//寻找ul标签下的a子和孙节点
result2 = html.xpath('//ul//a')
print(len(result), len(result2))
print(result, result2)

'''
运行结果：
3 3
[<Element a at 0x2963cc8>, <Element a at 0x2963d08>, <Element a at 0x2963d48>] [<Element a at 0x2963cc8>, <Element a at 0x2963d08>, <Element a at 0x2963d48>]
'''

4. 提取父节点

## 提取父节点
from lxml import etree

html = etree.HTML(text)
# 提取li节点中role属性为menuitem的节点的父节点的name属性内容
result = html.xpath('//li[@role="menuitem"]/../@name')
print(result)

'''
输出结果：
['liebiao']
'''

5. 属性匹配

## 属性匹配
html = etree.HTML(text)
# 匹配data-dataid为222878的节点
result = html.xpath('//a[@data-dataid="222878"]')
print(result)

'''
输出内容：
[<Element a at 0x2973c48>]
'''

6. 提取文本内容

## 提取文本内容
html = etree.HTML(text)
# 匹配data-dataid为222878的节点的文本内容
result = html.xpath('//a[@data-dataid="222878"]/text()')
print(result)

'''
输出内容：
['家电']
'''

7. 属性值获取

## 属性获取
from lxml import etree

html = etree.HTML(text)
result = html.xpath('//li/@aria-label')
print(result)

'''
输出内容：
['查看更多']
'''

8. 属性多值匹配

## 属性多值匹配
from lxml import etree

html = etree.HTML(text)
result = html.xpath('//li[@class="J_Cat"]')
result2 = html.xpath('//li[@class="J_Cat a-all"]//text()')
result3 = html.xpath('//li[contains(@class, "J_Cat")]//text()')
print(result, result2, result3)

'''
输出结果：
[] ['\n', '家电', '\n', '数码', '\n', '手机\n\n', '\ue62e', '\n'] ['\n', '家电', '\n', '数码', '\n', '手机\n\n', '\ue62e', '\n']
'''

9. 多属性匹配

## 多属性匹配
## 运算符介绍
# or        或
# and       与
# mod       除余
# |         返回节点集合
# +         加法
# -         减法
# *         乘法
# =         等于
# !=        不等于
# <         小于
# <=        小于或等于
# >         大于
# >=        大于或等于
from lxml import etree

html = etree.HTML(text)
result = html.xpath('//li[contains(@class, "J_Cat") and @role="menuitem"]/a/text()')
print(result)

'''
输出结果：
['家电', '数码', '手机\n\n', '\n']
'''

10. 按序选择，通过索引的方式进行选择

## 按序选择，通过索引的方式进行选择
from lxml import etree

html = etree.HTML(text)
# 提取li节点下第一个a节点的文本内容
print(html.xpath('//li/a[1]/text()'))
# 提取li节点下最后一个a节点的文本内容
print(html.xpath('//li/a[last()]/text()'))
# 提取li节点下位置小于3的a节点的文本内容
print(html.xpath('//li/a[position()<3]/text()'))
# 提取li节点下倒数第2个a节点的文本内容
print(html.xpath('//li/a[last()-1]/text()'))

'''
输出结果：
['手机\n\n', '\n']
['家电', '数码']
['数码']
'''

11. 节点轴选择

## 节点轴选择
# ancestor轴，可以节点获取所有的祖先节点
# attribute轴，可以获取节点的所有属性值
# child轴，可以获取节点的所有直接子节点
# descendant轴，可以获取节点的所有子孙节点
# following轴，可以获取节点后的所有节点
# following-sibling，可以获取当前节点的所有同级节点
from lxml import etree

html = etree.HTML(text)
print(html.xpath('//li/a[1]/ancestor::*'))
print(html.xpath('//li/a[1]/ancestor::ul'))
print(html.xpath('//li/a[1]/attribute::*'))
print(html.xpath('//li[1]/child::*'))
print(html.xpath('//ul[1]/descendant::a'))
print(html.xpath('//a[1]/following::*'))
print(html.xpath('//a[1]/following-sibling::*'))

'''
输出结果：
[<Element html at 0x2b53b88>, <Element body at 0x2b53b48>, <Element ul at 0x2b53d88>, <Element li at 0x2b53bc8>]
[<Element ul at 0x2b53b48>]
['1', '222878']
[<Element a at 0x2b53b48>, <Element a at 0x2b53d88>, <Element a at 0x2b53bc8>]
[<Element a at 0x2b53b48>, <Element a at 0x2b53d88>, <Element a at 0x2b53bc8>]
[<Element a at 0x2b53b48>, <Element a at 0x2b53d88>, <Element i at 0x2b53bc8>]
[<Element a at 0x2b53d88>, <Element a at 0x2b53bc8>]
'''

12. 用Xpath解析爬取豆瓣top250

### 用Xpath解析爬取豆瓣top250

from lxml import etree
import requests, json

def get_page(url):
    '''
    获取url网页代码
    :param url: 要爬取的网址
    :return: 网页代码
    '''

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36'
    }
    response = requests.get(url, headers=headers)
    if response.status_code == 200:
        print('get page success...')
        return response.text
    else:
        exit('get page fail...')

def parse_page(text):
    '''
    解析豆瓣电影top250网页代码
    :param html: 网页代码
    :return: data需要爬取的数据
    '''

    html = etree.HTML(text)
    items = html.xpath('//ol[@class="grid_view"]/li/div[@class="item"]')
    for item in items:
        #print(item.xpath('.//div[@class="bd"]/p[@class="quote"]/span/text()')[0])
        yield {
            'img': item.xpath('.//div[@class="pic"]//img/@src')[0],
            'details': item.xpath('.//div[@class="hd"]/a/@href')[0],
            'name': item.xpath('.//div[@class="hd"]//span[1]/text()')[0],
            'director': item.xpath('.//div[@class="bd"]/p[@class=""]/text()')[0].split()[1],
            'actor': item.xpath('.//div[@class="bd"]/p[@class=""]/text()')[0].split()[5] if len(item.xpath('.//div[@class="bd"]/p[@class=""]/text()')[0].split())>5 else 'None',
            'time': item.xpath('.//div[@class="bd"]/p[@class=""]/text()')[1].split()[0],
            'nation': item.xpath('.//div[@class="bd"]/p[@class=""]/text()')[1].split()[2],
            'type': item.xpath('.//div[@class="bd"]/p[@class=""]/text()')[1].split()[4:],
            'score': item.xpath('.//div[@class="bd"]/div/span[@class="rating_num"]/text()')[0],
            'introduction': item.xpath('.//div[@class="bd"]/p[@class="quote"]/span/text()') if item.xpath('.//div[@class="bd"]/p[@class="quote"]/span/text()') else 'None',
        }

    return items

def save_to_file(data):
    '''
    保存爬取到的数据到文本文件中
    :param data:
    :return:
    '''
    with open('豆瓣电影top250.txt', 'a', encoding='utf-8') as f:
        f.write(json.dumps(data, ensure_ascii=False) + '\n')

def main(start):
    url = 'https://movie.douban.com/top250?start=' + str(start)
    text = get_page(url)
    data = parse_page(text)
    for item in data:
        print(item)
        save_to_file(item)

if __name__ == '__main__':
    for i in range(10):
        start = i * 25
        main(start)