Python的scrapy之爬取51job网站的职位

今天老师讲解了Python中的爬虫框架--scrapy，然后带领我们做了一个小爬虫--爬取51job网的职位信息，并且保存到数据库中

用的是Python3.6  pycharm编辑器

爬虫主体：

import scrapy
from ..items import JobspidersItem

class JobsspiderSpider(scrapy.Spider):
    name = 'jobsspider'
    #allowed_domains = ['search.51job.com/list/010000,000000,0000,00,9,99,%2520,2,1.html']
    #start_urls = ['https://search.51job.com/list/010000,000000,0000,00,9,99,%2520,2,1.html/']
    start_urls = [
        'https://search.51job.com/list/010000,000000,0000,01,9,99,python,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=']

    def parse(self, response):
        currentPageItems = response.xpath('/html/body/div[@class="dw_wp"]/div[@class="dw_table"]/div[@class="el"]')
        print(currentPageItems)

        # currentPageItems = response.xpath('//div[@class="el"]')
        for jobItem in currentPageItems:
            print('----',jobItem)
            jobspidersItem = JobspidersItem()

            jobPosition = jobItem.xpath('p[@class="t1 "]/span/a/text()').extract()
            if jobPosition:
                #print(jobPosition[0].strip())
                jobspidersItem['jobPosition'] = jobPosition[0].strip()

            jobCompany = jobItem.xpath('span[@class="t2"]/a/text()').extract()
            if jobCompany:
                #print(jobCompany[0].strip())
                jobspidersItem['jobCompany'] = jobCompany[0].strip()

            jobArea = jobItem.xpath('span[@class="t3"]/text()').extract()
            if jobArea:
                #print(jobArea[0].strip())
                jobspidersItem['jobArea'] = jobArea[0].strip()

            jobSale = jobItem.xpath('span[@class="t4"]/text()').extract()
            if jobSale:
                # print(jobCompany[0].strip())
                jobspidersItem['jobSale'] = jobSale[0].strip()

            jobDate = jobItem.xpath('span[@class="t5"]/text()').extract()
            if jobDate:
                # print(jobCompany[0].strip())
                jobspidersItem['jobDate'] = jobDate[0].strip()

            yield jobspidersItem  # 通过yield 调用输出管道
            pass
        nextPageURL = response.xpath('//li[@class="bk"]/a/@href').extract()  # 取下一页的地址
        print(nextPageURL)
        if nextPageURL:
            url = response.urljoin(nextPageURL[-1])
            print('url', url)
            # 发送下一页请求并调用parse()函数继续解析
            yield scrapy.Request(url, self.parse, dont_filter=False)
            pass
        else:
            print("退出")
        pass

items.py   设置五个items

import scrapy

class JobspidersItem(scrapy.Item):
    # define the fields for your item here like:
    jobPosition = scrapy.Field()
    jobCompany = scrapy.Field()
    jobArea = scrapy.Field()
    jobSale = scrapy.Field()
    jobDate = scrapy.Field()
    pass

pipelines.py  输出管道

class JobspidersPipeline(object):
    def process_item(self, item, spider):
        print('职位:', item['jobPosition'])
        print('公司:', item['jobCompany'])
        print('工作地点:', item['jobArea'])
        print('薪资:', item['jobSale'])
        print('发布时间:', item['jobDate'])
        print('----------------------------')
        return item

pipelinesmysql.py  输出到mysql中  第一行的意思是使用了以前封装的数据库操作类

from week5_day04.dbutil import dbutil

# 作业： 自定义的管道，将完整的爬取数据，保存到MySql数据库中
class JobspidersPipeline(object):
    def process_item(self, item, spider):
        dbu = dbutil.MYSQLdbUtil()
        dbu.getConnection()  # 开启事物

        # 1.添加
        try:
            #sql = "insert into jobs (职位名,公司名,工作地点,薪资,发布时间)values(%s,%s,%s,%s,%s)"
            sql = "insert into t_job (jobname,jobcompany,jobarea,jobsale,jobdata)values(%s,%s,%s,%s,%s)"
            #date = []
            #dbu.execute(sql, date, True)
            dbu.execute(sql, (item['jobPosition'],item['jobCompany'],item['jobArea'],item['jobSale'],item['jobDate']),True)
            #dbu.execute(sql,True)
            dbu.commit()
            print('插入数据库成功！！')
        except:
            dbu.rollback()
            dbu.commit()  # 回滚后要提交
        finally:
            dbu.close()
        return item

最终结果：

通过这个最基础的51job爬虫，进入到scrapy框架的学习中，这东西挺好使