python 日产经销商

 '''

 ajaxCallSiteInfo: {1C8B2BC6-35E2-460E-A63D-3576F3039D79}
 '''

 import requests
 import json
 from dbutil.pgsql import PgsqlPipeline
 from datetime import date

 headers = {
     'Cookie': 'ASP.NET_SessionId=citapgvmwpvq4qscezfwqjwr; NTKF_T2D_CLIENTID=guest8EE3B5EE-2EEE-0022-B9B6-F4D65DCFD295; nTalk_CACHE_DATA={uid:kf_9255_ISME9754_guest8EE3B5EE-2EEE-00,tid:1567475785166484}; sc_ext_session=kr5rhbkw4ldkmlstifvmj152; nissan#lang=zh-CN; SC_ANALYTICS_GLOBAL_COOKIE=5b20d0477cfd4924913d8cce649b25ab|True; sc_ext_contact=5b20d0477cfd4924913d8cce649b25ab|True; NO_PAGE_DURATION=2019/9/3 21:18:07; no_screen=1280%7C800; Place=%7B%22province%22%3A%22%E6%B2%B3%E5%8C%97%22%2C%22city%22%3A%22%E7%9F%B3%E5%AE%B6%E5%BA%84%22%7D',
     'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36',
 }

 #读取cities.json文件取出省，市
 def get_data():
     with open("./dbutil/cities.json", 'r') as load_f:
         data_list = json.load(load_f)['provinces']
         return data_list

 #post请求 获取json文件
 def get_Dealers(citysName):
     url = 'https://www.dongfeng-nissan.com.cn/Nissan/ajax/Distributor/GetJsonDistributorList'

     data = {'city': citysName}

     response = requests.post(url, headers=headers, data=data, verify=False)
     dealers = json.loads(response.text)['data']['DealerInfos']
     return dealers

 #解析json文件

 def get_item(dealer, province_Name, citysName):
     item = {
         "dealer_name": dealer["StoreName"],
         "brand_id": None,
         "address": dealer["Address"],
         "brand": "日产",
         "province": province_Name,
         "city": citysName,
         "sale_call": dealer["SaleTel"],
         "customer_service_call": dealer["ServiceTel"],
         "update_time": date.today(),
         "longitude": dealer['Longitude'],
         "latitude": dealer['Latitude'],
         "dealer_type": None,
         "manufacturer_id": None,
         "manufacturer": "东风日产",
         "state": None,
         "opening_date": None,
         "close_date": None,
         "dealer_id_web": None,
         "controlling_shareholder": None,
         "other_shareholders": None,
         "status": None,
         "remarks": None,

     }
     return item

 #实例化函数返回items
 def get_items():
     items = []
     data_list = get_data()
     for data in data_list:
         # print(data)
         province_Name = data['provinceName']
         citys = data['citys']
         for city in citys:
             citysName = city['citysName']
             dealers = get_Dealers(citysName)
             for dealer in dealers:
                 item = get_item(dealer, province_Name, citysName)
                 items.append(item)
     return items

 #主进程存储
 def main():
     print('爬虫开始--------->')
     items = get_items()
     print('数据存储')
     pg = PgsqlPipeline()
     for item in items:
         pg.process_item(item)
     pg.close()
     print('数据存储完成')

 if __name__ == '__main__':
     main()

 import psycopg2

 class PgsqlPipeline(object):
     def __init__(self):
         self.conn = psycopg2.connect(database="dealer", user="postgres", password="postgres", host="127.0.0.1", port="")
         self.cursor = self.conn.cursor()
     def process_item(self, item):
         insert_sql = f"""INSERT INTO "201905.tDealer"
         ("sDealerName","nBrandID","sBrand","sProvince","sCity","sAddress","sSaleCall","sCustomerServiceCall","sDealerType","nManufacturerID","sManufacturer","nState","dOpeningDate","dCloseDate","dUpdateTime","nDealerIDWeb","sLongitude","sLatitude","sControllingShareholder","sOtherShareholders","sStatus","sRemarks")
         VALUES 

            (%(dealer_name)s, %(brand_id)s, %(brand)s, %(province)s, %(city)s, %(address)s, %(sale_call)s, %(customer_service_call)s, %(dealer_type)s, %(manufacturer_id)s,%(manufacturer)s, %(state)s, %(opening_date)s, %(close_date)s, %(update_time)s, %(dealer_id_web)s, %(longitude)s, %(latitude)s, %(controlling_shareholder)s, %(other_shareholders)s, %(status)s, %(remarks)s)"""

         self.cursor.execute(insert_sql, item)
         self.conn.commit()
     def close(self):
         self.cursor.close()
         self.conn.close()
 if __name__ == '__main__':
     pgsql = PgsqlPipeline()
     print(pgsql)