scala爬取指定地点的所有列车班次

需求介绍：

爬取指定地点的所有全国相关的列车班次详情。将结果写进mysql。

步骤及所遇到的问题：

1.寻取全国站点静态信息   https://kyfw.12306.cn/otn/resources/js/framework/station_name.js?station_version=1.9002 一个静态的js文件

2.借助web工具，本人谷歌浏览器F12在12306相关页面找取所需要的接口

3.寻找规律，注意去重写进mysql

直接上代码，看注释：

object TrainSchedulesMain {
  def main(args: Array[String]): Unit = {
    //    val dateStr = "2019-01-07"
    if (args == null || args.length < 2) {
      System.err.println("args is null or missing")
      System.exit(1)
    }
    val dateStrList = args(0).trim
    val station = args(1).trim

    assert(StringUtils.isNotBlank(dateStrList), "dateStrList is null or empty")
    assert(StringUtils.isNotBlank(station), "station is null or empty")

    // 打印参数
    println(args.mkString("  "))
    //获取全国所有的站点信息
    val allStationsMap = analysisAllStations()
    /*
    * 1.深圳
    * 2.深圳西
    * 3.深圳东
    * 4.深圳坪山
    * 5.深圳北
    * 6.福田
    */
    //    val fromStations = collection.mutable.ListBuffer("深圳", "深圳西", "深圳东", "深圳坪山", "深圳北", "福田")
    var dateStr = "";
    //出发站集合
    val fromStations = collection.mutable.ListBuffer(station.trim)
    dateStrList.split(",").foreach(date => {
      dateStr = date
      fromStations.foreach(r => {
        var n = 0
        //出发站
        val fromStationRequest = allStationsMap.get(r).get
        allStationsMap.foreach(d => {
          val toStationRequest = d._2 //train code
          //根据出发站和到达站请求12306
          excuteAnaly(fromStationRequest, toStationRequest)
          println(dateStr)
          println("进度：" + r)
          n += 1
          println(n)
        })

      })
    })

    def excuteAnaly(fromStationRequest: String, toStationRequest: String) = {
      val url1 =
        s"""https://kyfw.12306.cn/otn/leftTicket/queryZ?leftTicketDTO.train_date=${dateStr}&leftTicketDTO.from_station=${fromStationRequest}&leftTicketDTO.to_station=${toStationRequest}&purpose_codes=ADULT"""
          .stripMargin
      println("url1: " + url1)
      //获取所查询的 出发站-》到达站 的所有班次列车信息
      val responseStr = HttpRequest.sendGet(url1)
      println("url1Res: " + responseStr)
      if (StringUtils.isNotBlank(responseStr) && JSONUtil.isJson(responseStr)) {
        val allStationModel = JSONUtil.toJavaBean(responseStr, new AllStationTimes().getClass)
        //表示初始站为深圳  目的地为 toStationRequest 是有值
        if (allStationModel != null && allStationModel.getHttpstatus == 200 && allStationModel.getData.getResult != null && allStationModel.getData.getResult.size() > 0) {
          val resList = allStationModel.getData.getResult
          for (item <- 0 until resList.size()) {
            val str = resList.get(item)
            val indexNumStart = str.indexOf("预订")
            if (indexNumStart > -1) {
              val arrs = str.substring(indexNumStart).split("\\|")

              val trainNo = arrs(1)
              val trainCode = arrs(2)
              val startStation = arrs(3)
              val endStation = arrs(4)
              val fromStation = arrs(5)
              val toStation = arrs(6)
              // 并且目的地站必须是终点站才会当做一条 班次 写入msyql[否则是过站，导致重复写入mysql.并且站点还不全]
              if (toStation.trim.equals(endStation.trim)) {
                val url2 =s"""https://kyfw.12306.cn/otn/czxx/queryByTrainNo?train_no=${trainNo}&from_station_telecode=${fromStation}&to_station_telecode=${toStation}&depart_date=${dateStr}"""
                println("url2: " + url2)
                //查询上一步所有班次的详细各个站点顺序信息
                val res = HttpRequest.sendGet(url2)
                println("url2Res: " + res)
                if (StringUtils.isNotBlank(res) && JSONUtil.isJson(res)) {
                  val trainSchedulesModel = JSONUtil.toJavaBean(res, new TrainSchedules().getClass)
                  if (trainSchedulesModel != null && trainSchedulesModel.getData.getData.size() > 0) {
                    val data0 = trainSchedulesModel.getData.getData.get(0)

                    if (!MysqlHandleUtil(MysqlConnect.trainDB).isHasValue(
                      s"""select *
                         |
                         |from train_schedules
                         |where train_code='${data0.getStation_train_code}'
                         |and start_station_name='${data0.getStart_station_name}'
                         |and end_station_name='${data0.getEnd_station_name}'
                         |and start_time='${data0.getStart_time}'
                         |and dates='$dateStr'
                         |"""
                        .stripMargin)) {
                      val sql =
                        s"""insert into
                           |train_schedules(`train_code`,`start_station_name`,`end_station_name`,`start_time`,`arrive_time`,`dates`,`data`)
                           |values('${data0.getStation_train_code}','${data0.getStart_station_name}','${data0.getEnd_station_name}','${data0.getStart_time}','${trainSchedulesModel.getData.getData.get(trainSchedulesModel.getData.getData.size() - 1).getArrive_time}','${dateStr}','${JSONUtil.toJsonString(trainSchedulesModel.getData.getData)}')"""
                          .stripMargin
                      MysqlHandleUtil(MysqlConnect.trainDB).insertData(sql)
                    }
                  }
                }
              }
            }
          }
        }
      }
    }
  }

  /**
    * 解析全国所有站点（中文名字，编码）
    *
    * @author XXXX
    * @date 17:08
    * @param []
 * @return scala.collection.mutable.HashMap<java.lang.String,java.lang.String>
    */
  def analysisAllStations(): collection.mutable.HashMap[String, String] = {
    val hashMap = new mutable.HashMap[String, String]()
    val url = "https://kyfw.12306.cn/otn/resources/js/framework/station_name.js?station_version=1.9002"
    val response = HttpRequest.sendGet(url)
    assert(StringUtils.isNotBlank(response), "response is null or empty")
    val arrs = response.split("@")
    val len = arrs.length
    for (item <- 1 until len) {
      val ars = arrs(item).split("\\|")
      hashMap.put(ars(1), ars(2))
    }
    hashMap
  }
}

　　

效果：