重写JdbcRDD支持Sql命名参数和分区

Spark提供的JdbcRDD很不好用，没法指定命名参数，而且必须要提供两个Long类型的参数表示分区的范围，如果数据表没有long类型的字段或者不需要条件，那就不能用JdbcRDD了。这里我简单重写了一个GenericJdbcRDD，支持命名参数和分区，仅仅作为参考哦。

项目结构如下：

GenericJdbcRDD.scala

package yay.jdbc.rdd

import java.sql.{Connection, ResultSet}

import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.rdd.{JdbcRDD, RDD}
import org.apache.spark.{Logging, Partition, SparkContext, TaskContext}

import scala.reflect.ClassTag

/**
 * Created by ${yuananyun} on 2015/2/7.
 */
private[yay] class JdbcPartition(idx: Int, val lower: Long, val upper: Long) extends Partition {
  override def index = idx
}

class GenericJdbcRDD[T: ClassTag](
                                   sc: SparkContext,
                                   getConnection: () => Connection,
                                   sql: String,
                                   paramsMap: Map[String, Object],
                                   skip: Long,
                                   take: Long,
                                   numPartitions: Int,
                                   mapRow: (ResultSet) => T = JdbcRDD.resultSetToObjectArray _) extends RDD[T](sc, Nil) with Logging {
  @DeveloperApi
  override def compute(thePart: Partition, context: TaskContext) = new JdbcNextIterator[T] {
    context.addTaskCompletionListener { context => closeIfNeeded()}
    val part = thePart.asInstanceOf[JdbcPartition]
    val conn = getConnection()
    //    val stmt = conn.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY)
    var parsedSql = ""
    if (conn.getMetaData.getURL.matches("jdbc:mysql:.*")) {
      parsedSql = sql+" limit "+ part.lower+","+ part.upper
    }

    val stmt = new NamedParameterStatement(conn, parsedSql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY)

    // setFetchSize(Integer.MIN_VALUE) is a mysql driver specific way to force streaming results,
    // rather than pulling entire resultset into memory.
    // see http://dev.mysql.com/doc/refman/5.0/en/connector-j-reference-implementation-notes.html
    if (conn.getMetaData.getURL.matches("jdbc:mysql:.*")) {
      stmt.setFetchSize(Integer.MIN_VALUE)
      logInfo("statement fetch size set to: " + stmt.getFetchSize + " to force MySQL streaming ")
    }
    if (paramsMap != null && paramsMap.size > 0) {
      val paramsIter = paramsMap.iterator
      while (paramsIter.hasNext) {
        val (key, value) = paramsIter.next()
        stmt.setObject(key, value)
      }
    }
    val rs = stmt.executeQuery()

    override def getNext: T = {
      if (rs.next()) {
        mapRow(rs)
      } else {
        finished = true
        null.asInstanceOf[T]
      }
    }

    override def close() {
      try {
        if (null != rs && !rs.isClosed()) {
          rs.close()
        }
      } catch {
        case e: Exception => logWarning("Exception closing resultset", e)
      }
      try {
        if (null != stmt && !stmt.isClosed()) {
          stmt.close()
        }
      } catch {
        case e: Exception => logWarning("Exception closing statement", e)
      }
      try {
        if (null != conn && !conn.isClosed()) {
          conn.close()
        }
        logInfo("closed connection")
      } catch {
        case e: Exception => logWarning("Exception closing connection", e)
      }
    }
  }

  override protected def getPartitions: Array[Partition] = {
    take > 0 match {
      case false => throw new IllegalArgumentException("take 参数不能小于0")
      case _ => {
        val step = take / numPartitions
        (0 until numPartitions).map(i => {
          val start = (skip + i * step)
          val end = start + step
          new JdbcPartition(i, start, end)
        }).toArray
      }
    }
  }

}

JdbcNextIterator.scala

package yay.jdbc.rdd

/**
 * Created by ${yuananyun} on 2015/2/8.
 */
abstract private[yay] class JdbcNextIterator[U] extends Iterator[U] {
  private var gotNext = false
  private var nextValue: U = _
  private var closed = false
  protected var finished = false

  /**
   * Method for subclasses to implement to provide the next element.
   *
   * If no next element is available, the subclass should set `finished`
   * to `true` and may return any value (it will be ignored).
   *
   * This convention is required because `null` may be a valid value,
   * and using `Option` seems like it might create unnecessary Some/None
   * instances, given some iterators might be called in a tight loop.
   *
   * @return U, or set 'finished' when done
   */
  protected def getNext(): U

  /**
   * Method for subclasses to implement when all elements have been successfully
   * iterated, and the iteration is done.
   *
   * <b>Note:</b> `NextIterator` cannot guarantee that `close` will be
   * called because it has no control over what happens when an exception
   * happens in the user code that is calling hasNext/next.
   *
   * Ideally you should have another try/catch, as in HadoopRDD, that
   * ensures any resources are closed should iteration fail.
   */
  protected def close()

  /**
   * Calls the subclass-defined close method, but only once.
   *
   * Usually calling `close` multiple times should be fine, but historically
   * there have been issues with some InputFormats throwing exceptions.
   */
  def closeIfNeeded() {
    if (!closed) {
      close()
      closed = true
    }
  }

  override def hasNext: Boolean = {
    if (!finished) {
      if (!gotNext) {
        nextValue = getNext()
        if (finished) {
          closeIfNeeded()
        }
        gotNext = true
      }
    }
    !finished
  }

  override def next(): U = {
    if (!hasNext) {
      throw new NoSuchElementException("End of stream")
    }
    gotNext = false
    nextValue
  }
}

NamedParameterStatement.java

package yay.jdbc.rdd;

import java.io.InputStream;
import java.io.Reader;
import java.math.BigDecimal;
import java.net.URL;
import java.sql.*;
import java.sql.Date;
import java.util.*;

/**
 * This class wraps around a {@link PreparedStatement} and allows the programmer to set parameters by name instead
 * of by index. This eliminates any confusion as to which parameter index represents what. This also means that
 * rearranging the SQL statement or adding a parameter doesn't involve renumbering your indices.
 * Code such as this:
 * <p/>
 * <pre><code>
 * Connection conn = getConnection();
 * String sql = "select * from my_table where name=? or address=?";
 * PreparedStatement p = conn.prepareStatement(sql);
 * p.setString(1, "bob");
 * p.setString(2, "123");
 * ResultSet rs = p.executeQuery();
 * </code></pre>
 * <p/>
 * Can be replaced with:
 * <p/>
 * <pre><code>
 * Connection conn = getConnection();
 * String sql = "select * from my_table where name=:name or address=:address";
 * NamedParameterStatement p = new NamedParameterStatement(conn, sql);
 * p.setString("name", "bob");
 * p.setString("address", "123");
 * ResultSet rs = p.executeQuery();
 * </code></pre>
 */
public class NamedParameterStatement extends PreparedStatementWrapper
{
    private static final HashMap<String, Map<String, List<Integer>>> nameIndexCache = new HashMap<String, Map<String, List<Integer>>>();
    private static final HashMap<String, String> parsedSqlCache = new HashMap<String, String>();

    private final String parsedSql;
    private final Map<String, List<Integer>> nameIndexMap;

    /**
     * Creates a NamedParameterStatement. Wraps a call to
     * c.{@link Connection#prepareStatement(java.lang.String) prepareStatement}.
     *
     * @param conn the database connection
     * @param sql  the parameterized sql
     * @throws SQLException if the statement could not be created
     */
    public NamedParameterStatement(Connection conn, String sql,int resultSetType,int resultSetConcurrency) throws SQLException
    {
        if (nameIndexCache.containsKey(sql))
        {
            nameIndexMap = nameIndexCache.get(sql);
            parsedSql = parsedSqlCache.get(sql);
        } else
        {
            nameIndexMap = new HashMap<String, List<Integer>>();
            parsedSql = parseNamedSql(sql, nameIndexMap);

            nameIndexCache.put(sql, nameIndexMap);
            parsedSqlCache.put(sql, parsedSql);
        }
        ps = conn.prepareStatement(parsedSql,resultSetType,resultSetConcurrency);
    }

    /**
     * Returns the indexes for a parameter.
     *
     * @param name parameter name
     * @return parameter indexes
     * @throws IllegalArgumentException if the parameter does not exist
     */
    private List<Integer> getIndexes(String name)
    {
        List<Integer> indexes = nameIndexMap.get(name);
        if (indexes == null)
        {
            throw new IllegalArgumentException("Parameter not found: " + name);
        }
        return indexes;
    }

    /**
     * Parses a sql with named parameters. The parameter-index mappings
     * are put into the map, and the parsed sql is returned.
     *
     * @param sql sql with named parameters
     * @return the parsed sql
     */
    private static String parseNamedSql(String sql, Map<String, List<Integer>> nameIndexMap)
    {
        // I was originally using regular expressions, but they didn't work well for ignoring         // parameter-like strings inside quotes.
        int length = sql.length();
        StringBuffer parsedSql = new StringBuffer(length);
        boolean inSingleQuote = false;
        boolean inDoubleQuote = false;
        int index = 1;
        for (int i = 0; i < length; i++)
        {
            char c = sql.charAt(i);
            if (inSingleQuote)
            {
                if (c == '\'')
                {
                    inSingleQuote = false;
                }
            } else if (inDoubleQuote)
            {
                if (c == '"')
                {
                    inDoubleQuote = false;
                }
            } else
            {
                if (c == '\'')
                {
                    inSingleQuote = true;
                } else if (c == '"')
                {
                    inDoubleQuote = true;
                } else if (c == ':' && i + 1 < length && Character.isJavaIdentifierStart(sql.charAt(i + 1)))
                {
                    int j = i + 2;
                    while (j < length && Character.isJavaIdentifierPart(sql.charAt(j)))
                    {
                        j++;
                    }
                    String name = sql.substring(i + 1, j);
                    c = '?'; // replace the parameter with a question mark
                    i += name.length(); // skip past the end if the parameter
                    List<Integer> indexList = nameIndexMap.get(name);
                    if (indexList == null)
                    {
                        indexList = new LinkedList<Integer>();
                        nameIndexMap.put(name, indexList);
                    }
                    indexList.add(index);
                    index++;
                }
            }
            parsedSql.append(c);
        }

        return parsedSql.toString();
    }

    public void setArray(String name, Array value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setArray(index, value);
        }
    }

    public void setAsciiStream(String name, InputStream value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setAsciiStream(index, value);
        }
    }

    public void setAsciiStream(String name, InputStream value, int length) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setAsciiStream(index, value, length);
        }
    }

    public void setBigDecimal(String name, BigDecimal value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setBigDecimal(index, value);
        }
    }

    public void setBinaryStream(String name, InputStream value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setBinaryStream(index, value);
        }
    }

    public void setBinaryStream(String name, InputStream value, int length) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setBinaryStream(index, value, length);
        }
    }

    public void setBinaryStream(String name, InputStream value, long length) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setBinaryStream(index, value, length);
        }
    }

    public void setBlob(String name, Blob value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setBlob(index, value);
        }
    }

    public void setBlob(String name, InputStream value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setBlob(index, value);
        }
    }

    public void setBlob(String name, InputStream value, long length) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setBlob(index, value, length);
        }
    }

    public void setBoolean(String name, boolean value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setBoolean(index, value);
        }
    }

    public void setByte(String name, byte value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setByte(index, value);
        }
    }

    public void setBytes(String name, byte[] value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setBytes(index, value);
        }
    }

    public void setCharacterStream(String name, Reader value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setCharacterStream(index, value);
        }
    }

    public void setCharacterStream(String name, Reader value, int length) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setCharacterStream(index, value, length);
        }
    }

    public void setCharacterStream(String name, Reader value, long length) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setCharacterStream(index, value, length);
        }
    }

    public void setClob(String name, Clob value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setClob(index, value);
        }
    }

    public void setClob(String name, Reader value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setClob(index, value);
        }
    }

    public void setClob(String name, Reader value, long length) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setClob(index, value, length);
        }
    }

    public void setDate(String name, Date value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setDate(index, value);
        }
    }

    public void setDate(String name, Date value, Calendar cal) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setDate(index, value, cal);
        }
    }

    public void setDouble(String name, double value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setDouble(index, value);
        }
    }

    public void setFloat(String name, float value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setFloat(index, value);
        }
    }

    public void setInt(String name, int value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setInt(index, value);
        }
    }

    public void setLong(String name, long value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setLong(index, value);
        }
    }

    public void setNCharacterStream(String name, Reader value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setNCharacterStream(index, value);
        }
    }

    public void setNCharacterStream(String name, Reader value, long length) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setNCharacterStream(index, value, length);
        }
    }

    public void setNClob(String name, NClob value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setNClob(index, value);
        }
    }

    public void setNClob(String name, Reader value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setNClob(index, value);
        }
    }

    public void setNClob(String name, Reader value, long length) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setNClob(index, value, length);
        }
    }

    public void setNString(String name, String value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setNString(index, value);
        }
    }

    public void setNull(String name, int sqlType) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setNull(index, sqlType);
        }
    }

    public void setObject(String name, Object value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setObject(index, value);
        }
    }

    public void setObject(String name, Object value, int targetSqlType) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setObject(index, value, targetSqlType);
        }
    }

    public void setObject(String name, Object value, int targetSqlType, int scaleOrLength) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setObject(index, value, targetSqlType, scaleOrLength);
        }
    }

    public void setRef(String name, Ref value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setRef(index, value);
        }
    }

    public void setRowId(String name, RowId value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setRowId(index, value);
        }
    }

    public void setShort(String name, short value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setShort(index, value);
        }
    }

    public void setSQLXML(String name, SQLXML value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setSQLXML(index, value);
        }
    }

    public void setString(String name, String value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setString(index, value);
        }
    }

    public void setTime(String name, Time value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setTime(index, value);
        }
    }

    public void setTime(String name, Time value, Calendar cal) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setTime(index, value, cal);
        }
    }

    public void setTimestamp(String name, Timestamp value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setTimestamp(index, value);
        }
    }

    public void setTimestamp(String name, Timestamp value, Calendar cal) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setTimestamp(index, value, cal);
        }
    }

    @SuppressWarnings("deprecation")
    public void setUnicodeStream(String name, InputStream value, int length) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setUnicodeStream(index, value, length);
        }
    }

    public void setURL(String name, URL value) throws SQLException
    {
        for (Integer index : getIndexes(name))
        {
            ps.setURL(index, value);
        }
    }

}

PreparedStatementWrapper.java

package yay.jdbc.rdd;

import java.io.InputStream;
import java.io.Reader;
import java.math.BigDecimal;
import java.net.URL;
import java.sql.*;
import java.util.Calendar;

public class PreparedStatementWrapper implements PreparedStatement
{
    protected PreparedStatement ps;

    @Override
    public <T> T unwrap(Class<T> iface) throws SQLException
    {
        return ps.unwrap(iface);
    }

    @Override
    public ResultSet executeQuery(String sql) throws SQLException
    {
        return ps.executeQuery(sql);
    }

    @Override
    public ResultSet executeQuery() throws SQLException
    {
        return ps.executeQuery();
    }

    @Override
    public boolean isWrapperFor(Class<?> iface) throws SQLException
    {
        return ps.isWrapperFor(iface);
    }

    @Override
    public int executeUpdate(String sql) throws SQLException
    {
        return ps.executeUpdate(sql);
    }

    @Override
    public int executeUpdate() throws SQLException
    {
        return ps.executeUpdate();
    }

    @Override
    public void setNull(int parameterIndex, int sqlType) throws SQLException
    {
        ps.setNull(parameterIndex, sqlType);
    }

    @Override
    public void close() throws SQLException
    {
        ps.close();
    }

    @Override
    public int getMaxFieldSize() throws SQLException
    {
        return ps.getMaxFieldSize();
    }

    @Override
    public void setBoolean(int parameterIndex, boolean x) throws SQLException
    {
        ps.setBoolean(parameterIndex, x);
    }

    @Override
    public void setByte(int parameterIndex, byte x) throws SQLException
    {
        ps.setByte(parameterIndex, x);
    }

    @Override
    public void setMaxFieldSize(int max) throws SQLException
    {
        ps.setMaxFieldSize(max);
    }

    @Override
    public void setShort(int parameterIndex, short x) throws SQLException
    {
        ps.setShort(parameterIndex, x);
    }

    @Override
    public int getMaxRows() throws SQLException
    {
        return ps.getMaxRows();
    }

    @Override
    public void setInt(int parameterIndex, int x) throws SQLException
    {
        ps.setInt(parameterIndex, x);
    }

    @Override
    public void setMaxRows(int max) throws SQLException
    {
        ps.setMaxRows(max);
    }

    @Override
    public void setLong(int parameterIndex, long x) throws SQLException
    {
        ps.setLong(parameterIndex, x);
    }

    @Override
    public void setEscapeProcessing(boolean enable) throws SQLException
    {
        ps.setEscapeProcessing(enable);
    }

    @Override
    public void setFloat(int parameterIndex, float x) throws SQLException
    {
        ps.setFloat(parameterIndex, x);
    }

    @Override
    public void setDouble(int parameterIndex, double x) throws SQLException
    {
        ps.setDouble(parameterIndex, x);
    }

    @Override
    public int getQueryTimeout() throws SQLException
    {
        return ps.getQueryTimeout();
    }

    @Override
    public void setQueryTimeout(int seconds) throws SQLException
    {
        ps.setQueryTimeout(seconds);
    }

    @Override
    public void setBigDecimal(int parameterIndex, BigDecimal x) throws SQLException
    {
        ps.setBigDecimal(parameterIndex, x);
    }

    @Override
    public void setString(int parameterIndex, String x) throws SQLException
    {
        ps.setString(parameterIndex, x);
    }

    @Override
    public void setBytes(int parameterIndex, byte[] x) throws SQLException
    {
        ps.setBytes(parameterIndex, x);
    }

    @Override
    public void cancel() throws SQLException
    {
        ps.cancel();
    }

    @Override
    public SQLWarning getWarnings() throws SQLException
    {
        return ps.getWarnings();
    }

    @Override
    public void setDate(int parameterIndex, Date x) throws SQLException
    {
        ps.setDate(parameterIndex, x);
    }

    @Override
    public void setTime(int parameterIndex, Time x) throws SQLException
    {
        ps.setTime(parameterIndex, x);
    }

    @Override
    public void clearWarnings() throws SQLException
    {
        ps.clearWarnings();
    }

    @Override
    public void setCursorName(String name) throws SQLException
    {
        ps.setCursorName(name);
    }

    @Override
    public void setTimestamp(int parameterIndex, Timestamp x) throws SQLException
    {
        ps.setTimestamp(parameterIndex, x);
    }

    @Override
    public void setAsciiStream(int parameterIndex, InputStream x, int length) throws SQLException
    {
        ps.setAsciiStream(parameterIndex, x, length);
    }

    @Override
    public boolean execute(String sql) throws SQLException
    {
        return ps.execute(sql);
    }

    @Override
    @SuppressWarnings("deprecation")
    public void setUnicodeStream(int parameterIndex, InputStream x, int length) throws SQLException
    {
        ps.setUnicodeStream(parameterIndex, x, length);
    }

    @Override
    public ResultSet getResultSet() throws SQLException
    {
        return ps.getResultSet();
    }

    @Override
    public void setBinaryStream(int parameterIndex, InputStream x, int length) throws SQLException
    {
        ps.setBinaryStream(parameterIndex, x, length);
    }

    @Override
    public int getUpdateCount() throws SQLException
    {
        return ps.getUpdateCount();
    }

    @Override
    public boolean getMoreResults() throws SQLException
    {
        return ps.getMoreResults();
    }

    @Override
    public void clearParameters() throws SQLException
    {
        ps.clearParameters();
    }

    @Override
    public void setObject(int parameterIndex, Object x, int targetSqlType) throws SQLException
    {
        ps.setObject(parameterIndex, x, targetSqlType);
    }

    @Override
    public void setFetchDirection(int direction) throws SQLException
    {
        ps.setFetchDirection(direction);
    }

    @Override
    public int getFetchDirection() throws SQLException
    {
        return ps.getFetchDirection();
    }

    @Override
    public void setObject(int parameterIndex, Object x) throws SQLException
    {
        ps.setObject(parameterIndex, x);
    }

    @Override
    public void setFetchSize(int rows) throws SQLException
    {
        ps.setFetchSize(rows);
    }

    @Override
    public int getFetchSize() throws SQLException
    {
        return ps.getFetchSize();
    }

    @Override
    public int getResultSetConcurrency() throws SQLException
    {
        return ps.getResultSetConcurrency();
    }

    @Override
    public boolean execute() throws SQLException
    {
        return ps.execute();
    }

    @Override
    public int getResultSetType() throws SQLException
    {
        return ps.getResultSetType();
    }

    @Override
    public void addBatch(String sql) throws SQLException
    {
        ps.addBatch(sql);
    }

    @Override
    public void clearBatch() throws SQLException
    {
        ps.clearBatch();
    }

    @Override
    public void addBatch() throws SQLException
    {
        ps.addBatch();
    }

    @Override
    public int[] executeBatch() throws SQLException
    {
        return ps.executeBatch();
    }

    @Override
    public void setCharacterStream(int parameterIndex, Reader reader, int length) throws SQLException
    {
        ps.setCharacterStream(parameterIndex, reader, length);
    }

    @Override
    public void setRef(int parameterIndex, Ref x) throws SQLException
    {
        ps.setRef(parameterIndex, x);
    }

    @Override
    public void setBlob(int parameterIndex, Blob x) throws SQLException
    {
        ps.setBlob(parameterIndex, x);
    }

    @Override
    public void setClob(int parameterIndex, Clob x) throws SQLException
    {
        ps.setClob(parameterIndex, x);
    }

    @Override
    public Connection getConnection() throws SQLException
    {
        return ps.getConnection();
    }

    @Override
    public void setArray(int parameterIndex, Array x) throws SQLException
    {
        ps.setArray(parameterIndex, x);
    }

    @Override
    public ResultSetMetaData getMetaData() throws SQLException
    {
        return ps.getMetaData();
    }

    @Override
    public boolean getMoreResults(int current) throws SQLException
    {
        return ps.getMoreResults(current);
    }

    @Override
    public void setDate(int parameterIndex, Date x, Calendar cal) throws SQLException
    {
        ps.setDate(parameterIndex, x, cal);
    }

    @Override
    public ResultSet getGeneratedKeys() throws SQLException
    {
        return ps.getGeneratedKeys();
    }

    @Override
    public void setTime(int parameterIndex, Time x, Calendar cal) throws SQLException
    {
        ps.setTime(parameterIndex, x, cal);
    }

    @Override
    public int executeUpdate(String sql, int autoGeneratedKeys) throws SQLException
    {
        return ps.executeUpdate(sql, autoGeneratedKeys);
    }

    @Override
    public void setTimestamp(int parameterIndex, Timestamp x, Calendar cal) throws SQLException
    {
        ps.setTimestamp(parameterIndex, x, cal);
    }

    @Override
    public void setNull(int parameterIndex, int sqlType, String typeName) throws SQLException
    {
        ps.setNull(parameterIndex, sqlType, typeName);
    }

    @Override
    public int executeUpdate(String sql, int[] columnIndexes) throws SQLException
    {
        return ps.executeUpdate(sql, columnIndexes);
    }

    @Override
    public void setURL(int parameterIndex, URL x) throws SQLException
    {
        ps.setURL(parameterIndex, x);
    }

    @Override
    public int executeUpdate(String sql, String[] columnNames) throws SQLException
    {
        return ps.executeUpdate(sql, columnNames);
    }

    @Override
    public ParameterMetaData getParameterMetaData() throws SQLException
    {
        return ps.getParameterMetaData();
    }

    @Override
    public void setRowId(int parameterIndex, RowId x) throws SQLException
    {
        ps.setRowId(parameterIndex, x);
    }

    @Override
    public void setNString(int parameterIndex, String value) throws SQLException
    {
        ps.setNString(parameterIndex, value);
    }

    @Override
    public boolean execute(String sql, int autoGeneratedKeys) throws SQLException
    {
        return ps.execute(sql, autoGeneratedKeys);
    }

    @Override
    public void setNCharacterStream(int parameterIndex, Reader value, long length) throws SQLException
    {
        ps.setNCharacterStream(parameterIndex, value, length);
    }

    @Override
    public void setNClob(int parameterIndex, NClob value) throws SQLException
    {
        ps.setNClob(parameterIndex, value);
    }

    @Override
    public void setClob(int parameterIndex, Reader reader, long length) throws SQLException
    {
        ps.setClob(parameterIndex, reader, length);
    }

    @Override
    public boolean execute(String sql, int[] columnIndexes) throws SQLException
    {
        return ps.execute(sql, columnIndexes);
    }

    @Override
    public void setBlob(int parameterIndex, InputStream inputStream, long length) throws SQLException
    {
        ps.setBlob(parameterIndex, inputStream, length);
    }

    @Override
    public void setNClob(int parameterIndex, Reader reader, long length) throws SQLException
    {
        ps.setNClob(parameterIndex, reader, length);
    }

    @Override
    public boolean execute(String sql, String[] columnNames) throws SQLException
    {
        return ps.execute(sql, columnNames);
    }

    @Override
    public void setSQLXML(int parameterIndex, SQLXML xmlObject) throws SQLException
    {
        ps.setSQLXML(parameterIndex, xmlObject);
    }

    @Override
    public void setObject(int parameterIndex, Object x, int targetSqlType, int scaleOrLength) throws SQLException
    {
        ps.setObject(parameterIndex, x, targetSqlType, scaleOrLength);
    }

    @Override
    public int getResultSetHoldability() throws SQLException
    {
        return ps.getResultSetHoldability();
    }

    @Override
    public boolean isClosed() throws SQLException
    {
        return ps.isClosed();
    }

    @Override
    public void setPoolable(boolean poolable) throws SQLException
    {
        ps.setPoolable(poolable);
    }

    @Override
    public boolean isPoolable() throws SQLException
    {
        return ps.isPoolable();
    }

    @Override
    public void closeOnCompletion() throws SQLException
    {
        ps.closeOnCompletion();
    }

    @Override
    public void setAsciiStream(int parameterIndex, InputStream x, long length) throws SQLException
    {
        ps.setAsciiStream(parameterIndex, x, length);
    }

    @Override
    public boolean isCloseOnCompletion() throws SQLException
    {
        return ps.isCloseOnCompletion();
    }

    @Override
    public void setBinaryStream(int parameterIndex, InputStream x, long length) throws SQLException
    {
        ps.setBinaryStream(parameterIndex, x, length);
    }

    @Override
    public void setCharacterStream(int parameterIndex, Reader reader, long length) throws SQLException
    {
        ps.setCharacterStream(parameterIndex, reader, length);
    }

    @Override
    public void setAsciiStream(int parameterIndex, InputStream x) throws SQLException
    {
        ps.setAsciiStream(parameterIndex, x);
    }

    @Override
    public void setBinaryStream(int parameterIndex, InputStream x) throws SQLException
    {
        ps.setBinaryStream(parameterIndex, x);
    }

    @Override
    public void setCharacterStream(int parameterIndex, Reader reader) throws SQLException
    {
        ps.setCharacterStream(parameterIndex, reader);
    }

    @Override
    public void setNCharacterStream(int parameterIndex, Reader value) throws SQLException
    {
        ps.setNCharacterStream(parameterIndex, value);
    }

    @Override
    public void setClob(int parameterIndex, Reader reader) throws SQLException
    {
        ps.setClob(parameterIndex, reader);
    }

    @Override
    public void setBlob(int parameterIndex, InputStream inputStream) throws SQLException
    {
        ps.setBlob(parameterIndex, inputStream);
    }

    @Override
    public void setNClob(int parameterIndex, Reader reader) throws SQLException
    {
        ps.setNClob(parameterIndex, reader);
    }

}

具体的用法：

object SparkToJDBC {
  def main(args: Array[String]): Unit = {
    val sc = new SparkContext("local", "mySql")
    val paramsMap = Map[String, Object]("msgType" -> "99", "sendTime" -> "1419821878146")
    val rdd = new GenericJdbcRDD(sc, () => {
      Class.forName("com.mysql.jdbc.Driver").newInstance()
      DriverManager.getConnection("jdbc:mysql://xxxxxxxx?characterEncoding=utf-8", "root", "xxxxx")
    }, "SELECT * FROM emb_message where msg_type=:msgType and send_time>=:sendTime", 

paramsMap, 0, 1000, 3, r => (r.getString(6),r.getString(11)))

//    rdd.foreach(x => println(x))
        rdd.saveAsTextFile("c:\\temp\\test")
    sc.stop()
  }

}

红色部分：其中paramsMap表示要传给sql的命名参数，0和1000分别表示skip和take，3表示生成的Rdd的partition数。