partitioner

(K.hashcode & Integer.MAX_VALE) % (reducer number)
hashpartitioner 相同key的数据一定会在同一个reducer中，但一个reducer中不就只有一个key

class HostPartitioner(ins:Array[String]) extends Partitioner {
    val parMap = new mutable.HashMap[String, Int]()
    val count = 0
    for (i <- ins) {
        parMap += (i -> count)
        count += 1
    }

    override def numPartitions:Int = ins.length
    verride def getPartition(key: Any):Int =  {
         parMap.getOrElse(key.toString, 0)
    }
}

调用的时候

val hostPartitioner = new HostPartitioner()
rdd.partitionBy(hostPartitioner)


rdd.partitionBy(new HashPartitioner(6))