Spark入门

安装：https://www.jianshu.com/p/0f4cc9ecc14c

第一个项目：Word Count

import org.apache.spark.{SparkConf, SparkContext}

object SparkWordCount {
  def FILE_NAME: String = "word_count_results_";

  def main(args: Array[String]) {
    val conf = new SparkConf().setAppName("Demo").setMaster("local");
    val sc = new SparkContext(conf);
    val textFile = sc.textFile("test.txt");
    val wordCounts = textFile.flatMap(line => line.split(" "))
                             .map(word => (word, 1)).reduceByKey((a, b) => a + b)

    println("Word Count program running results:");
    wordCounts.collect().foreach(e => {
      val (k, v) = e
      println(k + "=" + v)
    });

    wordCounts.saveAsTextFile(FILE_NAME + System.currentTimeMillis());
    println("Word Count program running results are successfully saved.");
  }
}