spark版hello word | Bruce's Blog

分类目录

现在位置: 首页＞大数据＞正文

spark版hello word

2022-05-09 16:20 大数据暂无评论阅读(77)

import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.SparkConf
object WordCount {
def main(args: Array[String]) {
val inputFile = "/Users/artefact/software/spark-3.1.3-bin-hadoop3.2/data/wordcount.txt"
val conf = new SparkConf().setAppName("WordCount").setMaster("local")
val sc = new SparkContext(conf)
val textFile = sc.textFile(inputFile)
val wordCount = textFile
.flatMap(_.split(" ")) //.flatMap(line => line.split(" "))
.filter(f=> !f.equals(""))
.map(word => (word, 1))
.reduceByKey((a, b) => a + b)
.sortBy(key=>key._1) //_1=ASC, _2=DESC
wordCount.foreach(println)
}
}