springboot2.0 操作 spark2.x

一、首先很少有java连接spark的情况，一般是python, scala 直接上了。
二、话说，python也要引入pyspark，还有下载spark的lib, 也挺麻烦的。只有scala可以直接调用 spark-shell, 还算好用。废话不多说，直接上代码
三、完整的pom，不多不少刚刚好。强调下：版本号什么的都不能错，错了编译一定失败，MAVEN打包也失败。

    4.0.0
    
        org.springframework.boot
        spring-boot-starter-parent
        2.1.9.RELEASE
         
    
    com.example
    demo
    0.0.1-SNAPSHOT
    demo
    Demo project for Spring Boot
        1.8
        2.1.0
        2.7.4
        2.0.0.RELEASE
        1.8
        1.8
    



    
        org.projectlombok
        lombok
        true
    
    
        org.springframework.boot
        spring-boot-starter-test
        test
    

    
    
        org.apache.spark
        spark-core_2.11
        1.2.2
        
            
                org.slf4j
                slf4j-log4j12
            
        
    
    
        org.apache.spark
        spark-sql_2.11
        ${spark.version}
    
    
        org.apache.spark
        spark-hive_2.11
        ${spark.version}
    
    
        org.apache.spark
        spark-streaming_2.11
        ${spark.version}
    
    
        org.apache.hadoop
        hadoop-client
        ${hadoop.version}
        
            
                org.slf4j
                slf4j-log4j12
            
        
    
    

    
    
        org.springframework.boot
        spring-boot-starter
        ${springboot.version}
    
    
        org.springframework.boot
        spring-boot-configuration-processor
        ${springboot.version}
        true
    
    
        org.springframework.boot
        spring-boot-autoconfigure
        ${springboot.version}
    
    
        org.springframework.boot
        spring-boot-test
        ${springboot.version}
    
    

    
        mysql
        mysql-connector-java
        5.1.6
    
    
    
        javax.validation
        validation-api
        1.1.0.Final
    
    
    
        org.hibernate
        hibernate-validator
        5.2.4.Final
    



    
        
            org.springframework.boot
            spring-boot-maven-plugin
        

        
            net.alchim31.maven
            scala-maven-plugin
            3.3.1
            
                
                    scala-compile-first
                    process-resources
                    
                        add-source
                        compile
                    
                

                
                    compile
                    
                        compile
                        testCompile
                    
                
            
            

            
        

        
            org.scala-tools
            maven-scala-plugin
            2.15.2
            
                
                    
                        compile
                        testCompile
                    
                
            
            
                2.11.8
                
                    -target:jvm-1.8
                
            
        
    

四、完整代码

package com.example.demo;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

@SpringBootApplication
public class DemoApplication implements CommandLineRunner {

    @Autowired
    private DoWork doWork;

    public static void main(String[] args) {
        SparkConf conf = new SparkConf().setAppName("Simple Application");
        JavaSparkContext sc = new JavaSparkContext(conf);
        JavaRDD rdd = sc.textFile("D:\\workspace_java\\demo_spark1\\dat\\20191003.txt");
        System.out.println("=====>" + rdd.count());

    }
}