【十四】SparkSQL外部数据源操作之Mysql

SparkMysql.scala代码

package com.sid.com

import org.apache.spark.sql.SparkSession
import java.util.Properties


/**
  * 操作mysql的数据
  * */
object SparkMysql {
  def main(args: Array[String]): Unit = {
    val spark = SparkSession.builder().master("local[2]").appName("SparkMysql").getOrCreate()

    //第一种方法
    val jdbcDF = spark.read.format("jdbc")
      .option("url","jdbc:mysql://localhost:3306/sid")
      .option("dbtable","wordcount")
      .option("user","root").option("password","Liyijie331")
      .option("driver","com.mysql.jdbc.Driver")load()

    //第二种方法
    //val connectionProperties = new Properties()
   // connectionProperties.put("user","root")
    //connectionProperties.put("password","Liyijie331")
   // connectionProperties.put("driver","com.mysql.jdbc.Driver")
    //val jdbc2DF = spark.read.jdbc("jdbc:mysql://localhost:3306/sid","wordcount",connectionProperties)


    jdbcDF.printSchema()
    jdbcDF.show()
  }

}

pom.xml

  4.0.0
  com.sid.com
  sparksqltrain
  1.0-SNAPSHOT
  2008
  
    2.11.8
    2.2.0
  

  
    
      scala-tools.org
      Scala-Tools Maven2 Repository
      http://scala-tools.org/repo-releases
    
  

  
    
      scala-tools.org
      Scala-Tools Maven2 Repository
      http://scala-tools.org/repo-releases
    
  

  
    
    
      org.scala-lang
      scala-library
      ${scala.version}
    
    
    
      org.apache.spark
      spark-sql_2.11
      ${spark.version}
    
    
    
      org.apache.spark
      spark-hive_2.11
      ${spark.version}
    

    
      org.spark-project.hive
      hive-jdbc
      1.2.1.spark2
    

    
    
      mysql
      mysql-connector-java
      5.1.38
    


  

  
    src/main/scala
    src/test/scala
    
      
        org.scala-tools
        maven-scala-plugin
        
          
            
              compile
              testCompile
            
          
        
        
          ${scala.version}
          
            -target:jvm-1.5
          
        
      
      
        org.apache.maven.plugins
        maven-eclipse-plugin
        
          true
          
            ch.epfl.lamp.sdt.core.scalabuilder
          
          
            ch.epfl.lamp.sdt.core.scalanature
          
          
            org.eclipse.jdt.launching.JRE_CONTAINER
            ch.epfl.lamp.sdt.launching.SCALA_CONTAINER
          
        
      
    
  
  
    
      
        org.scala-tools
        maven-scala-plugin
        
          ${scala.version}
        
      
    
  

运行结果

官网介绍

Spark所有第三方外部数据源的介绍