pyspark学习_RDD转为DataFrame

#方法1：反射机制推断RDD模式
people.txt 
Tom 12
Jack 13
Janny 14
from pyspark.sql import SparkSession,Row 

spark = SparkSession.builder.getOrCreate()
lines = spark.sparkContext.textFile("people.txt")
people = lines.map(lambda x:x.split(",")).map(lambda x:Row(name=x[0],age=int(x[1])))
peopleDF = spark.createDataFrame(people) #RDD注册为DataFrame
peopleDF.createOrReplaceTempView("people") #DataFrame注册people表
result = spark.sql("select * from peopleDF")
result.rdd.map(lambda x:'name:'+x['name']+' '+'age:'+str(x['age'])).foreach(print)
#方法2：编程方式定义RDD模式
from pyspark.sql import SparkSession,Row
from pyspark.sql.types import StructType,StructField,StringType,IntegerType

spark = SparkSession.builder.getOrCreate()
#创建schema
schema = StructType([StructField('name',StringType(),True),StructField('age',IntegerType(),True)])
#创建content
content = spark.sparkContext.textFile("people.txt").map(lambda x:x.split(" ")).map(lambda x:Row(x[0],x[1]))
peopleDF = spark.createDataFrame(content,schema)
peopleDF.createOrReplaceTempView("people")
result = spark.sql("select * from peopleDF")
result.rdd.map(lambda x:'name:'+x['name']+' '+'age:'+str(x['age'])).foreach(print)
#方法3：toDF()方法
from pyspark.sql import SparkSession
spark = SparkSession.builder.getOrCreate()
peopleDF = spark.sparkContext.textFile("people.txt").map(lambda x:x.split(",")).toDF(schema=['name','age'])
peopleDF.show()