[spark]pyspark的PYTHONPATH配置

参考http://www.cnblogs.com/chenfool/p/3858930.html

软件准备

spark-1.3.0-bin-hadoop1.tgz

jdk-8u45-linux-x64.tar.gz

scala

export SPARK_HOME=~/spark-1.3.0-bin-hadoop1
export PYTHONPATH=${SPARK_HOME}/python/:${SPARK_HOME}/python/lib/py4j-0.8.1-src.zip;

cp $SPARK_HOME/conf/spark-env.sh.template $SPARK_HOME/conf/spark-env.sh
vi spark-env.sh

export JAVA_HOME=#your java home
export HADOOP_HOME=#your hadoop home
export SCALA_HOME=#your scala home
export HADOOP_CONF=#your hadoop configure directory
export SPARK_WORKER_MEMORY=1G# you can edit
export SPARK_MASTER_IP=localhost #you can edit
export MASTER=spark://localhost:7077 #you  can edit

 

"test.py"
#!/sur/bin/env python
# -*- coding: utf8 -*-
import pyspark
if __name__ == '__main__':
    pass

python test.py