如何在pyspark中处理多余空格 —— regex_replace/trim

我们在做数据清洗时经常用和一些多余无用的空格打交道，例如移除开头和末尾的空格，或将多个连续空格替换成一个等。以下提供几个在pypark中处理字段中多余空格的例子。

import pyspark.sql.functions as F
from pyspark.sql.functions import col
def single_space(col):
    return F.trim(F.regexp_replace(col, " +", " "))


def remove_all_whitespace(col):
    return F.regexp_replace(col, "\\s+", "")


df = sc.parallelize([
    (1, "a  b"), (2, " ab  c d "), (3, " ab c  d e  f  ")
]).toDF(["k", "v"])

df.show()
'''
+---+---------------+
|  k|              v|
+---+---------------+
|  1|           a  b|
|  2|       ab  c d |
|  3| ab c  d e  f  |
+---+---------------+
'''
df.select(single_space(col("v"))).show()
'''
+------------------------------+
|trim(regexp_replace(v,  +,  ))|
+------------------------------+
|                           a b|
|                        ab c d|
|                    ab c d e f|
+------------------------------+
'''
df.select(remove_all_whitespace(col("v"))).show()
'''
+------------------------+
|regexp_replace(v, \s+, )|
+------------------------+
|                      ab|
|                    abcd|
|                  abcdef|
+------------------------+
'''