python modin加速

更新：数据量大内存小的情况下可能会报错。

modin
参考

BASE = '../data/output/grid_part_1.pkl'

# Read data with Pandas
import pandas as pd

start_time = time.time()
df = pd.read_pickle(BASE)
print("Pandas Loading Time = {}".format(time.time() - start_time))


# Read data with Modin
import modin.pandas as pd

start_time = time.time()
df = pd.read_pickle(BASE)
print("Modin Loading Time = {}".format(time.time() - start_time))

Pandas Loading Time = 7.24544095993042
Modin Loading Time = 5.5456836223602295