scikit-learn 学习笔记-1-加载文本语料库

先上官方文档：
http://scikit-learn.org/stable/user_guide.html
API：
http://scikit-learn.org/stable/modules/classes.html

加载文本语料的方法doc文档为
http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_files.html#sklearn.datasets.load_files

语料库目录结构：

container_folder/
    category_1_folder/
        file_1.txt file_2.txt ... file_42.txt
    category_2_folder/
        file_43.txt file_44.txt ...

源码分析：

def load_files(container_path, description=None, categories=None, load_content=True, shuffle=True, encoding=None, decode_error='strict', random_state=0):
    #用于记录每个语料对应的标签是哪一个
    target = []
    #存放标签名
    target_names = []
    #存放语料文件路径名
    filenames = []
#获取所有子目录的名，其实就是标签名
    folders = [f for f in sorted(listdir(container_path))
               if isdir(join(container_path, f))]
#限定加载的类的种类
    if categories is not None:
        folders = [f for f in folders if f in categories]
#初始化target_names 和 filenames 列表
    for label, folder in enumerate(folders):
        target_names.append(folder)
        folder_path = join(container_path, folder)
        documents = [join(folder_path, d)
                     for d in sorted(listdir(folder_path))]
        target.extend(len(documents) * [label])
        filenames.extend(documents)

    # 转换为array方便花式检索
    filenames = np.array(filenames)
    target = np.array(target)

    #是否打乱顺序
    if shuffle:
        random_state = check_random_state(random_state)
        indices = np.arange(filenames.shape[0])
        random_state.shuffle(indices)
        filenames = filenames[indices]
        target = target[indices]
#是否加载内容，如果加载则需要指定编码方式，如果不指定则会以byte方式读入
    if load_content:
        data = []
        for filename in filenames:
            with open(filename, 'rb') as f:
                data.append(f.read())
        if encoding is not None:
            data = [d.decode(encoding, decode_error) for d in data]
        return Bunch(data=data,
                     filenames=filenames,
                     target_names=target_names,
                     target=target,
                     DESCR=description)
#最终返回的是Bunch对象
    return Bunch(filenames=filenames,
                 target_names=target_names,
                 target=target,
                 DESCR=description)