9. 获得图片路径，构造出训练集和验证集，同时构造出相同人脸和不同人脸的测试集，将结果存储为.csv格式 1.random.shuffle(数据清洗) 2.random.sample(从数据集中随机选取2个数据) 3. random.choice(从数据集中抽取一个数据) 4.pickle.dump(将数据集写成.pkl数据)

# -*- coding: utf-8 -*-
‘‘‘ 
Created on 2019/7/8/0008 9:29
    
@Author : Sheng1994
 ‘‘‘

import os
import numpy as np
import random
import pickle


def test_pair_generate(test_image_list, each_k=5):

    test_paris_list = []

    test_images_length = len(test_image_list)

    for people_index, people_images in enumerate(test_image_list):

        # 生成相同一对的脸
        for _ in range(each_k):
            same_paris = random.sample(people_images, 2)
            test_paris_list.append((same_paris[0], same_paris[1], 1))

        # 生成不同的一对脸
        for _ in range(each_k):
            index_random = people_index
            while index_random == people_index:
                index_random = random.randint(0, test_images_length)
            diff_one = random.choice(test_image_list[people_index])
            diff_another = random.choice(test_image_list[index_random])
            test_paris_list.append((diff_one, diff_another, 0))

    return test_paris_list


def save_to_pkl(path, v1, v2):

    pkl_file = open(path, ‘wb‘)
    pickle.dump((v1, v2), pkl_file, pickle.HIGHEST_PROTOCOL)
    pkl_file.close()

def build_dataset(source_folder):
    # 第一步：将数据的路径进行添加，对于训练集和验证集的数据其标签使用label+ 来表示，对于测试集的数据使用相同和不同人脸数据集进行表示
    label = 1
    train_dataset, valid_dataset, test_dataset = [], [], []
    counter = 0

    test_pair_counter = 0
    train_counter = 0

    for people_folder in os.listdir(source_folder):
        people_images = []
        people_folder_path = source_folder + os.sep + people_folder
        for vedio_folder in os.listdir(people_folder_path):
            vedio_folder_path = people_folder_path + os.sep + vedio_folder
            for vedio_file_name in os.listdir(vedio_folder_path):
                full_path = vedio_folder_path + os.sep + vedio_file_name
                people_images.append(full_path)

        random.shuffle(people_images)
        if len(people_images) < 100:
             test_dataset.append(people_images)
             test_pair_counter += 1

        else:
            valid_dataset.extend(zip(people_images[0:50], [label]*50))
            test_dataset.extend(zip(people_images[50:600], [label]*550))
            label += 1
            train_counter += 1

        print(people_folder +‘: id--->‘ + str(counter))

        counter += 1
    # 将train和test数据集的个数表示下来
    save_to_pkl(‘image/train_test_number.pkl‘, train_counter, test_pair_counter)
    # 第二步：对测试数据进行生成，产生各5组的相同人脸数据集和不同人脸数据集
    test_pairs_dataset = test_pair_generate(test_dataset, each_k=5)

    random.shuffle(train_dataset)
    random.shuffle(valid_dataset)
    random.shuffle(test_pairs_dataset)

    return train_dataset, valid_dataset, test_pairs_dataset


def save_to_csv(dataset, file_name):
    with open(file_name, "w") as f:
        for item in dataset:
            f.write(",".join(map(str, item)) + "\n")



def run():

    random.seed(7)

    train_dataset, valid_dataset, test_dataset = build_dataset(‘image\\result‘)
    # 第三步：数据清洗，并将数据集存储到train_dataset_path 
    train_dataset_path = ‘image\\train_dataset.csv‘
    valid_dataset_path = ‘image\\valid_dataset.csv‘
    test_dataset_path = ‘image\\test_dataset.csv‘
    save_to_csv(train_dataset, train_dataset_path)
    save_to_csv(valid_dataset, valid_dataset_path)
    save_to_csv(test_dataset, test_dataset_path)


if __name__ == ‘__main__‘:
    run()