进程池和线程池配合使用

from concurrent.futures.thread import ThreadPoolExecutor
from concurrent.futures.process import ProcessPoolExecutor
from multiprocessing import Manager, cpu_count

import time


def process_callback(*args):
    print("process_callback",*args)


def thread_func(range,mg_list):
    print(range)
    time.sleep(2)
    mg_list.append(range)
    return mg_list


def process_func(range,mg_list):
    # mg_list.append(range)
    p = ThreadPoolExecutor(100)
    for j in range:
        p.submit(thread_func, j, mg_list)
    p.shutdown()
    return mg_list


if __name__ == ‘__main__‘:
    start_time = time.time()
    # p = Pool()
    p = ProcessPoolExecutor()
    mg = Manager()
    mg_list = mg.list()
    data_len = 20000
    thread_number = data_len//cpu_count()
    range_index = [index for index in range(data_len) if index % thread_number == 0]
    range_index.append(data_len)
    print(range_index)
    for index, i in enumerate(range_index, 0):
        if index == len(range_index) - 1:
            continue
        range_data = range(i, range_index[index + 1])
    #     p.apply_async(func=process_func, args=(range_data, mg_list))   # 提交任务的方法 异步提交
    # p.close()
    # p.join()
        p.submit(process_func, range_data, mg_list)
    p.shutdown()
    print("mg_list", mg_list)
    print("mg_list", len(mg_list))
    print("use_time", time.time() - start_time)

进程池和线程池配合使用

原文：https://www.cnblogs.com/qsnack/p/14826708.html