python的concurrent.futures.ThreadPoolExecutor

Quickstart

from concurrent import futures


def fib(n):
    if n == 0 or n == 1:
        return 1
    return fib(n-1) + fib(n-2)

# base
with futures.ThreadPoolExecutor() as executor:
    f1 = executor.submit(fib, 4) # return future
    f2 = executor.submit(fib, 5)

print(f1.result())
print(f2.result())


FIBS = [3, 7, 10, 20]
# advance
with futures.ThreadPoolExecutor(max_workers=5) as executor:
    future_to_num = {executor.submit(fib, num): num for num in FIBS}
    for future in futures.as_completed(future_to_num):
        print(future.result(), future_to_num[future])

# map
with futures.ThreadPoolExecutor(max_workers=5) as executor:
    for num, res in zip(FIBS, executor.map(fib, FIBS)):
        print(num, res)

HOW-TO

通过改变max_workers 来提速

通过改变chunksize 来加速

chunksize表示每个进程完成多少次函数计算之后 会被杀死掉 默认是1个

FIBS = [1, 3, 5, 7] * 5


def do_calculate(chunksize=1):
    with futures.ProcessPoolExecutor(max_workers=5) as executor:
        for num, ret in zip(FIBS, executor.map(fib, FIBS, chunksize=chunksize)):
            pass


def main():
    for num in FIBS:
        fib(num)
        
(3.6)  ~/Documents/code/zhangyu/learn_python/test_concurrent   master ●  python -m perf timeit 'from concur import do_calculate,main;do_calculate(5)' --quiet
Mean +- std dev: 28.4 ms +- 0.8 ms
(3.6)  ~/Documents/code/zhangyu/learn_python/test_concurrent   master ●  python -m perf timeit 'from concur import do_calculate,main;do_calculate(1)' --quiet
Mean +- std dev: 19.1 ms +- 1.3 ms
(3.6)  ~/Documents/code/zhangyu/learn_python/test_concurrent   master ●  python -m perf timeit 'from concur import do_calculate,main;do_calculate(3)' --quiet
Mean +- std dev: 15.2 ms +- 0.7 ms
(3.6)  ~/Documents/code/zhangyu/learn_python/test_concurrent   master ●  python -m perf timeit 'from concur import do_calculate,main;do_calculate(30)' --quiet
Mean +- std dev: 13.8 ms +- 0.6 ms
(3.6)  ~/Documents/code/zhangyu/learn_python/test_concurrent   master ●  python -m perf timeit 'from concur import do_calculate,main;main()' --quiet
Mean +- std dev: 64.9 us +- 1.6 us