Python多进程

1. 多进程实现斐波那契

对于耗费cpu的操作，多进程由于多线程, 我拿斐波那契进行比较发现多进程速度远远高于多线程

# 多线程
import time
from concurrent.futures import ThreadPoolExecutor, as_completed

def fib(n):
    if n<=2:
        return 1
    return fib(n-1)+fib(n-2)

if __name__ == "__main__":
    with ThreadPoolExecutor(2) as executor:
        all_task = [executor.submit(fib, (num)) for num in range(38)]      # 39s
        start_time = time.time()
        for future in as_completed(all_task):
            data = future.result()
            print("exe result: {}".format(data))

        print("last time is: {}".format(time.time()-start_time))

# 多进程
import time
from concurrent.futures import ThreadPoolExecutor, as_completed


def fib(n):
    if n<=2:
        return 1
    return fib(n-1)+fib(n-2)

if __name__ == "__main__":
    with ProcessPoolExecutor(2) as executor:
        all_task = [executor.submit(fib, (num)) for num in range(38)]      # 19
        start_time = time.time()
        for future in as_completed(all_task):
            data = future.result()
            print("exe result: {}".format(data))

        print("last time is: {}".format(time.time()-start_time))
    start_time = time.time()

2. 使用进程池

import multiprocessing

#多进程编程
import time
def get_html(n):
    time.sleep(n)
    print("sub_progress success")
    return n

if __name__ == "__main__":
pool = multiprocessing.Pool(multiprocessing.cpu_count())
 #使用线程池**********************************************************
    result = pool.apply_async(get_html, args=(3,))
    #等待所有任务完成
    pool.close()
    pool.join()

    print(result.get())   # sub_progress success
                          #3
# ********************************************************************
# imap 会按顺序打印
    for result in pool.imap(get_html, [1,5,3]):
        print("{} sleep success".format(result))
# sub_progress success
# 1 sleep success
# sub_progress success
# sub_progress success
# 5 sleep success
# 3 sleep success
# ********************************************************************
# imap_unordered 按时间顺序打印
    for result in pool.imap_unordered(get_html, [1,5,3]):
        print("{} sleep success".format(result))

# sub_progress success
# 1 sleep success
# sub_progress success
# 3 sleep success
# sub_progress success
# 5 sleep success

3. 进程使用Queue

进程的Queue在multiprocessing下才好用

import time
from multiprocessing import Process, Queue, Manager, Pool, Pipe
def producer(queue):
    queue.put("a")
    time.sleep(2)

def consumer(queue):
    time.sleep(2)
    data = queue.get()
    print(data)

if __name__=="__main__":
    queue = Queue(10)
    my_producter = Process(target=producer, args=(queue,))
    my_consumer = Process(target=consumer, args=(queue,))
    my_producter.start()
    my_consumer.start()
    my_producter.join()
    my_consumer.join()  # a

4. 共享变量不适用于多进程

#共享全局变量不能适用于多进程编程，可以适用于多线程
import time
from multiprocessing import Process, Queue, Manager, Pool, Pipe

def producer(a):
    a += 100
    time.sleep(2)

def consumer(a):
    time.sleep(2)
    print(a)

if __name__ == "__main__":
    a = 1
    my_producer = Process(target=producer, args=(a,))
    my_consumer = Process(target=consumer, args=(a,))
    my_producer.start()
    my_consumer.start()
    my_producer.join()
    my_consumer.join()  
    # 1

5. 通过pipe实现进程间通信

# pipe性能高于queue

def producer(pipe):
    pipe.send("fz")


def consumer(pipe):
    print(pipe.recv())


if __name__== "__main__":
    receice_pipe, send_pipe = Pipe()
    # pipe只能适用于两个进程
    my_producer = Process(target=producer, args=(send_pipe,))
    my_consumer = Process(target=consumer, args=(receice_pipe,))

    my_producer.start()
    my_consumer.start()
    my_producer.join()
    my_consumer.join()
    # fz

6. 两个进程共同操作一个dict

除了dict managers包下包含了所有python的类型可以选择

import time
from multiprocessing import Process, Queue, Manager, Pool, Pipe

def add_data(p_dict, key, value):
    p_dict[key]=value

if __name__=="__main__":
    progress_dict = Manager().dict()

    first_progress = Process(target=add_data, args=(progress_dict, "fz",21))
    second_progress = Process(target=add_data, args=(progress_dict, "xk",19))

    first_progress.start()
    second_progress.start()
    first_progress.join()
    second_progress.join()
    print(progress_dict)