Template Code for Thread Parallelism and Process Parallelism in Python

Python
2024-07-03 09:23 (4 months ago) ytyng

The only difference is whether to use ThreadPoolExecutor or ProcessPoolExecutor.

Thread Parallelism

Code

from concurrent.futures import ThreadPoolExecutor
import time
import uuid


_uuid = str(uuid.uuid4())
print(f'Code evaluated. uuid={_uuid}')


def task(a, *, b):
    print(f'Task {a} started')
    time.sleep(2)
    print(f'Task {a} completed')
    return f'worker result({a}) = {a * b}, uuid = {_uuid}'


def main():
    _task_args = range(10)

    workers = []

    with ThreadPoolExecutor(max_workers=3) as executor:
        for _task_arg in _task_args:
            print(f'Worker append task {_task_arg}')
            # Add tasks and execute them sequentially
            workers.append(
                executor.submit(
                    task, _task_arg, b=_task_arg+1
                )
            )

    # This point is reached when all tasks are completed
    print('Contextmanager exited.')

    for worker in workers:
        print('Worker result:', worker.result())


if __name__ == '__main__':
    main()

Result

Code evaluated. uuid=e7fe32bc-83d6-4b14-ba07-e0b7b410f8eb
Worker append task 0
Task 0 startedWorker append task 1

Task 1 started
Worker append task 2
Task 2 started
Worker append task 3
Worker append task 4
Worker append task 5
Worker append task 6
Worker append task 7
Worker append task 8
Worker append task 9
Task 2 completed
Task 3 started
Task 0 completed
Task 4 started
Task 1 completed
Task 5 started
Task 3 completed
Task 6 started
Task 4 completed
Task 5 completed
Task 7 started
Task 8 started
Task 6 completed
Task 9 started
Task 7 completed
Task 8 completed
Task 9 completed
Contextmanager exited.
Worker result: worker result(0) = 0, uuid = e7fe32bc-83d6-4b14-ba07-e0b7b410f8eb
Worker result: worker result(1) = 2, uuid = e7fe32bc-83d6-4b14-ba07-e0b7b410f8eb
Worker result: worker result(2) = 6, uuid = e7fe32bc-83d6-4b14-ba07-e0b7b410f8eb
Worker result: worker result(3) = 12, uuid = e7fe32bc-83d6-4b14-ba07-e0b7b410f8eb
Worker result: worker result(4) = 20, uuid = e7fe32bc-83d6-4b14-ba07-e0b7b410f8eb
Worker result: worker result(5) = 30, uuid = e7fe32bc-83d6-4b14-ba07-e0b7b410f8eb
Worker result: worker result(6) = 42, uuid = e7fe32bc-83d6-4b14-ba07-e0b7b410f8eb
Worker result: worker result(7) = 56, uuid = e7fe32bc-83d6-4b14-ba07-e0b7b410f8eb
Worker result: worker result(8) = 72, uuid = e7fe32bc-83d6-4b14-ba07-e0b7b410f8eb
Worker result: worker result(9) = 90, uuid = e7fe32bc-83d6-4b14-ba07-e0b7b410f8eb

Process Parallelism

Code

from concurrent.futures import ProcessPoolExecutor
import uuid
import time

_uuid = str(uuid.uuid4())
print(f'Code evaluated. uuid={_uuid}')


def task(a, *, b):
    print(f'Task {a} started')
    time.sleep(2)
    print(f'Task {a} completed')
    return f'worker result({a}) = {a * b}, uuid = {_uuid}'


def main():
    _task_args = range(10)

    workers = []

    with ProcessPoolExecutor(max_workers=3) as executor:
        for _task_arg in _task_args:
            print(f'Worker append task {_task_arg}')
            # Add tasks and execute them sequentially
            workers.append(
                executor.submit(
                    task, _task_arg, b=_task_arg+1
                )
            )
    # This point is reached when all tasks are completed
    print('Contextmanager exited.')

    for worker in workers:
        print('Worker result:', worker.result())


if __name__ == '__main__':
    main()

Result

Code evaluated. uuid=036d3d4f-52df-4a56-a2f3-e965496887ff
Worker append task 0
Worker append task 1
Worker append task 2
Worker append task 3
Worker append task 4
Worker append task 5
Worker append task 6
Worker append task 7
Worker append task 8
Worker append task 9
Code evaluated. uuid=9341bfa9-3851-4ba0-bfaf-189eb9a438fe
Code evaluated. uuid=7b9c1dbb-bcd8-4c79-bf4b-75f2178f2bf4
Task 0 started
Task 1 started
Code evaluated. uuid=cab23013-d3c1-4520-851a-172cdcfaacf6
Task 2 started
Task 0 completed
Task 3 started
Task 2 completedTask 1 completed

Task 4 started
Task 5 started
Task 3 completed
Task 6 started
Task 5 completed
Task 7 started
Task 4 completed
Task 8 started
Task 8 completed
Task 9 started
Task 6 completed
Task 7 completed
Task 9 completed
Contextmanager exited.
Worker result: worker result(0) = 0, uuid = 9341bfa9-3851-4ba0-bfaf-189eb9a438fe
Worker result: worker result(1) = 2, uuid = 7b9c1dbb-bcd8-4c79-bf4b-75f2178f2bf4
Worker result: worker result(2) = 6, uuid = cab23013-d3c1-4520-851a-172cdcfaacf6
Worker result: worker result(3) = 12, uuid = 9341bfa9-3851-4ba0-bfaf-189eb9a438fe
Worker result: worker result(4) = 20, uuid = cab23013-d3c1-4520-851a-172cdcfaacf6
Worker result: worker result(5) = 30, uuid = 7b9c1dbb-bcd8-4c79-bf4b-75f2178f2bf4
Worker result: worker result(6) = 42, uuid = 9341bfa9-3851-4ba0-bfaf-189eb9a438fe
Worker result: worker result(7) = 56, uuid = 7b9c1dbb-bcd8-4c79-bf4b-75f2178f2bf4
Worker result: worker result(8) = 72, uuid = cab23013-d3c1-4520-851a-172cdcfaacf6
Worker result: worker result(9) = 90, uuid = cab23013-d3c1-4520-851a-172cdcfaacf6

The key point is that when using ProcessPoolExecutor, the code evaluation is performed for the number of max_workers, and these are reused subsequently. The code is not re-evaluated for each process.

Currently unrated

Comments

Archive

2024
2023
2022
2021
2020
2019
2018
2017
2016
2015
2014
2013
2012
2011