PyTorch Multiprocessing Tutorial

Click on the above“Mechanical and Electronic Engineering Technology” to follow us

Multiprocessing is a term in computer science that refers to running multiple processes simultaneously, where these processes can execute different tasks at the same time. In computer operating systems, a process is the basic unit of resource allocation, and each process has its own independent memory space and system resources, which do not interfere with each other.

Multiprocessing technology can be used to achieve parallel computing and distributed computing, where each process can independently execute different tasks, thus allowing multiple tasks to be processed simultaneously, improving the processing efficiency of the computer.

PyTorch supports multiprocessing training using the torch.multiprocessing module. This module provides functionality similar to the multiprocessing module in the Python standard library, but has been extended in PyTorch to better support distributed training.

Using the torch.multiprocessing module, you can create multiple processes, each of which can have its own PyTorch tensor and model parameters. This way, you can distribute data to different processes, allowing them to execute the training process in parallel.

1. Create and Run Parallel Processes

import torch.multiprocessing as mp
def action(name,times):  init = 0   for i in range(times):    init += i  print("this process is " + name)

if __name__ =='__main__':  process1 = mp.Process(target=action,args=('process1',10000000))  process2 = mp.Process(target=action,args=('process2',1000))
  process1.start()  process2.start()  # Wait for process2 to finish before continuing  # process2.join()    print("main process")

main process

this process is process2

this process is process1

2. Use Queue to Share Data

import torch.multiprocessing as mp
def action(q,name,times):  init = 0   for i in range(times):    init += i  print("this process is " + name)  q.put(init)
if __name__ =='__main__':  q = mp.Queue()  process1 = mp.Process(target=action,args=(q,'process1',10000000))  process2 = mp.Process(target=action,args=(q,'process2',1000))
  process1.start()  process2.start()  # Wait for process1 to finish  process1.join()  # Wait for process2 to finish  process2.join()  # Get the result from process1  result1 = q.get()  # Get the result from process2  result2 = q.get()
  print(result1)  print(result2)  print("main process")

this process is process2

this process is process1

499500

49999995000000

main process

3. Process Pool

import torch.multiprocessing as mp
def action(times):  init = 0   for i in range(times):    init += i  return init

if __name__ =='__main__':  times = [1000,1000000]  # Create a process pool with two processes  pool = mp.Pool(processes=2)  res = pool.map(action,times)  print(res)

[499500, 499999500000]

4. Process Lock

import torch.multiprocessing as mpimport time
def action(v,num,lock):  lock.acquire()  for i in range(5):    time.sleep(0.1)    v.value += num    print(v.value)  lock.release()

if __name__ =='__main__':  # Create a new lock object  lock = mp.Lock()  # Create a new shared variable v, initialized to 0, data type 'i' (integer)  v = mp.Value('i',0)  p1 = mp.Process(target=action,args=(v,1,lock))  p2 = mp.Process(target=action,args=(v,2,lock))  p1.start()  p2.start()  p1.join()  p2.join()

5. Compare Time Consumption Between Multiprocessing and Single Process

import torch.multiprocessing as mpimport time
def action(name,times):  init = 0   for i in range(times):    init += i  print("this process is " + name)
def mpfun():  process1 = mp.Process(target=action,args=('process1',100000000))  process2 = mp.Process(target=action,args=('process2',100000000))
  process1.start()  process2.start()
  process1.join()  process2.join()
def spfun():  action('main process',100000000)  action('main process',100000000)
if __name__ =='__main__':  start_time = time.time()  mpfun()  end_time = time.time()  print(end_time-start_time)    start_time2 = time.time()  spfun()  end_time2 = time.time()  print(end_time2-start_time2)

this process is process1

this process is process2

8.2586669921875

this process is main process

7.6229119300842285

6. Shared Variables

import torch.multiprocessing as mpimport torch
def action(element,t):  t[element] += (element+1) * 1000
if __name__ == "__main__":  t = torch.zeros(2)  t.share_memory_()  print('before mp: t=')  print(t)
  p0 = mp.Process(target=action,args=(0,t))  p1 = mp.Process(target=action,args=(1,t))  p0.start()  p1.start()  p0.join()  p1.join()  print('after mp: t=')  print(t)

before mp: t=

tensor([0., 0.])

after mp: t=

tensor([1000., 2000.])

multigpu_lenet

multigpu_test

Want to learn more?

Quickly scan the code to follow

Leave a Comment Cancel reply