use pytorch Do more GPU Training , Just learn to change the code of single card training slightly . Don't make it too much trouble . Through one demo It's the quickest way to start .

1. You know how many cards the machine has ： 

nvidia-smi

 2. For model DataParallel The packing ：

device_ids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] # 10 The card machine
model = torch.nn.DataParallel(model, device_ids=device_ids) # Specify the device to be used
model = model.cuda(device=device_ids[0]) # The model is loaded into the device 0

3. Data also specifies the device ：

X_train, y_train = X_train.cuda(device=device_ids[0]), y_train.cuda(device=device_ids[0])

You just need to use device_ids[0] Just define a style , There's no need to specify devices card by card . But without this step, there will be an error . 

4. Last , Come and have a look at complete demo, The notes are different from single card training ：

（ Code from https://www.cnblogs.com/fanghao/p/10334631.html, The original code runs in my environment with bug, The following program is a modified version of ）

import torch
from torchvision import datasets, transforms
import torchvision
from tqdm import tqdm
device_ids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] # You can use GPU
BATCH_SIZE = 64
transform = transforms.Compose([transforms.ToTensor()])
data_train = datasets.MNIST(root = "./data/",
transform=transform,
train=True,
download=True)
data_test = datasets.MNIST(root="./data/",
transform=transform,
train=False)
data_loader_train = torch.utils.data.DataLoader(dataset=data_train,
# Single card batch size * Number of cards
batch_size=BATCH_SIZE * len(device_ids),
shuffle=True,
num_workers=2)
data_loader_test = torch.utils.data.DataLoader(dataset=data_test,
batch_size=BATCH_SIZE * len(device_ids),
shuffle=True,
num_workers=2)
class Model(torch.nn.Module):
def __init__(self):
super(Model, self).__init__()
self.conv1 = torch.nn.Sequential(
torch.nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=1),
torch.nn.ReLU(),
torch.nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1),
torch.nn.ReLU(),
torch.nn.MaxPool2d(stride=2, kernel_size=2),
)
self.dense = torch.nn.Sequential(
torch.nn.Linear(14 * 14 * 128, 1024),
torch.nn.ReLU(),
torch.nn.Dropout(p=0.5),
torch.nn.Linear(1024, 10)
)
def forward(self, x):
x = self.conv1(x)
x = x.view(-1, 14 * 14 * 128)
x = self.dense(x)
return x
model = Model()
# Specify the device to be used
model = torch.nn.DataParallel(model, device_ids=device_ids)
# The model is loaded into the device 0
model = model.cuda(device=device_ids[0])
cost = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters())
from time import sleep
n_epochs = 50
for epoch in range(n_epochs):
running_loss = 0.0
running_correct = 0
print("Epoch {}/{}".format(epoch, n_epochs))
print("-"*10)
for data in tqdm(data_loader_train):
X_train, y_train = data
# Designated equipment 0
X_train, y_train = X_train.cuda(device=device_ids[0]), y_train.cuda(device=device_ids[0])
outputs = model(X_train)
_,pred = torch.max(outputs.data, 1)
optimizer.zero_grad()
loss = cost(outputs, y_train)
loss.backward()
optimizer.step()
running_loss += loss.data.item()
running_correct += torch.sum(pred == y_train.data)
testing_correct = 0
for data in data_loader_test:
X_test, y_test = data
# Designated equipment 1
X_test, y_test = X_test.cuda(device=device_ids[0]), y_test.cuda(device=device_ids[0])
outputs = model(X_test)
_, pred = torch.max(outputs.data, 1)
testing_correct += torch.sum(pred == y_test.data)
print("Loss is:{:.4f}, Train Accuracy is:{:.4f}%, Test Accuracy is:{:.4f}".format(torch.true_divide(running_loss, len(data_train)),
torch.true_divide(100*running_correct, len(data_train)),
torch.true_divide(100*testing_correct, len(data_test))))
torch.save(model.state_dict(), "model_parameter.pkl")

Through the above program transformation method , We can transform single card training into multi card training . Of course , Some customized sampler,dataloader It may have to be changed accordingly , Specific problems should be dealt with in detail .

I'm on another training mission , It's fully activated 10 Card training at the same time ：10 individual 2080ti, It doesn't mean flashcard ( flee