PyTorch Mastery - Practical Deep Learning

PyTorch patterns, debugging, ו-GPU optimization.

Quick Start - Complete Training Pipeline

import torch
import torch.nn as nn
from torch.utils.data import DataLoader, TensorDataset

# Device setup
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(f"Using device: {device}")

# Data
dataset = TensorDataset(torch.tensor(X).float(), torch.tensor(y).float())
train_loader = DataLoader(dataset, batch_size=32, shuffle=True)

# Model
model = nn.Sequential(
    nn.Linear(input_dim, 64),
    nn.ReLU(),
    nn.Linear(64, 1)
).to(device)

# Training
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

for epoch in range(100):
    model.train()
    for X_batch, y_batch in train_loader:
        X_batch, y_batch = X_batch.to(device), y_batch.to(device)

        optimizer.zero_grad()
        outputs = model(X_batch)
        loss = criterion(outputs, y_batch)
        loss.backward()
        optimizer.step()

    if epoch % 10 == 0:
        print(f"Epoch {epoch}, Loss: {loss.item():.4f}")

When This Skill Activates

Use this skill when:

• Writing PyTorch training code
• Creating custom Dataset/DataLoader
• Debugging tensor shape mismatches
• Fixing CUDA out of memory errors
• Optimizing GPU utilization
• Saving/loading model checkpoints

Core Patterns

Pattern 1: Tensor Shapes (NCHW Format)

# Image tensors: [Batch, Channels, Height, Width]
images = torch.randn(32, 3, 224, 224)
#                    │   │   │    │
#                    │   │   │    └── Width
#                    │   │   └── Height
#                    │   └── Channels (RGB=3, Grayscale=1)
#                    └── Batch size

# Feature tensors: [Batch, Features]
features = torch.randn(32, 128)
#                       │   │
#                       │   └── Number of features
#                       └── Batch size

# Sequence tensors: [Batch, Sequence, Features]
sequences = torch.randn(32, 100, 256)
#                        │   │    │
#                        │   │    └── Feature/embedding dim
#                        │   └── Sequence length
#                        └── Batch size

Pattern 2: Custom Dataset Class

from torch.utils.data import Dataset, DataLoader

class CustomDataset(Dataset):
    def __init__(self, X, y, transform=None):
        self.X = torch.tensor(X, dtype=torch.float32)
        self.y = torch.tensor(y, dtype=torch.float32)
        self.transform = transform

    def __len__(self):
        return len(self.X)

    def __getitem__(self, idx):
        x = self.X[idx]
        y = self.y[idx]

        if self.transform:
            x = self.transform(x)

        return x, y

# Usage
dataset = CustomDataset(X_train, y_train)
train_loader = DataLoader(
    dataset,
    batch_size=32,
    shuffle=True,        # Shuffle for training
    num_workers=4,       # Parallel loading
    pin_memory=True      # Faster GPU transfer
)

Pattern 3: DataLoader Parameters

# Training loader (shuffle, augmentation)
train_loader = DataLoader(
    train_dataset,
    batch_size=32,
    shuffle=True,         # Random order each epoch
    num_workers=4,        # Parallel data loading
    pin_memory=True,      # Faster CPU→GPU transfer
    drop_last=True        # Drop incomplete last batch
)

# Validation/Test loader (no shuffle, no augmentation)
val_loader = DataLoader(
    val_dataset,
    batch_size=64,        # Can be larger (no gradients)
    shuffle=False,        # Keep order for reproducibility
    num_workers=4,
    pin_memory=True
)

Pattern 4: Device Management

# Setup
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# Move model to device
model = model.to(device)

# Move data to device (in training loop)
for X_batch, y_batch in train_loader:
    X_batch = X_batch.to(device)
    y_batch = y_batch.to(device)
    # ... training code

# Check model device
print(next(model.parameters()).device)

# Multi-GPU (DataParallel)
if torch.cuda.device_count() > 1:
    model = nn.DataParallel(model)

Pattern 5: Training Loop with Validation

def train_epoch(model, train_loader, criterion, optimizer, device):
    model.train()
    total_loss = 0

    for X_batch, y_batch in train_loader:
        X_batch, y_batch = X_batch.to(device), y_batch.to(device)

        optimizer.zero_grad()
        outputs = model(X_batch)
        loss = criterion(outputs, y_batch)
        loss.backward()
        optimizer.step()

        total_loss += loss.item()

    return total_loss / len(train_loader)


def validate(model, val_loader, criterion, device):
    model.eval()
    total_loss = 0

    with torch.no_grad():  # No gradient computation
        for X_batch, y_batch in val_loader:
            X_batch, y_batch = X_batch.to(device), y_batch.to(device)
            outputs = model(X_batch)
            loss = criterion(outputs, y_batch)
            total_loss += loss.item()

    return total_loss / len(val_loader)


# Full training
best_val_loss = float('inf')
for epoch in range(num_epochs):
    train_loss = train_epoch(model, train_loader, criterion, optimizer, device)
    val_loss = validate(model, val_loader, criterion, device)

    print(f"Epoch {epoch}: Train={train_loss:.4f}, Val={val_loss:.4f}")

    # Save best model
    if val_loss < best_val_loss:
        best_val_loss = val_loss
        torch.save(model.state_dict(), 'best_model.pt')

Pattern 6: Save/Load Checkpoints

# Save checkpoint (full state)
checkpoint = {
    'epoch': epoch,
    'model_state_dict': model.state_dict(),
    'optimizer_state_dict': optimizer.state_dict(),
    'loss': loss,
    'best_val_loss': best_val_loss
}
torch.save(checkpoint, 'checkpoint.pt')

# Load checkpoint
checkpoint = torch.load('checkpoint.pt')
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
start_epoch = checkpoint['epoch']

# Save just model weights
torch.save(model.state_dict(), 'model_weights.pt')

# Load just model weights
model.load_state_dict(torch.load('model_weights.pt'))
model.eval()  # Set to evaluation mode

Pattern 7: GPU Memory Management

# Check GPU memory
print(f"Allocated: {torch.cuda.memory_allocated() / 1e9:.2f} GB")
print(f"Cached: {torch.cuda.memory_reserved() / 1e9:.2f} GB")

# Clear cache
torch.cuda.empty_cache()

# Gradient checkpointing (trade compute for memory)
from torch.utils.checkpoint import checkpoint
# Use in forward: output = checkpoint(self.layer, input)

# Delete tensors explicitly
del large_tensor
torch.cuda.empty_cache()

# Reduce batch size if OOM
# Or use gradient accumulation:
accumulation_steps = 4
optimizer.zero_grad()
for i, (X, y) in enumerate(train_loader):
    loss = criterion(model(X), y) / accumulation_steps
    loss.backward()

    if (i + 1) % accumulation_steps == 0:
        optimizer.step()
        optimizer.zero_grad()

Pattern 8: Model Accuracy Function

def model_accuracy(model, dataloader):
    """Calculate accuracy with proper eval mode."""
    model.eval()
    device = next(model.parameters()).device  # Auto-detect device
    correct = 0
    total = 0

    with torch.no_grad():
        for X_batch, y_batch in dataloader:
            X_batch, y_batch = X_batch.to(device), y_batch.to(device)
            outputs = model(X_batch)

            # For classification
            _, predicted = torch.max(outputs, 1)
            total += y_batch.size(0)
            correct += (predicted == y_batch).sum().item()

    return correct / total  # Returns Python float

Reference Navigation

For detailed content, see:

• Tensors & Operations: reference/tensors_operations.md - Creation, Broadcasting, NCHW format
• Training Patterns: reference/training_patterns.md - Loops, Validation, Early stopping
• Memory Management: reference/memory_management.md - GPU optimization, Gradient accumulation
• Debugging Tips: reference/debugging_tips.md - Shape errors, CUDA issues
• Code Templates: reference/code_templates.md - Ready-to-use snippets
• Environment Setup: reference/environment_setup.md - GPU/CUDA verification, System health check

Common Mistakes to Avoid

1. Forgetting model.eval() for Inference

# WRONG: Dropout still active, BatchNorm uses batch stats
predictions = model(X_test)

# CORRECT: Switch to eval mode
model.eval()
with torch.no_grad():
    predictions = model(X_test)

2. Not Using torch.no_grad() for Inference

# WRONG: Wastefully computes gradients
model.eval()
predictions = model(X_test)  # Still tracking gradients!

# CORRECT: Disable gradient computation
model.eval()
with torch.no_grad():  # Saves memory and time
    predictions = model(X_test)

3. Shape Mismatch Debugging

# Add shape prints for debugging
print(f"Input shape: {x.shape}")
x = self.conv1(x)
print(f"After conv1: {x.shape}")
x = self.pool(x)
print(f"After pool: {x.shape}")
x = x.view(x.size(0), -1)  # Flatten
print(f"After flatten: {x.shape}")

# Common fix: Wrong flatten size
# Calculate: channels × height × width after convolutions

4. GPU/CPU Tensor Mismatch

# WRONG: Mixing devices
model = model.cuda()
X = torch.randn(32, 10)  # On CPU!
output = model(X)  # Error!

# CORRECT: Same device
model = model.to(device)
X = X.to(device)
output = model(X)

5. In-place Operations Breaking Gradients

# WRONG: In-place modification
x = model(input)
x += 1  # In-place! May break gradient computation

# CORRECT: Create new tensor
x = model(input)
x = x + 1  # New tensor

6. Not Converting NumPy to Tensor Properly

# WRONG: Wrong dtype
X_tensor = torch.tensor(X_numpy)  # May be float64

# CORRECT: Specify dtype
X_tensor = torch.tensor(X_numpy, dtype=torch.float32)
# Or
X_tensor = torch.from_numpy(X_numpy).float()

Teaching Mode

When explaining PyTorch concepts:

Tensor Visual

Tensor = N-dimensional array

0D: scalar       42
1D: vector      [1, 2, 3]
2D: matrix      [[1,2], [3,4]]
3D: volume      [[[1,2],[3,4]], [[5,6],[7,8]]]
4D: batch+3D    Images: [batch, channels, height, width]

Training Loop Visual

┌─────────────────────────────────────────┐
│           TRAINING LOOP                  │
├─────────────────────────────────────────┤
│                                          │
│  for epoch in epochs:                    │
│      for batch in dataloader:            │
│                                          │
│          ① optimizer.zero_grad()         │
│             ↓                            │
│          ② output = model(X)     Forward │
│             ↓                            │
│          ③ loss = criterion(output, y)   │
│             ↓                            │
│          ④ loss.backward()       Backward│
│             ↓                            │
│          ⑤ optimizer.step()      Update  │
│                                          │
└─────────────────────────────────────────┘

GPU Memory Analogy

GPU Memory = Hotel

- Tensors = Guests checking in
- .to(device) = Booking a room
- .cpu() = Checking out
- torch.cuda.empty_cache() = Cleaning empty rooms
- OOM Error = Hotel full!

Solutions:
- Smaller batch = fewer guests at once
- Gradient accumulation = guests take turns
- del tensor = force checkout