Data handling and loading

• With a random number generator, either with numpy or with torch

rng = np.random.default_rng(seed=0)

a = rng.uniform(size=(3,))

torch.random.manual_seed(0)

a = torch.rand(size=(3,))

from torch.utils.data import Dataset

class Simple1DRandomDataset(Dataset):

def __init__(self, samples: np.ndarray):

# initialization of the dataset

def __getitem__(self, index):

# return one sample with the sample itself an the belonging index

def __len__(self):

# return the total numbers of samples in the dataset

from torch.utils.data import DataLoader

our_dataloader = DataLoader( our_dataset, shuffle=True,

batch_size=4, num_workers=0)

• The batch size is a hyperparameter we need to set

• Number of workers refers to the option of multiple processing, only works with the name = main statement

from torch.utils.data import Subset

n_samples = len(our_dataset)

shuffled_indices = rng.permutation(n_samples)

test_set_indices = shuffled_indices[:int(n_samples / 5)]

validation_set_indices = shuffled_indices [int(n_samples / 5):int(n_samples / 5) * 2]

training_set_indices = shuffled_indices[int(n_samples / 5) * 2:]

• You need to create seperate Dataloaders afterwards

• With the argument collate_fn

training_loader = DataLoader(training_set, shuffle=False,

batch_size=4, collate_fn=no_stack_collate_fn)

• The corresponding sample, specified by the index

• It specifies how the samples are combined into minibatches

• They create minibatches from the samples returned by a torch.utils.data.Dataset instance