torchosr/example_outlier.py at main · w4k2/torchosr · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
from torchosr.data import configure_division_outlier, get_train_test_outlier
from torch.utils.data import DataLoader
import torch
from tqdm import tqdm
from torchosr.architectures.architectures import fc_lower_stack
from torchosr.data.base_datasets import MNIST_base, Omniglot_base
from torchosr.models import Openmax, TSoftmax
from torchvision import transforms
from torchosr.utils.base import get_openmax_epsilon, get_softmax_epsilon, inverse_transform

t_mnist = transforms.Compose([
        transforms.Resize(28),
        transforms.ToTensor()])

t_omni = transforms.Compose([
            transforms.Resize(28),
            transforms.ToTensor(),
            inverse_transform()])

# Modelling parameters
learning_rate = 1e-3
batch_size = 64
epochs = 128

# Evaluation parameters
repeats = 5         # openness repeats
n_splits = 5        # classical validation splits
n_openness = 5      # openness values
root='data'

# Load dataset
base = MNIST_base(root=root, download=True, transform=t_mnist)
out = Omniglot_base(root=root, download=True, transform=t_omni)

config, openness = configure_division_outlier(base, out, n_openness, repeats, seed=1233)

n_methods = 2
n_measures = 4
results = torch.full((n_measures, len(config), n_splits, n_methods, epochs), torch.nan)

pbar = tqdm(total=len(config)*n_splits*n_methods*epochs)

# Iterating configurations
for config_idx, (kkc, uuc) in enumerate(config):
        print('# Configuration %i [openness %.3f]' % (config_idx, openness[config_idx // repeats]),
              'known:', kkc.numpy(),
              'unknown:', uuc.numpy())
        # Iterate divisions
        for fold in range(n_splits):
            train_data, test_data = get_train_test_outlier(base, out,
                                                   kkc, uuc,
                                                   root='data',
                                                   tunning=False,
                                                   fold=fold,
                                                   seed=1411,
                                                   n_folds=n_splits)

            train_data_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True)
            test_data_loader = DataLoader(test_data, batch_size=batch_size, shuffle=True)

            methods = [
            TSoftmax(lower_stack=fc_lower_stack(depth=1, img_size_x=28, n_out_channels=64),
                                n_known=len(kkc),
                                epsilon=get_softmax_epsilon(len(kkc))),
            Openmax(lower_stack=fc_lower_stack(depth=1, img_size_x=28, n_out_channels=64),
                               n_known=len(kkc),
                               epsilon=get_openmax_epsilon(len(kkc))),
            ]

            for model_id, model in enumerate(methods):
                # Initialize loss function
                loss_fn = torch.nn.CrossEntropyLoss()

                # Initialize optimizer
                optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

                for t in range(epochs):
                    # Train
                    model.train(train_data_loader, loss_fn, optimizer)

                    # Test
                    inner_score, outer_score, hp_score, overall_score = model.test(test_data_loader, loss_fn)
                    results[0, config_idx, fold, model_id, t] = inner_score
                    results[1, config_idx, fold, model_id, t] = outer_score
                    results[2, config_idx, fold, model_id, t] = hp_score
                    results[3, config_idx, fold, model_id, t] = overall_score


                    pbar.update(1)

                print(config_idx,fold, model_id, '\n', results[:,config_idx, fold, model_id])
pbar.close()