Commit a5f23fd4 authored by Václav Novák's avatar Václav Novák
Browse files

fix: added normalization to predictions, modify suggestions

parent 2247c0a8

dummy_classifier.py

+45 −28
Original line number Diff line number Diff line
@@ -2,29 +2,53 @@ from src.builders import build_dataset, build_dataloader 
from sklearn.dummy import DummyClassifier

from sklearn.metrics import precision_recall_curve

from sklearn.metrics import auc

from sklearn.datasets import make_classification

from sklearn.model_selection import train_test_split



from matplotlib import pyplot



import platform



NUM_WORKERS = 0 if platform.system() == "Windows" else 4





# train images

train_X = []

# train labels

train_y = []

def evaluate_and_plot(train_X, train_y, test_X, test_y, name):

    model = DummyClassifier(strategy="prior")

    model.fit(train_X, train_y)



    predicted_probs = model.predict_proba(test_X)[:, 1]



    precision, recall, _ = precision_recall_curve(test_y, predicted_probs)

    auc_score = auc(recall, precision)



    print("AUC PRC of {}: {}".format(name, auc_score))



    pyplot.plot(recall, precision, label=name)

    pyplot.xlabel('Recall')

    pyplot.ylabel('Precision')



    pyplot.xlim(0.0, 1.0)

    pyplot.ylim(0.0, 1.0)



# test images

test_X = []

# test labels

test_y = []

    pyplot.legend()

    pyplot.show()





# initialize datasets

train_dataset = build_dataset("dataset", True, 128)

# train images - sigcomp

train_X_sigcomp = []

# train labels - sigcomp

train_y_sigcomp = []



# test images - sigcomp

test_X_sigcomp = []

# test labels - sigcomp

test_y_sigcomp = []



# initialize dataloaders - sigcomp

train_dataset = build_dataset("dataset", True, 8)

train_dataloader = build_dataloader(train_dataset, 1, False, NUM_WORKERS)



test_dataset = build_dataset("dataset", False, 128)

test_dataset = build_dataset("dataset", False, 8)

test_dataloader = build_dataloader(test_dataset, 1, False, NUM_WORKERS)



# enumerate through datasets and sort into lists
@@ -32,28 +56,21 @@ for _, data in enumerate(train_dataloader, 0): 
    image1, image2, label = data



    # input is ignored

    test_X.append(0)

    test_y.append(label.item())

    train_X_sigcomp.append(0)

    train_y_sigcomp.append(label.item())



for _, data in enumerate(test_dataloader, 0):

    image1, image2, label = data



    train_X.append(0)

    train_y.append(label.item())

    test_X_sigcomp.append(0)

    test_y_sigcomp.append(label.item())





# try all strategies

strategies = ["most_frequent", "prior", "stratified", "uniform"]



for strategy in strategies:

    model = DummyClassifier(strategy=strategy)

    model.fit(train_X, train_y)



    predicted = model.predict_proba(test_X)

    pos_probs = predicted[:, 1]



    precision, recall, _ = precision_recall_curve(test_y, pos_probs)

    auc_score = auc(recall, precision)

# create train, test samples of random balanced dataset

X, y = make_classification(n_samples=1000, n_classes=2, random_state=42)

train_X_random, test_X_random, train_y_random, test_y_random = train_test_split(X, y, test_size=0.5, random_state=42)



    print("Used strategy: {}, score: {}".format(strategy, auc_score))



# predict and get auc prc, graph

evaluate_and_plot(train_X_sigcomp, train_y_sigcomp, test_X_sigcomp, test_y_sigcomp, "SigComp2011 dataset")

evaluate_and_plot(train_X_random, train_y_random, test_X_random, test_y_random, "Balanced dataset")

src/builders.py

+0 −4
Original line number Diff line number Diff line
@@ -114,14 +114,12 @@ def build_c_layers(trial: Trial) -> nn.Sequential: 


            c_layers.append(maxpool_layer)





        # Build dropout if should be present

        if trial.params["{}_contains_dropout".format(curr_name_prefix)]:

            dropout_layer = nn.Dropout(p=trial.params["{}_dropout_rate".format(curr_name_prefix)])



            c_layers.append(dropout_layer)





    return nn.Sequential(*c_layers)
@@ -153,7 +151,6 @@ def build_l_layers(trial: Trial, first_input_size: int) -> nn.Sequential: 
        if i != trial.params["l_layers"] - 1:

            l_layers.append(nn.ReLU(inplace=True))

        

        

        # Build dropout

        if i != trial.params["l_layers"] - 1 and trial.params["{}_contains_dropout".format(curr_name_prefix)]:

            dropout_layer = nn.Dropout(
@@ -161,5 +158,4 @@ def build_l_layers(trial: Trial, first_input_size: int) -> nn.Sequential: 


            l_layers.append(dropout_layer)





    return nn.Sequential(*l_layers)

src/evaluation.py

+2 −0
Original line number Diff line number Diff line
@@ -44,6 +44,8 @@ def evaluate(model: SiameseNetwork, trial: Trial, cuda_capability: bool, num_wor 
            predicted.append(distance)

            actual.append(label.item())



    predicted = [float(prediction) / max(predicted) for prediction in predicted]



    # https://towardsdatascience.com/optimal-threshold-for-imbalanced-classification-5884e870c293

    precision, recall, thresholds = precision_recall_curve(actual, predicted)

    auc_prc = auc(recall, precision)

src/suggestions.py

+25 −25
Original line number Diff line number Diff line 
from optuna import Trial

from typing import Tuple

from typing import Tuple, List





def suggest_params_general(trial: Trial) -> None:
@@ -20,11 +20,11 @@ def suggest_params_general(trial: Trial) -> None: 
    trial.suggest_int("batch_size", 4, 80, step=4)



    # Parameters of loss function

    trial.suggest_float("margin", 1, 3)

    trial.suggest_float("margin", 1.5, 2.5)



    # Parameters of optimizer

    trial.suggest_float("lr", 1e-5, 0.1)

    trial.suggest_float("weight_decay", 1e-6, 1e-2, log=True)

    trial.suggest_float("lr", 1e-5, 0.01)

    trial.suggest_float("weight_decay", 1e-10, 1e-5, log=True)

    trial.suggest_float("beta1", 0.65, 0.95)

    trial.suggest_float("beta2", 0.945, 0.999)
@@ -40,7 +40,7 @@ def suggest_params_general(trial: Trial) -> None: 
def suggest_c_layer(trial: Trial, n: int, kernel_size: Tuple[int, int],

                    stride: Tuple[int, int],

                    padding: Tuple[int, int],

                    outputs: Tuple[int, int, int]) -> None:

                    outputs: List[int]) -> None:

    """

    Suggest convolutional layer.
@@ -49,7 +49,7 @@ def suggest_c_layer(trial: Trial, n: int, kernel_size: Tuple[int, int], 
    @param kernel_size: list, list of possible kernel sizes

    @param stride: list, list of possible strides

    @param padding: list, list of possible paddings

    @param outputs: tuple, limits of output size and step taken

    @param outputs: list, limits of possible output layers



    @return: None

    """
@@ -59,8 +59,7 @@ def suggest_c_layer(trial: Trial, n: int, kernel_size: Tuple[int, int], 
                      kernel_size[1], step=2)

    trial.suggest_int("{}_stride".format(name_prefix), stride[0], stride[1])

    trial.suggest_int("{}_padding".format(name_prefix), padding[0], padding[1])

    trial.suggest_int("{}_output".format(name_prefix), outputs[0], outputs[1],

                      step=outputs[2])

    trial.suggest_categorical("{}_output".format(name_prefix), outputs)





def suggest_maxpool(trial: Trial, n: int) -> None:
@@ -96,16 +95,17 @@ def suggest_dropout(trial: Trial, n: int, is_in_fc: bool) -> None: 
    @return: None

    """



    chances = [False, False, False] if is_in_fc else [True, True, False]

    name_prefix = "l_layers_{}_layer_{}".format(trial.params["l_layers"],

                                                n) if is_in_fc \

        else "c_layers_{}_layer_{}".format(trial.params["c_layers"], n)



    contains_dropout = trial.suggest_categorical(

        "{}_contains_dropout".format(name_prefix), [False, True, True])

        "{}_contains_dropout".format(name_prefix), chances)



    if contains_dropout:

        rate_low = 0.3 if is_in_fc else 0.1

        rate_high = 0.7 if is_in_fc else 0.3

        rate_low = 0.3 if is_in_fc else 0.3

        rate_high = 0.7 if is_in_fc else 0.8



        trial.suggest_float("{}_dropout_rate".format(name_prefix), rate_low,

                            rate_high, step=0.05)
@@ -125,30 +125,30 @@ def suggest_c_layers(trial: Trial) -> None: 
        kernel_sizes = [(7, 13), (3, 9), (3, 5)]

        strides = [(1, 3), (1, 2), (1, 1)]

        paddings = [(1, 6), (1, 4), (1, 2)]

        # outputs = [[16, 32, 64, 96, 128], [64, 128, 192, 224, 256],

        # [128, 256, 320, 384, 448, 512]]

        outputs = [(16, 128, 16), (64, 256, 64), (128, 640, 64)]

        outputs = [[16, 32, 64, 96, 128], [64, 128, 192, 224, 256],

                   [128, 256, 320, 384, 448, 512]]

        # outputs = [(16, 128, 16), (64, 256, 64), (128, 640, 64)]



    elif trial.params["c_layers"] == 4:

        kernel_sizes = [(7, 13), (3, 9), (3, 5), (3, 3)]

        strides = [(1, 3), (1, 2), (1, 1), (1, 1)]

        paddings = [(1, 6), (1, 4), (1, 2), (1, 1)]

        # outputs = [[32, 64, 96, 128, 160, 192],

        # [96, 128, 160, 192, 256, 320, 384],

        # [128, 160, 192, 256, 320, 384, 448],

        # [192, 256, 320, 448, 512]]

        outputs = [(32, 192, 32), (96, 384, 32), (128, 448, 32),

                   (192, 512, 64)]

        outputs = [[32, 64, 96, 128, 160, 192],

                   [96, 128, 160, 192, 256, 320, 384],

                   [128, 160, 192, 256, 320, 384, 448],

                   [192, 256, 320, 448, 512]]

        # outputs = [(32, 192, 32), (96, 384, 32), (128, 448, 32),

        # (192, 512, 64)]



    else:

        kernel_sizes = [(7, 13), (3, 9), (3, 5), (3, 3), (3, 3)]

        strides = [(1, 3), (1, 2), (1, 1), (1, 1), (1, 1)]

        paddings = [(1, 6), (1, 4), (1, 2), (1, 1), (1, 1)]

        # outputs = [[32, 64, 96, 128, 160, 192],

        # [96, 128, 160, 192, 256, 320], [128, 160, 192, 256, 320, 384],

        # [192, 256, 320, 384, 448], [192, 256, 320, 384, 448, 512]]

        outputs = [(32, 192, 16), (96, 320, 32), (128, 384, 32),

                   (192, 448, 32), (192, 512, 64)]

        outputs = [[32, 64, 96, 128, 160, 192],

                   [96, 128, 160, 192, 256, 320], [128, 160, 192, 256, 320, 384],

                   [192, 256, 320, 384, 448], [192, 256, 320, 384, 448, 512]]

        # outputs = [(32, 192, 16), (96, 320, 32), (128, 384, 32),

        # (192, 448, 32), (192, 512, 64)]



    # Suggest each convolutional layer

    for i in range(0, trial.params["c_layers"]):