File size: 4,626 Bytes
9157432 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 |
import numpy as np
from sklearn.metrics import accuracy_score, f1_score
__all__ = ['MetricsTop']
class MetricsTop():
def __init__(self, train_mode):
if train_mode == "regression":
self.metrics_dict = {
'MOSI': self.__eval_mosi_regression,
'MOSEI': self.__eval_mosei_regression,
}
else:
self.metrics_dict = {
'MOSI': self.__eval_mosi_classification,
'MOSEI': self.__eval_mosei_classification,
}
def __eval_mosi_classification(self, y_pred, y_true):
y_pred = y_pred.cpu().detach().numpy()
y_true = y_true.cpu().detach().numpy()
# three classes
y_pred_3 = np.argmax(y_pred, axis=1)
Mult_acc_3 = accuracy_score(y_pred_3, y_true)
F1_score_3 = f1_score(y_true, y_pred_3, average='weighted')
# two classes
y_pred = np.array([[v[0], v[2]] for v in y_pred])
# with 0 (<= 0 or > 0)
y_pred_2 = np.argmax(y_pred, axis=1)
y_true_2 = []
for v in y_true:
y_true_2.append(0 if v <= 1 else 1)
y_true_2 = np.array(y_true_2)
Has0_acc_2 = accuracy_score(y_pred_2, y_true_2)
Has0_F1_score = f1_score(y_true_2, y_pred_2, average='weighted')
# without 0 (< 0 or > 0)
non_zeros = np.array([i for i, e in enumerate(y_true) if e != 1])
y_pred_2 = y_pred[non_zeros]
y_pred_2 = np.argmax(y_pred_2, axis=1)
y_true_2 = y_true[non_zeros]
Non0_acc_2 = accuracy_score(y_pred_2, y_true_2)
Non0_F1_score = f1_score(y_true_2, y_pred_2, average='weighted')
eval_results = {
"Has0_acc_2": round(Has0_acc_2, 4),
"Has0_F1_score": round(Has0_F1_score, 4),
"Non0_acc_2": round(Non0_acc_2, 4),
"Non0_F1_score": round(Non0_F1_score, 4),
"Acc_3": round(Mult_acc_3, 4),
"F1_score_3": round(F1_score_3, 4)
}
return eval_results
def __eval_mosei_classification(self, y_pred, y_true):
return self.__eval_mosi_classification(y_pred, y_true)
def __multiclass_acc(self, y_pred, y_true):
"""
Compute the multiclass accuracy w.r.t. groundtruth
:param preds: Float array representing the predictions, dimension (N,)
:param truths: Float/int array representing the groundtruth classes, dimension (N,)
:return: Classification accuracy
"""
return np.sum(np.round(y_pred) == np.round(y_true)) / float(len(y_true))
def __eval_mosei_regression(self, y_pred, y_true, exclude_zero=False):
test_preds = y_pred.view(-1).cpu().detach().numpy()
test_truth = y_true.view(-1).cpu().detach().numpy()
test_preds_a7 = np.clip(test_preds, a_min=-3., a_max=3.)
test_truth_a7 = np.clip(test_truth, a_min=-3., a_max=3.)
test_preds_a5 = np.clip(test_preds, a_min=-2., a_max=2.)
test_truth_a5 = np.clip(test_truth, a_min=-2., a_max=2.)
test_preds_a3 = np.clip(test_preds, a_min=-1., a_max=1.)
test_truth_a3 = np.clip(test_truth, a_min=-1., a_max=1.)
mae = np.mean(np.absolute(test_preds - test_truth)).astype(np.float64) # Average L1 distance between preds and truths
corr = np.corrcoef(test_preds, test_truth)[0][1]
mult_a7 = self.__multiclass_acc(test_preds_a7, test_truth_a7)
mult_a5 = self.__multiclass_acc(test_preds_a5, test_truth_a5)
mult_a3 = self.__multiclass_acc(test_preds_a3, test_truth_a3)
non_zeros = np.array([i for i, e in enumerate(test_truth) if e != 0])
non_zeros_binary_truth = (test_truth[non_zeros] > 0)
non_zeros_binary_preds = (test_preds[non_zeros] > 0)
non_zeros_acc2 = accuracy_score(non_zeros_binary_preds, non_zeros_binary_truth)
non_zeros_f1_score = f1_score(non_zeros_binary_truth, non_zeros_binary_preds, average='weighted')
binary_truth = (test_truth >= 0)
binary_preds = (test_preds >= 0)
acc2 = accuracy_score(binary_preds, binary_truth)
f_score = f1_score(binary_truth, binary_preds, average='weighted')
eval_results = {
"Acc_2": round(non_zeros_acc2, 4),
"F1_score": round(non_zeros_f1_score, 4),
"Acc_7": round(mult_a7, 4),
"MAE": round(mae, 4),
}
return eval_results
def __eval_mosi_regression(self, y_pred, y_true):
return self.__eval_mosei_regression(y_pred, y_true)
def getMetics(self, datasetName):
return self.metrics_dict[datasetName.upper()] |