| import torch |
| import numpy as np |
| from collections import OrderedDict |
| import pandas as pd |
| import os |
| from tqdm import tqdm |
| import cv2 |
| from utils.misc import split_np_imgrid, get_np_imgrid |
|
|
|
|
| def cal_ber(tn, tp, fn, fp): |
| return 0.5*(fp/(tn+fp) + fn/(fn+tp)) |
|
|
| def cal_acc(tn, tp, fn, fp): |
| return (tp + tn) / (tp + tn + fp + fn) |
|
|
|
|
| def get_binary_classification_metrics(pred, gt, threshold=None): |
| if threshold is not None: |
| gt = (gt > threshold) |
| pred = (pred > threshold) |
| TP = np.logical_and(gt, pred).sum() |
| TN = np.logical_and(np.logical_not(gt), np.logical_not(pred)).sum() |
| FN = np.logical_and(gt, np.logical_not(pred)).sum() |
| FP = np.logical_and(np.logical_not(gt), pred).sum() |
| BER = cal_ber(TN, TP, FN, FP) |
| ACC = cal_acc(TN, TP, FN, FP) |
| return OrderedDict( [('TP', TP), |
| ('TN', TN), |
| ('FP', FP), |
| ('FN', FN), |
| ('BER', BER), |
| ('ACC', ACC)] |
| ) |
|
|
|
|
| def evaluate(res_root, pred_id, gt_id, nimg, nrow, threshold): |
| img_names = os.listdir(res_root) |
| score_dict = OrderedDict() |
|
|
| for img_name in img_names: |
| im_grid_path = os.path.join(res_root, img_name) |
| im_grid = cv2.imread(im_grid_path) |
| ims = split_np_imgrid(im_grid, nimg, nrow) |
| pred = ims[pred_id] |
| gt = ims[gt_id] |
| score_dict[img_name] = get_binary_classification_metrics(pred, |
| gt, |
| threshold) |
| |
| df = pd.DataFrame(score_dict) |
| df['ave'] = df.mean(axis=1) |
|
|
| tn = df['ave']['TN'] |
| tp = df['ave']['TP'] |
| fn = df['ave']['FN'] |
| fp = df['ave']['FP'] |
|
|
| pos_err = (1 - tp / (tp + fn)) * 100 |
| neg_err = (1 - tn / (tn + fp)) * 100 |
| ber = (pos_err + neg_err) / 2 |
| acc = (tn + tp) / (tn + tp + fn + fp) |
|
|
| return pos_err, neg_err, ber, acc, df |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| |
|
|
| class AverageMeter(object): |
| """Computes and stores the average and current value""" |
| def __init__(self): |
| self.sum = 0 |
| self.count = 0 |
|
|
| def update(self, val, weight=1): |
| self.sum += val * weight |
| self.count += weight |
|
|
| def average(self): |
| if self.count == 0: |
| return 0 |
| else: |
| return self.sum / self.count |
|
|
| def clear(self): |
| self.sum = 0 |
| self.count = 0 |
|
|
| def compute_cm_torch(y_pred, y_label, n_class): |
| mask = (y_label >= 0) & (y_label < n_class) |
| hist = torch.bincount(n_class * y_label[mask] + y_pred[mask], |
| minlength=n_class**2).reshape(n_class, n_class) |
| return hist |
|
|
| class MyConfuseMatrixMeter(AverageMeter): |
| """More Clear Confusion Matrix Meter""" |
| def __init__(self, n_class): |
| super(MyConfuseMatrixMeter, self).__init__() |
| self.n_class = n_class |
|
|
| def update_cm(self, y_pred, y_label, weight=1): |
| y_label = y_label.type(torch.int64) |
| val = compute_cm_torch(y_pred=y_pred.flatten(), y_label=y_label.flatten(), |
| n_class=self.n_class) |
| self.update(val, weight) |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| def get_scores_binary(self): |
| assert self.n_class == 2, "this function can only be called for binary calssification problem" |
| tn, fp, fn, tp = self.sum.flatten() |
| eps = torch.finfo(torch.float32).eps |
| pos_err = (1 - tp / (tp + fn + eps)) * 100 |
| neg_err = (1 - tn / (tn + fp + eps)) * 100 |
| ber = (pos_err + neg_err) / 2 |
| acc = (tn + tp) / (tn + tp + fn + fp + eps) |
| score_dict = {} |
| score_dict['pos_err'] = pos_err |
| score_dict['neg_err'] = neg_err |
| score_dict['ber'] = ber |
| score_dict['acc'] = acc |
| return score_dict |
|
|
