Coverage for uqmodels/evaluation/evaluation.py: 24%

1import numpy as np 

2 

3import uqmodels.evaluation.base_metrics as metrics 

4from uqmodels.utils import compute_born 

5 

6# Evaluation anaylsis : 

7 

8 

9def var_explain(y, pred, var_A, var_E, set_): 

10 for n in list(set(set_)): 

11 flag_set = set_ == n 

12 if set_.sum() == 0: 

13 print(n, "%data", 0) 

14 return [0, 0, 0, 0, 0, 0] 

15 w = flag_set.mean() 

16 y_ = y[flag_set] 

17 y_b = np.mean(y[flag_set]) 

18 y_h = pred[flag_set] 

19 va = var_A[flag_set] 

20 ve = var_E[flag_set] 

21 r = y_ - y_h 

22 y_ - y_b 

23 vloc_unexplain = w * y_.var() 

24 if y_.var() < 10e-5: 

25 vloc_unexplain = 10e-5 

26 vloc_explain = w * np.power(y_b - y.mean(), 2).mean() 

27 

28 vtot = np.power(y - y.mean(), 2).mean() 

29 array = np.array( 

30 [ 

31 np.power(y_h - y_b, 2).mean() + (2 * r * (y_h - y_b)).mean(), 

32 np.power(r, 2).mean(), 

33 va.mean(), 

34 ve.mean(), 

35 ] 

36 ) 

37 

38 array_bis = np.array( 

39 [ 

40 (vloc_unexplain + vloc_explain) / w, 

41 vloc_explain / w, 

42 np.power(y_h - y_b, 2).mean() + (2 * r * (y_h - y_b)).mean(), 

43 np.power(r, 2).mean(), 

44 va.mean(), 

45 ve.mean(), 

46 ] 

47 ) 

48 print( 

49 n, 

50 "%data", 

51 np.round(w * 100, 1), 

52 "Explain_ctx :", 

53 np.round(array * w / (vloc_unexplain), 3), 

54 "Explain_glob :", 

55 np.round(array_bis * w / vtot, 3), 

56 ) 

57 return () 

58 

59 

60def evaluate(y, output, list_metrics, list_sets=None, context=None, verbose=False): 

61 list_perf = [] 

62 if list_sets is None: 

63 list_sets = [np.ones(len(y)) == 1] 

64 for metrics_ in list_metrics: 

65 list_perf.append(metrics_.compute(y, output, list_sets, context)) 

66 if verbose: 

67 print(metrics_.name, list_perf[-1]) 

68 

69 return list_perf 

70 

71 

72def compute_perf(Y_, pred, sigma, test): 

73 # Compare mutli-performance for MSE et UQ coverage compare to naive baseline 

74 perf_baseline = [] 

75 perf_baseline1 = [] 

76 perf_modele = [] 

77 pref_cov = [] 

78 pref_sharp = [] 

79 for i in range(pred.shape[1]): 

80 std_min = Y_[test, i].std() + 0.000001 

81 env_bot, env_top = compute_born(pred, sigma, 0.1) 

82 

83 perf_baseline.append( 

84 metrics.mean_absolute_error(Y_[test, i], np.roll(Y_[test, i], 1)) / std_min 

85 ) 

86 mean_MA = ( 

87 np.roll(Y_[test, i], -1) 

88 + np.roll(Y_[test, i], -2) 

89 + np.roll(Y_[test, i], -3) 

90 + np.roll(Y_[test, i], -4) 

91 ) / 4 

92 

93 perf_baseline1.append( 

94 metrics.mean_absolute_error(Y_[test, i], mean_MA) / std_min 

95 ) 

96 perf_modele.append( 

97 metrics.mean_absolute_error(Y_[test, i], pred[test, i]) / std_min 

98 ) 

99 pref_cov.append( 

100 np.round( 

101 metrics.average_coverage( 

102 Y_[test, i], env_bot[test, i], env_top[test, i] 

103 ), 

104 2, 

105 ) 

106 ) 

107 pref_sharp.append( 

108 np.round( 

109 metrics.sharpness(env_bot[test, i], env_top[test, i]) 

110 / Y_[test, i].std(), 

111 2, 

112 ) 

113 ) 

114 

115 print("temoin0_mae_r", np.round(np.array(perf_baseline), 2).tolist()) 

116 print("temoin1_mae_r", np.round(np.array(perf_baseline1), 2).tolist()) 

117 print("mae_r", np.round(np.array(perf_modele), 2).tolist()) 

118 print("cover", pref_cov) 

119 print("sharp", pref_sharp)