Ranked Probability Scores

[1]:

import sys

sys.path.append("../")

import penaltyblog as pb

Examples

Examples below taken from Solving the problem of inadequate scoring rules for assessing probabilistic football forecast models

[2]:

predictions = [
    [1, 0, 0],
    [0.9, 0.1, 0],
    [0.8, 0.1, 0.1],
    [0.5, 0.25, 0.25],
    [0.35, 0.3, 0.35],
    [0.6, 0.3, 0.1],
    [0.6, 0.25, 0.15],
    [0.6, 0.15, 0.25],
    [0.57, 0.33, 0.1],
    [0.6, 0.2, 0.2],
]

observed = [0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0]

[3]:

for p, o in zip(predictions, observed):
    rps = pb.metrics.rps(p, o)
    print(round(rps, 4))