diff --git a/ci/eval/compare/cmp-stats.py b/ci/eval/compare/cmp-stats.py
index 0b6bbb55af90..8d256d10eb9c 100644
--- a/ci/eval/compare/cmp-stats.py
+++ b/ci/eval/compare/cmp-stats.py
@@ -3,6 +3,7 @@ import json
 import numpy as np
 import os
 import pandas as pd
+import warnings
 
 from pathlib import Path
 from scipy.stats import ttest_rel
@@ -149,6 +150,9 @@ def main():
 
     options = parser.parse_args()
 
+    # Turn warnings into errors
+    warnings.simplefilter("error")
+
     before_stats = Path(options.before)
     after_stats = Path(options.after)