diff --git a/run_project_tests.py b/run_project_tests.py
index 57a8188..648ec47 100755
--- a/run_project_tests.py
+++ b/run_project_tests.py
@@ -962,12 +962,6 @@
     if not t.category.endswith('frameworks'):
         return True
 
-    # For the moment, all skips in jobs which don't set MESON_CI_JOBNAME are
-    # treated as expected.  In the future, we should make it mandatory to set
-    # MESON_CI_JOBNAME for all CI jobs.
-    if ci_jobname is None:
-        return True
-
     return False
 
 def skip_csharp(backend: Backend) -> bool:
@@ -1490,6 +1484,9 @@
             mesonlib.windows_proof_rm(str(d))
 
 if __name__ == '__main__':
+    if under_ci and not ci_jobname:
+        raise SystemExit('Running under CI but MESON_CI_JOBNAME is not set')
+
     setup_vsenv()
 
     try: