commit acb550f4410ae861e53cae27a9feb4b11d45769f
author Kevin Cheng <kevin.cheng@arm.com> Tue Jun 29 15:32:19 2021 -0700
committer Kevin Cheng <kevin.cheng@arm.com> Fri Aug 20 18:07:06 2021 +0100
tree ae2f4ec558c2cdf1afa020b80a09d7ab4be5ef6d
parent 68e7aee65bda5ac03fa7def753b7dc7462554793

Replace node level check ASSERT_MSG_NODE()/FATAL_ERROR_NODE() with REQUIRE() or ERROR_IF()

- Adding return code enum class: {VALID, UNPREDICTABLE, ERROR}
- Runtime errors (e.g. memory allocation failure) will abort immediately, or will return one of the three return codes
  Part of the codes are re-written to pass REQUIRE() to the top-level (e.g. apply_scale_32/16())
- Update setExpectedFailure() to setExpectedReturnCode() on test generation script
- Update test regression script to interface with reference model change

Signed-off-by: Kevin Cheng <kevin.cheng@arm.com>
Change-Id: Ia063c936bcb2a54d6e379a5bb6801aa72d1186f1

verif/tosa_ref_run.py

diff --git a/verif/tosa_ref_run.py b/verif/tosa_ref_run.py
index 098f39b..499513b 100644
--- a/verif/tosa_ref_run.py
+++ b/verif/tosa_ref_run.py
@@ -1,5 +1,3 @@
-import os
-
 # Copyright (c) 2020-2021, ARM Limited.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,9 +16,17 @@
 import json
 import shlex
 import subprocess
+from enum import Enum, IntEnum, unique
 from tosa_test_runner import TosaTestRunner, run_sh_command
 
 
+@unique
+class TosaReturnCode(IntEnum):
+    VALID = 0
+    UNPREDICTABLE = 1
+    ERROR = 2
+
+
 class TosaRefRunner(TosaTestRunner):
     def __init__(self, args, runnerArgs, testDir):
         super().__init__(args, runnerArgs, testDir)
@@ -41,18 +47,29 @@
         if args.ref_intermediates:
             ref_cmd.extend(["-Ddump_intermediates=1"])
 
-        expectedFailure = self.testDesc["expected_failure"]
+        expectedReturnCode = self.testDesc["expected_return_code"]
 
         try:
-            run_sh_command(self.args, ref_cmd)
-            if expectedFailure:
-                result = TosaTestRunner.Result.UNEXPECTED_PASS
+            rc = run_sh_command(self.args, ref_cmd)
+            if rc == TosaReturnCode.VALID:
+                if expectedReturnCode == TosaReturnCode.VALID:
+                    result = TosaTestRunner.Result.EXPECTED_PASS
+                else:
+                    result = TosaTestRunner.Result.UNEXPECTED_PASS
+            elif rc == TosaReturnCode.ERROR:
+                if expectedReturnCode == TosaReturnCode.ERROR:
+                    result = TosaTestRunner.Result.EXPECTED_FAILURE
+                else:
+                    result = TosaTestRunner.Result.UNEXPECTED_FAILURE
+            elif rc == TosaReturnCode.UNPREDICTABLE:
+                if expectedReturnCode == TosaReturnCode.UNPREDICTABLE:
+                    result = TosaTestRunner.Result.EXPECTED_FAILURE
+                else:
+                    result = TosaTestRunner.Result.UNEXPECTED_FAILURE
             else:
-                result = TosaTestRunner.Result.EXPECTED_PASS
+                raise Exception("Return code unknown.")
+
         except Exception as e:
-            if expectedFailure:
-                result = TosaTestRunner.Result.EXPECTED_FAILURE
-            else:
-                result = TosaTestRunner.Result.UNEXPECTED_FAILURE
+            raise Exception("Runtime Error when running: {}".format(" ".join(ref_cmd)))
 
         return result