COMPMID-417: Fix validation
Change-Id: I7a745037136bc6e02d177f65fe4f4cd43873b98e
Reviewed-on: http://mpd-gerrit.cambridge.arm.com/87406
Tested-by: Kaizen <jeremy.johnson+kaizengerrit@arm.com>
Reviewed-by: Anthony Barbier <anthony.barbier@arm.com>
diff --git a/tests/framework/Framework.cpp b/tests/framework/Framework.cpp
index 343b7a8..31e5243 100644
--- a/tests/framework/Framework.cpp
+++ b/tests/framework/Framework.cpp
@@ -164,7 +164,8 @@
{
if(_log_level >= error.level() && _printer != nullptr)
{
- _printer->print_error(error);
+ constexpr bool expected_error = true;
+ _printer->print_error(error, expected_error);
}
if(_current_test_result != nullptr)
@@ -232,6 +233,8 @@
_printer->print_errors_header();
}
+ const bool is_expected_failure = test_factory.status() == TestCaseFactory::Status::EXPECTED_FAILURE;
+
try
{
std::unique_ptr<TestCase> test_case = test_factory.make();
@@ -265,7 +268,7 @@
{
if(_log_level >= error.level() && _printer != nullptr)
{
- _printer->print_error(error);
+ _printer->print_error(error, is_expected_failure);
}
result.status = TestResult::Status::FAILED;
@@ -282,7 +285,8 @@
{
std::stringstream stream;
stream << "Error code: " << error.err();
- _printer->print_error(TestError(error.what(), LogLevel::ERRORS, stream.str()));
+ TestError test_error(error.what(), LogLevel::ERRORS, stream.str());
+ _printer->print_error(test_error, is_expected_failure);
}
result.status = TestResult::Status::FAILED;
@@ -297,7 +301,7 @@
{
if(_log_level >= LogLevel::ERRORS && _printer != nullptr)
{
- _printer->print_error(error);
+ _printer->print_error(error, is_expected_failure);
}
result.status = TestResult::Status::CRASHED;
@@ -311,7 +315,7 @@
{
if(_log_level >= LogLevel::ERRORS && _printer != nullptr)
{
- _printer->print_error(TestError("Received unknown exception"));
+ _printer->print_error(TestError("Received unknown exception"), is_expected_failure);
}
result.status = TestResult::Status::CRASHED;
@@ -326,7 +330,7 @@
{
if(_log_level >= LogLevel::ERRORS && _printer != nullptr)
{
- _printer->print_error(error);
+ _printer->print_error(error, is_expected_failure);
}
result.status = TestResult::Status::CRASHED;
@@ -340,7 +344,7 @@
{
if(_log_level >= LogLevel::ERRORS && _printer != nullptr)
{
- _printer->print_error(TestError("Received unknown exception"));
+ _printer->print_error(TestError("Received unknown exception"), is_expected_failure);
}
result.status = TestResult::Status::CRASHED;
diff --git a/tests/framework/printers/JSONPrinter.cpp b/tests/framework/printers/JSONPrinter.cpp
index ae19cae..4f17e62 100644
--- a/tests/framework/printers/JSONPrinter.cpp
+++ b/tests/framework/printers/JSONPrinter.cpp
@@ -46,6 +46,31 @@
}
}
+template <typename T>
+void JSONPrinter::print_strings(T &&first, T &&last)
+{
+ bool first_entry = true;
+ std::stringstream log;
+
+ while(first != last)
+ {
+ print_separator(first_entry);
+
+ *_stream << R"(")";
+
+ log.str(*first);
+
+ for(std::string line; !std::getline(log, line).fail();)
+ {
+ *_stream << line << "; ";
+ }
+
+ *_stream << R"(")";
+
+ ++first;
+ }
+}
+
void JSONPrinter::print_entry(const std::string &name, const std::string &value)
{
print_separator(_first_entry);
@@ -90,38 +115,43 @@
void JSONPrinter::print_errors_header()
{
- print_separator(_first_test_entry);
-
- _first_error = true;
- *_stream << R"("errors" : [)";
+ _errors.clear();
+ _expected_errors.clear();
+ _infos.clear();
}
void JSONPrinter::print_errors_footer()
{
+ print_separator(_first_test_entry);
+
+ *_stream << R"("errors" : [)";
+ print_strings(_errors.begin(), _errors.end());
+ *_stream << "]";
+
+ *_stream << R"(, "expected_errors" : [)";
+ print_strings(_expected_errors.begin(), _expected_errors.end());
+ *_stream << "]";
+
+ *_stream << R"(, "infos" : [)";
+ print_strings(_infos.begin(), _infos.end());
*_stream << "]";
}
-void JSONPrinter::print_error(const std::exception &error)
+void JSONPrinter::print_error(const std::exception &error, bool expected)
{
- std::stringstream error_log;
- error_log.str(error.what());
-
- for(std::string line; !std::getline(error_log, line).fail();)
+ if(expected)
{
- print_separator(_first_error);
-
- *_stream << R"(")" << line << R"(")";
+ _expected_errors.emplace_back(error.what());
+ }
+ else
+ {
+ _errors.emplace_back(error.what());
}
}
void JSONPrinter::print_info(const std::string &info)
{
- std::istringstream iss(info);
- for(std::string line; !std::getline(iss, line).fail();)
- {
- print_separator(_first_error);
- *_stream << R"(")" << line << R"(")";
- }
+ _infos.push_back(info);
}
void JSONPrinter::print_measurements(const Profiler::MeasurementsMap &measurements)
diff --git a/tests/framework/printers/JSONPrinter.h b/tests/framework/printers/JSONPrinter.h
index 18bd438..a2811ea 100644
--- a/tests/framework/printers/JSONPrinter.h
+++ b/tests/framework/printers/JSONPrinter.h
@@ -26,6 +26,8 @@
#include "Printer.h"
+#include <list>
+
namespace arm_compute
{
namespace test
@@ -47,17 +49,22 @@
void print_test_footer() override;
void print_errors_header() override;
void print_errors_footer() override;
- void print_error(const std::exception &error) override;
+ void print_error(const std::exception &error, bool expected) override;
void print_info(const std::string &info) override;
void print_measurements(const Profiler::MeasurementsMap &measurements) override;
private:
void print_separator(bool &flag);
+ template <typename T>
+ void print_strings(T &&first, T &&last);
+
+ std::list<std::string> _infos{};
+ std::list<std::string> _errors{};
+ std::list<std::string> _expected_errors{};
bool _first_entry{ true };
bool _first_test{ true };
bool _first_test_entry{ true };
- bool _first_error{ true };
};
} // namespace framework
} // namespace test
diff --git a/tests/framework/printers/PrettyPrinter.cpp b/tests/framework/printers/PrettyPrinter.cpp
index 2f7df18..5eec72a 100644
--- a/tests/framework/printers/PrettyPrinter.cpp
+++ b/tests/framework/printers/PrettyPrinter.cpp
@@ -102,9 +102,10 @@
*_stream << begin_color("1") << "INFO: " << info << end_color() << "\n";
}
-void PrettyPrinter::print_error(const std::exception &error)
+void PrettyPrinter::print_error(const std::exception &error, bool expected)
{
- *_stream << begin_color("1") << "ERROR: " << error.what() << end_color() << "\n";
+ std::string prefix = expected ? "EXPECTED ERROR: " : "ERROR: ";
+ *_stream << begin_color("1") << prefix << error.what() << end_color() << "\n";
}
void PrettyPrinter::print_measurements(const Profiler::MeasurementsMap &measurements)
diff --git a/tests/framework/printers/PrettyPrinter.h b/tests/framework/printers/PrettyPrinter.h
index 3e2bebd..f72a613 100644
--- a/tests/framework/printers/PrettyPrinter.h
+++ b/tests/framework/printers/PrettyPrinter.h
@@ -53,7 +53,7 @@
void print_test_footer() override;
void print_errors_header() override;
void print_errors_footer() override;
- void print_error(const std::exception &error) override;
+ void print_error(const std::exception &error, bool expected) override;
void print_info(const std::string &info) override;
void print_measurements(const Profiler::MeasurementsMap &measurements) override;
diff --git a/tests/framework/printers/Printer.h b/tests/framework/printers/Printer.h
index 16a4170..c2a4424 100644
--- a/tests/framework/printers/Printer.h
+++ b/tests/framework/printers/Printer.h
@@ -104,9 +104,10 @@
/** Print test error.
*
- * @param[in] error Description of the error.
+ * @param[in] error Description of the error.
+ * @param[in] expected Whether the error was expected or not.
*/
- virtual void print_error(const std::exception &error) = 0;
+ virtual void print_error(const std::exception &error, bool expected) = 0;
/** Print test log info.
*