COMPMID-2986: Extend test framework options.
Add the following option to the test framework:
- delay: Allows run delay between tests in seconds.
Signed-off-by: Georgios Pinitas <georgios.pinitas@arm.com>
Change-Id: Ib1b7a1d1ad83c9a99b23f353b1800db075b32bce
Reviewed-on: https://review.mlplatform.org/c/2525
Reviewed-by: Michele Di Giorgio <michele.digiorgio@arm.com>
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Gian Marco Iodice <gianmarco.iodice@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
diff --git a/tests/benchmark_examples/RunExample.cpp b/tests/benchmark_examples/RunExample.cpp
index f3de308..613e985 100644
--- a/tests/benchmark_examples/RunExample.cpp
+++ b/tests/benchmark_examples/RunExample.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018-2019 ARM Limited.
+ * Copyright (c) 2018-2020 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -156,7 +156,13 @@
}
}
- framework.init(options.instruments->value(), options.iterations->value(), framework::DatasetMode::ALL, "", "", options.log_level->value());
+ // Initialize framework
+ framework::FrameworkConfig fconfig;
+ fconfig.instruments = options.instruments->value();
+ fconfig.num_iterations = options.iterations->value();
+ fconfig.log_level = options.log_level->value();
+ framework.init(fconfig);
+
for(auto &p : printers)
{
framework.add_printer(p.get());
diff --git a/tests/framework/Framework.cpp b/tests/framework/Framework.cpp
index 44887a9..4d66b14 100644
--- a/tests/framework/Framework.cpp
+++ b/tests/framework/Framework.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2019 ARM Limited.
+ * Copyright (c) 2017-2020 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -125,14 +125,14 @@
return instance;
}
-void Framework::init(const std::vector<framework::InstrumentsDescription> &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, const std::string &id_filter,
- LogLevel log_level)
+void Framework::init(const FrameworkConfig &config)
{
- _test_filter = TestFilter(mode, name_filter, id_filter);
- _num_iterations = num_iterations;
- _log_level = log_level;
+ _test_filter = TestFilter(config.mode, config.name_filter, config.id_filter);
+ _num_iterations = config.num_iterations;
+ _log_level = config.log_level;
+ _cooldown_sec = config.cooldown_sec;
- _instruments = std::set<framework::InstrumentsDescription>(instruments.begin(), instruments.end());
+ _instruments = std::set<framework::InstrumentsDescription>(std::begin(config.instruments), std::end(config.instruments));
}
std::string Framework::current_suite_name() const
@@ -579,6 +579,9 @@
run_test(test_info, *test_factory);
++id_run_test;
+
+ // Run test delay
+ sleep_in_seconds(_cooldown_sec);
}
++id;
diff --git a/tests/framework/Framework.h b/tests/framework/Framework.h
index c02416f..fb52fd8 100644
--- a/tests/framework/Framework.h
+++ b/tests/framework/Framework.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2019 ARM Limited.
+ * Copyright (c) 2017-2020 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -54,6 +54,18 @@
{
namespace framework
{
+/** Framework configuration structure */
+struct FrameworkConfig
+{
+ std::vector<framework::InstrumentsDescription> instruments{}; /**< Instrument types that will be used for benchmarking. */
+ std::string name_filter{}; /**< Regular expression to filter tests by name. Only matching tests will be executed. */
+ std::string id_filter{}; /**< String to match selected test ids. Only matching tests will be executed. */
+ DatasetMode mode{ DatasetMode::ALL }; /**< Dataset mode. */
+ int num_iterations{ 1 }; /**< Number of iterations per test. */
+ float cooldown_sec{ -1.f }; /**< Delay between tests in seconds. */
+ LogLevel log_level{ LogLevel::NONE }; /**< Verbosity of the output. */
+};
+
/** Information about a test case.
*
* A test can be identified either via its id or via its name. Additionally
@@ -99,14 +111,9 @@
*
* @see TestFilter::TestFilter for the format of the string to filter ids.
*
- * @param[in] instruments Instrument types that will be used for benchmarking.
- * @param[in] num_iterations Number of iterations per test.
- * @param[in] mode Dataset mode.
- * @param[in] name_filter Regular expression to filter tests by name. Only matching tests will be executed.
- * @param[in] id_filter String to match selected test ids. Only matching tests will be executed.
- * @param[in] log_level Verbosity of the output.
+ * @param[in] config Framework configuration meta-data.
*/
- void init(const std::vector<framework::InstrumentsDescription> &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, const std::string &id_filter, LogLevel log_level);
+ void init(const FrameworkConfig &config);
/** Add a new test suite.
*
@@ -329,6 +336,7 @@
std::vector<std::unique_ptr<TestCaseFactory>> _test_factories{};
std::map<TestInfo, TestResult> _test_results{};
int _num_iterations{ 1 };
+ float _cooldown_sec{ -1.f };
bool _throw_errors{ false };
bool _stop_on_error{ false };
bool _error_on_missing_assets{ false };
diff --git a/tests/framework/Utils.cpp b/tests/framework/Utils.cpp
new file mode 100644
index 0000000..589d274
--- /dev/null
+++ b/tests/framework/Utils.cpp
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2019-2020 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "Utils.h"
+
+#ifndef NO_MULTI_THREADING
+#include <chrono>
+#include <thread>
+#endif /* NO_MULTI_THREADING */
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+void sleep_in_seconds(float seconds)
+{
+ // Early return on non-positive input
+ if(seconds <= 0.f)
+ {
+ return;
+ }
+
+#ifndef NO_MULTI_THREADING
+ const int64_t us = static_cast<int64_t>(seconds * 1e6);
+ std::this_thread::sleep_for(std::chrono::microseconds(us));
+#endif /* NO_MULTI_THREADING */
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
diff --git a/tests/framework/Utils.h b/tests/framework/Utils.h
index c0442cf..0280daa 100644
--- a/tests/framework/Utils.h
+++ b/tests/framework/Utils.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017 ARM Limited.
+ * Copyright (c) 2017-2020 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -170,6 +170,11 @@
return ss.str();
}
+/** Makes the calling thread to sleep for a specified number of seconds
+ *
+ * @param[in] seconds Amount of seconds to sleep. Will return immediately if less or equal to zero.
+ */
+void sleep_in_seconds(float seconds);
} // namespace framework
} // namespace test
} // namespace arm_compute
diff --git a/tests/main.cpp b/tests/main.cpp
index 604a51f..441e28b 100644
--- a/tests/main.cpp
+++ b/tests/main.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2019 ARM Limited.
+ * Copyright (c) 2017-2020 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -142,6 +142,8 @@
#endif /* ARM_COMPUTE_CL */
auto threads = parser.add_option<utils::SimpleOption<int>>("threads", 1);
threads->set_help("Number of threads to use");
+ auto cooldown_sec = parser.add_option<utils::SimpleOption<float>>("delay", -1.f);
+ cooldown_sec->set_help("Delay to add between test executions in seconds");
try
{
@@ -261,12 +263,15 @@
framework.set_instruments_info(instruments_info);
// Initialize framework
- framework.init(options.instruments->value(),
- options.iterations->value(),
- dataset_mode->value(),
- filter->value(),
- filter_id->value(),
- options.log_level->value());
+ framework::FrameworkConfig fconfig;
+ fconfig.instruments = options.instruments->value();
+ fconfig.name_filter = filter->value();
+ fconfig.id_filter = filter_id->value();
+ fconfig.num_iterations = options.iterations->value();
+ fconfig.mode = dataset_mode->value();
+ fconfig.log_level = options.log_level->value();
+ fconfig.cooldown_sec = cooldown_sec->value();
+ framework.init(fconfig);
for(auto &p : printers)
{
diff --git a/tests/validate_examples/RunExample.cpp b/tests/validate_examples/RunExample.cpp
index f00460a..5d5291a 100644
--- a/tests/validate_examples/RunExample.cpp
+++ b/tests/validate_examples/RunExample.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018-2019 ARM Limited.
+ * Copyright (c) 2018-2020 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -172,7 +172,13 @@
}
}
- framework.init(options.instruments->value(), options.iterations->value(), framework::DatasetMode::ALL, "", "", options.log_level->value());
+ // Initialize framework
+ framework::FrameworkConfig fconfig;
+ fconfig.instruments = options.instruments->value();
+ fconfig.num_iterations = options.iterations->value();
+ fconfig.log_level = options.log_level->value();
+ framework.init(fconfig);
+
for(auto &p : printers)
{
framework.add_printer(p.get());