| // |
| // Copyright © 2017 Arm Ltd. All rights reserved. |
| // See LICENSE file in the project root for full license information. |
| // |
| |
| #define LOG_TAG "ArmnnDriver" |
| |
| #include "ArmnnDriver.hpp" |
| #include "ArmnnPreparedModel.hpp" |
| #include "ModelToINetworkConverter.hpp" |
| #include "Utils.hpp" |
| |
| #include <log/log.h> |
| #include "SystemPropertiesUtils.hpp" |
| |
| #include "OperationsUtils.h" |
| |
| #include <boost/algorithm/string/predicate.hpp> |
| #include <boost/program_options.hpp> |
| |
| #include <cassert> |
| #include <functional> |
| #include <string> |
| #include <sstream> |
| |
| using namespace android; |
| using namespace std; |
| |
| namespace |
| { |
| |
| const char *g_Float32PerformanceExecTimeName = "ArmNN.float32Performance.execTime"; |
| const char *g_Float32PerformancePowerUsageName = "ArmNN.float32Performance.powerUsage"; |
| const char *g_Quantized8PerformanceExecTimeName = "ArmNN.quantized8Performance.execTime"; |
| const char *g_Quantized8PerformancePowerUsageName = "ArmNN.quantized8Performance.powerUsage"; |
| |
| }; //namespace |
| |
| namespace armnn_driver |
| { |
| |
| DriverOptions::DriverOptions(armnn::Compute computeDevice) |
| : m_ComputeDevice(computeDevice) |
| , m_VerboseLogging(false) |
| , m_UseAndroidNnCpuExecutor(false) |
| , m_ClTunedParametersMode(armnn::IClTunedParameters::Mode::UseTunedParameters) |
| { |
| } |
| |
| DriverOptions::DriverOptions(int argc, char** argv) |
| : m_ComputeDevice(armnn::Compute::GpuAcc) |
| , m_VerboseLogging(false) |
| , m_UseAndroidNnCpuExecutor(false) |
| , m_ClTunedParametersMode(armnn::IClTunedParameters::Mode::UseTunedParameters) |
| { |
| namespace po = boost::program_options; |
| |
| std::string computeDeviceAsString; |
| std::string unsupportedOperationsAsString; |
| std::string clTunedParametersModeAsString; |
| |
| po::options_description optionsDesc("Options"); |
| optionsDesc.add_options() |
| ("compute,c", |
| po::value<std::string>(&computeDeviceAsString)->default_value("GpuAcc"), |
| "Which device to run layers on by default. Possible values are: CpuRef, CpuAcc, GpuAcc") |
| |
| ("verbose-logging,v", |
| po::bool_switch(&m_VerboseLogging), |
| "Turns verbose logging on") |
| |
| ("use-androidnn-cpu-executor,e", |
| po::bool_switch(&m_UseAndroidNnCpuExecutor), |
| "Forces the driver to satisfy requests via the Android-provided CpuExecutor") |
| |
| ("request-inputs-and-outputs-dump-dir,d", |
| po::value<std::string>(&m_RequestInputsAndOutputsDumpDir)->default_value(""), |
| "If non-empty, the directory where request inputs and outputs should be dumped") |
| |
| ("unsupported-operations,u", |
| po::value<std::string>(&unsupportedOperationsAsString)->default_value(""), |
| "If non-empty, a comma-separated list of operation indices which the driver will forcibly " |
| "consider unsupported") |
| |
| ("cl-tuned-parameters-file,t", |
| po::value<std::string>(&m_ClTunedParametersFile)->default_value(""), |
| "If non-empty, the given file will be used to load/save CL tuned parameters. " |
| "See also --cl-tuned-parameters-mode") |
| |
| ("cl-tuned-parameters-mode,m", |
| po::value<std::string>(&clTunedParametersModeAsString)->default_value("UseTunedParameters"), |
| "If 'UseTunedParameters' (the default), will read CL tuned parameters from the file specified by " |
| "--cl-tuned-parameters-file. " |
| "If 'UpdateTunedParameters', will also find the optimum parameters when preparing new networks and update " |
| "the file accordingly."); |
| |
| |
| po::variables_map variablesMap; |
| try |
| { |
| po::store(po::parse_command_line(argc, argv, optionsDesc), variablesMap); |
| po::notify(variablesMap); |
| } |
| catch (const po::error& e) |
| { |
| ALOGW("An error occurred attempting to parse program options: %s", e.what()); |
| } |
| |
| if (computeDeviceAsString == "CpuRef") |
| { |
| m_ComputeDevice = armnn::Compute::CpuRef; |
| } |
| else if (computeDeviceAsString == "GpuAcc") |
| { |
| m_ComputeDevice = armnn::Compute::GpuAcc; |
| } |
| else if (computeDeviceAsString == "CpuAcc") |
| { |
| m_ComputeDevice = armnn::Compute::CpuAcc; |
| } |
| else |
| { |
| ALOGW("Requested unknown compute device %s. Defaulting to compute id %s", |
| computeDeviceAsString.c_str(), GetComputeDeviceAsCString(m_ComputeDevice)); |
| } |
| |
| if (!unsupportedOperationsAsString.empty()) |
| { |
| std::istringstream argStream(unsupportedOperationsAsString); |
| |
| std::string s; |
| while (!argStream.eof()) |
| { |
| std::getline(argStream, s, ','); |
| try |
| { |
| unsigned int operationIdx = std::stoi(s); |
| m_ForcedUnsupportedOperations.insert(operationIdx); |
| } |
| catch (const std::invalid_argument&) |
| { |
| ALOGW("Ignoring invalid integer argument in -u/--unsupported-operations value: %s", s.c_str()); |
| } |
| } |
| } |
| |
| if (!m_ClTunedParametersFile.empty()) |
| { |
| // The mode is only relevant if the file path has been provided |
| if (clTunedParametersModeAsString == "UseTunedParameters") |
| { |
| m_ClTunedParametersMode = armnn::IClTunedParameters::Mode::UseTunedParameters; |
| } |
| else if (clTunedParametersModeAsString == "UpdateTunedParameters") |
| { |
| m_ClTunedParametersMode = armnn::IClTunedParameters::Mode::UpdateTunedParameters; |
| } |
| else |
| { |
| ALOGW("Requested unknown cl-tuned-parameters-mode '%s'. Defaulting to UseTunedParameters", |
| clTunedParametersModeAsString.c_str()); |
| } |
| } |
| } |
| |
| ArmnnDriver::ArmnnDriver(DriverOptions options) |
| : m_Runtime(nullptr, nullptr) |
| , m_ClTunedParameters(nullptr, nullptr) |
| , m_Options(std::move(options)) |
| { |
| ALOGV("ArmnnDriver::ArmnnDriver()"); |
| |
| armnn::ConfigureLogging(false, m_Options.IsVerboseLoggingEnabled(), armnn::LogSeverity::Trace); |
| if (m_Options.IsVerboseLoggingEnabled()) |
| { |
| SetMinimumLogSeverity(base::VERBOSE); |
| } |
| else |
| { |
| SetMinimumLogSeverity(base::INFO); |
| } |
| |
| try |
| { |
| armnn::IRuntime::CreationOptions options(m_Options.GetComputeDevice()); |
| options.m_UseCpuRefAsFallback = false; |
| if (!m_Options.GetClTunedParametersFile().empty()) |
| { |
| m_ClTunedParameters = armnn::IClTunedParameters::Create(m_Options.GetClTunedParametersMode()); |
| try |
| { |
| m_ClTunedParameters->Load(m_Options.GetClTunedParametersFile().c_str()); |
| } |
| catch (const armnn::Exception& error) |
| { |
| // This is only a warning because the file won't exist the first time you are generating it. |
| ALOGW("ArmnnDriver: Failed to load CL tuned parameters file '%s': %s", |
| m_Options.GetClTunedParametersFile().c_str(), error.what()); |
| } |
| options.m_ClTunedParameters = m_ClTunedParameters.get(); |
| } |
| m_Runtime = armnn::IRuntime::Create(options); |
| } |
| catch (const armnn::ClRuntimeUnavailableException& error) |
| { |
| ALOGE("ArmnnDriver: Failed to setup CL runtime: %s. Device will be unavailable.", error.what()); |
| } |
| } |
| |
| Return<void> ArmnnDriver::getCapabilities(getCapabilities_cb cb) |
| { |
| ALOGV("ArmnnDriver::getCapabilities()"); |
| |
| Capabilities capabilities; |
| if (m_Runtime) |
| { |
| capabilities.float32Performance.execTime = |
| ParseSystemProperty(g_Float32PerformanceExecTimeName, 1.0f); |
| |
| capabilities.float32Performance.powerUsage = |
| ParseSystemProperty(g_Float32PerformancePowerUsageName, 1.0f); |
| |
| capabilities.quantized8Performance.execTime = |
| ParseSystemProperty(g_Quantized8PerformanceExecTimeName, 1.0f); |
| |
| capabilities.quantized8Performance.powerUsage = |
| ParseSystemProperty(g_Quantized8PerformancePowerUsageName, 1.0f); |
| |
| cb(ErrorStatus::NONE, capabilities); |
| } |
| else |
| { |
| capabilities.float32Performance.execTime = 0; |
| capabilities.float32Performance.powerUsage = 0; |
| capabilities.quantized8Performance.execTime = 0; |
| capabilities.quantized8Performance.powerUsage = 0; |
| |
| cb(ErrorStatus::DEVICE_UNAVAILABLE, capabilities); |
| } |
| |
| return Void(); |
| } |
| |
| Return<void> ArmnnDriver::getSupportedOperations(const Model& model, getSupportedOperations_cb cb) |
| { |
| ALOGV("ArmnnDriver::getSupportedOperations()"); |
| |
| std::vector<bool> result; |
| |
| if (!m_Runtime) |
| { |
| cb(ErrorStatus::DEVICE_UNAVAILABLE, result); |
| return Void(); |
| } |
| |
| // Run general model validation, if this doesn't pass we shouldn't analyse the model anyway |
| if (!android::nn::validateModel(model)) |
| { |
| cb(ErrorStatus::INVALID_ARGUMENT, result); |
| return Void(); |
| } |
| |
| // Attempt to convert the model to an ArmNN input network (INetwork). |
| ModelToINetworkConverter modelConverter(m_Runtime->GetDeviceSpec().DefaultComputeDevice, model, |
| m_Options.GetForcedUnsupportedOperations()); |
| |
| if (modelConverter.GetConversionResult() != ConversionResult::Success |
| && modelConverter.GetConversionResult() != ConversionResult::UnsupportedFeature) |
| { |
| cb(ErrorStatus::GENERAL_FAILURE, result); |
| return Void(); |
| } |
| |
| // Check each operation if it was converted successfully and copy the flags |
| // into the result (vector<bool>) that we need to return to Android |
| result.reserve(model.operations.size()); |
| for (uint32_t operationIdx = 0; operationIdx < model.operations.size(); operationIdx++) |
| { |
| bool operationSupported = modelConverter.IsOperationSupported(operationIdx); |
| result.push_back(operationSupported); |
| } |
| |
| cb(ErrorStatus::NONE, result); |
| return Void(); |
| } |
| |
| namespace |
| { |
| |
| void NotifyCallbackAndCheck(const sp<IPreparedModelCallback>& callback, ErrorStatus errorStatus, |
| const ::android::sp<IPreparedModel>& preparedModelPtr) |
| { |
| Return<void> returned = callback->notify(errorStatus, preparedModelPtr); |
| // This check is required, if the callback fails and it isn't checked it will bring down the service |
| if (!returned.isOk()) |
| { |
| ALOGE("ArmnnDriver::prepareModel: hidl callback failed to return properly: %s ", |
| returned.description().c_str()); |
| } |
| } |
| |
| Return<ErrorStatus> FailPrepareModel(ErrorStatus error, |
| const std::string& message, |
| const sp<IPreparedModelCallback>& callback) |
| { |
| ALOGW("ArmnnDriver::prepareModel: %s", message.c_str()); |
| NotifyCallbackAndCheck(callback, error, nullptr); |
| return error; |
| } |
| |
| } |
| |
| Return<ErrorStatus> ArmnnDriver::prepareModel(const Model& model, |
| const sp<IPreparedModelCallback>& cb) |
| { |
| ALOGV("ArmnnDriver::prepareModel()"); |
| |
| if (cb.get() == nullptr) |
| { |
| ALOGW("ArmnnDriver::prepareModel: Invalid callback passed to prepareModel"); |
| return ErrorStatus::INVALID_ARGUMENT; |
| } |
| |
| if (!m_Runtime) |
| { |
| return FailPrepareModel(ErrorStatus::DEVICE_UNAVAILABLE, "ArmnnDriver::prepareModel: Device unavailable", cb); |
| } |
| |
| if (!android::nn::validateModel(model)) |
| { |
| return FailPrepareModel(ErrorStatus::INVALID_ARGUMENT, |
| "ArmnnDriver::prepareModel: Invalid model passed as input", cb); |
| } |
| |
| if (m_Options.UseAndroidNnCpuExecutor()) |
| { |
| sp<AndroidNnCpuExecutorPreparedModel> preparedModel = new AndroidNnCpuExecutorPreparedModel(model, |
| m_Options.GetRequestInputsAndOutputsDumpDir()); |
| if (preparedModel->Initialize()) |
| { |
| NotifyCallbackAndCheck(cb, ErrorStatus::NONE, preparedModel); |
| return ErrorStatus::NONE; |
| } |
| else |
| { |
| NotifyCallbackAndCheck(cb, ErrorStatus::INVALID_ARGUMENT, preparedModel); |
| return ErrorStatus::INVALID_ARGUMENT; |
| } |
| } |
| |
| // Deliberately ignore any unsupported operations requested by the options - |
| // at this point we're being asked to prepare a model that we've already declared support for |
| // and the operation indices may be different to those in getSupportedOperations anyway. |
| std::set<unsigned int> unsupportedOperations; |
| ModelToINetworkConverter modelConverter(m_Runtime->GetDeviceSpec().DefaultComputeDevice, model, |
| unsupportedOperations); |
| |
| if (modelConverter.GetConversionResult() != ConversionResult::Success) |
| { |
| return FailPrepareModel(ErrorStatus::GENERAL_FAILURE, "ModelToINetworkConverter failed", cb); |
| } |
| |
| // optimize the network |
| armnn::IOptimizedNetworkPtr optNet(nullptr, nullptr); |
| try |
| { |
| optNet = armnn::Optimize(*modelConverter.GetINetwork(), m_Runtime->GetDeviceSpec()); |
| } |
| catch (armnn::Exception& e) |
| { |
| std::stringstream message; |
| message << "armnn::Exception ("<<e.what()<<") caught from optimize."; |
| return FailPrepareModel(ErrorStatus::GENERAL_FAILURE, message.str(), cb); |
| } |
| |
| // Check that the optimized network is valid. |
| if (!optNet) |
| { |
| return FailPrepareModel(ErrorStatus::GENERAL_FAILURE, |
| "ArmnnDriver::prepareModel: Invalid optimized network", cb); |
| } |
| |
| // Export the optimized network graph to a dot file if an output dump directory |
| // has been specified in the drivers' arguments. |
| ExportNetworkGraphToDotFile(*optNet, |
| m_Options.GetRequestInputsAndOutputsDumpDir(), |
| model); |
| |
| // load it into the runtime |
| armnn::NetworkId netId = 0; |
| try |
| { |
| if (m_Runtime->LoadNetwork(netId, std::move(optNet)) != armnn::Status::Success) |
| { |
| return FailPrepareModel(ErrorStatus::GENERAL_FAILURE, |
| "ArmnnDriver::prepareModel: Network could not be loaded", cb); |
| } |
| } |
| catch (armnn::Exception& e) |
| { |
| std::stringstream message; |
| message << "armnn::Exception (" << e.what()<< ") caught from LoadNetwork."; |
| return FailPrepareModel(ErrorStatus::GENERAL_FAILURE, message.str(), cb); |
| } |
| |
| std::unique_ptr<ArmnnPreparedModel> preparedModel(new ArmnnPreparedModel( |
| netId, |
| m_Runtime.get(), |
| model, |
| m_Options.GetRequestInputsAndOutputsDumpDir() |
| )); |
| |
| // Run a single 'dummy' inference of the model. This means that CL kernels will get compiled (and tuned if |
| // this is enabled) before the first 'real' inference which removes the overhead of the first inference. |
| preparedModel->ExecuteWithDummyInputs(); |
| |
| if (m_ClTunedParameters && |
| m_Options.GetClTunedParametersMode() == armnn::IClTunedParameters::Mode::UpdateTunedParameters) |
| { |
| // Now that we've done one inference the CL kernel parameters will have been tuned, so save the updated file. |
| try |
| { |
| m_ClTunedParameters->Save(m_Options.GetClTunedParametersFile().c_str()); |
| } |
| catch (const armnn::Exception& error) |
| { |
| ALOGE("ArmnnDriver: Failed to save CL tuned parameters file '%s': %s", |
| m_Options.GetClTunedParametersFile().c_str(), error.what()); |
| } |
| } |
| |
| NotifyCallbackAndCheck(cb, ErrorStatus::NONE, preparedModel.release()); |
| |
| return ErrorStatus::NONE; |
| } |
| |
| Return<DeviceStatus> ArmnnDriver::getStatus() |
| { |
| ALOGV("ArmnnDriver::getStatus()"); |
| return DeviceStatus::AVAILABLE; |
| } |
| |
| } |