utils/inference_runner/inference_runner.cpp - ml/ethos-u/ethos-u-linux-driver-stack - Gitiles

 /*
  * SPDX-FileCopyrightText: Copyright 2020-2024 Arm Limited and/or its affiliates <open-source-office@arm.com>
  * SPDX-License-Identifier: Apache-2.0
  *
  * Licensed under the Apache License, Version 2.0 (the License); you may
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  * www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an AS IS BASIS, WITHOUT
  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 #include <ethosu.hpp>
 #include <uapi/ethosu.h>

 #include <fstream>
 #include <iomanip>
 #include <iostream>
 #include <list>
 #include <stdio.h>
 #include <string>
 #include <unistd.h>
 #include <utility>

 using namespace std;
 using namespace EthosU;

 namespace {
 int64_t defaultTimeout = 60000000000;

 void help(const string &exe) {
     cerr << "Usage: " << exe << " [ARGS]\n";
     cerr << "\n";
     cerr << "Arguments:\n";
     cerr << "    -h --help       Print this help message.\n";
     cerr << "    -n --network    File to read network from.\n";
     cerr << "       --index      Network model index, stored in firmware binary.\n";
     cerr << "    -i --ifm        File to read IFM from.\n";
     cerr << "    -o --ofm        File to write IFM to.\n";
     cerr << "    -P --pmu [0.." << Inference::getMaxPmuEventCounters() << "] eventid.\n";
     cerr << "                    PMU counter to enable followed by eventid, can be passed multiple times.\n";
     cerr << "    -C --cycles     Enable cycle counter for inference.\n";
     cerr << "    -t --timeout    Timeout in nanoseconds (default " << defaultTimeout << ").\n";
     cerr << "    -p              Print OFM.\n";
     cerr << endl;
 }

 void rangeCheck(const int i, const int argc, const string &arg) {
     if (i >= argc) {
         cerr << "Error: Missing argument to '" << arg << "'" << endl;
         exit(1);
     }
 }

 pair<unique_ptr<unsigned char[]>, size_t> getNetworkData(const string &filename) {
     ifstream stream(filename, ios::binary);
     if (!stream.is_open()) {
         cerr << "Error: Failed to open '" << filename << "'" << endl;
         exit(1);
     }

     stream.seekg(0, ios_base::end);
     size_t size = stream.tellg();
     stream.seekg(0, ios_base::beg);

     unique_ptr<unsigned char[]> data = std::make_unique<unsigned char[]>(size);
     stream.read(reinterpret_cast<char *>(data.get()), size);

     return make_pair(std::move(data), size);
 }

 shared_ptr<Inference> createInference(Device &device,
                                       shared_ptr<Network> &network,
                                       const string &filename,
                                       const std::vector<uint8_t> &counters,
                                       bool enableCycleCounter) {
     // Open IFM file
     ifstream stream(filename, ios::binary);
     if (!stream.is_open()) {
         cerr << "Error: Failed to open '" << filename << "'" << endl;
         exit(1);
     }

     // Get IFM file size
     stream.seekg(0, ios_base::end);
     size_t size = stream.tellg();
     stream.seekg(0, ios_base::beg);

     if (size != network->getIfmSize()) {
         cerr << "Error: IFM size does not match network size. filename=" << filename << ", size=" << size
              << ", network=" << network->getIfmSize() << endl;
         exit(1);
     }

     // Create IFM buffers
     vector<shared_ptr<Buffer>> ifm;
     for (auto size : network->getIfmDims()) {
         shared_ptr<Buffer> buffer = make_shared<Buffer>(device, size);
         stream.read(buffer->data(), size);

         if (!stream) {
             cerr << "Error: Failed to read IFM" << endl;
             exit(1);
         }

         ifm.push_back(buffer);
     }

     // Create OFM buffers
     vector<shared_ptr<Buffer>> ofm;
     for (auto size : network->getOfmDims()) {
         ofm.push_back(make_shared<Buffer>(device, size));
     }

     return make_shared<Inference>(
         network, ifm.begin(), ifm.end(), ofm.begin(), ofm.end(), counters, enableCycleCounter);
 }

 ostream &operator<<(ostream &os, Buffer &buf) {
     char *c         = buf.data();
     const char *end = c + buf.size();

     while (c < end) {
         os << hex << setw(2) << static_cast<int>(*c++) << " " << dec;
     }

     return os;
 }

 } // namespace

 int main(int argc, char *argv[]) {
     const string exe = argv[0];
     string networkArg;
     int networkIndex = -1;
     list<string> ifmArg;
     vector<uint8_t> enabledCounters(Inference::getMaxPmuEventCounters());
     string ofmArg;
     int64_t timeout         = defaultTimeout;
     bool print              = false;
     bool enableCycleCounter = false;

     for (int i = 1; i < argc; ++i) {
         const string arg(argv[i]);

         if (arg == "-h" || arg == "--help") {
             help(exe);
             exit(1);
         } else if (arg == "--network" || arg == "-n") {
             rangeCheck(++i, argc, arg);
             networkArg = argv[i];
         } else if (arg == "--index") {
             rangeCheck(++i, argc, arg);
             networkIndex = stoi(argv[i]);
         } else if (arg == "--ifm" || arg == "-i") {
             rangeCheck(++i, argc, arg);
             ifmArg.push_back(argv[i]);
         } else if (arg == "--ofm" || arg == "-o") {
             rangeCheck(++i, argc, arg);
             ofmArg = argv[i];
         } else if (arg == "--timeout" || arg == "-t") {
             rangeCheck(++i, argc, arg);
             timeout = stoll(argv[i]);
         } else if (arg == "--pmu" || arg == "-P") {
             unsigned pmu = 0, event = 0;
             rangeCheck(++i, argc, arg);
             pmu = stoi(argv[i]);

             rangeCheck(++i, argc, arg);
             event = stoi(argv[i]);

             if (pmu >= enabledCounters.size()) {
                 cerr << "PMU out of bounds!" << endl;
                 help(exe);
                 exit(1);
             }
             cout << argv[i] << " -> Enabling " << pmu << " with event " << event << endl;
             enabledCounters[pmu] = event;
         } else if (arg == "--cycles" || arg == "-C") {
             enableCycleCounter = true;
         } else if (arg == "-p") {
             print = true;
         } else {
             cerr << "Error: Invalid argument '" << arg << "'" << endl;
             help(exe);
             exit(1);
         }
     }

     if (networkArg.empty()) {
         cerr << "Error: Missing 'network' argument" << endl;
         exit(1);
     }

     if (ifmArg.empty()) {
         cerr << "Error: Missing 'ifm' argument" << endl;
         exit(1);
     }

     if (ofmArg.empty()) {
         cerr << "Error: Missing 'ofm' argument" << endl;
         exit(1);
     }

     try {
         cout << "Driver library version:" << getLibraryVersion() << endl;

         Device device;
         cout << "Kernel driver version:" << device.getDriverVersion() << endl;

         cout << "Send Ping" << endl;
         device.ioctl(ETHOSU_IOCTL_PING);

         cout << "Send capabilities request" << endl;
         Capabilities capabilities = device.capabilities();

         cout << "Capabilities:" << endl
              << "\tversion_status:" << unsigned(capabilities.hwId.versionStatus) << endl
              << "\tversion:" << capabilities.hwId.version << endl
              << "\tproduct:" << capabilities.hwId.product << endl
              << "\tarchitecture:" << capabilities.hwId.architecture << endl
              << "\tdriver:" << capabilities.driver << endl
              << "\tmacs_per_cc:" << unsigned(capabilities.hwCfg.macsPerClockCycle) << endl
              << "\tcmd_stream_version:" << unsigned(capabilities.hwCfg.cmdStreamVersion) << endl
              << "\ttype:" << capabilities.hwCfg.type << endl
              << "\tcustom_dma:" << std::boolalpha << capabilities.hwCfg.customDma << endl;

         /* Create network */
         cout << "Create network" << endl;

         shared_ptr<Network> network;

         if (networkIndex < 0) {
             auto networkData = getNetworkData(networkArg);
             network          = make_shared<Network>(device, networkData.first.get(), networkData.second);
         } else {
             network = make_shared<Network>(device, networkIndex);
         }

         /* Create one inference per IFM */
         list<shared_ptr<Inference>> inferences;
         for (auto &filename : ifmArg) {
             cout << "Create inference" << endl;
             inferences.push_back(createInference(device, network, filename, enabledCounters, enableCycleCounter));
         }

         cout << "Wait for inferences" << endl;

         int ofmIndex = 0;
         for (auto &inference : inferences) {
             cout << "Inference status: " << inference->status() << endl;

             /* make sure the wait completes ok */
             try {
                 cout << "Wait for inference" << endl;
                 bool timedout = inference->wait(timeout);
                 if (timedout) {
                     cout << "Inference timed out, cancelling it" << endl;
                     bool aborted = inference->cancel();
                     if (!aborted || inference->status() != InferenceStatus::ABORTED) {
                         cout << "Inference cancellation failed" << endl;
                     }
                 }
             } catch (std::exception &e) {
                 cout << "Failed to wait for or to cancel inference: " << e.what() << endl;
                 exit(1);
             }

             cout << "Inference status: " << inference->status() << endl;

             if (inference->status() == InferenceStatus::OK) {
                 string ofmFilename = ofmArg + "." + to_string(ofmIndex);
                 ofstream ofmStream(ofmFilename, ios::binary);
                 if (!ofmStream.is_open()) {
                     cerr << "Error: Failed to open '" << ofmFilename << "'" << endl;
                     exit(1);
                 }

                 /* The inference completed and has ok status */
                 for (auto &ofmBuffer : inference->getOfmBuffers()) {
                     cout << "OFM size: " << ofmBuffer->size() << endl;

                     if (print) {
                         cout << "OFM data: " << *ofmBuffer << endl;
                     }

                     ofmStream.write(ofmBuffer->data(), ofmBuffer->size());
                 }

                 ofmStream.flush();

                 /* Read out PMU counters if configured */
                 if (std::count(enabledCounters.begin(), enabledCounters.end(), 0) <
                     Inference::getMaxPmuEventCounters()) {

                     const std::vector<uint64_t> pmus = inference->getPmuCounters();
                     cout << "PMUs : [";
                     for (auto p : pmus) {
                         cout << " " << p;
                     }
                     cout << " ]" << endl;
                 }
                 if (enableCycleCounter)
                     cout << "Cycle counter: " << inference->getCycleCounter() << endl;
             }

             ofmIndex++;
         }
     } catch (Exception &e) {
         cerr << "Error: " << e.what() << endl;
         return 1;
     }

     return 0;
 }
	/*
	* SPDX-FileCopyrightText: Copyright 2020-2024 Arm Limited and/or its affiliates <open-source-office@arm.com>
	* SPDX-License-Identifier: Apache-2.0
	*
	* Licensed under the Apache License, Version 2.0 (the License); you may
	* not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an AS IS BASIS, WITHOUT
	* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	#include <ethosu.hpp>
	#include <uapi/ethosu.h>

	#include <fstream>
	#include <iomanip>
	#include <iostream>
	#include <list>
	#include <stdio.h>
	#include <string>
	#include <unistd.h>
	#include <utility>

	using namespace std;
	using namespace EthosU;

	namespace {
	int64_t defaultTimeout = 60000000000;

	void help(const string &exe) {
	cerr << "Usage: " << exe << " [ARGS]\n";
	cerr << "\n";
	cerr << "Arguments:\n";
	cerr << " -h --help Print this help message.\n";
	cerr << " -n --network File to read network from.\n";
	cerr << " --index Network model index, stored in firmware binary.\n";
	cerr << " -i --ifm File to read IFM from.\n";
	cerr << " -o --ofm File to write IFM to.\n";
	cerr << " -P --pmu [0.." << Inference::getMaxPmuEventCounters() << "] eventid.\n";
	cerr << " PMU counter to enable followed by eventid, can be passed multiple times.\n";
	cerr << " -C --cycles Enable cycle counter for inference.\n";
	cerr << " -t --timeout Timeout in nanoseconds (default " << defaultTimeout << ").\n";
	cerr << " -p Print OFM.\n";
	cerr << endl;
	}

	void rangeCheck(const int i, const int argc, const string &arg) {
	if (i >= argc) {
	cerr << "Error: Missing argument to '" << arg << "'" << endl;
	exit(1);
	}
	}

	pair<unique_ptr<unsigned char[]>, size_t> getNetworkData(const string &filename) {
	ifstream stream(filename, ios::binary);
	if (!stream.is_open()) {
	cerr << "Error: Failed to open '" << filename << "'" << endl;
	exit(1);
	}

	stream.seekg(0, ios_base::end);
	size_t size = stream.tellg();
	stream.seekg(0, ios_base::beg);

	unique_ptr<unsigned char[]> data = std::make_unique<unsigned char[]>(size);
	stream.read(reinterpret_cast<char *>(data.get()), size);

	return make_pair(std::move(data), size);
	}

	shared_ptr<Inference> createInference(Device &device,
	shared_ptr<Network> &network,
	const string &filename,
	const std::vector<uint8_t> &counters,
	bool enableCycleCounter) {
	// Open IFM file
	ifstream stream(filename, ios::binary);
	if (!stream.is_open()) {
	cerr << "Error: Failed to open '" << filename << "'" << endl;
	exit(1);
	}

	// Get IFM file size
	stream.seekg(0, ios_base::end);
	size_t size = stream.tellg();
	stream.seekg(0, ios_base::beg);

	if (size != network->getIfmSize()) {
	cerr << "Error: IFM size does not match network size. filename=" << filename << ", size=" << size
	<< ", network=" << network->getIfmSize() << endl;
	exit(1);
	}

	// Create IFM buffers
	vector<shared_ptr<Buffer>> ifm;
	for (auto size : network->getIfmDims()) {
	shared_ptr<Buffer> buffer = make_shared<Buffer>(device, size);
	stream.read(buffer->data(), size);

	if (!stream) {
	cerr << "Error: Failed to read IFM" << endl;
	exit(1);
	}

	ifm.push_back(buffer);
	}

	// Create OFM buffers
	vector<shared_ptr<Buffer>> ofm;
	for (auto size : network->getOfmDims()) {
	ofm.push_back(make_shared<Buffer>(device, size));
	}

	return make_shared<Inference>(
	network, ifm.begin(), ifm.end(), ofm.begin(), ofm.end(), counters, enableCycleCounter);
	}

	ostream &operator<<(ostream &os, Buffer &buf) {
	char *c = buf.data();
	const char *end = c + buf.size();

	while (c < end) {
	os << hex << setw(2) << static_cast<int>(*c++) << " " << dec;
	}

	return os;
	}

	} // namespace

	int main(int argc, char *argv[]) {
	const string exe = argv[0];
	string networkArg;
	int networkIndex = -1;
	list<string> ifmArg;
	vector<uint8_t> enabledCounters(Inference::getMaxPmuEventCounters());
	string ofmArg;
	int64_t timeout = defaultTimeout;
	bool print = false;
	bool enableCycleCounter = false;

	for (int i = 1; i < argc; ++i) {
	const string arg(argv[i]);

	if (arg == "-h" \|\| arg == "--help") {
	help(exe);
	exit(1);
	} else if (arg == "--network" \|\| arg == "-n") {
	rangeCheck(++i, argc, arg);
	networkArg = argv[i];
	} else if (arg == "--index") {
	rangeCheck(++i, argc, arg);
	networkIndex = stoi(argv[i]);
	} else if (arg == "--ifm" \|\| arg == "-i") {
	rangeCheck(++i, argc, arg);
	ifmArg.push_back(argv[i]);
	} else if (arg == "--ofm" \|\| arg == "-o") {
	rangeCheck(++i, argc, arg);
	ofmArg = argv[i];
	} else if (arg == "--timeout" \|\| arg == "-t") {
	rangeCheck(++i, argc, arg);
	timeout = stoll(argv[i]);
	} else if (arg == "--pmu" \|\| arg == "-P") {
	unsigned pmu = 0, event = 0;
	rangeCheck(++i, argc, arg);
	pmu = stoi(argv[i]);

	rangeCheck(++i, argc, arg);
	event = stoi(argv[i]);

	if (pmu >= enabledCounters.size()) {
	cerr << "PMU out of bounds!" << endl;
	help(exe);
	exit(1);
	}
	cout << argv[i] << " -> Enabling " << pmu << " with event " << event << endl;
	enabledCounters[pmu] = event;
	} else if (arg == "--cycles" \|\| arg == "-C") {
	enableCycleCounter = true;
	} else if (arg == "-p") {
	print = true;
	} else {
	cerr << "Error: Invalid argument '" << arg << "'" << endl;
	help(exe);
	exit(1);
	}
	}

	if (networkArg.empty()) {
	cerr << "Error: Missing 'network' argument" << endl;
	exit(1);
	}

	if (ifmArg.empty()) {
	cerr << "Error: Missing 'ifm' argument" << endl;
	exit(1);
	}

	if (ofmArg.empty()) {
	cerr << "Error: Missing 'ofm' argument" << endl;
	exit(1);
	}

	try {
	cout << "Driver library version:" << getLibraryVersion() << endl;

	Device device;
	cout << "Kernel driver version:" << device.getDriverVersion() << endl;

	cout << "Send Ping" << endl;
	device.ioctl(ETHOSU_IOCTL_PING);

	cout << "Send capabilities request" << endl;
	Capabilities capabilities = device.capabilities();

	cout << "Capabilities:" << endl
	<< "\tversion_status:" << unsigned(capabilities.hwId.versionStatus) << endl
	<< "\tversion:" << capabilities.hwId.version << endl
	<< "\tproduct:" << capabilities.hwId.product << endl
	<< "\tarchitecture:" << capabilities.hwId.architecture << endl
	<< "\tdriver:" << capabilities.driver << endl
	<< "\tmacs_per_cc:" << unsigned(capabilities.hwCfg.macsPerClockCycle) << endl
	<< "\tcmd_stream_version:" << unsigned(capabilities.hwCfg.cmdStreamVersion) << endl
	<< "\ttype:" << capabilities.hwCfg.type << endl
	<< "\tcustom_dma:" << std::boolalpha << capabilities.hwCfg.customDma << endl;

	/* Create network */
	cout << "Create network" << endl;

	shared_ptr<Network> network;

	if (networkIndex < 0) {
	auto networkData = getNetworkData(networkArg);
	network = make_shared<Network>(device, networkData.first.get(), networkData.second);
	} else {
	network = make_shared<Network>(device, networkIndex);
	}

	/* Create one inference per IFM */
	list<shared_ptr<Inference>> inferences;
	for (auto &filename : ifmArg) {
	cout << "Create inference" << endl;
	inferences.push_back(createInference(device, network, filename, enabledCounters, enableCycleCounter));
	}

	cout << "Wait for inferences" << endl;

	int ofmIndex = 0;
	for (auto &inference : inferences) {
	cout << "Inference status: " << inference->status() << endl;

	/* make sure the wait completes ok */
	try {
	cout << "Wait for inference" << endl;
	bool timedout = inference->wait(timeout);
	if (timedout) {
	cout << "Inference timed out, cancelling it" << endl;
	bool aborted = inference->cancel();
	if (!aborted \|\| inference->status() != InferenceStatus::ABORTED) {
	cout << "Inference cancellation failed" << endl;
	}
	}
	} catch (std::exception &e) {
	cout << "Failed to wait for or to cancel inference: " << e.what() << endl;
	exit(1);
	}

	cout << "Inference status: " << inference->status() << endl;

	if (inference->status() == InferenceStatus::OK) {
	string ofmFilename = ofmArg + "." + to_string(ofmIndex);
	ofstream ofmStream(ofmFilename, ios::binary);
	if (!ofmStream.is_open()) {
	cerr << "Error: Failed to open '" << ofmFilename << "'" << endl;
	exit(1);
	}

	/* The inference completed and has ok status */
	for (auto &ofmBuffer : inference->getOfmBuffers()) {
	cout << "OFM size: " << ofmBuffer->size() << endl;

	if (print) {
	cout << "OFM data: " << *ofmBuffer << endl;
	}

	ofmStream.write(ofmBuffer->data(), ofmBuffer->size());
	}

	ofmStream.flush();

	/* Read out PMU counters if configured */
	if (std::count(enabledCounters.begin(), enabledCounters.end(), 0) <
	Inference::getMaxPmuEventCounters()) {

	const std::vector<uint64_t> pmus = inference->getPmuCounters();
	cout << "PMUs : [";
	for (auto p : pmus) {
	cout << " " << p;
	}
	cout << " ]" << endl;
	}
	if (enableCycleCounter)
	cout << "Cycle counter: " << inference->getCycleCounter() << endl;
	}

	ofmIndex++;
	}
	} catch (Exception &e) {
	cerr << "Error: " << e.what() << endl;
	return 1;
	}

	return 0;
	}