IVGCVSW-5468 Rewrite QuantizationDataSet.cpp to avoid use of CsvReader
* Remove armnnUtils/CsvReader and usage
* Remove armnn/CsvReaderTest and usage
* Replace functionality in QuantizationDataSet.cpp
Signed-off-by: James Ward <james.ward@arm.com>
Change-Id: I7213904482afa93ae6d607aa5e69117c8c34ea81
diff --git a/Android.mk b/Android.mk
index 0f23cf9..2af7be5 100644
--- a/Android.mk
+++ b/Android.mk
@@ -128,7 +128,6 @@
src/armnn/TypesUtils.cpp \
src/armnn/Utils.cpp \
src/armnn/WallClockTimer.cpp \
- src/armnnUtils/CsvReader.cpp \
src/armnnUtils/DataLayoutIndexed.cpp \
src/armnnUtils/DotSerializer.cpp \
src/armnnUtils/FloatingPointConverter.cpp \
@@ -238,8 +237,7 @@
src/profiling/backends/BackendProfiling.cpp
LOCAL_STATIC_LIBRARIES := \
- arm_compute_library \
- libboost_system
+ arm_compute_library
LOCAL_SHARED_LIBRARIES := \
liblog
@@ -354,7 +352,6 @@
LOCAL_SRC_FILES := \
$(ARMNN_BACKEND_TEST_SOURCES) \
src/armnn/test/ConstTensorLayerVisitor.cpp \
- src/armnn/test/CsvReaderTest.cpp \
src/armnn/test/EndToEndTest.cpp \
src/armnn/ExecutionFrame.cpp \
src/armnn/test/ExecutionFrameTest.cpp \
diff --git a/CMakeLists.txt b/CMakeLists.txt
index abe9bf1..a6d88e0 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -59,8 +59,6 @@
src/armnnUtils/LeakChecking.hpp
src/armnnUtils/ModelAccuracyChecker.cpp
src/armnnUtils/ModelAccuracyChecker.hpp
- src/armnnUtils/CsvReader.cpp
- src/armnnUtils/CsvReader.hpp
src/armnnUtils/FloatingPointConverter.cpp
src/armnnUtils/VerificationHelpers.hpp
src/armnnUtils/VerificationHelpers.cpp
@@ -177,6 +175,7 @@
target_include_directories(armnnQuantizer PRIVATE include/armnnDeserializer)
target_include_directories(armnnQuantizer PRIVATE src/armnnUtils)
target_include_directories(armnnQuantizer PRIVATE src/armnn)
+ target_include_directories(armnnQuantizer PRIVATE third-party)
include_directories(SYSTEM "${FLATBUFFERS_INCLUDE_PATH}")
set_target_properties(armnnQuantizer PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR})
@@ -626,7 +625,6 @@
src/armnn/test/ConstTensorLayerVisitor.hpp
src/armnn/test/ConstTensorLayerVisitor.cpp
src/armnn/test/CreateWorkload.hpp
- src/armnn/test/CsvReaderTest.cpp
src/armnn/test/EndToEndTest.cpp
src/armnn/test/ExecutionFrameTest.cpp
src/armnn/test/FloatingPointConverterTest.cpp
diff --git a/src/armnn/test/CsvReaderTest.cpp b/src/armnn/test/CsvReaderTest.cpp
deleted file mode 100644
index 96185c8..0000000
--- a/src/armnn/test/CsvReaderTest.cpp
+++ /dev/null
@@ -1,130 +0,0 @@
-//
-// Copyright © 2017 Arm Ltd. All rights reserved.
-// SPDX-License-Identifier: MIT
-//
-#include "CsvReader.hpp"
-#include "armnn/Optional.hpp"
-#include <Filesystem.hpp>
-
-#include <boost/test/unit_test.hpp>
-
-#include <iostream>
-#include <string>
-
-using namespace armnnUtils;
-
-namespace {
-struct TestHelper {
-
- TestHelper()
- {
- BOOST_TEST_MESSAGE("setup fixture");
- }
-
- ~TestHelper()
- {
- BOOST_TEST_MESSAGE("teardown fixture");
- TearDown();
- }
-
- std::string CreateTempCsvFile()
- {
- fs::path p = armnnUtils::Filesystem::NamedTempFile("Armnn-CreateTempCsvFileTest-TempFile.csv");
- try
- {
- std::ofstream ofs{p};
- ofs << "airplane, bicycle , bird , \"m,o,n,k,e,y\"\n";
- ofs << "banana, shoe, \"ice\"";
- ofs.close();
- } catch (std::exception &e)
- {
- std::cerr << "Unable to write to file at location [" << p.c_str() << "] : " << e.what() << std::endl;
- BOOST_TEST(false);
- }
-
- m_CsvFile = p;
- return p.string();
- }
-
- int CheckStringsMatch(CsvRow &row, unsigned int index, std::string expectedValue)
- {
- return row.values.at(index).compare(expectedValue);
- }
-
- void TearDown()
- {
- RemoveCsvFile();
- }
-
- void RemoveCsvFile()
- {
- if (m_CsvFile)
- {
- try
- {
- fs::remove(m_CsvFile.value());
- }
- catch (std::exception &e)
- {
- std::cerr << "Unable to delete file [" << m_CsvFile.value() << "] : " << e.what() << std::endl;
- BOOST_TEST(false);
- }
- }
- }
-
- armnn::Optional<fs::path> m_CsvFile;
-};
-}
-
-BOOST_AUTO_TEST_SUITE(CsvReaderTest)
-
-BOOST_FIXTURE_TEST_CASE(TestParseVector, TestHelper)
-{
- CsvReader reader;
- std::vector<std::string> csvStrings;
- csvStrings.reserve(2);
- csvStrings.push_back("airplane, automobile , bird , \"c,a,t\"");
- csvStrings.push_back("banana, shoe, \"ice\"");
-
- std::vector<CsvRow> row = reader.ParseVector(csvStrings);
- CsvRow row1 = row[0];
- CsvRow row2 = row[1];
-
- BOOST_CHECK(row.size() == 2);
-
- BOOST_CHECK(row1.values.size() == 4);
- BOOST_CHECK(CheckStringsMatch(row1, 0, "airplane") == 0);
- BOOST_CHECK(CheckStringsMatch(row1, 1, "automobile") == 0);
- BOOST_CHECK(CheckStringsMatch(row1, 2, "bird") == 0);
- BOOST_CHECK(CheckStringsMatch(row1, 3, "c,a,t") == 0);
-
- BOOST_CHECK(row2.values.size() == 3);
- BOOST_CHECK(CheckStringsMatch(row2, 0, "banana") == 0);
- BOOST_CHECK(CheckStringsMatch(row2, 1, "shoe") == 0);
- BOOST_CHECK(CheckStringsMatch(row2, 2, "ice") == 0);
-}
-
-BOOST_FIXTURE_TEST_CASE(TestLoadingFileFromDisk, TestHelper)
-{
- CsvReader reader;
- std::string theFilePath = TestHelper::CreateTempCsvFile();
-
- std::vector<CsvRow> row = reader.ParseFile(theFilePath);
- CsvRow row1 = row[0];
- CsvRow row2 = row[1];
-
- BOOST_CHECK(row.size() == 2);
-
- BOOST_CHECK(row1.values.size() == 4);
- BOOST_CHECK(CheckStringsMatch(row1, 0, "airplane") == 0);
- BOOST_CHECK(CheckStringsMatch(row1, 1, "bicycle") == 0);
- BOOST_CHECK(CheckStringsMatch(row1, 2, "bird") == 0);
- BOOST_CHECK(CheckStringsMatch(row1, 3, "m,o,n,k,e,y") == 0);
-
- BOOST_CHECK(row2.values.size() == 3);
- BOOST_CHECK(CheckStringsMatch(row2, 0, "banana") == 0);
- BOOST_CHECK(CheckStringsMatch(row2, 1, "shoe") == 0);
- BOOST_CHECK(CheckStringsMatch(row2, 2, "ice") == 0);
-}
-
-BOOST_AUTO_TEST_SUITE_END()
diff --git a/src/armnnQuantizer/QuantizationDataSet.cpp b/src/armnnQuantizer/QuantizationDataSet.cpp
index 819b437..acd301a 100644
--- a/src/armnnQuantizer/QuantizationDataSet.cpp
+++ b/src/armnnQuantizer/QuantizationDataSet.cpp
@@ -4,8 +4,10 @@
//
#include "QuantizationDataSet.hpp"
-#include "CsvReader.hpp"
+#include <fmt/format.h>
+
+#include <armnn/utility/StringUtils.hpp>
#include <armnn/utility/IgnoreUnused.hpp>
#include <Filesystem.hpp>
@@ -67,49 +69,49 @@
}
-unsigned int GetPassIdFromCsvRow(std::vector<armnnUtils::CsvRow> csvRows, unsigned int rowIndex)
+unsigned int GetPassIdFromCsvRow(std::vector<std::string> tokens, unsigned int lineIndex)
{
unsigned int passId;
try
{
- passId = static_cast<unsigned int>(std::stoi(csvRows[rowIndex].values[0]));
+ passId = static_cast<unsigned int>(std::stoi(tokens[0]));
}
catch (const std::invalid_argument&)
{
- throw armnn::ParseException("Pass ID [" + csvRows[rowIndex].values[0] + "]" +
- " is not correct format on CSV row " + std::to_string(rowIndex));
+ throw armnn::ParseException(fmt::format("Pass ID [{}] is not correct format on CSV row {}",
+ tokens[0], lineIndex));
}
return passId;
}
-armnn::LayerBindingId GetBindingIdFromCsvRow(std::vector<armnnUtils::CsvRow> csvRows, unsigned int rowIndex)
+armnn::LayerBindingId GetBindingIdFromCsvRow(std::vector<std::string> tokens, unsigned int lineIndex)
{
armnn::LayerBindingId bindingId;
try
{
- bindingId = std::stoi(csvRows[rowIndex].values[1]);
+ bindingId = std::stoi(tokens[1]);
}
catch (const std::invalid_argument&)
{
- throw armnn::ParseException("Binding ID [" + csvRows[rowIndex].values[0] + "]" +
- " is not correct format on CSV row " + std::to_string(rowIndex));
+ throw armnn::ParseException(fmt::format("Binding ID [{}] is not correct format on CSV row {}",
+ tokens[1], lineIndex));
}
return bindingId;
}
-std::string GetFileNameFromCsvRow(std::vector<armnnUtils::CsvRow> csvRows, unsigned int rowIndex)
+std::string GetFileNameFromCsvRow(std::vector<std::string> tokens, unsigned int lineIndex)
{
- std::string fileName = csvRows[rowIndex].values[2];
+ std::string fileName = armnn::stringUtils::StringTrim(tokens[2]);
if (!fs::exists(fileName))
{
- throw armnn::ParseException("File [ " + fileName + "] provided on CSV row " + std::to_string(rowIndex) +
- " does not exist.");
+ throw armnn::ParseException(fmt::format("File [{}] provided on CSV row {} does not exist.",
+ fileName, lineIndex));
}
if (fileName.empty())
{
- throw armnn::ParseException("Filename cannot be empty on CSV row " + std::to_string(rowIndex));
+ throw armnn::ParseException(fmt::format("Filename cannot be empty on CSV row {} ", lineIndex));
}
return fileName;
}
@@ -118,34 +120,41 @@
void QuantizationDataSet::ParseCsvFile()
{
std::map<unsigned int, QuantizationInput> passIdToQuantizationInput;
- armnnUtils::CsvReader reader;
if (m_CsvFilePath == "")
{
throw armnn::Exception("CSV file not specified.");
}
- // Parse CSV file and extract data
- std::vector<armnnUtils::CsvRow> csvRows = reader.ParseFile(m_CsvFilePath);
- if (csvRows.empty())
+ std::ifstream inf (m_CsvFilePath.c_str());
+ std::string line;
+ std::vector<std::string> tokens;
+ unsigned int lineIndex = 0;
+
+ if (!inf)
{
- throw armnn::Exception("CSV file [" + m_CsvFilePath + "] is empty.");
+ throw armnn::Exception(fmt::format("CSV file {} not found.", m_CsvFilePath));
}
- for (unsigned int i = 0; i < csvRows.size(); ++i)
+ while (getline(inf, line))
{
- if (csvRows[i].values.size() != 3)
+ tokens = armnn::stringUtils::StringTokenizer(line, ",");
+
+ if (tokens.size() != 3)
{
- throw armnn::Exception("CSV file [" + m_CsvFilePath + "] does not have correct number of entries " +
- "on line " + std::to_string(i) + ". Expected 3 entries " +
- "but was " + std::to_string(csvRows[i].values.size()));
+ throw armnn::Exception(fmt::format("CSV file [{}] does not have correct number of entries" \
+ "on line {}. Expected 3 entries but was {}.",
+ m_CsvFilePath, lineIndex, tokens.size()));
+
}
- unsigned int passId = GetPassIdFromCsvRow(csvRows, i);
- armnn::LayerBindingId bindingId = GetBindingIdFromCsvRow(csvRows, i);
- std::string rawFileName = GetFileNameFromCsvRow(csvRows, i);
+ unsigned int passId = GetPassIdFromCsvRow(tokens, lineIndex);
+ armnn::LayerBindingId bindingId = GetBindingIdFromCsvRow(tokens, lineIndex);
+ std::string rawFileName = GetFileNameFromCsvRow(tokens, lineIndex);
AddInputData(passId, bindingId, rawFileName, passIdToQuantizationInput);
+
+ ++lineIndex;
}
if (passIdToQuantizationInput.empty())
diff --git a/src/armnnUtils/CsvReader.cpp b/src/armnnUtils/CsvReader.cpp
deleted file mode 100644
index feee4d1..0000000
--- a/src/armnnUtils/CsvReader.cpp
+++ /dev/null
@@ -1,63 +0,0 @@
-//
-// Copyright © 2017 Arm Ltd. All rights reserved.
-// SPDX-License-Identifier: MIT
-//
-
-#include "CsvReader.hpp"
-#include "armnn/utility/StringUtils.hpp"
-
-#include <boost/tokenizer.hpp>
-
-#include <fstream>
-#include <string>
-#include <vector>
-
-using Tokenizer = boost::tokenizer<boost::escaped_list_separator<char>>;
-
-namespace armnnUtils
-{
-
-CsvRow ParseLine(const std::string& csvLine)
-{
- Tokenizer tokenizer(csvLine);
- CsvRow entry;
-
- for (const auto &token : tokenizer)
- {
- entry.values.push_back(armnn::stringUtils::StringTrimCopy(token));
- }
- return entry;
-}
-
-std::vector<CsvRow> CsvReader::ParseFile(const std::string& csvFile)
-{
- std::vector<CsvRow> result;
-
- std::ifstream in(csvFile.c_str());
- if (!in.is_open())
- return result;
-
- std::string line;
- while (getline(in, line))
- {
- if(!line.empty())
- {
- CsvRow entry = ParseLine(line);
- result.push_back(entry);
- }
- }
- return result;
-}
-
-std::vector<CsvRow> CsvReader::ParseVector(const std::vector<std::string>& csvVector)
-{
- std::vector<CsvRow> result;
-
- for (auto const& line: csvVector)
- {
- CsvRow entry = ParseLine(line);
- result.push_back(entry);
- }
- return result;
-}
-} // namespace armnnUtils
\ No newline at end of file
diff --git a/src/armnnUtils/CsvReader.hpp b/src/armnnUtils/CsvReader.hpp
deleted file mode 100644
index d95f4f7..0000000
--- a/src/armnnUtils/CsvReader.hpp
+++ /dev/null
@@ -1,25 +0,0 @@
-//
-// Copyright © 2017 Arm Ltd. All rights reserved.
-// SPDX-License-Identifier: MIT
-//
-#pragma once
-
-#include <vector>
-#include <string>
-
-namespace armnnUtils
-{
-
-struct CsvRow
-{
- std::vector<std::string> values;
-};
-
-class CsvReader
-{
-public:
- static std::vector<CsvRow> ParseFile(const std::string& csvFile);
-
- static std::vector<CsvRow> ParseVector(const std::vector<std::string>& csvVector);
-};
-} // namespace armnnUtils
diff --git a/tests/NetworkExecutionUtils/NetworkExecutionUtils.hpp b/tests/NetworkExecutionUtils/NetworkExecutionUtils.hpp
index d101d4a..9d9e616 100644
--- a/tests/NetworkExecutionUtils/NetworkExecutionUtils.hpp
+++ b/tests/NetworkExecutionUtils/NetworkExecutionUtils.hpp
@@ -5,7 +5,6 @@
#pragma once
-#include "CsvReader.hpp"
#include <armnn/IRuntime.hpp>
#include <armnn/Types.hpp>