Isabella Gottardi | 883bad7 | 2019-07-15 17:33:07 +0100 | [diff] [blame] | 1 | /* |
Matthew Bentham | 9204646 | 2020-03-07 22:15:55 +0000 | [diff] [blame^] | 2 | * Copyright (c) 2019-2020 ARM Limited. |
Isabella Gottardi | 883bad7 | 2019-07-15 17:33:07 +0100 | [diff] [blame] | 3 | * |
| 4 | * SPDX-License-Identifier: MIT |
| 5 | * |
| 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy |
| 7 | * of this software and associated documentation files (the "Software"), to |
| 8 | * deal in the Software without restriction, including without limitation the |
| 9 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or |
| 10 | * sell copies of the Software, and to permit persons to whom the Software is |
| 11 | * furnished to do so, subject to the following conditions: |
| 12 | * |
| 13 | * The above copyright notice and this permission notice shall be included in all |
| 14 | * copies or substantial portions of the Software. |
| 15 | * |
| 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 22 | * SOFTWARE. |
| 23 | */ |
| 24 | #include "arm_compute/core/CPP/kernels/CPPNonMaximumSuppressionKernel.h" |
| 25 | |
| 26 | #include "arm_compute/core/Error.h" |
| 27 | #include "arm_compute/core/Helpers.h" |
| 28 | #include "arm_compute/core/Validate.h" |
Isabella Gottardi | 883bad7 | 2019-07-15 17:33:07 +0100 | [diff] [blame] | 29 | |
Georgios Pinitas | b94c368 | 2019-09-27 19:07:15 +0100 | [diff] [blame] | 30 | #include <algorithm> |
Isabella Gottardi | 883bad7 | 2019-07-15 17:33:07 +0100 | [diff] [blame] | 31 | |
| 32 | namespace arm_compute |
| 33 | { |
| 34 | namespace |
| 35 | { |
| 36 | Status validate_arguments(const ITensorInfo *bboxes, const ITensorInfo *scores, const ITensorInfo *output_indices, unsigned int max_output_size, |
| 37 | const float score_threshold, const float iou_threshold) |
| 38 | { |
| 39 | ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(bboxes, scores, output_indices); |
| 40 | ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(bboxes, 1, DataType::F32); |
| 41 | ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(output_indices, 1, DataType::S32); |
| 42 | ARM_COMPUTE_RETURN_ERROR_ON_MSG(bboxes->num_dimensions() > 2, "The bboxes tensor must be a 2-D float tensor of shape [4, num_boxes]."); |
| 43 | ARM_COMPUTE_RETURN_ERROR_ON_MSG(scores->num_dimensions() > 1, "The scores tensor must be a 1-D float tensor of shape [num_boxes]."); |
| 44 | ARM_COMPUTE_RETURN_ERROR_ON_MSG(output_indices->num_dimensions() > 1, "The indices must be 1-D integer tensor of shape [M], where max_output_size <= M"); |
| 45 | ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(bboxes, scores); |
| 46 | ARM_COMPUTE_RETURN_ERROR_ON_MSG(output_indices->dimension(0) == 0, "Indices tensor must be bigger than 0"); |
| 47 | ARM_COMPUTE_RETURN_ERROR_ON_MSG(max_output_size == 0, "Max size cannot be 0"); |
| 48 | ARM_COMPUTE_RETURN_ERROR_ON_MSG(iou_threshold < 0.f || iou_threshold > 1.f, "IOU threshold must be in [0,1]"); |
| 49 | ARM_COMPUTE_RETURN_ERROR_ON_MSG(score_threshold < 0.f || score_threshold > 1.f, "Score threshold must be in [0,1]"); |
| 50 | |
| 51 | return Status{}; |
| 52 | } |
| 53 | } // namespace |
| 54 | |
| 55 | CPPNonMaximumSuppressionKernel::CPPNonMaximumSuppressionKernel() |
Georgios Pinitas | b94c368 | 2019-09-27 19:07:15 +0100 | [diff] [blame] | 56 | : _input_bboxes(nullptr), _input_scores(nullptr), _output_indices(nullptr), _max_output_size(0), _score_threshold(0.f), _iou_threshold(0.f), _num_boxes(0) |
Isabella Gottardi | 883bad7 | 2019-07-15 17:33:07 +0100 | [diff] [blame] | 57 | { |
| 58 | } |
| 59 | |
Georgios Pinitas | b94c368 | 2019-09-27 19:07:15 +0100 | [diff] [blame] | 60 | void CPPNonMaximumSuppressionKernel::configure(const ITensor *input_bboxes, const ITensor *input_scores, ITensor *output_indices, |
| 61 | unsigned int max_output_size, const float score_threshold, const float iou_threshold) |
Isabella Gottardi | 883bad7 | 2019-07-15 17:33:07 +0100 | [diff] [blame] | 62 | { |
| 63 | ARM_COMPUTE_ERROR_ON_NULLPTR(input_bboxes, input_scores, output_indices); |
| 64 | ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input_bboxes->info(), input_scores->info(), output_indices->info(), max_output_size, score_threshold, iou_threshold)); |
| 65 | |
| 66 | auto_init_if_empty(*output_indices->info(), TensorShape(max_output_size), 1, DataType::U8, QuantizationInfo()); |
| 67 | |
| 68 | _input_bboxes = input_bboxes; |
| 69 | _input_scores = input_scores; |
| 70 | _output_indices = output_indices; |
| 71 | _score_threshold = score_threshold; |
| 72 | _iou_threshold = iou_threshold; |
| 73 | _max_output_size = max_output_size; |
| 74 | _num_boxes = input_scores->info()->dimension(0); |
| 75 | |
Isabella Gottardi | 883bad7 | 2019-07-15 17:33:07 +0100 | [diff] [blame] | 76 | // Configure kernel window |
| 77 | Window win = calculate_max_window(*output_indices->info(), Steps()); |
| 78 | |
| 79 | // The CPPNonMaximumSuppressionKernel doesn't need padding so update_window_and_padding() can be skipped |
| 80 | ICPPKernel::configure(win); |
| 81 | } |
| 82 | |
Georgios Pinitas | b94c368 | 2019-09-27 19:07:15 +0100 | [diff] [blame] | 83 | Status CPPNonMaximumSuppressionKernel::validate(const ITensorInfo *bboxes, const ITensorInfo *scores, const ITensorInfo *output_indices, |
| 84 | unsigned int max_output_size, const float score_threshold, const float iou_threshold) |
Isabella Gottardi | 883bad7 | 2019-07-15 17:33:07 +0100 | [diff] [blame] | 85 | { |
| 86 | ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(bboxes, scores, output_indices, max_output_size, score_threshold, iou_threshold)); |
| 87 | return Status{}; |
| 88 | } |
| 89 | |
| 90 | void CPPNonMaximumSuppressionKernel::run(const Window &window, const ThreadInfo &info) |
| 91 | { |
| 92 | ARM_COMPUTE_UNUSED(info); |
| 93 | ARM_COMPUTE_UNUSED(window); |
| 94 | ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this); |
| 95 | ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(ICPPKernel::window(), window); |
| 96 | |
Georgios Pinitas | b94c368 | 2019-09-27 19:07:15 +0100 | [diff] [blame] | 97 | // Auxiliary tensors |
| 98 | std::vector<int> indices_above_thd; |
| 99 | std::vector<float> scores_above_thd; |
Isabella Gottardi | 883bad7 | 2019-07-15 17:33:07 +0100 | [diff] [blame] | 100 | for(unsigned int i = 0; i < _num_boxes; ++i) |
| 101 | { |
| 102 | const float score_i = *(reinterpret_cast<float *>(_input_scores->ptr_to_element(Coordinates(i)))); |
| 103 | if(score_i >= _score_threshold) |
| 104 | { |
Georgios Pinitas | b94c368 | 2019-09-27 19:07:15 +0100 | [diff] [blame] | 105 | scores_above_thd.emplace_back(score_i); |
| 106 | indices_above_thd.emplace_back(i); |
Isabella Gottardi | 883bad7 | 2019-07-15 17:33:07 +0100 | [diff] [blame] | 107 | } |
| 108 | } |
| 109 | |
| 110 | // Sort selected indices based on scores |
Georgios Pinitas | b94c368 | 2019-09-27 19:07:15 +0100 | [diff] [blame] | 111 | const unsigned int num_above_thd = indices_above_thd.size(); |
| 112 | std::vector<unsigned int> sorted_indices; |
| 113 | sorted_indices.resize(num_above_thd); |
| 114 | std::iota(sorted_indices.data(), sorted_indices.data() + num_above_thd, 0); |
| 115 | std::sort(std::begin(sorted_indices), |
| 116 | std::end(sorted_indices), |
Isabella Gottardi | 883bad7 | 2019-07-15 17:33:07 +0100 | [diff] [blame] | 117 | [&](unsigned int first, unsigned int second) |
| 118 | { |
Georgios Pinitas | b94c368 | 2019-09-27 19:07:15 +0100 | [diff] [blame] | 119 | return scores_above_thd[first] > scores_above_thd[second]; |
Isabella Gottardi | 883bad7 | 2019-07-15 17:33:07 +0100 | [diff] [blame] | 120 | }); |
| 121 | |
| 122 | // Number of output is the minimum between max_detection and the scores above the threshold |
| 123 | const unsigned int num_output = std::min(_max_output_size, num_above_thd); |
| 124 | unsigned int output_idx = 0; |
Georgios Pinitas | b94c368 | 2019-09-27 19:07:15 +0100 | [diff] [blame] | 125 | std::vector<bool> visited(num_above_thd, false); |
Isabella Gottardi | 883bad7 | 2019-07-15 17:33:07 +0100 | [diff] [blame] | 126 | |
Georgios Pinitas | b94c368 | 2019-09-27 19:07:15 +0100 | [diff] [blame] | 127 | // Keep only boxes with small IoU |
Isabella Gottardi | 883bad7 | 2019-07-15 17:33:07 +0100 | [diff] [blame] | 128 | for(unsigned int i = 0; i < num_above_thd; ++i) |
| 129 | { |
| 130 | // Check if the output is full |
| 131 | if(output_idx >= num_output) |
| 132 | { |
| 133 | break; |
| 134 | } |
| 135 | |
| 136 | // Check if it was already visited, if not add it to the output and update the indices counter |
Georgios Pinitas | b94c368 | 2019-09-27 19:07:15 +0100 | [diff] [blame] | 137 | if(!visited[sorted_indices[i]]) |
Isabella Gottardi | 883bad7 | 2019-07-15 17:33:07 +0100 | [diff] [blame] | 138 | { |
Georgios Pinitas | b94c368 | 2019-09-27 19:07:15 +0100 | [diff] [blame] | 139 | *(reinterpret_cast<int *>(_output_indices->ptr_to_element(Coordinates(output_idx)))) = indices_above_thd[sorted_indices[i]]; |
| 140 | visited[sorted_indices[i]] = true; |
Isabella Gottardi | 883bad7 | 2019-07-15 17:33:07 +0100 | [diff] [blame] | 141 | ++output_idx; |
| 142 | } |
| 143 | else |
| 144 | { |
| 145 | continue; |
| 146 | } |
| 147 | |
| 148 | // Once added one element at the output check if the next ones overlap and can be skipped |
| 149 | for(unsigned int j = i + 1; j < num_above_thd; ++j) |
| 150 | { |
Georgios Pinitas | b94c368 | 2019-09-27 19:07:15 +0100 | [diff] [blame] | 151 | if(!visited[sorted_indices[j]]) |
Isabella Gottardi | 883bad7 | 2019-07-15 17:33:07 +0100 | [diff] [blame] | 152 | { |
| 153 | // Calculate IoU |
Georgios Pinitas | b94c368 | 2019-09-27 19:07:15 +0100 | [diff] [blame] | 154 | const unsigned int i_index = indices_above_thd[sorted_indices[i]]; |
| 155 | const unsigned int j_index = indices_above_thd[sorted_indices[j]]; |
Isabella Gottardi | 883bad7 | 2019-07-15 17:33:07 +0100 | [diff] [blame] | 156 | // Box-corner format: xmin, ymin, xmax, ymax |
| 157 | const auto box_i_xmin = *(reinterpret_cast<float *>(_input_bboxes->ptr_to_element(Coordinates(0, i_index)))); |
| 158 | const auto box_i_ymin = *(reinterpret_cast<float *>(_input_bboxes->ptr_to_element(Coordinates(1, i_index)))); |
| 159 | const auto box_i_xmax = *(reinterpret_cast<float *>(_input_bboxes->ptr_to_element(Coordinates(2, i_index)))); |
| 160 | const auto box_i_ymax = *(reinterpret_cast<float *>(_input_bboxes->ptr_to_element(Coordinates(3, i_index)))); |
| 161 | |
| 162 | const auto box_j_xmin = *(reinterpret_cast<float *>(_input_bboxes->ptr_to_element(Coordinates(0, j_index)))); |
| 163 | const auto box_j_ymin = *(reinterpret_cast<float *>(_input_bboxes->ptr_to_element(Coordinates(1, j_index)))); |
| 164 | const auto box_j_xmax = *(reinterpret_cast<float *>(_input_bboxes->ptr_to_element(Coordinates(2, j_index)))); |
| 165 | const auto box_j_ymax = *(reinterpret_cast<float *>(_input_bboxes->ptr_to_element(Coordinates(3, j_index)))); |
| 166 | |
| 167 | const float area_i = (box_i_xmax - box_i_xmin) * (box_i_ymax - box_i_ymin); |
| 168 | const float area_j = (box_j_xmax - box_j_xmin) * (box_j_ymax - box_j_ymin); |
| 169 | float overlap; |
| 170 | if(area_i <= 0 || area_j <= 0) |
| 171 | { |
| 172 | overlap = 0.0f; |
| 173 | } |
| 174 | else |
| 175 | { |
| 176 | const auto y_min_intersection = std::max<float>(box_i_ymin, box_j_ymin); |
| 177 | const auto x_min_intersection = std::max<float>(box_i_xmin, box_j_xmin); |
| 178 | const auto y_max_intersection = std::min<float>(box_i_ymax, box_j_ymax); |
| 179 | const auto x_max_intersection = std::min<float>(box_i_xmax, box_j_xmax); |
| 180 | const auto area_intersection = std::max<float>(y_max_intersection - y_min_intersection, 0.0f) * std::max<float>(x_max_intersection - x_min_intersection, 0.0f); |
| 181 | overlap = area_intersection / (area_i + area_j - area_intersection); |
| 182 | } |
| 183 | |
| 184 | if(overlap > _iou_threshold) |
| 185 | { |
Georgios Pinitas | b94c368 | 2019-09-27 19:07:15 +0100 | [diff] [blame] | 186 | visited[sorted_indices[j]] = true; |
Isabella Gottardi | 883bad7 | 2019-07-15 17:33:07 +0100 | [diff] [blame] | 187 | } |
| 188 | } |
| 189 | } |
| 190 | } |
| 191 | // The output could be full but not the output indices tensor |
| 192 | // Instead return values not valid we put -1 |
| 193 | for(; output_idx < _max_output_size; ++output_idx) |
| 194 | { |
| 195 | *(reinterpret_cast<int *>(_output_indices->ptr_to_element(Coordinates(output_idx)))) = -1; |
| 196 | } |
| 197 | } |
| 198 | } // namespace arm_compute |