Blame - src/core/cpu/kernels/pooling/neon/fp16.cpp - ml/ComputeLibrary

void pooling2_f16_maxpool_indices(const ITensor *src, ITensor *dst0, ITensor *dst1, PoolingLayerInfo &pool_info, const Window &window_src, const Window &window)

41

{

42

const int window_start_x = window.x().start();

43

const int window_end_x = window.x().end();

44

const int window_step_x = 8;

45

46

Window window_out = window;

47

window_out.set(Window::DimX, Window::Dimension(0, 1, 1));

48

49

Iterator in(src, window_src);

50

Iterator out(dst0, window_out);

51

Iterator indices(dst1, window_out);

52

53

const int pool_pad_top = pool_info.pad_stride_info.pad_top();

54

const int pool_pad_left = pool_info.pad_stride_info.pad_left();

55

56

int pool_stride_x = 0;

57

int pool_stride_y = 0;

58

std::tie(pool_stride_x, pool_stride_y) = pool_info.pad_stride_info.stride();

59

60

const int pad_right = src->info()->padding().right;

61

const int in_stride_y = static_cast<int>(src->info()->strides_in_bytes().y());

62

const int in_stride_z = static_cast<int>(src->info()->strides_in_bytes().z());

63

64

execute_window_loop(window_out, [&](const Coordinates & id)

65

{

66

const int idx_width = id.y() * pool_stride_x;

67

const int idx_height = id.z() * pool_stride_y;

68

const int pool_limit_y = pool_pad_top - idx_height;

69

const int pool_limit_x = pool_pad_left - idx_width;

70

71

const int pool_start_y = std::max(0, window_src.z().start() + pool_limit_y);

72

const int pool_start_x = std::max(0, window_src.y().start() + pool_limit_x);

73

const int in_x0_offset = (pool_start_x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (pool_start_y - pool_pad_top) * static_cast<int>(src->info()->strides_in_bytes().z());

74

const int in_x1_offset = (pool_start_x + 1 - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (pool_start_y - pool_pad_top) * static_cast<int>

75

(src->info()->strides_in_bytes().z());

76

const int in_x2_offset = (pool_start_x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (pool_start_y + 1 - pool_pad_top) * static_cast<int>

77

(src->info()->strides_in_bytes().z());

78

const int in_x3_offset = (pool_start_x + 1 - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (pool_start_y + 1 - pool_pad_top) * static_cast<int>

79

(src->info()->strides_in_bytes().z());

80

81

int x_off = window_start_x;

82

for(; x_off <= (window_end_x - window_step_x); x_off += window_step_x)

83

{

84

const auto in_x0_ptr = reinterpret_cast<const float16_t *>(in.ptr() + in_x0_offset) + x_off;

85

const auto in_x1_ptr = reinterpret_cast<const float16_t *>(in.ptr() + in_x1_offset) + x_off;

86

const auto in_x2_ptr = reinterpret_cast<const float16_t *>(in.ptr() + in_x2_offset) + x_off;

87

const auto in_x3_ptr = reinterpret_cast<const float16_t *>(in.ptr() + in_x3_offset) + x_off;

88

const auto v_x0 = vld1q_f16(in_x0_ptr);

89

const auto v_x1 = vld1q_f16(in_x1_ptr);

90

const auto v_x2 = vld1q_f16(in_x2_ptr);

91

const auto v_x3 = vld1q_f16(in_x3_ptr);

92

float16x8_t vres = vmaxq_f16(vmaxq_f16(v_x2, v_x3), vmaxq_f16(v_x0, v_x1));

93

// Store result

94

vst1q_f16(reinterpret_cast<float16_t *>(out.ptr()) + x_off, vres);

95

Manuel Bottini

ca62c6f

2021-03-23 11:50:34 +0000

[diff] [blame^]

96

const uint32_t offset_base = offset_no_padding<float16_t>(in.offset(), id, *src->info(), pool_stride_x, pool_stride_y, DataLayout::NHWC);

Sheri Zhang

79144a6

2021-02-08 17:43:04 +0000

[diff] [blame]

97

const uint32_t offset_x0 = (uint32_t)offset_base / sizeof(float16_t) + x_off;

98

const uint32_t offset_x1 = (uint32_t)offset_x0 + in_stride_y / sizeof(float16_t) - pad_right;

99

const uint32_t offset_x2 = (uint32_t)offset_x0 + in_stride_z / sizeof(float16_t) - pad_right * src->info()->tensor_shape()[1];

100

const uint32_t offset_x3 = (uint32_t)offset_x2 + in_stride_y / sizeof(float16_t) - pad_right;

101

const uint32x4_t voffset_x0_0 = { offset_x0, offset_x0 + 1, offset_x0 + 2, offset_x0 + 3 };

102

const uint32x4_t voffset_x0_1 = { offset_x0 + 4, offset_x0 + 5, offset_x0 + 6, offset_x0 + 7 };

103

const uint16x8_t voffset_x0 = vcombine_u16(vmovn_u32(voffset_x0_0), vmovn_u32(voffset_x0_1));

104

const uint32x4_t voffset_x1_0 = { offset_x1, offset_x1 + 1, offset_x1 + 2, offset_x1 + 3 };

105

const uint32x4_t voffset_x1_1 = { offset_x1 + 4, offset_x1 + 5, offset_x1 + 6, offset_x1 + 7 };

106

const uint16x8_t voffset_x1 = vcombine_u16(vmovn_u32(voffset_x1_0), vmovn_u32(voffset_x1_1));

107

const uint32x4_t voffset_x2_0 = { offset_x2, offset_x2 + 1, offset_x2 + 2, offset_x2 + 3 };

108

const uint32x4_t voffset_x2_1 = { offset_x2 + 4, offset_x2 + 5, offset_x2 + 6, offset_x2 + 7 };

109

const uint16x8_t voffset_x2 = vcombine_u16(vmovn_u32(voffset_x2_0), vmovn_u32(voffset_x2_1));

110

const uint32x4_t voffset_x3_0 = { offset_x3, offset_x3 + 1, offset_x3 + 2, offset_x3 + 3 };

111

const uint32x4_t voffset_x3_1 = { offset_x3 + 4, offset_x3 + 5, offset_x3 + 6, offset_x3 + 7 };

112

const uint16x8_t voffset_x3 = vcombine_u16(vmovn_u32(voffset_x3_0), vmovn_u32(voffset_x3_1));

113

const uint16x8_t tmp_indices0 = vbslq_u16(vcgeq_f16(v_x0, v_x1), voffset_x0, voffset_x1);

114

const uint16x8_t tmp_indices1 = vbslq_u16(vcgeq_f16(v_x2, v_x3), voffset_x2, voffset_x3);

115

const uint16x8_t tmp_indices2 = vbslq_u16(vcgeq_f16(vmaxq_f16(v_x0, v_x1), vmaxq_f16(v_x2, v_x3)), tmp_indices0, tmp_indices1);

116

const uint32x4_t tmp_indeces3_0 = vmovl_u16(vget_low_u16(tmp_indices2));

117

const uint32x4_t tmp_indeces3_1 = vmovl_u16(vget_high_u16(tmp_indices2));

118

// Store indicies

119

vst1q_u32(reinterpret_cast<uint32_t *>(indices.ptr()) + x_off, tmp_indeces3_0);

120

vst1q_u32(reinterpret_cast<uint32_t *>(indices.ptr() + 16) + x_off, tmp_indeces3_1);

}

// Left-overs loop

for(; x_off < window_end_x; ++x_off)

125

{

126

const auto x0 = *(reinterpret_cast<const float16_t *>(in.ptr() + in_x0_offset) + x_off);

127

const auto x1 = *(reinterpret_cast<const float16_t *>(in.ptr() + in_x1_offset) + x_off);

128

const auto x2 = *(reinterpret_cast<const float16_t *>(in.ptr() + in_x2_offset) + x_off);

129

const auto x3 = *(reinterpret_cast<const float16_t *>(in.ptr() + in_x3_offset) + x_off);

130

float16_t res = std::max(std::max(x2, x3), std::max(x0, x1));

131

132

// Store result

133

*(reinterpret_cast<float16_t *>(out.ptr()) + x_off) = res;

134

Manuel Bottini

ca62c6f

2021-03-23 11:50:34 +0000

[diff] [blame^]

135

const uint32_t offset_base = offset_no_padding<float16_t>(in.offset(), id, *src->info(), pool_stride_x, pool_stride_y, DataLayout::NHWC);

Sheri Zhang

79144a6

2021-02-08 17:43:04 +0000

[diff] [blame]

136

const uint32_t offset_x0 = (uint32_t)offset_base / sizeof(float16_t) + x_off;

137

const uint32_t offset_x1 = (uint32_t)offset_x0 + in_stride_y / sizeof(float16_t) - pad_right;

138

const uint32_t offset_x2 = (uint32_t)offset_x0 + in_stride_z / sizeof(float16_t) - pad_right * src->info()->tensor_shape()[1];

139

const uint32_t offset_x3 = (uint32_t)offset_x2 + in_stride_y / sizeof(float16_t) - pad_right;

140

const uint32_t tmp_idx0 = (x0 >= x1) ? offset_x0 : offset_x1;

141

const uint32_t tmp_idx1 = (x2 >= x3) ? offset_x2 : offset_x3;

142

const uint32_t tmp_idx2 = (std::max(x0, x1) >= std::max(x2, x3)) ? tmp_idx0 : tmp_idx1;

143

144

// Store indices

145

*(reinterpret_cast<uint32_t *>(indices.ptr()) + x_off) = tmp_idx2;

}

},

in, out, indices);

}

}

void poolingMxN_fp16_neon_nhwc(const ITensor *src, ITensor *dst0, ITensor *dst1, PoolingLayerInfo &pool_info, const Window &window_src, const Window &window)

153

{

154

if(pool_info.pool_size == Size2D(2, 2) && pool_info.pool_type == PoolingType::MAX && dst1)

155

{

156

pooling2_f16_maxpool_indices(src, dst0, dst1, pool_info, window_src, window);

157

}

158

const int window_start_x = window.x().start();

159

const int window_end_x = window.x().end();

160

const int window_step_x = 8;

161

162

Window window_out = window;

163

window_out.set(Window::DimX, Window::Dimension(0, 1, 1));

164

165

Iterator in(src, window_src);

166

Iterator out(dst0, window_out);

167

168

const int pool_size_x = pool_info.is_global_pooling ? src->info()->tensor_shape().y() : pool_info.pool_size.width;

169

const int pool_size_y = pool_info.is_global_pooling ? src->info()->tensor_shape().z() : pool_info.pool_size.height;

170

const int pool_pad_right = pool_info.pad_stride_info.pad_right();

171

const int pool_pad_top = pool_info.pad_stride_info.pad_top();

172

const int pool_pad_left = pool_info.pad_stride_info.pad_left();

173

const int pool_pad_bottom = pool_info.pad_stride_info.pad_bottom();

174

int pool_stride_x = 0;

175

int pool_stride_y = 0;

176

std::tie(pool_stride_x, pool_stride_y) = pool_info.pad_stride_info.stride();

177

const int upper_bound_w = src->info()->dimension(1) + (pool_info.exclude_padding ? 0 : pool_pad_right);

178

const int upper_bound_h = src->info()->dimension(2) + (pool_info.exclude_padding ? 0 : pool_pad_bottom);

float16x8_t vres;

execute_window_loop(window_out, [&](const Coordinates & id)

183

{

184

const int idx_width = id.y() * pool_stride_x;

185

const int idx_height = id.z() * pool_stride_y;

186

const int pool_limit_y = pool_pad_top - idx_height;

187

const int pool_limit_x = pool_pad_left - idx_width;

188

189

const int pool_start_y = std::max(0, window_src.z().start() + pool_limit_y);

190

const int pool_end_y = std::min(pool_size_y, window_src.z().end() + pool_limit_y);

191

const int pool_start_x = std::max(0, window_src.y().start() + pool_limit_x);

192

const int pool_end_x = std::min(pool_size_x, window_src.y().end() + pool_limit_x);

193

194

int x_off = window_start_x;

195

for(; x_off <= (window_end_x - window_step_x); x_off += window_step_x)

196

{

197

if(pool_info.pool_type != PoolingType::MAX)

198

{

199

// Calculate scale

200

const float scale = calculate_avg_scale(pool_info.exclude_padding, DataLayout::NHWC, id, pool_size_x, pool_size_y, upper_bound_w, upper_bound_h, pool_pad_left, pool_pad_top, pool_stride_x,

201

pool_stride_y);

202

const float16x8_t scale_v = vdupq_n_f16(scale);

203

204

// Perform pooling

205

vres = vdupq_n_f16(0.0f);

206

for(int y = pool_start_y; y < pool_end_y; ++y)

207

{

208

for(int x = pool_start_x; x < pool_end_x; ++x)

209

{

210

const float16x8_t data = vld1q_f16(reinterpret_cast<const float16_t *>(in.ptr() + (x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (y - pool_pad_top) * static_cast<int>

211

(src->info()->strides_in_bytes().z())) + x_off);

212

213

// Get power of 2 in case of l2 pooling and accumulate

214

if(pool_info.pool_type == PoolingType::L2)

215

{

216

vres = vaddq_f16(vres, vmulq_f16(data, data));

}

else

{

vres = vaddq_f16(vres, data);

}

}

}

// Divide by scale

vres = vmulq_f16(vres, scale_v);

}

else

{

vres = vdupq_n_f16(std::numeric_limits<float>::lowest());

230

231

for(int y = pool_start_y; y < pool_end_y; ++y)

232

{

233

for(int x = pool_start_x; x < pool_end_x; ++x)

234

{

235

const float16x8_t data = vld1q_f16(reinterpret_cast<const float16_t *>(in.ptr() + (x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (y - pool_pad_top) * static_cast<int>

236

(src->info()->strides_in_bytes().z())) + x_off);

237

vres = vmaxq_f16(vres, data);

}

}

}

// Calculate square-root in case of l2 pooling

243

if(pool_info.pool_type == PoolingType::L2)

244

{

245

float16x8_t sqrt_reciprocal = vrsqrteq_f16(vres);

246

vres = vmulq_f16(vres, vmulq_f16(vrsqrtsq_f16(vmulq_f16(vres, sqrt_reciprocal), sqrt_reciprocal), sqrt_reciprocal));

}

// Store result

vst1q_f16(reinterpret_cast<float16_t *>(out.ptr()) + x_off, vres);

}

// Left-overs loop

for(; x_off < window_end_x; ++x_off)

255

{

256

float16_t res = 0.0f;

257

258

if(pool_info.pool_type != PoolingType::MAX)

259

{

260

// Calculate scale

261

const float16_t scale = calculate_avg_scale(pool_info.exclude_padding, DataLayout::NHWC, id, pool_size_x, pool_size_y, upper_bound_w, upper_bound_h, pool_pad_left, pool_pad_top, pool_stride_x,

262

pool_stride_y);

263

264

for(int y = pool_start_y; y < pool_end_y; ++y)

265

{

266

for(int x = pool_start_x; x < pool_end_x; ++x)

267

{

268

const float data = *(reinterpret_cast<const float16_t *>(in.ptr() + (x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (y - pool_pad_top) * static_cast<int>

269

(src->info()->strides_in_bytes().z())) + x_off);

270

271

// Get power of 2 in case of l2 pooling and accumulate

272

if(pool_info.pool_type == PoolingType::L2)

{

res += data * data;

}

else

{

res += data;

}

}

}

// Divide by scale

res *= scale;

}

else

{

res = std::numeric_limits<float>::lowest();

289

for(int y = pool_start_y; y < pool_end_y; ++y)

290

{

291

for(int x = pool_start_x; x < pool_end_x; ++x)

292

{

293

const float16_t data = *(reinterpret_cast<const float16_t *>(in.ptr() + (x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (y - pool_pad_top) * static_cast<int>

294

(src->info()->strides_in_bytes().z())) + x_off);

295

res = std::max(res, data);

}

}

}