Blame - src/core/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedOnlyRhsKernel.cpp - ml/ComputeLibrary

std::pair<Status, Window> validate_and_configure_window(ITensorInfo *src0, ITensorInfo *src1, ITensorInfo *src2, ITensorInfo *dst, const GEMMLHSMatrixInfo &lhs_info,

122

const GEMMRHSMatrixInfo &rhs_info, const GEMMKernelInfo &gemm_info, ElementsProcessed &num_elements_processed)

123

{

124

unsigned int &num_elems_processed_per_iteration_x = num_elements_processed[0];

125

unsigned int &num_elems_processed_per_iteration_y = num_elements_processed[1];

126

bool reinterpret_input_as_3d = gemm_info.reinterpret_input_as_3d;

127

bool reinterpret_output_as_3d = gemm_info.depth_output_gemm3d != 0;

Window win{};

Window win_out{};

bool window_changed = false;

132

133

// In case both input and dst have to be reinterpreted as 3D tensors,

134

// force reinterpret_input_as_3d and reinterpret_output_as_3d to be false.

135

// This approach should only be used when the input/dst tensors have pad on the y direction

136

if((reinterpret_input_as_3d == reinterpret_output_as_3d) && gemm_info.has_pad_y)

137

{

138

reinterpret_output_as_3d = false;

139

}

140

141

// dst tensor auto initialization if not yet initialized

142

auto_init_if_empty(*dst, src0->clone()->set_tensor_shape(misc::shape_calculator::compute_mm_shape(*src0, *src1, gemm_info)));

143

144

TensorInfo tmp_info(*dst);

145

146

if(reinterpret_output_as_3d)

147

{

148

// Since the dst tensor has to be reinterpreted as 3D and the execute window is based on a 2D GEMM,

149

// the window needs to be constructed on the 2D collapsed version of the tensor

150

TensorShape tmp_shape(dst->tensor_shape());

151

tmp_shape.collapse(2U, 1U);

152

tmp_info.set_tensor_shape(tmp_shape);

153

}

154

155

// Configure kernel window

156

num_elems_processed_per_iteration_x = rhs_info.n0;

157

num_elems_processed_per_iteration_y = lhs_info.m0;

158

159

win = calculate_max_window(tmp_info, Steps(num_elems_processed_per_iteration_x, num_elems_processed_per_iteration_y));

160

win_out = calculate_max_window(*dst, Steps(num_elems_processed_per_iteration_x, num_elems_processed_per_iteration_y));

if(src2 != nullptr)

{

const int bias_processed_per_iteration_x = num_elems_processed_per_iteration_x;

165

166

AccessWindowStatic src2_access(src2, 0, 0,

167

ceil_to_multiple(src2->dimension(0), bias_processed_per_iteration_x),

168

src2->dimension(1));

169

170

window_changed = update_window_and_padding(win, src2_access);

171

}

172

173

// Collapse along the Z direction

174

// This collapse needs to be here in order to tune the Z dimension of LWS

175

Window collapsed = win;

176

const unsigned int dimension_to_collapse = std::min(static_cast<unsigned int>(dst->num_dimensions()), 2u);

177

collapsed = win.collapse(win, dimension_to_collapse);

178

179

Status err = (window_changed) ? ARM_COMPUTE_CREATE_ERROR(ErrorCode::RUNTIME_ERROR, "Insufficient Padding!") : Status{};

180

return std::make_pair(err, collapsed);

}

} // namespace

Giorgio Arena

2021-06-28 11:00:27 +0100

[diff] [blame]

184

ClGemmMatrixMultiplyReshapedOnlyRhsKernel::ClGemmMatrixMultiplyReshapedOnlyRhsKernel()

185

{

186

_type = CLKernelType::GEMM;

187

}

188

Georgios Pinitas

856f66e

2021-04-22 21:13:21 +0100

[diff] [blame]

189

void ClGemmMatrixMultiplyReshapedOnlyRhsKernel::configure(const CLCompileContext &compile_context,

190

ITensorInfo *src0, ITensorInfo *src1, ITensorInfo *src2, ITensorInfo *dst, float alpha, float beta,

191

const GEMMLHSMatrixInfo &lhs_info, const GEMMRHSMatrixInfo &rhs_info, const GEMMKernelInfo &gemm_info)

192

{

193

ARM_COMPUTE_ERROR_ON_NULLPTR(src0, src1, dst);

194

195

ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(src0, src1, src2, dst, alpha, beta, lhs_info, rhs_info, gemm_info));

196

197

_reinterpret_input_as_3d = gemm_info.reinterpret_input_as_3d;

198

_reinterpret_output_as_3d = gemm_info.depth_output_gemm3d != 0;

199

_use_dummy_work_items = preferred_dummy_work_items_support(CLKernelLibrary::get().get_device());

200

_add_bias = src2 != nullptr;

201

_export_to_cl_image = rhs_info.export_to_cl_image;

202

_has_pad_y = gemm_info.has_pad_y;

203

204

auto padding_info = get_padding_info({ src0, src1, dst });

205

206

// In case both input and dst have to be reinterpreted as 3D tensors,

207

// force reinterpret_input_as_3d and reinterpret_output_as_3d to be false.

208

if((_reinterpret_input_as_3d == _reinterpret_output_as_3d) && _has_pad_y)

209

{

210

_reinterpret_input_as_3d = false;

211

_reinterpret_output_as_3d = false;

212

}

213

214

// Check if we need to slide the matrix B

215

const unsigned int num_dimensions_src0 = src0->num_dimensions();

216

_slide_matrix_b = (src1->num_dimensions() >= num_dimensions_src0);

217

218

ElementsProcessed num_elements_processed{};

219

220

// Configure kernel window

221

auto win_config = validate_and_configure_window(src0, src1, src2, dst, lhs_info, rhs_info, gemm_info, num_elements_processed);

222

ARM_COMPUTE_ERROR_THROW_ON(win_config.first);

223

ICLKernel::configure_internal(win_config.second);

224

225

// If _reinterpret_input_as_3d = reinterpret_output_as_3d = true,

226

// we will dispatch a batched-GEMM to reduce the complexity of the address calculation within the OpenCL kernel.

227

// This means that the actual m used by the kernel is given by dst->dimension(1) and not by gemm_info.m

228

const unsigned int internal_m = _reinterpret_output_as_3d ? gemm_info.m : dst->dimension(1);

229

230

// These variables are used only if gemm_info.has_pad_y == true

231

const unsigned int h_gemm_3d = _reinterpret_output_as_3d ? dst->dimension(1) : src0->dimension(1);

232

const unsigned int d_gemm_3d = _reinterpret_output_as_3d ? dst->dimension(2) : src0->dimension(2);

233

234

// Shrink M0 to be always <= M (internal_m) to prevent out-of-bounds reads.

235

// NOTE: This might have implications on heuristics and performance

236

const unsigned int internal_m0 = std::min(internal_m, lhs_info.m0);

237

238

// Calculate partial (store instead of load) M0 and partial N0 for the partial blocks at the end of a row/column if any. This is to avoid padding.

239

const unsigned int partial_store_m0 = internal_m % internal_m0;

240

const unsigned int partial_store_n0 = gemm_info.n % rhs_info.n0;

241

242

// Create build options

243

CLBuildOptions build_opts;

244

build_opts.add_option("-DDATA_TYPE=" + get_cl_type_from_data_type(src0->data_type()));

245

build_opts.add_option_if(!(helpers::float_ops::is_one(alpha)), "-DALPHA=" + float_to_string_with_full_precision(alpha));

246

build_opts.add_option_if(src2 != nullptr, "-DBETA=" + float_to_string_with_full_precision(beta));

247

build_opts.add_option_if(helpers::float_ops::is_one(beta), "-DUNIT_BETA");

248

build_opts.add_option_if(gemm_info.broadcast_bias, "-DBROADCAST_BIAS");

249

build_opts.add_option_if(!_slide_matrix_b, "-DMATRIX_B_DEPTH=" + support::cpp11::to_string(src1->dimension(2)));

250

build_opts.add_option_if(rhs_info.interleave, "-DRHS_INTERLEAVE");

251

build_opts.add_option_if(_use_dummy_work_items, "-DDUMMY_WORK_ITEMS");

252

build_opts.add_option_if(rhs_info.export_to_cl_image, "-DOPENCL_IMAGE_SUPPORT");

253

build_opts.add_option("-DRHS_HEIGHT=" + support::cpp11::to_string(src1->dimension(1)));

254

build_opts.add_option("-DM=" + support::cpp11::to_string(internal_m));

255

build_opts.add_option("-DN=" + support::cpp11::to_string(gemm_info.n));

256

build_opts.add_option("-DK=" + support::cpp11::to_string(gemm_info.k));

257

build_opts.add_option("-DM0=" + support::cpp11::to_string(internal_m0));

258

build_opts.add_option("-DN0=" + support::cpp11::to_string(rhs_info.n0));

259

build_opts.add_option("-DK0=" + support::cpp11::to_string(rhs_info.k0));

260

build_opts.add_option("-DH0=" + support::cpp11::to_string(rhs_info.h0));

261

build_opts.add_option("-DPARTIAL_STORE_M0=" + support::cpp11::to_string(partial_store_m0));

262

build_opts.add_option("-DPARTIAL_STORE_N0=" + support::cpp11::to_string(partial_store_n0));

263

build_opts.add_option_if(gemm_info.activation_info.enabled(), "-DACTIVATION_TYPE=" + lower_string(string_from_activation_func(gemm_info.activation_info.activation())));

264

build_opts.add_option_if(gemm_info.activation_info.enabled(), "-DA_VAL=" + float_to_string_with_full_precision(gemm_info.activation_info.a()));

265

build_opts.add_option_if(gemm_info.activation_info.enabled(), "-DB_VAL=" + float_to_string_with_full_precision(gemm_info.activation_info.b()));

266

if(_has_pad_y)

267

{

268

build_opts.add_option_if(_reinterpret_input_as_3d, "-DREINTERPRET_INPUT_AS_3D");

269

build_opts.add_option_if(_reinterpret_output_as_3d, "-DREINTERPRET_OUTPUT_AS_3D");

270

build_opts.add_option_if(_reinterpret_input_as_3d || _reinterpret_output_as_3d, "-DHEIGHT_GEMM3D=" + support::cpp11::to_string(h_gemm_3d));

271

build_opts.add_option_if(_reinterpret_input_as_3d || _reinterpret_output_as_3d, "-DDEPTH_GEMM3D=" + support::cpp11::to_string(d_gemm_3d));

272

}

273

274

std::string kernel_name("gemm_mm_reshaped_only_rhs_");

275

kernel_name += rhs_info.transpose ? "t" : "nt";

276

kernel_name += rhs_info.export_to_cl_image ? "_texture" : "";

277

278

// Create kernel

279

_kernel = create_kernel(compile_context, kernel_name, build_opts.options());

280

281

// Set config_id for enabling LWS tuning

282

_config_id = kernel_name;

283

_config_id += "_";

284

_config_id += (_has_pad_y ? "" : "no_pad_y_");

285

_config_id += (_add_bias ? "add_bias_" : "");

286

_config_id += (gemm_info.broadcast_bias ? "broadcast_bias_" : "");

287

_config_id += (_reinterpret_input_as_3d ? "3di_" : "");

288

_config_id += (_reinterpret_output_as_3d ? "3do_" : "");

289

_config_id += (gemm_info.activation_info.enabled() ? "fused_activation_" : "");

290

_config_id += lower_string(string_from_data_type(src0->data_type()));

291

_config_id += "_";

292

_config_id += support::cpp11::to_string(dst->dimension(1));

293

_config_id += "_";

294

_config_id += support::cpp11::to_string(dst->dimension(0));

295

_config_id += "_";

296

_config_id += support::cpp11::to_string(gemm_info.k);

297

_config_id += "_";

298

_config_id += support::cpp11::to_string(dst->dimension(2));

299

_config_id += "_";

300

_config_id += support::cpp11::to_string(lhs_info.m0);

301

_config_id += "_";

302

_config_id += support::cpp11::to_string(rhs_info.n0);

303

_config_id += "_";

304

_config_id += support::cpp11::to_string(rhs_info.k0);

305

_config_id += "_";

306

_config_id += support::cpp11::to_string(rhs_info.h0);

307

_config_id += "_";

308

_config_id += support::cpp11::to_string(rhs_info.interleave);

309

310

ARM_COMPUTE_ERROR_ON(has_padding_changed(padding_info));

311

}

312

313

Status ClGemmMatrixMultiplyReshapedOnlyRhsKernel::validate(const ITensorInfo *src0, const ITensorInfo *src1, const ITensorInfo *src2, const ITensorInfo *dst, float alpha, float beta,

314

const GEMMLHSMatrixInfo &lhs_info,

315

const GEMMRHSMatrixInfo &rhs_info, const GEMMKernelInfo &gemm_info)

316

{

317

ElementsProcessed num_elements_processed{};

318

ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(src0, src1, src2, dst, alpha, beta, lhs_info, rhs_info, gemm_info));

319

ARM_COMPUTE_RETURN_ON_ERROR(validate_and_configure_window(src0->clone().get(),

320

src1->clone().get(),

321

src2 != nullptr ? src2->clone().get() : nullptr,

dst->clone().get(),

lhs_info,

rhs_info,

gemm_info,

num_elements_processed)

.first);

return Status{};

}

void ClGemmMatrixMultiplyReshapedOnlyRhsKernel::run_op(ITensorPack &tensors, const Window &window, cl::CommandQueue &queue)

333

{

334

ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);

335

ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(ICLKernel::window(), window);

336

337

const auto src0 = utils::cast::polymorphic_downcast<const ICLTensor *>(tensors.get_const_tensor(TensorType::ACL_SRC_0));

338

const auto src1 = utils::cast::polymorphic_downcast<const ICLTensor *>(tensors.get_const_tensor(TensorType::ACL_SRC_1));

339

const auto src2 = utils::cast::polymorphic_downcast<const ICLTensor *>(tensors.get_const_tensor(TensorType::ACL_SRC_2));

340

auto dst = utils::cast::polymorphic_downcast<ICLTensor *>(tensors.get_tensor(TensorType::ACL_DST));

341

342

ARM_COMPUTE_ERROR_ON_NULLPTR(src0, src1, dst);

343

ARM_COMPUTE_ERROR_ON(_add_bias && src2 == nullptr);

344

345

if(src1->info()->num_dimensions() < 3)

346

{

347

// The stride_z for matrix B must be zero if we do not slice

348

ARM_COMPUTE_ERROR_ON(src1->info()->strides_in_bytes()[3] != 0);

349

}

350

351

const size_t lhs_idx_batch_size = _reinterpret_input_as_3d && !_has_pad_y ? 3u : 2u;

352

const size_t rhs_idx_batch_size = 2u;

353

const size_t bia_idx_batch_size = 2u;

354

const size_t out_idx_batch_size = _reinterpret_output_as_3d && !_has_pad_y ? 3u : 2u;

355

356

Window slice = window.first_slice_window_3D();

357

Window slice_matrix_b = slice;

358

359

slice_matrix_b.set(Window::DimX, Window::Dimension(0, 1, 1));

360

slice_matrix_b.set(Window::DimY, Window::Dimension(0, 1, 1));

361

362

// Get cross plane pads

363

const unsigned int total_cross_plane_pad_lhs = src0->info()->padding().top + src0->info()->padding().bottom;

364

const unsigned int total_cross_plane_pad_out = dst->info()->padding().top + dst->info()->padding().bottom;

365

366

// The execution should fail if we try to run with has_pad_y = false but we have padding in either the LHS or DST tensor

367

ARM_COMPUTE_ERROR_ON(!_has_pad_y && ((total_cross_plane_pad_lhs != 0) || (total_cross_plane_pad_out != 0)));

368

369

cl::Image2D src1_image2d;

370

371

if(_export_to_cl_image)

372

{

373

const TensorShape shape2d(src1->info()->dimension(0) / 4, src1->info()->dimension(1) * src1->info()->dimension(2));

374

const size_t image_row_pitch = src1->info()->strides_in_bytes()[1];

375

376

src1_image2d = create_image2d_from_buffer(CLKernelLibrary::get().context(), src1->cl_buffer(), shape2d, src1->info()->data_type(), image_row_pitch);

}

do

{

Window slice_b = slice;

382

// Don't slice matrix B along the z dimension if matrix B has just 2 dimensions and matrix A more than 2

383

// This scenario can happen when the matrix multiplication is used to perform a convolution operation

384

if(!_slide_matrix_b)

385

{

386

slice_b = slice_matrix_b;

387

}

388

389

unsigned int idx = 0;

390

391

// LHS buffer

392

add_2D_tensor_argument(idx, src0, slice);

393

394

// RHS buffer or RHS OpenCL image (_export_to_cl_image == true)

395

if(_export_to_cl_image)

396

{

397

_kernel.setArg(idx++, src1_image2d);

}

else

{

add_2D_tensor_argument(idx, src1, slice_b);

402

}

403

404

// Bias buffer (_add_bias == true)

405

add_2D_tensor_argument_if(_add_bias, idx, src2, slice);

406

407

// dst buffer

408

add_2D_tensor_argument(idx, dst, slice);

409

410

// LHS stride_z

411

_kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(src0->info()->strides_in_bytes()[lhs_idx_batch_size]));

412

413

// RHS stride_z (not used if _export_to_cl_image == true)

414

_kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(src1->info()->strides_in_bytes()[rhs_idx_batch_size]));

415

416

// Bias stride_z (if _add_bias == true)

417

if(_add_bias)

418

{

419

_kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(src2->info()->strides_in_bytes()[bia_idx_batch_size]));

}

// dst stride_z

_kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(dst->info()->strides_in_bytes()[out_idx_batch_size]));

424

425

// Cross-plan padding (if _reinterpret_input_as_3d = true)

426

if(_reinterpret_input_as_3d && _has_pad_y)

427

{

428

_kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(total_cross_plane_pad_lhs));

429

}

430

431

// Cross-plan padding (if reinterpret_output_as_3d = true)

432

if(_reinterpret_output_as_3d && _has_pad_y)

433

{

434

_kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(total_cross_plane_pad_out));

435

}

436

437

enqueue(queue, *this, slice, lws_hint(), _use_dummy_work_items);

438

}

439

while(window.slide_window_slice_3D(slice));

440

}

441

} // namespace kernels

442

} // namespace opencl

443

} // namespace arm_compute