Blame - src/core/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedOnlyRhsKernel.cpp - ml/ComputeLibrary

std::pair<Status, Window> validate_and_configure_window(ITensorInfo *src0, ITensorInfo *src1, ITensorInfo *src2, ITensorInfo *dst, const GEMMLHSMatrixInfo &lhs_info,

122

const GEMMRHSMatrixInfo &rhs_info, const GEMMKernelInfo &gemm_info, ElementsProcessed &num_elements_processed)

123

{

124

unsigned int &num_elems_processed_per_iteration_x = num_elements_processed[0];

125

unsigned int &num_elems_processed_per_iteration_y = num_elements_processed[1];

126

bool reinterpret_input_as_3d = gemm_info.reinterpret_input_as_3d;

127

bool reinterpret_output_as_3d = gemm_info.depth_output_gemm3d != 0;

Window win{};

Window win_out{};

bool window_changed = false;

132

133

// In case both input and dst have to be reinterpreted as 3D tensors,

134

// force reinterpret_input_as_3d and reinterpret_output_as_3d to be false.

135

// This approach should only be used when the input/dst tensors have pad on the y direction

136

if((reinterpret_input_as_3d == reinterpret_output_as_3d) && gemm_info.has_pad_y)

137

{

138

reinterpret_output_as_3d = false;

139

}

140

141

// dst tensor auto initialization if not yet initialized

142

auto_init_if_empty(*dst, src0->clone()->set_tensor_shape(misc::shape_calculator::compute_mm_shape(*src0, *src1, gemm_info)));

143

144

TensorInfo tmp_info(*dst);

145

146

if(reinterpret_output_as_3d)

147

{

148

// Since the dst tensor has to be reinterpreted as 3D and the execute window is based on a 2D GEMM,

149

// the window needs to be constructed on the 2D collapsed version of the tensor

150

TensorShape tmp_shape(dst->tensor_shape());

151

tmp_shape.collapse(2U, 1U);

152

tmp_info.set_tensor_shape(tmp_shape);

153

}

154

155

// Configure kernel window

156

num_elems_processed_per_iteration_x = rhs_info.n0;

157

num_elems_processed_per_iteration_y = lhs_info.m0;

158

159

win = calculate_max_window(tmp_info, Steps(num_elems_processed_per_iteration_x, num_elems_processed_per_iteration_y));

160

win_out = calculate_max_window(*dst, Steps(num_elems_processed_per_iteration_x, num_elems_processed_per_iteration_y));

if(src2 != nullptr)

{

const int bias_processed_per_iteration_x = num_elems_processed_per_iteration_x;

165

166

AccessWindowStatic src2_access(src2, 0, 0,

167

ceil_to_multiple(src2->dimension(0), bias_processed_per_iteration_x),

168

src2->dimension(1));

169

170

window_changed = update_window_and_padding(win, src2_access);

171

}

172

173

// Collapse along the Z direction

174

// This collapse needs to be here in order to tune the Z dimension of LWS

175

Window collapsed = win;

176

const unsigned int dimension_to_collapse = std::min(static_cast<unsigned int>(dst->num_dimensions()), 2u);

177

collapsed = win.collapse(win, dimension_to_collapse);

178

179

Status err = (window_changed) ? ARM_COMPUTE_CREATE_ERROR(ErrorCode::RUNTIME_ERROR, "Insufficient Padding!") : Status{};

180

return std::make_pair(err, collapsed);

}

} // namespace

void ClGemmMatrixMultiplyReshapedOnlyRhsKernel::configure(const CLCompileContext &compile_context,

185

ITensorInfo *src0, ITensorInfo *src1, ITensorInfo *src2, ITensorInfo *dst, float alpha, float beta,

186

const GEMMLHSMatrixInfo &lhs_info, const GEMMRHSMatrixInfo &rhs_info, const GEMMKernelInfo &gemm_info)

187

{

188

ARM_COMPUTE_ERROR_ON_NULLPTR(src0, src1, dst);

189

190

ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(src0, src1, src2, dst, alpha, beta, lhs_info, rhs_info, gemm_info));

191

192

_reinterpret_input_as_3d = gemm_info.reinterpret_input_as_3d;

193

_reinterpret_output_as_3d = gemm_info.depth_output_gemm3d != 0;

194

_use_dummy_work_items = preferred_dummy_work_items_support(CLKernelLibrary::get().get_device());

195

_add_bias = src2 != nullptr;

196

_export_to_cl_image = rhs_info.export_to_cl_image;

197

_has_pad_y = gemm_info.has_pad_y;

198

199

auto padding_info = get_padding_info({ src0, src1, dst });

200

201

// In case both input and dst have to be reinterpreted as 3D tensors,

202

// force reinterpret_input_as_3d and reinterpret_output_as_3d to be false.

203

if((_reinterpret_input_as_3d == _reinterpret_output_as_3d) && _has_pad_y)

204

{

205

_reinterpret_input_as_3d = false;

206

_reinterpret_output_as_3d = false;

207

}

208

209

// Check if we need to slide the matrix B

210

const unsigned int num_dimensions_src0 = src0->num_dimensions();

211

_slide_matrix_b = (src1->num_dimensions() >= num_dimensions_src0);

212

213

ElementsProcessed num_elements_processed{};

214

215

// Configure kernel window

216

auto win_config = validate_and_configure_window(src0, src1, src2, dst, lhs_info, rhs_info, gemm_info, num_elements_processed);

217

ARM_COMPUTE_ERROR_THROW_ON(win_config.first);

218

ICLKernel::configure_internal(win_config.second);

219

220

// If _reinterpret_input_as_3d = reinterpret_output_as_3d = true,

221

// we will dispatch a batched-GEMM to reduce the complexity of the address calculation within the OpenCL kernel.

222

// This means that the actual m used by the kernel is given by dst->dimension(1) and not by gemm_info.m

223

const unsigned int internal_m = _reinterpret_output_as_3d ? gemm_info.m : dst->dimension(1);

224

225

// These variables are used only if gemm_info.has_pad_y == true

226

const unsigned int h_gemm_3d = _reinterpret_output_as_3d ? dst->dimension(1) : src0->dimension(1);

227

const unsigned int d_gemm_3d = _reinterpret_output_as_3d ? dst->dimension(2) : src0->dimension(2);

228

229

// Shrink M0 to be always <= M (internal_m) to prevent out-of-bounds reads.

230

// NOTE: This might have implications on heuristics and performance

231

const unsigned int internal_m0 = std::min(internal_m, lhs_info.m0);

232

233

// Calculate partial (store instead of load) M0 and partial N0 for the partial blocks at the end of a row/column if any. This is to avoid padding.

234

const unsigned int partial_store_m0 = internal_m % internal_m0;

235

const unsigned int partial_store_n0 = gemm_info.n % rhs_info.n0;

236

237

// Create build options

238

CLBuildOptions build_opts;

239

build_opts.add_option("-DDATA_TYPE=" + get_cl_type_from_data_type(src0->data_type()));

240

build_opts.add_option_if(!(helpers::float_ops::is_one(alpha)), "-DALPHA=" + float_to_string_with_full_precision(alpha));

241

build_opts.add_option_if(src2 != nullptr, "-DBETA=" + float_to_string_with_full_precision(beta));

242

build_opts.add_option_if(helpers::float_ops::is_one(beta), "-DUNIT_BETA");

243

build_opts.add_option_if(gemm_info.broadcast_bias, "-DBROADCAST_BIAS");

244

build_opts.add_option_if(!_slide_matrix_b, "-DMATRIX_B_DEPTH=" + support::cpp11::to_string(src1->dimension(2)));

245

build_opts.add_option_if(rhs_info.interleave, "-DRHS_INTERLEAVE");

246

build_opts.add_option_if(_use_dummy_work_items, "-DDUMMY_WORK_ITEMS");

247

build_opts.add_option_if(rhs_info.export_to_cl_image, "-DOPENCL_IMAGE_SUPPORT");

248

build_opts.add_option("-DRHS_HEIGHT=" + support::cpp11::to_string(src1->dimension(1)));

249

build_opts.add_option("-DM=" + support::cpp11::to_string(internal_m));

250

build_opts.add_option("-DN=" + support::cpp11::to_string(gemm_info.n));

251

build_opts.add_option("-DK=" + support::cpp11::to_string(gemm_info.k));

252

build_opts.add_option("-DM0=" + support::cpp11::to_string(internal_m0));

253

build_opts.add_option("-DN0=" + support::cpp11::to_string(rhs_info.n0));

254

build_opts.add_option("-DK0=" + support::cpp11::to_string(rhs_info.k0));

255

build_opts.add_option("-DH0=" + support::cpp11::to_string(rhs_info.h0));

256

build_opts.add_option("-DPARTIAL_STORE_M0=" + support::cpp11::to_string(partial_store_m0));

257

build_opts.add_option("-DPARTIAL_STORE_N0=" + support::cpp11::to_string(partial_store_n0));

258

build_opts.add_option_if(gemm_info.activation_info.enabled(), "-DACTIVATION_TYPE=" + lower_string(string_from_activation_func(gemm_info.activation_info.activation())));

259

build_opts.add_option_if(gemm_info.activation_info.enabled(), "-DA_VAL=" + float_to_string_with_full_precision(gemm_info.activation_info.a()));

260

build_opts.add_option_if(gemm_info.activation_info.enabled(), "-DB_VAL=" + float_to_string_with_full_precision(gemm_info.activation_info.b()));

261

if(_has_pad_y)

262

{

263

build_opts.add_option_if(_reinterpret_input_as_3d, "-DREINTERPRET_INPUT_AS_3D");

264

build_opts.add_option_if(_reinterpret_output_as_3d, "-DREINTERPRET_OUTPUT_AS_3D");

265

build_opts.add_option_if(_reinterpret_input_as_3d || _reinterpret_output_as_3d, "-DHEIGHT_GEMM3D=" + support::cpp11::to_string(h_gemm_3d));

266

build_opts.add_option_if(_reinterpret_input_as_3d || _reinterpret_output_as_3d, "-DDEPTH_GEMM3D=" + support::cpp11::to_string(d_gemm_3d));

267

}

268

269

std::string kernel_name("gemm_mm_reshaped_only_rhs_");

270

kernel_name += rhs_info.transpose ? "t" : "nt";

271

kernel_name += rhs_info.export_to_cl_image ? "_texture" : "";

272

273

// Create kernel

274

_kernel = create_kernel(compile_context, kernel_name, build_opts.options());

275

276

// Set config_id for enabling LWS tuning

277

_config_id = kernel_name;

278

_config_id += "_";

279

_config_id += (_has_pad_y ? "" : "no_pad_y_");

280

_config_id += (_add_bias ? "add_bias_" : "");

281

_config_id += (gemm_info.broadcast_bias ? "broadcast_bias_" : "");

282

_config_id += (_reinterpret_input_as_3d ? "3di_" : "");

283

_config_id += (_reinterpret_output_as_3d ? "3do_" : "");

284

_config_id += (gemm_info.activation_info.enabled() ? "fused_activation_" : "");

285

_config_id += lower_string(string_from_data_type(src0->data_type()));

286

_config_id += "_";

287

_config_id += support::cpp11::to_string(dst->dimension(1));

288

_config_id += "_";

289

_config_id += support::cpp11::to_string(dst->dimension(0));

290

_config_id += "_";

291

_config_id += support::cpp11::to_string(gemm_info.k);

292

_config_id += "_";

293

_config_id += support::cpp11::to_string(dst->dimension(2));

294

_config_id += "_";

295

_config_id += support::cpp11::to_string(lhs_info.m0);

296

_config_id += "_";

297

_config_id += support::cpp11::to_string(rhs_info.n0);

298

_config_id += "_";

299

_config_id += support::cpp11::to_string(rhs_info.k0);

300

_config_id += "_";

301

_config_id += support::cpp11::to_string(rhs_info.h0);

302

_config_id += "_";

303

_config_id += support::cpp11::to_string(rhs_info.interleave);

304

305

ARM_COMPUTE_ERROR_ON(has_padding_changed(padding_info));

306

}

307

308

Status ClGemmMatrixMultiplyReshapedOnlyRhsKernel::validate(const ITensorInfo *src0, const ITensorInfo *src1, const ITensorInfo *src2, const ITensorInfo *dst, float alpha, float beta,

309

const GEMMLHSMatrixInfo &lhs_info,

310

const GEMMRHSMatrixInfo &rhs_info, const GEMMKernelInfo &gemm_info)

311

{

312

ElementsProcessed num_elements_processed{};

313

ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(src0, src1, src2, dst, alpha, beta, lhs_info, rhs_info, gemm_info));

314

ARM_COMPUTE_RETURN_ON_ERROR(validate_and_configure_window(src0->clone().get(),

315

src1->clone().get(),

316

src2 != nullptr ? src2->clone().get() : nullptr,

dst->clone().get(),

lhs_info,

rhs_info,

gemm_info,

num_elements_processed)

.first);

return Status{};

}

void ClGemmMatrixMultiplyReshapedOnlyRhsKernel::run_op(ITensorPack &tensors, const Window &window, cl::CommandQueue &queue)

328

{

329

ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);

330

ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(ICLKernel::window(), window);

331

332

const auto src0 = utils::cast::polymorphic_downcast<const ICLTensor *>(tensors.get_const_tensor(TensorType::ACL_SRC_0));

333

const auto src1 = utils::cast::polymorphic_downcast<const ICLTensor *>(tensors.get_const_tensor(TensorType::ACL_SRC_1));

334

const auto src2 = utils::cast::polymorphic_downcast<const ICLTensor *>(tensors.get_const_tensor(TensorType::ACL_SRC_2));

335

auto dst = utils::cast::polymorphic_downcast<ICLTensor *>(tensors.get_tensor(TensorType::ACL_DST));

336

337

ARM_COMPUTE_ERROR_ON_NULLPTR(src0, src1, dst);

338

ARM_COMPUTE_ERROR_ON(_add_bias && src2 == nullptr);

339

340

if(src1->info()->num_dimensions() < 3)

341

{

342

// The stride_z for matrix B must be zero if we do not slice

343

ARM_COMPUTE_ERROR_ON(src1->info()->strides_in_bytes()[3] != 0);

344

}

345

346

const size_t lhs_idx_batch_size = _reinterpret_input_as_3d && !_has_pad_y ? 3u : 2u;

347

const size_t rhs_idx_batch_size = 2u;

348

const size_t bia_idx_batch_size = 2u;

349

const size_t out_idx_batch_size = _reinterpret_output_as_3d && !_has_pad_y ? 3u : 2u;

350

351

Window slice = window.first_slice_window_3D();

352

Window slice_matrix_b = slice;

353

354

slice_matrix_b.set(Window::DimX, Window::Dimension(0, 1, 1));

355

slice_matrix_b.set(Window::DimY, Window::Dimension(0, 1, 1));

356

357

// Get cross plane pads

358

const unsigned int total_cross_plane_pad_lhs = src0->info()->padding().top + src0->info()->padding().bottom;

359

const unsigned int total_cross_plane_pad_out = dst->info()->padding().top + dst->info()->padding().bottom;

360

361

// The execution should fail if we try to run with has_pad_y = false but we have padding in either the LHS or DST tensor

362

ARM_COMPUTE_ERROR_ON(!_has_pad_y && ((total_cross_plane_pad_lhs != 0) || (total_cross_plane_pad_out != 0)));

363

364

cl::Image2D src1_image2d;

365

366

if(_export_to_cl_image)

367

{

368

const TensorShape shape2d(src1->info()->dimension(0) / 4, src1->info()->dimension(1) * src1->info()->dimension(2));

369

const size_t image_row_pitch = src1->info()->strides_in_bytes()[1];

370

371

src1_image2d = create_image2d_from_buffer(CLKernelLibrary::get().context(), src1->cl_buffer(), shape2d, src1->info()->data_type(), image_row_pitch);

}

do

{

Window slice_b = slice;

377

// Don't slice matrix B along the z dimension if matrix B has just 2 dimensions and matrix A more than 2

378

// This scenario can happen when the matrix multiplication is used to perform a convolution operation

379

if(!_slide_matrix_b)

380

{

381

slice_b = slice_matrix_b;

382

}

383

384

unsigned int idx = 0;

385

386

// LHS buffer

387

add_2D_tensor_argument(idx, src0, slice);

388

389

// RHS buffer or RHS OpenCL image (_export_to_cl_image == true)

390

if(_export_to_cl_image)

391

{

392

_kernel.setArg(idx++, src1_image2d);

}

else

{

add_2D_tensor_argument(idx, src1, slice_b);

397

}

398

399

// Bias buffer (_add_bias == true)

400

add_2D_tensor_argument_if(_add_bias, idx, src2, slice);

401

402

// dst buffer

403

add_2D_tensor_argument(idx, dst, slice);

404

405

// LHS stride_z

406

_kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(src0->info()->strides_in_bytes()[lhs_idx_batch_size]));

407

408

// RHS stride_z (not used if _export_to_cl_image == true)

409

_kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(src1->info()->strides_in_bytes()[rhs_idx_batch_size]));

410

411

// Bias stride_z (if _add_bias == true)

412

if(_add_bias)

413

{

414

_kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(src2->info()->strides_in_bytes()[bia_idx_batch_size]));

}

// dst stride_z

_kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(dst->info()->strides_in_bytes()[out_idx_batch_size]));

419

420

// Cross-plan padding (if _reinterpret_input_as_3d = true)

421

if(_reinterpret_input_as_3d && _has_pad_y)

422

{

423

_kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(total_cross_plane_pad_lhs));

424

}

425

426

// Cross-plan padding (if reinterpret_output_as_3d = true)

427

if(_reinterpret_output_as_3d && _has_pad_y)

428

{

429

_kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(total_cross_plane_pad_out));

430

}

431

432

enqueue(queue, *this, slice, lws_hint(), _use_dummy_work_items);

433

}

434

while(window.slide_window_slice_3D(slice));

435

}

436

} // namespace kernels

437

} // namespace opencl

438

} // namespace arm_compute