src/core/GLES_COMPUTE/cs_shaders/softmax_layer.cs - ml/ComputeLibrary - Gitiles

 /*
  * Copyright (c) 2017 ARM Limited.
  *
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to
  * deal in the Software without restriction, including without limitation the
  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
  * The above copyright notice and this permission notice shall be included in all
  * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */

 layout(local_size_x = LOCAL_SIZE_X, local_size_y = LOCAL_SIZE_Y, local_size_z = LOCAL_SIZE_Z) in;

 #include "helpers_cs.h"

 #if defined(DATA_TYPE_FP16)
 precision mediump float;
 #endif // DATA_TYPE_FP16

 // Common definitions
 #define MAX_OP(x, y) max((x), (y))
 #define ADD_OP(x, y) ((x) + (y))
 #define SUB_OP(x, y) ((x) - (y))
 #define DIV_OP(x, y) ((x) / (y))
 #define EXP_OP(x) exp((x))

 const float float_min = -1.0 / 0.0;
 const vec4  vec4_min  = vec4(float_min);

 #ifdef SOFTMAX_LAYER_MAX

 /** Identifies the maximum value across the 1st dimension.
  *
  * @note The data type must be passed at compile time using "#define DATA_TYPE_NAME". e.g. "#define DATA_TYPE_FP32"
  * @note In case the input is not multiple of 8 NON_MULTIPLE_OF_8 must be passed.
  *
  * @param[in]  src_ptr   Pointer to the source tensor slice. Supported data types: F16/F32
  * @param[in]  src_attrs The attributes of the source tensor
  * @param[out] dst_ptr   Pointer to the destination tensor slice. Supported data types: same as @p src_ptr
  * @param[in]  dst_attrs The attributes of the destination tensor
  * @param[in]  width     Input image width
  */
 SHADER_PARAMS_DECLARATION
 {
     Tensor3DAttributes src_attrs;
     Tensor3DAttributes dst_attrs;
     uint               width;
 };

 #if defined(DATA_TYPE_FP32)

 TENSOR_DECLARATION(1, srcBuffer, vec4[2], src_ptr, src_shift, 5, readonly);
 TENSOR_DECLARATION(2, dstBuffer, float, dst_ptr, dst_shift, 2, writeonly);

 void main(void)
 {
     ImageIterator src_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(src_attrs, src_shift);
     ImageIterator dst_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(dst_attrs, dst_shift);

     // Initialize local maximum
     vec4 max_val = vec4_min;

     // Calculate max of row
     uint width3 = width >> 3;
     for(int i = 0; i < int(width3); i++)
     {
         vec4 data[2] = LOAD(src_ptr, IMAGE_OFFSET(src_iter, i << 3, 0));
         max_val      = MAX_OP(data[0], max_val);
         max_val      = MAX_OP(data[1], max_val);
     }

 #ifdef NON_MULTIPLE_OF_8
     // Handle non multiple of 8
     vec4 data[2] = LOAD(src_ptr, IMAGE_OFFSET(src_iter, width3 << 3, 0));
     int  idx     = 0;
     if(width >> 2 != width3 << 1)
     {
         max_val = MAX_OP(data[0], max_val);
         idx     = 1;
     }
     for(int i = 0; i < int(width) % 4; i++)
     {
         max_val.x = MAX_OP(data[idx][i], max_val.x);
     }
 #endif /* NON_MULTIPLE_OF_8 */

     // Perform max reduction
     max_val.xy = MAX_OP(max_val.xy, max_val.zw);
     max_val.x  = MAX_OP(max_val.x, max_val.y);

     // Store result
     STORE_CURRENT_ITEM(dst_ptr, dst_iter, max_val.x);
 }
 #elif defined(DATA_TYPE_FP16)

 TENSOR_DECLARATION(1, srcBuffer, uvec4, src_ptr, src_shift, 4, readonly);
 TENSOR_DECLARATION(2, dstBuffer, uint, dst_ptr, dst_shift, 2, writeonly);

 void main(void)
 {
     ImageIterator src_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(src_attrs, src_shift);
     ImageIterator dst_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(dst_attrs, dst_shift);

     // Initialize local maximum
     vec4 max_val = vec4_min;

     // Calculate max of row
     uint width3 = width >> 3;
     for(int i = 0; i < int(width3); i++)
     {
         vec4 data[2] = LOAD_UNPACK8_HALF(src_ptr, IMAGE_OFFSET(src_iter, i << 3, 0));
         max_val      = MAX_OP(data[0], max_val);
         max_val      = MAX_OP(data[1], max_val);
     }

 #ifdef NON_MULTIPLE_OF_8
     // Handle non multiple of 8
     vec4 data[2] = LOAD_UNPACK8_HALF(src_ptr, IMAGE_OFFSET(src_iter, width3 << 3, 0));
     int  idx     = 0;
     if(width >> 2 != width3 << 1)
     {
         max_val = MAX_OP(data[0], max_val);
         idx     = 1;
     }
     for(int i = 0; i < int(width) % 4; i++)
     {
         max_val.x = MAX_OP(data[idx][i], max_val.x);
     }
 #endif /* NON_MULTIPLE_OF_8 */

     // Perform max reduction
     max_val.xy = MAX_OP(max_val.xy, max_val.zw);
     max_val.x  = MAX_OP(max_val.x, max_val.y);

     STORE_PACK2_CURRENT_ITEM_HALF(dst_ptr, dst_iter, max_val.xy);
 }
 #else  // DATA_TYPE_FP32
 #error Data type not supported
 #endif // DATA_TYPE_FP32
 #elif defined(SOFTMAX_LAYER_SHIFT_EXP_SUM)

 /** Shifts the values of the input tensor by the max calculated in softmax_layer_max kernel,
  * then gets the exponent of each element as sums all elements across each row.
  *
  * @note The data type must be passed at compile time using "#define DATA_TYPE_NAME". e.g. "#define DATA_TYPE_FP32"
  * @note In case the input is not multiple of 8 NON_MULTIPLE_OF_8 must be passed.
  *
  * @param[in]  src_ptr   Pointer to the source tensor slice. Supported data types: F16/F32
  * @param[in]  src_attrs The attributes of the source tensor
  * @param[in]  max_ptr   Pointer to the max values tensor slice. Supported data types: same as @p src_ptr
  * @param[in]  max_attrs The attributes of the max values tensor
  * @param[out] dst_ptr   Pointer to the destination tensor slice. Supported data types: same as @p src_ptr
  * @param[in]  dst_attrs The attributes of the destination tensor
  * @param[out] sum_ptr   Pointer to the sum values tensor slice. Supported data types: same as @p src_ptr
  * @param[in]  sum_attrs The attributes of the sum values tensor
  * @param[in]  width     Input image width
  */
 SHADER_PARAMS_DECLARATION
 {
     Tensor3DAttributes src_attrs;
     Tensor3DAttributes max_attrs;
     Tensor3DAttributes dst_attrs;
     Tensor3DAttributes sum_attrs;
     uint               width;
 };
 #if defined(DATA_TYPE_FP32)

 TENSOR_DECLARATION(1, srcBuffer, vec4[2], src_ptr, src_shift, 5, readonly);
 TENSOR_DECLARATION(2, maxBuffer, float, max_ptr, max_shift, 2, readonly);
 TENSOR_DECLARATION(3, dstBuffer, vec4[2], dst_ptr, dst_shift, 5, writeonly);
 TENSOR_DECLARATION(4, sumBuffer, float, sum_ptr, sum_shift, 2, writeonly);

 void main(void)
 {
     ImageIterator src_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(src_attrs, src_shift);
     ImageIterator dst_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(dst_attrs, dst_shift);
     ImageIterator max_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(max_attrs, max_shift);
     ImageIterator sum_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(sum_attrs, sum_shift);

     // Load max value of 1D logits vector (row)
     vec4 max_val = vec4(LOAD_CURRENT_ITEM(max_ptr, max_iter));

     // Set sum vector
     vec4 sum1D = vec4(0);

     // Shift values, exp and sum
     uint width3 = width >> 3;
     for(int i = 0; i < int(width3); i++)
     {
         vec4 data[2];
         data    = LOAD(src_ptr, IMAGE_OFFSET(src_iter, i << 3, 0));
         data[0] = SUB_OP(data[0], max_val);
         data[1] = SUB_OP(data[1], max_val);
         data[0] = EXP_OP(data[0]);
         data[1] = EXP_OP(data[1]);
         STORE(dst_ptr, IMAGE_OFFSET(dst_iter, i << 3, 0), data);
         sum1D = ADD_OP(sum1D, data[0]);
         sum1D = ADD_OP(sum1D, data[1]);
     }

 #ifdef NON_MULTIPLE_OF_8
     // Handle non multiple of 8
     vec4 data[2] = LOAD(src_ptr, IMAGE_OFFSET(src_iter, width3 << 3, 0));
     int  idx     = 0;
     if(width >> 2 != width3 << 1)
     {
         data[0] = SUB_OP(data[0], max_val);
         data[0] = EXP_OP(data[0]);
         sum1D   = ADD_OP(sum1D, data[0]);
         idx     = 1;
     }
     for(int i = 0; i < int(width) % 4; i++)
     {
         data[idx][i] = SUB_OP(data[idx][i], max_val.x);
         data[idx][i] = EXP_OP(data[idx][i]);
         sum1D.x      = ADD_OP(sum1D.x, data[idx][i]);
     }
     STORE(dst_ptr, IMAGE_OFFSET(dst_iter, width3 << 3, 0), data);
 #endif /* NON_MULTIPLE_OF_8 */

     // Perform min/max reduction
     sum1D.xy = ADD_OP(sum1D.xy, sum1D.zw);
     sum1D.x  = ADD_OP(sum1D.x, sum1D.y);

     // Calculate and store result
     STORE_CURRENT_ITEM(sum_ptr, sum_iter, sum1D.x);
 }
 #elif defined(DATA_TYPE_FP16)

 TENSOR_DECLARATION(1, srcBuffer, uvec4, src_ptr, src_shift, 4, readonly);
 TENSOR_DECLARATION(2, maxBuffer, uint, max_ptr, max_shift, 2, readonly);
 TENSOR_DECLARATION(3, dstBuffer, uvec4, dst_ptr, dst_shift, 4, writeonly);
 TENSOR_DECLARATION(4, sumBuffer, uint, sum_ptr, sum_shift, 2, writeonly);

 void main(void)
 {
     ImageIterator src_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(src_attrs, src_shift);
     ImageIterator dst_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(dst_attrs, dst_shift);
     ImageIterator max_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(max_attrs, max_shift);
     ImageIterator sum_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(sum_attrs, sum_shift);

     // Load max value of 1D logits vector (row)
     vec2 datamaxinit = LOAD_UNPACK2_CURRENT_ITEM_HALF(max_ptr, max_iter);
     vec4 max_val     = vec4(datamaxinit.x);

     // Set sum vector
     vec4 sum1D = vec4(0.f);

     // Shift values, exp and sum
     uint width3 = width >> 3;
     for(int i = 0; i < int(width3); i++)
     {
         vec4 data[2] = LOAD_UNPACK8_HALF(src_ptr, IMAGE_OFFSET(src_iter, i << 3, 0));
         data[0]      = SUB_OP(data[0], max_val);
         data[1]      = SUB_OP(data[1], max_val);
         data[0]      = EXP_OP(data[0]);
         data[1]      = EXP_OP(data[1]);
         STORE_PACK8_HALF(dst_ptr, IMAGE_OFFSET(dst_iter, i << 3, 0), data);
         sum1D = ADD_OP(sum1D, data[0]);
         sum1D = ADD_OP(sum1D, data[1]);
     }

 #ifdef NON_MULTIPLE_OF_8
     // Handle non multiple of 8
     vec4 data[2] = LOAD_UNPACK8_HALF(src_ptr, IMAGE_OFFSET(src_iter, width3 << 3, 0));
     int  idx     = 0;
     if(width >> 2 != width3 << 1)
     {
         data[0] = SUB_OP(data[0], max_val);
         data[0] = EXP_OP(data[0]);
         sum1D   = ADD_OP(sum1D, data[0]);
         idx     = 1;
     }
     for(int i = 0; i < int(width) % 4; i++)
     {
         data[idx][i] = SUB_OP(data[idx][i], max_val.x);
         data[idx][i] = EXP_OP(data[idx][i]);
         sum1D.x      = ADD_OP(sum1D.x, data[idx][i]);
     }
     STORE_PACK8_HALF(dst_ptr, IMAGE_OFFSET(dst_iter, width3 << 3, 0), data);
 #endif /* NON_MULTIPLE_OF_8 */
     // Perform min/max reduction
     sum1D.xy = ADD_OP(sum1D.xy, sum1D.zw);
     sum1D.x  = ADD_OP(sum1D.x, sum1D.y);

     // Calculate and store result
     STORE_PACK2_CURRENT_ITEM_HALF(sum_ptr, sum_iter, sum1D.xy);
 }
 #else  // DATA_TYPE_FP32
 #error Data type not supported
 #endif // DATA_TYPE_FP32
 #elif defined(SOFTMAX_LAYER_NORM)

 /** Divides all the values of the input tensor by the sum calculated from softmax_layer_shift_exp_sum kernel.
  *
  * @note The data type must be passed at compile time using "#define DATA_TYPE_NAME". e.g. "#define DATA_TYPE_FP32"
  *
  * @param[in]  src_ptr   Pointer to the source tensor slice. Supported data types: F16/F32
  * @param[in]  src_attrs The attributes of the source tensor
  * @param[in]  sum_ptr   Pointer to the sum values tensor slice. Supported data types: same as @p src_ptr
  * @param[in]  sum_attrs The attributes of the sum values tensor
  * @param[out] dst_ptr   Pointer to the destination tensor slice. Supported data types: same as @p src_ptr
  * @param[in]  dst_attrs The attributes of the destination tensor
  */
 SHADER_PARAMS_DECLARATION
 {
     Tensor3DAttributes src_attrs;
     Tensor3DAttributes sum_attrs;
     Tensor3DAttributes dst_attrs;
 };
 #if defined(DATA_TYPE_FP32)
 TENSOR_DECLARATION(1, srcBuffer, vec4[2], src_ptr, src_shift, 5, readonly);
 TENSOR_DECLARATION(2, sumBuffer, float, sum_ptr, sum_shift, 2, readonly);
 TENSOR_DECLARATION(3, dstBuffer, vec4[2], dst_ptr, dst_shift, 5, writeonly);
 void main(void)
 {
     ImageIterator src_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(src_attrs, src_shift);
     ImageIterator dst_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(dst_attrs, dst_shift);
     ImageIterator sum_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR_NO_STEP(sum_attrs, sum_shift);

     // Load max value of 1D logits vector (row)
     vec4 sum_val = vec4(LOAD(sum_ptr, IMAGE_OFFSET(sum_iter, 0, gl_GlobalInvocationID.y)));

     vec4 data[2] = LOAD_CURRENT_ITEM(src_ptr, src_iter);
     data[0]      = DIV_OP(data[0], sum_val);
     data[1]      = DIV_OP(data[1], sum_val);
     STORE_CURRENT_ITEM(dst_ptr, dst_iter, data);
 }
 #elif defined(DATA_TYPE_FP16)
 TENSOR_DECLARATION(1, srcBuffer, uvec4, src_ptr, src_shift, 4, readonly);
 TENSOR_DECLARATION(2, sumBuffer, uint, sum_ptr, sum_shift, 2, readonly);
 TENSOR_DECLARATION(3, dstBuffer, uvec4, dst_ptr, dst_shift, 4, writeonly);
 void main(void)
 {
     ImageIterator src_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(src_attrs, src_shift);
     ImageIterator dst_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(dst_attrs, dst_shift);
     ImageIterator sum_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR_NO_STEP(sum_attrs, sum_shift);

     // Load max value of 1D logits vector (row)
     vec4 sum_val = vec4(LOAD_UNPACK2_HALF(sum_ptr, IMAGE_OFFSET(sum_iter, 0, gl_GlobalInvocationID.y)).x);

     vec4 data[2] = LOAD_UNPACK8_CURRENT_ITEM_HALF(src_ptr, src_iter);
     data[0]      = DIV_OP(data[0], sum_val);
     data[1]      = DIV_OP(data[1], sum_val);
     STORE_PACK8_CURRENT_ITEM_HALF(dst_ptr, dst_iter, data);
 }
 #else // DATA_TYPE_FP32
 #error Data type not supported
 #endif // DATA_TYPE_FP32
 #endif // SOFTMAX_LAYER_MAX
	/*
	* Copyright (c) 2017 ARM Limited.
	*
	* SPDX-License-Identifier: MIT
	*
	* Permission is hereby granted, free of charge, to any person obtaining a copy
	* of this software and associated documentation files (the "Software"), to
	* deal in the Software without restriction, including without limitation the
	* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
	* sell copies of the Software, and to permit persons to whom the Software is
	* furnished to do so, subject to the following conditions:
	*
	* The above copyright notice and this permission notice shall be included in all
	* copies or substantial portions of the Software.
	*
	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
	* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	* SOFTWARE.
	*/

	layout(local_size_x = LOCAL_SIZE_X, local_size_y = LOCAL_SIZE_Y, local_size_z = LOCAL_SIZE_Z) in;

	#include "helpers_cs.h"

	#if defined(DATA_TYPE_FP16)
	precision mediump float;
	#endif // DATA_TYPE_FP16

	// Common definitions
	#define MAX_OP(x, y) max((x), (y))
	#define ADD_OP(x, y) ((x) + (y))
	#define SUB_OP(x, y) ((x) - (y))
	#define DIV_OP(x, y) ((x) / (y))
	#define EXP_OP(x) exp((x))

	const float float_min = -1.0 / 0.0;
	const vec4 vec4_min = vec4(float_min);

	#ifdef SOFTMAX_LAYER_MAX

	/** Identifies the maximum value across the 1st dimension.
	*
	* @note The data type must be passed at compile time using "#define DATA_TYPE_NAME". e.g. "#define DATA_TYPE_FP32"
	* @note In case the input is not multiple of 8 NON_MULTIPLE_OF_8 must be passed.
	*
	* @param[in] src_ptr Pointer to the source tensor slice. Supported data types: F16/F32
	* @param[in] src_attrs The attributes of the source tensor
	* @param[out] dst_ptr Pointer to the destination tensor slice. Supported data types: same as @p src_ptr
	* @param[in] dst_attrs The attributes of the destination tensor
	* @param[in] width Input image width
	*/
	SHADER_PARAMS_DECLARATION
	{
	Tensor3DAttributes src_attrs;
	Tensor3DAttributes dst_attrs;
	uint width;
	};

	#if defined(DATA_TYPE_FP32)

	TENSOR_DECLARATION(1, srcBuffer, vec4[2], src_ptr, src_shift, 5, readonly);
	TENSOR_DECLARATION(2, dstBuffer, float, dst_ptr, dst_shift, 2, writeonly);

	void main(void)
	{
	ImageIterator src_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(src_attrs, src_shift);
	ImageIterator dst_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(dst_attrs, dst_shift);

	// Initialize local maximum
	vec4 max_val = vec4_min;

	// Calculate max of row
	uint width3 = width >> 3;
	for(int i = 0; i < int(width3); i++)
	{
	vec4 data[2] = LOAD(src_ptr, IMAGE_OFFSET(src_iter, i << 3, 0));
	max_val = MAX_OP(data[0], max_val);
	max_val = MAX_OP(data[1], max_val);
	}

	#ifdef NON_MULTIPLE_OF_8
	// Handle non multiple of 8
	vec4 data[2] = LOAD(src_ptr, IMAGE_OFFSET(src_iter, width3 << 3, 0));
	int idx = 0;
	if(width >> 2 != width3 << 1)
	{
	max_val = MAX_OP(data[0], max_val);
	idx = 1;
	}
	for(int i = 0; i < int(width) % 4; i++)
	{
	max_val.x = MAX_OP(data[idx][i], max_val.x);
	}
	#endif /* NON_MULTIPLE_OF_8 */

	// Perform max reduction
	max_val.xy = MAX_OP(max_val.xy, max_val.zw);
	max_val.x = MAX_OP(max_val.x, max_val.y);

	// Store result
	STORE_CURRENT_ITEM(dst_ptr, dst_iter, max_val.x);
	}
	#elif defined(DATA_TYPE_FP16)

	TENSOR_DECLARATION(1, srcBuffer, uvec4, src_ptr, src_shift, 4, readonly);
	TENSOR_DECLARATION(2, dstBuffer, uint, dst_ptr, dst_shift, 2, writeonly);

	void main(void)
	{
	ImageIterator src_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(src_attrs, src_shift);
	ImageIterator dst_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(dst_attrs, dst_shift);

	// Initialize local maximum
	vec4 max_val = vec4_min;

	// Calculate max of row
	uint width3 = width >> 3;
	for(int i = 0; i < int(width3); i++)
	{
	vec4 data[2] = LOAD_UNPACK8_HALF(src_ptr, IMAGE_OFFSET(src_iter, i << 3, 0));
	max_val = MAX_OP(data[0], max_val);
	max_val = MAX_OP(data[1], max_val);
	}

	#ifdef NON_MULTIPLE_OF_8
	// Handle non multiple of 8
	vec4 data[2] = LOAD_UNPACK8_HALF(src_ptr, IMAGE_OFFSET(src_iter, width3 << 3, 0));
	int idx = 0;
	if(width >> 2 != width3 << 1)
	{
	max_val = MAX_OP(data[0], max_val);
	idx = 1;
	}
	for(int i = 0; i < int(width) % 4; i++)
	{
	max_val.x = MAX_OP(data[idx][i], max_val.x);
	}
	#endif /* NON_MULTIPLE_OF_8 */

	// Perform max reduction
	max_val.xy = MAX_OP(max_val.xy, max_val.zw);
	max_val.x = MAX_OP(max_val.x, max_val.y);

	STORE_PACK2_CURRENT_ITEM_HALF(dst_ptr, dst_iter, max_val.xy);
	}
	#else // DATA_TYPE_FP32
	#error Data type not supported
	#endif // DATA_TYPE_FP32
	#elif defined(SOFTMAX_LAYER_SHIFT_EXP_SUM)

	/** Shifts the values of the input tensor by the max calculated in softmax_layer_max kernel,
	* then gets the exponent of each element as sums all elements across each row.
	*
	* @note The data type must be passed at compile time using "#define DATA_TYPE_NAME". e.g. "#define DATA_TYPE_FP32"
	* @note In case the input is not multiple of 8 NON_MULTIPLE_OF_8 must be passed.
	*
	* @param[in] src_ptr Pointer to the source tensor slice. Supported data types: F16/F32
	* @param[in] src_attrs The attributes of the source tensor
	* @param[in] max_ptr Pointer to the max values tensor slice. Supported data types: same as @p src_ptr
	* @param[in] max_attrs The attributes of the max values tensor
	* @param[out] dst_ptr Pointer to the destination tensor slice. Supported data types: same as @p src_ptr
	* @param[in] dst_attrs The attributes of the destination tensor
	* @param[out] sum_ptr Pointer to the sum values tensor slice. Supported data types: same as @p src_ptr
	* @param[in] sum_attrs The attributes of the sum values tensor
	* @param[in] width Input image width
	*/
	SHADER_PARAMS_DECLARATION
	{
	Tensor3DAttributes src_attrs;
	Tensor3DAttributes max_attrs;
	Tensor3DAttributes dst_attrs;
	Tensor3DAttributes sum_attrs;
	uint width;
	};
	#if defined(DATA_TYPE_FP32)

	TENSOR_DECLARATION(1, srcBuffer, vec4[2], src_ptr, src_shift, 5, readonly);
	TENSOR_DECLARATION(2, maxBuffer, float, max_ptr, max_shift, 2, readonly);
	TENSOR_DECLARATION(3, dstBuffer, vec4[2], dst_ptr, dst_shift, 5, writeonly);
	TENSOR_DECLARATION(4, sumBuffer, float, sum_ptr, sum_shift, 2, writeonly);

	void main(void)
	{
	ImageIterator src_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(src_attrs, src_shift);
	ImageIterator dst_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(dst_attrs, dst_shift);
	ImageIterator max_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(max_attrs, max_shift);
	ImageIterator sum_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(sum_attrs, sum_shift);

	// Load max value of 1D logits vector (row)
	vec4 max_val = vec4(LOAD_CURRENT_ITEM(max_ptr, max_iter));

	// Set sum vector
	vec4 sum1D = vec4(0);

	// Shift values, exp and sum
	uint width3 = width >> 3;
	for(int i = 0; i < int(width3); i++)
	{
	vec4 data[2];
	data = LOAD(src_ptr, IMAGE_OFFSET(src_iter, i << 3, 0));
	data[0] = SUB_OP(data[0], max_val);
	data[1] = SUB_OP(data[1], max_val);
	data[0] = EXP_OP(data[0]);
	data[1] = EXP_OP(data[1]);
	STORE(dst_ptr, IMAGE_OFFSET(dst_iter, i << 3, 0), data);
	sum1D = ADD_OP(sum1D, data[0]);
	sum1D = ADD_OP(sum1D, data[1]);
	}

	#ifdef NON_MULTIPLE_OF_8
	// Handle non multiple of 8
	vec4 data[2] = LOAD(src_ptr, IMAGE_OFFSET(src_iter, width3 << 3, 0));
	int idx = 0;
	if(width >> 2 != width3 << 1)
	{
	data[0] = SUB_OP(data[0], max_val);
	data[0] = EXP_OP(data[0]);
	sum1D = ADD_OP(sum1D, data[0]);
	idx = 1;
	}
	for(int i = 0; i < int(width) % 4; i++)
	{
	data[idx][i] = SUB_OP(data[idx][i], max_val.x);
	data[idx][i] = EXP_OP(data[idx][i]);
	sum1D.x = ADD_OP(sum1D.x, data[idx][i]);
	}
	STORE(dst_ptr, IMAGE_OFFSET(dst_iter, width3 << 3, 0), data);
	#endif /* NON_MULTIPLE_OF_8 */

	// Perform min/max reduction
	sum1D.xy = ADD_OP(sum1D.xy, sum1D.zw);
	sum1D.x = ADD_OP(sum1D.x, sum1D.y);

	// Calculate and store result
	STORE_CURRENT_ITEM(sum_ptr, sum_iter, sum1D.x);
	}
	#elif defined(DATA_TYPE_FP16)

	TENSOR_DECLARATION(1, srcBuffer, uvec4, src_ptr, src_shift, 4, readonly);
	TENSOR_DECLARATION(2, maxBuffer, uint, max_ptr, max_shift, 2, readonly);
	TENSOR_DECLARATION(3, dstBuffer, uvec4, dst_ptr, dst_shift, 4, writeonly);
	TENSOR_DECLARATION(4, sumBuffer, uint, sum_ptr, sum_shift, 2, writeonly);

	void main(void)
	{
	ImageIterator src_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(src_attrs, src_shift);
	ImageIterator dst_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(dst_attrs, dst_shift);
	ImageIterator max_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(max_attrs, max_shift);
	ImageIterator sum_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(sum_attrs, sum_shift);

	// Load max value of 1D logits vector (row)
	vec2 datamaxinit = LOAD_UNPACK2_CURRENT_ITEM_HALF(max_ptr, max_iter);
	vec4 max_val = vec4(datamaxinit.x);

	// Set sum vector
	vec4 sum1D = vec4(0.f);

	// Shift values, exp and sum
	uint width3 = width >> 3;
	for(int i = 0; i < int(width3); i++)
	{
	vec4 data[2] = LOAD_UNPACK8_HALF(src_ptr, IMAGE_OFFSET(src_iter, i << 3, 0));
	data[0] = SUB_OP(data[0], max_val);
	data[1] = SUB_OP(data[1], max_val);
	data[0] = EXP_OP(data[0]);
	data[1] = EXP_OP(data[1]);
	STORE_PACK8_HALF(dst_ptr, IMAGE_OFFSET(dst_iter, i << 3, 0), data);
	sum1D = ADD_OP(sum1D, data[0]);
	sum1D = ADD_OP(sum1D, data[1]);
	}

	#ifdef NON_MULTIPLE_OF_8
	// Handle non multiple of 8
	vec4 data[2] = LOAD_UNPACK8_HALF(src_ptr, IMAGE_OFFSET(src_iter, width3 << 3, 0));
	int idx = 0;
	if(width >> 2 != width3 << 1)
	{
	data[0] = SUB_OP(data[0], max_val);
	data[0] = EXP_OP(data[0]);
	sum1D = ADD_OP(sum1D, data[0]);
	idx = 1;
	}
	for(int i = 0; i < int(width) % 4; i++)
	{
	data[idx][i] = SUB_OP(data[idx][i], max_val.x);
	data[idx][i] = EXP_OP(data[idx][i]);
	sum1D.x = ADD_OP(sum1D.x, data[idx][i]);
	}
	STORE_PACK8_HALF(dst_ptr, IMAGE_OFFSET(dst_iter, width3 << 3, 0), data);
	#endif /* NON_MULTIPLE_OF_8 */
	// Perform min/max reduction
	sum1D.xy = ADD_OP(sum1D.xy, sum1D.zw);
	sum1D.x = ADD_OP(sum1D.x, sum1D.y);

	// Calculate and store result
	STORE_PACK2_CURRENT_ITEM_HALF(sum_ptr, sum_iter, sum1D.xy);
	}
	#else // DATA_TYPE_FP32
	#error Data type not supported
	#endif // DATA_TYPE_FP32
	#elif defined(SOFTMAX_LAYER_NORM)

	/** Divides all the values of the input tensor by the sum calculated from softmax_layer_shift_exp_sum kernel.
	*
	* @note The data type must be passed at compile time using "#define DATA_TYPE_NAME". e.g. "#define DATA_TYPE_FP32"
	*
	* @param[in] src_ptr Pointer to the source tensor slice. Supported data types: F16/F32
	* @param[in] src_attrs The attributes of the source tensor
	* @param[in] sum_ptr Pointer to the sum values tensor slice. Supported data types: same as @p src_ptr
	* @param[in] sum_attrs The attributes of the sum values tensor
	* @param[out] dst_ptr Pointer to the destination tensor slice. Supported data types: same as @p src_ptr
	* @param[in] dst_attrs The attributes of the destination tensor
	*/
	SHADER_PARAMS_DECLARATION
	{
	Tensor3DAttributes src_attrs;
	Tensor3DAttributes sum_attrs;
	Tensor3DAttributes dst_attrs;
	};
	#if defined(DATA_TYPE_FP32)
	TENSOR_DECLARATION(1, srcBuffer, vec4[2], src_ptr, src_shift, 5, readonly);
	TENSOR_DECLARATION(2, sumBuffer, float, sum_ptr, sum_shift, 2, readonly);
	TENSOR_DECLARATION(3, dstBuffer, vec4[2], dst_ptr, dst_shift, 5, writeonly);
	void main(void)
	{
	ImageIterator src_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(src_attrs, src_shift);
	ImageIterator dst_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(dst_attrs, dst_shift);
	ImageIterator sum_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR_NO_STEP(sum_attrs, sum_shift);

	// Load max value of 1D logits vector (row)
	vec4 sum_val = vec4(LOAD(sum_ptr, IMAGE_OFFSET(sum_iter, 0, gl_GlobalInvocationID.y)));

	vec4 data[2] = LOAD_CURRENT_ITEM(src_ptr, src_iter);
	data[0] = DIV_OP(data[0], sum_val);
	data[1] = DIV_OP(data[1], sum_val);
	STORE_CURRENT_ITEM(dst_ptr, dst_iter, data);
	}
	#elif defined(DATA_TYPE_FP16)
	TENSOR_DECLARATION(1, srcBuffer, uvec4, src_ptr, src_shift, 4, readonly);
	TENSOR_DECLARATION(2, sumBuffer, uint, sum_ptr, sum_shift, 2, readonly);
	TENSOR_DECLARATION(3, dstBuffer, uvec4, dst_ptr, dst_shift, 4, writeonly);
	void main(void)
	{
	ImageIterator src_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(src_attrs, src_shift);
	ImageIterator dst_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(dst_attrs, dst_shift);
	ImageIterator sum_iter = CONVERT_TENSOR3D_TO_IMAGE_ITERATOR_NO_STEP(sum_attrs, sum_shift);

	// Load max value of 1D logits vector (row)
	vec4 sum_val = vec4(LOAD_UNPACK2_HALF(sum_ptr, IMAGE_OFFSET(sum_iter, 0, gl_GlobalInvocationID.y)).x);

	vec4 data[2] = LOAD_UNPACK8_CURRENT_ITEM_HALF(src_ptr, src_iter);
	data[0] = DIV_OP(data[0], sum_val);
	data[1] = DIV_OP(data[1], sum_val);
	STORE_PACK8_CURRENT_ITEM_HALF(dst_ptr, dst_iter, data);
	}
	#else // DATA_TYPE_FP32
	#error Data type not supported
	#endif // DATA_TYPE_FP32
	#endif // SOFTMAX_LAYER_MAX