Refactor TOSA Common to be built without enabling the TOSA Reference backend

 * The only way to build TOSA common was to enable ARMNNTOSAREF.
 * Only include and link the required headers/libraries for TOSA Common.
 * Remove duplicate include_directories calls.

Signed-off-by: Matthew Sloyan <matthew.sloyan@arm.com>
Change-Id: Ia5a19bf5b0bb903be0248ac550439df1e8ac5aa7
diff --git a/src/backends/tosaReference/CMakeLists.txt b/src/backends/tosaReference/CMakeLists.txt
index c053def..35393cc 100644
--- a/src/backends/tosaReference/CMakeLists.txt
+++ b/src/backends/tosaReference/CMakeLists.txt
@@ -1,13 +1,8 @@
 #
-# Copyright © 2022 Arm Ltd and Contributors. All rights reserved.
+# Copyright © 2022-2023 Arm Ltd and Contributors. All rights reserved.
 # SPDX-License-Identifier: MIT
 #
 
-include_directories(SYSTEM ${FLATBUFFERS_INCLUDE_PATH})
-include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third-party/half)
-include_directories(SYSTEM ${TOSA_SERIALIZATION_LIB_INCLUDE})
-include_directories(SYSTEM ${TOSA_REFERENCE_MODEL_INCLUDE})
-
 if(ARMNNTOSAREF)
     list(APPEND armnnTosaRefBackend_sources
         TosaRefBackend.cpp