James Conroy | 919ec71 | 2022-07-13 12:57:53 +0100 | [diff] [blame] | 1 | #!/bin/bash |
| 2 | # |
| 3 | # Copyright © 2022 Arm Ltd and Contributors. All rights reserved. |
| 4 | # SPDX-License-Identifier: MIT |
| 5 | # |
| 6 | |
| 7 | # Script which downloads and builds Arm NN dependencies |
| 8 | # Perquisite to running build-armnn.sh |
| 9 | |
| 10 | set -o nounset # Catch references to undefined variables. |
| 11 | set -o pipefail # Catch non zero exit codes within pipelines. |
| 12 | set -o errexit # Catch and propagate non zero exit codes. |
| 13 | |
| 14 | rel_path=$(dirname "$0") # relative path from where script is executed to script location |
| 15 | |
| 16 | # Download an archive using wget and extract using tar |
| 17 | # Takes three arguments: |
| 18 | # 1. Name of dependency being downloaded e.g. Flatbuffers |
| 19 | # 2. Link to archive |
| 20 | # 3. Filename given to archive upon downloading |
| 21 | download_and_extract() |
| 22 | { |
| 23 | cd "$SOURCE_DIR" |
| 24 | |
James Conroy | 210897d | 2022-08-04 16:55:05 +0100 | [diff] [blame] | 25 | echo -e "\n***** Downloading $1 *****\n" |
James Conroy | 919ec71 | 2022-07-13 12:57:53 +0100 | [diff] [blame] | 26 | wget -O "$3" "$2" |
| 27 | |
| 28 | echo -e "\n***** Extracting archive *****" |
| 29 | tar -xzf "$3" |
| 30 | |
| 31 | echo -e "\n***** Removing archive *****" |
| 32 | rm "$3" |
| 33 | |
| 34 | echo -e "\n***** $1 downloaded *****" |
| 35 | } |
| 36 | |
| 37 | download_protobuf() |
| 38 | { |
| 39 | download_and_extract \ |
| 40 | "Protobuf" \ |
| 41 | "https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-all-$PROTOBUF_VERSION.tar.gz" \ |
| 42 | "protobuf-all-$PROTOBUF_VERSION.tar.gz" |
| 43 | } |
| 44 | |
| 45 | build_protobuf() |
| 46 | { |
| 47 | local native_build=$1 |
| 48 | local build_dir="$PROTOBUF_BUILD_TARGET" |
| 49 | local cmake_flags="" |
| 50 | local target_arch="$TARGET_ARCH" |
| 51 | local additional_cmds="" |
| 52 | |
| 53 | if [ "$native_build" -eq 0 ]; then |
| 54 | mkdir -p "$PROTOBUF_BUILD_TARGET" |
| 55 | additional_cmds+="--with-protoc=$PROTOCOL_COMPILER_HOST " |
| 56 | if [ "$TARGET_ARCH" == "aarch64" ]; then |
| 57 | cmake_flags+="$AARCH64_COMPILER_FLAGS" |
| 58 | additional_cmds+="--host=aarch64-linux " |
James Conroy | 919ec71 | 2022-07-13 12:57:53 +0100 | [diff] [blame] | 59 | fi |
| 60 | else |
| 61 | target_arch="$HOST_ARCH" |
| 62 | mkdir -p "$PROTOBUF_BUILD_HOST" |
| 63 | build_dir="$PROTOBUF_BUILD_HOST" |
| 64 | fi |
| 65 | |
| 66 | echo -e "\n***** Building Protobuf for $target_arch ***** " |
| 67 | |
| 68 | cd "$PROTOBUF_BUILD_ROOT" |
| 69 | |
| 70 | # Cleanup any previous cmake files, except actual builds which we keep |
| 71 | find . -mindepth 1 -name "*_build" -prune -o -exec rm -rf {} + |
| 72 | |
| 73 | eval "$cmake_flags" \ |
| 74 | "$PROTOBUF_SRC"/configure --prefix="$build_dir" "$additional_cmds" |
| 75 | make install -j "$NUM_THREADS" |
| 76 | |
| 77 | echo -e "\n***** Protobuf built for $target_arch ***** " |
| 78 | } |
| 79 | |
| 80 | download_flatbuffers() |
| 81 | { |
| 82 | download_and_extract \ |
| 83 | "Flatbuffers" \ |
| 84 | "https://github.com/google/flatbuffers/archive/v$FLATBUFFERS_VERSION.tar.gz" \ |
| 85 | "flatbuffers-$FLATBUFFERS_VERSION.tar.gz" |
| 86 | } |
| 87 | |
| 88 | build_flatbuffers() |
| 89 | { |
| 90 | local native_build=$1 |
| 91 | local build_dir="$FLATBUFFERS_BUILD_TARGET" |
| 92 | local target_arch="$TARGET_ARCH" |
| 93 | |
| 94 | local cmake_flags="CXXFLAGS=-fPIC " |
| 95 | |
| 96 | if [ "$native_build" -eq 0 ]; then |
| 97 | mkdir -p "$FLATBUFFERS_BUILD_TARGET" |
| 98 | if [ "$TARGET_ARCH" == "aarch64" ]; then |
| 99 | cmake_flags+="$AARCH64_COMPILER_FLAGS" |
James Conroy | 919ec71 | 2022-07-13 12:57:53 +0100 | [diff] [blame] | 100 | fi |
| 101 | else |
| 102 | target_arch="$HOST_ARCH" |
| 103 | mkdir -p "$FLATBUFFERS_BUILD_HOST" |
| 104 | build_dir="$FLATBUFFERS_BUILD_HOST" |
| 105 | fi |
| 106 | |
| 107 | echo -e "\n***** Building flatbuffers for $target_arch *****" |
| 108 | |
| 109 | mkdir -p "$FLATBUFFERS_BUILD_ROOT" |
| 110 | cd "$FLATBUFFERS_BUILD_ROOT" |
| 111 | |
| 112 | # Cleanup any previous cmake files, except actual builds which we keep |
| 113 | find . -mindepth 1 -name "*_build" -prune -o -exec rm -rf {} + |
| 114 | |
| 115 | eval "$cmake_flags" \ |
James Conroy | 2e950f4 | 2022-11-01 15:01:06 +0000 | [diff] [blame] | 116 | cmake -DFLATBUFFERS_BUILD_FLATC="$native_build" \ |
James Conroy | 919ec71 | 2022-07-13 12:57:53 +0100 | [diff] [blame] | 117 | -DCMAKE_INSTALL_PREFIX:PATH="$build_dir" \ |
| 118 | -DFLATBUFFERS_BUILD_TESTS=0 \ |
| 119 | "$FLATBUFFERS_SRC" |
| 120 | make all install -j "$NUM_THREADS" |
| 121 | |
| 122 | echo -e "\n***** Built flatbuffers for $target_arch *****" |
| 123 | } |
| 124 | |
| 125 | download_tensorflow() |
| 126 | { |
| 127 | cd "$SOURCE_DIR" |
| 128 | |
| 129 | echo -e "\n***** Downloading TensorFlow *****" |
| 130 | git clone https://github.com/tensorflow/tensorflow.git |
| 131 | cd "$TENSORFLOW_SRC" |
| 132 | |
James Conroy | 210897d | 2022-08-04 16:55:05 +0100 | [diff] [blame] | 133 | git checkout "$TENSORFLOW_VERSION" |
James Conroy | 919ec71 | 2022-07-13 12:57:53 +0100 | [diff] [blame] | 134 | echo -e "\n***** TensorFlow downloaded *****" |
| 135 | } |
| 136 | |
| 137 | build_tflite() |
| 138 | { |
| 139 | mkdir -p "$TFLITE_BUILD_TARGET" |
| 140 | cd "$TFLITE_BUILD_TARGET" |
| 141 | |
| 142 | local target_arch_cmd="" # default is native, no command needed |
| 143 | local cmake_flags="" |
| 144 | |
| 145 | case "$TARGET_ARCH" in |
| 146 | "aarch64") |
| 147 | cmake_flags+="$AARCH64_COMPILER_FLAGS" |
James Conroy | 2e950f4 | 2022-11-01 15:01:06 +0000 | [diff] [blame] | 148 | target_arch_cmd="-DCMAKE_SYSTEM_PROCESSOR=aarch64 \ |
| 149 | -DCMAKE_SYSTEM_NAME=Linux " |
James Conroy | 919ec71 | 2022-07-13 12:57:53 +0100 | [diff] [blame] | 150 | |
| 151 | if [ "$NATIVE_BUILD" -eq 0 ]; then |
| 152 | cmake_flags+="ARMCC_FLAGS='-funsafe-math-optimizations' " |
| 153 | fi |
| 154 | ;; |
James Conroy | 919ec71 | 2022-07-13 12:57:53 +0100 | [diff] [blame] | 155 | esac |
| 156 | |
| 157 | echo -e "\n***** Building TF Lite for $TARGET_ARCH *****" |
| 158 | |
| 159 | # Cleanup any previous cmake files, except actual builds which we keep |
| 160 | find . -mindepth 1 -name "*_build" -prune -o -exec rm -rf {} + |
| 161 | |
| 162 | eval "$cmake_flags" \ |
| 163 | cmake -DTFLITE_ENABLE_XNNPACK=OFF \ |
| 164 | "$target_arch_cmd" \ |
| 165 | "$TFLITE_SRC" |
| 166 | cmake --build . -j "$NUM_THREADS" |
| 167 | |
| 168 | echo -e "\n***** Built TF Lite for $TARGET_ARCH *****" |
| 169 | } |
| 170 | |
| 171 | generate_tflite_schema() |
| 172 | { |
| 173 | echo -e "\n***** Generating TF Lite Schema *****" |
| 174 | mkdir -p "$TFLITE_BUILD_ROOT" |
| 175 | cd "$TFLITE_BUILD_ROOT" |
| 176 | |
| 177 | cp "$SCHEMA_SRC" . |
| 178 | |
| 179 | $FLATC -c --gen-object-api --reflect-types --reflect-names schema.fbs |
| 180 | |
| 181 | echo -e "\n***** Generated TF Lite Schema *****" |
| 182 | } |
| 183 | |
| 184 | download_onnx() |
| 185 | { |
| 186 | download_and_extract \ |
| 187 | "ONNX" \ |
| 188 | "https://github.com/onnx/onnx/releases/download/v$ONNX_VERSION/onnx-$ONNX_VERSION.tar.gz" \ |
| 189 | "onnx-$ONNX_VERSION.tar.gz" |
| 190 | } |
| 191 | |
| 192 | generate_onnx_sources() |
| 193 | { |
| 194 | mkdir -p "$ONNX_BUILD_TARGET" |
| 195 | cd "$ONNX_SRC" |
| 196 | |
| 197 | echo -e "\n***** Generating ONNX sources for $TARGET_ARCH *****" |
| 198 | |
| 199 | export LD_LIBRARY_PATH="$PROTOBUF_BUILD_HOST"/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} |
| 200 | |
| 201 | eval "$PROTOCOL_COMPILER_HOST" onnx/onnx.proto \ |
| 202 | --proto_path=. \ |
| 203 | --proto_path="$ONNX_SRC" \ |
| 204 | --proto_path="$PROTOBUF_BUILD_HOST"/include \ |
| 205 | --cpp_out "$ONNX_BUILD_TARGET" |
| 206 | |
| 207 | echo -e "\n***** Generated ONNX sources for $TARGET_ARCH *****" |
| 208 | } |
| 209 | |
James Conroy | 919ec71 | 2022-07-13 12:57:53 +0100 | [diff] [blame] | 210 | usage() |
| 211 | { |
| 212 | cat <<EOF |
| 213 | setup-armnn.sh - Download and build Arm NN dependencies in the current directory (ROOT_DIR) |
| 214 | setup-armnn.sh [OPTION]... |
| 215 | --tflite-delegate |
| 216 | setup dependencies for the Arm NN TF Lite Delegate |
| 217 | --tflite-parser |
| 218 | setup dependencies for the Arm NN TF Lite Parser |
| 219 | --onnx-parser |
| 220 | setup dependencies for the Arm NN ONNX parser |
| 221 | --all |
| 222 | setup dependencies for all Arm NN components listed above |
James Conroy | e6f30ad | 2022-09-08 12:04:26 +0100 | [diff] [blame] | 223 | --target-arch=[aarch64|x86_64] |
James Conroy | 919ec71 | 2022-07-13 12:57:53 +0100 | [diff] [blame] | 224 | specify a target architecture (mandatory) |
| 225 | --num-threads=<INTEGER> |
| 226 | specify number of threads/cores to build dependencies with (optional: defaults to number of online CPU cores on host) |
| 227 | -h, --help |
| 228 | print brief usage information and exit |
| 229 | -x |
| 230 | enable shell tracing in this script |
| 231 | |
| 232 | At least one dependency flag (e.g. --tflite-delegate) must be provided or else provide --all to setup all dependencies. |
| 233 | Directories called "source" and "build" will be generated in the current directory (ROOT_DIR) from which this script is called. |
| 234 | It's recommended to call this script in a directory outside of this Arm NN source repo, to avoid nested repositories. |
| 235 | |
James Conroy | 919ec71 | 2022-07-13 12:57:53 +0100 | [diff] [blame] | 236 | Examples: |
| 237 | Setup for aarch64 with all Arm NN dependencies: |
| 238 | <PATH_TO>/setup-armnn.sh --target-arch=aarch64 --all |
| 239 | Setup for aarch64 with TF Lite Delegate and TF Lite Parser dependencies only: |
| 240 | <PATH_TO>/setup-armnn.sh --target-arch=aarch64 --tflite-delegate --tflite-parser |
James Conroy | 919ec71 | 2022-07-13 12:57:53 +0100 | [diff] [blame] | 241 | EOF |
| 242 | } |
| 243 | |
| 244 | # This will catch in validation.sh if not set |
| 245 | target_arch="" |
| 246 | |
| 247 | # Default flag values |
| 248 | flag_tflite_delegate=0 |
| 249 | flag_tflite_parser=0 |
| 250 | flag_onnx_parser=0 |
| 251 | |
| 252 | # If --num-threads is not set, the default NUM_THREADS value in common.sh will be used |
| 253 | num_threads=0 |
| 254 | |
| 255 | name=$(basename "$0") |
| 256 | |
| 257 | # If no options provided, show help |
| 258 | if [ $# -eq 0 ]; then |
| 259 | usage |
| 260 | exit 1 |
| 261 | fi |
| 262 | |
| 263 | args=$(getopt -ohx -l tflite-delegate,tflite-parser,onnx-parser,all,target-arch:,num-threads:,help -n "$name" -- "$@") |
| 264 | eval set -- "$args" |
| 265 | while [ $# -gt 0 ]; do |
| 266 | if [ -n "${opt_prev:-}" ]; then |
| 267 | eval "$opt_prev=\$1" |
| 268 | opt_prev= |
| 269 | shift 1 |
| 270 | continue |
| 271 | elif [ -n "${opt_append:-}" ]; then |
| 272 | if [ -n "$1" ]; then |
| 273 | eval "$opt_append=\"\${$opt_append:-} \$1\"" |
| 274 | fi |
| 275 | opt_append= |
| 276 | shift 1 |
| 277 | continue |
| 278 | fi |
| 279 | case $1 in |
| 280 | --tflite-parser) |
| 281 | flag_tflite_parser=1 |
| 282 | ;; |
| 283 | |
| 284 | --tflite-delegate) |
| 285 | flag_tflite_delegate=1 |
| 286 | ;; |
| 287 | |
| 288 | --onnx-parser) |
| 289 | flag_onnx_parser=1 |
| 290 | ;; |
| 291 | |
| 292 | --all) |
| 293 | flag_tflite_delegate=1 |
| 294 | flag_tflite_parser=1 |
| 295 | flag_onnx_parser=1 |
| 296 | ;; |
| 297 | |
| 298 | --target-arch) |
| 299 | opt_prev=target_arch |
| 300 | ;; |
| 301 | |
| 302 | --num-threads) |
| 303 | opt_prev=num_threads |
| 304 | ;; |
| 305 | |
| 306 | -h | --help) |
| 307 | usage |
| 308 | exit 0 |
| 309 | ;; |
| 310 | |
| 311 | -x) |
| 312 | set -x |
| 313 | ;; |
| 314 | |
| 315 | --) |
| 316 | shift |
| 317 | break 2 |
| 318 | ;; |
| 319 | |
| 320 | esac |
| 321 | shift 1 |
| 322 | done |
| 323 | |
| 324 | # shellcheck source=common.sh |
| 325 | source "$rel_path"/common.sh |
| 326 | |
| 327 | echo -e "\nINFO: Displaying configuration information before execution of $name" |
| 328 | echo " target-arch: $TARGET_ARCH" |
| 329 | echo " host-arch: $HOST_ARCH" |
| 330 | echo " tflite-delegate: $flag_tflite_delegate" |
| 331 | echo " tflite-parser: $flag_tflite_parser" |
| 332 | echo " onnx-parser: $flag_onnx_parser" |
| 333 | echo " num-threads: $NUM_THREADS" |
| 334 | echo " root directory: $ROOT_DIR" |
| 335 | echo "source directory: $SOURCE_DIR" |
| 336 | echo " build directory: $BUILD_DIR" |
| 337 | |
James Conroy | 210897d | 2022-08-04 16:55:05 +0100 | [diff] [blame] | 338 | if check_if_repository .; then |
James Conroy | 919ec71 | 2022-07-13 12:57:53 +0100 | [diff] [blame] | 339 | echo -e "\n***** WARNING: Running script inside a git repository. To avoid nested repos, call this script from outside of this repo. *****" |
| 340 | fi |
| 341 | |
| 342 | echo -e "\nScript execution will begin in 10 seconds..." |
| 343 | |
| 344 | sleep 10 |
| 345 | |
| 346 | mkdir -p "$SOURCE_DIR" |
| 347 | mkdir -p "$BUILD_DIR" |
| 348 | |
James Conroy | 919ec71 | 2022-07-13 12:57:53 +0100 | [diff] [blame] | 349 | if [ "$flag_tflite_delegate" -eq 1 ] || [ "$flag_tflite_parser" -eq 1 ]; then |
| 350 | download_flatbuffers |
| 351 | |
| 352 | # Host build |
| 353 | build_flatbuffers 1 |
| 354 | |
| 355 | # Target build for cross compile |
| 356 | if [ "$NATIVE_BUILD" -eq 0 ]; then |
| 357 | build_flatbuffers 0 |
| 358 | fi |
| 359 | |
| 360 | download_tensorflow |
| 361 | fi |
| 362 | |
| 363 | if [ "$flag_tflite_parser" -eq 1 ]; then |
| 364 | generate_tflite_schema |
| 365 | fi |
| 366 | |
| 367 | if [ "$flag_tflite_delegate" -eq 1 ]; then |
| 368 | build_tflite |
| 369 | fi |
| 370 | |
| 371 | if [ "$flag_onnx_parser" -eq 1 ]; then |
| 372 | download_protobuf |
| 373 | |
| 374 | # Host build |
| 375 | build_protobuf 1 |
| 376 | |
| 377 | # Target build for cross compile |
| 378 | if [ "$NATIVE_BUILD" -eq 0 ]; then |
| 379 | build_protobuf 0 |
| 380 | fi |
| 381 | |
| 382 | download_onnx |
| 383 | generate_onnx_sources |
| 384 | fi |
| 385 | |
| 386 | echo -e "\n***** Arm NN setup complete. Now build with build-armnn.sh. *****\n" |
| 387 | |
| 388 | exit 0 |