Bazel and CMake builds
Resolves: ONCPUML-1110, ONCPUML-1109
Co-authored-by: Georgios Pinitas <georgios.pinitas@arm.com>
Co-authored-by: Joe Ramsay <joe.ramsay@arm.com>
Signed-off-by: David Svantesson <david.svantesson@arm.com>
Change-Id: Iea693dbe53bf0af87867d6a9e0d1fd9fbe59ef3a
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/8981
Benchmark: Arm Jenkins <bsgcomp@arm.com>
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Jakub Sujak <jakub.sujak@arm.com>
Reviewed-by: Gunes Bayir <gunes.bayir@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
diff --git a/scripts/generate_build_files.py b/scripts/generate_build_files.py
new file mode 100644
index 0000000..8f8ed0e
--- /dev/null
+++ b/scripts/generate_build_files.py
@@ -0,0 +1,296 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2023 Arm Limited.
+#
+# SPDX-License-Identifier: MIT
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""Generates build files for either bazel or cmake experimental builds using filelist.json
+Usage
+ python scripts/generate_build_files.py --bazel
+ python scripts/generate_build_files.py --cmake
+
+Writes generated file to the bazel BUILD file located under src/ if using --bazel flag.
+Writes generated file to the CMake CMakeLists.txt file located under src/ if using --cmake flag.
+"""
+
+import argparse
+import json
+import glob
+
+
+def get_operator_backend_files(filelist, operators, backend='', techs=[], attrs=[]):
+ files = {"common": []}
+
+ # Early return if filelist is empty
+ if backend not in filelist:
+ return files
+
+ # Iterate over operators and create the file lists to compiler
+ for operator in operators:
+ if operator in filelist[backend]['operators']:
+ files['common'] += filelist[backend]['operators'][operator]["files"]["common"]
+ for tech in techs:
+ if tech in filelist[backend]['operators'][operator]["files"]:
+ # Add tech as a key to dictionary if not there
+ if tech not in files:
+ files[tech] = []
+
+ # Add tech files to the tech file list
+ tech_files = filelist[backend]['operators'][operator]["files"][tech]
+ files[tech] += tech_files.get('common', [])
+ for attr in attrs:
+ files[tech] += tech_files.get(attr, [])
+
+ # Remove duplicates if they exist
+ return {k: list(set(v)) for k, v in files.items()}
+
+
+def collect_operators(filelist, operators, backend=''):
+ ops = set()
+ for operator in operators:
+ if operator in filelist[backend]['operators']:
+ ops.add(operator)
+ if 'deps' in filelist[backend]['operators'][operator]:
+ ops.update(filelist[backend]['operators'][operator]['deps'])
+ else:
+ print("Operator {0} is unsupported on {1} backend!".format(
+ operator, backend))
+
+ return ops
+
+
+def resolve_operator_dependencies(filelist, operators, backend=''):
+ resolved_operators = collect_operators(filelist, operators, backend)
+
+ are_ops_resolved = False
+ while not are_ops_resolved:
+ resolution_pass = collect_operators(
+ filelist, resolved_operators, backend)
+ if len(resolution_pass) != len(resolved_operators):
+ resolved_operators.update(resolution_pass)
+ else:
+ are_ops_resolved = True
+
+ return resolved_operators
+
+def get_template_header():
+ return """# Copyright (c) 2023 Arm Limited.
+#
+# SPDX-License-Identifier: MIT
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE."""
+
+def build_from_template_bazel(srcs_graph, srcs_sve, srcs_sve2, srcs_core):
+
+ line_separator = '",\n\t"'
+
+ template = f"""{get_template_header()}
+
+filegroup(
+ name = "arm_compute_graph_srcs",
+ srcs = ["{line_separator.join(srcs_graph)}"] +
+ glob(["**/*.h",
+ "**/*.hpp",
+ "**/*.inl"]),
+ visibility = ["//visibility:public"]
+)
+
+filegroup(
+ name = "arm_compute_sve2_srcs",
+ srcs = ["{line_separator.join(srcs_sve2)}"] +
+ glob(["**/*.h",
+ "**/*.hpp",
+ "**/*.inl"]),
+ visibility = ["//visibility:public"]
+)
+
+filegroup(
+ name = "arm_compute_sve_srcs",
+ srcs = ["{line_separator.join(srcs_sve)}"] +
+ glob(["**/*.h",
+ "**/*.hpp",
+ "**/*.inl"]),
+ visibility = ["//visibility:public"]
+)
+
+filegroup(
+ name = "arm_compute_srcs",
+ srcs = ["{line_separator.join(srcs_core)}"] +
+ glob(["**/*.h",
+ "**/*.hpp",
+ "**/*.inl"]),
+ visibility = ["//visibility:public"]
+)
+"""
+
+ return template
+
+
+def build_from_template_cmake(srcs_graph, srcs_sve, srcs_sve2, srcs_core):
+
+ line_separator = '\n\t'
+
+ template = f"""{get_template_header()}
+
+target_sources(
+ arm_compute_graph
+ PRIVATE
+ {line_separator.join(srcs_graph)}
+)
+
+target_sources(
+ arm_compute_sve
+ PRIVATE
+ {line_separator.join(srcs_sve)}
+)
+
+target_sources(
+ arm_compute_sve2
+ PRIVATE
+ {line_separator.join(srcs_sve2)}
+)
+
+target_sources(
+ arm_compute_core
+ PRIVATE
+ {line_separator.join(srcs_core)}
+)
+ """
+ return template
+
+
+def gather_sources():
+
+ # Source file list
+ with open("filelist.json") as fp:
+ filelist = json.load(fp)
+
+ # Common backend files
+ lib_files = filelist['common']
+
+ # TODO Add Fixed format GEMM kernels ?
+
+ # Logging files
+ lib_files += filelist['logging']
+
+ # C API files
+ lib_files += filelist['c_api']['common']
+ lib_files += filelist['c_api']['operators']
+
+ # Scheduler infrastructure
+ lib_files += filelist['scheduler']['single']
+ # Add both cppthreads and omp sources for now
+ lib_files += filelist['scheduler']['threads']
+ lib_files += filelist['scheduler']['omp']
+
+ # Graph files
+ graph_files = glob.glob('src/graph/*.cpp')
+ graph_files += glob.glob('src/graph/*/*.cpp')
+
+ lib_files_sve = []
+ lib_files_sve2 = []
+
+ # -------------------------------------
+ # NEON files
+ lib_files += filelist['cpu']['common']
+ simd = ['neon', 'sve', 'sve2']
+
+ # Get attributes
+ data_types = ["qasymm8", "qasymm8_signed", "qsymm16",
+ "fp16", "fp32", "integer"] # Are all needed?
+ data_layouts = ["nhwc", "nchw"] # Are both needed?
+ experimental_fixed_format_kernels = ["experimental_fixed_format_kernels"]
+ attrs = data_types + data_layouts + \
+ experimental_fixed_format_kernels + ["estate64"]
+
+ # Setup data-type and data-layout files to include
+ cpu_operators = filelist['cpu']['operators'].keys()
+ cpu_ops_to_build = resolve_operator_dependencies(
+ filelist, cpu_operators, 'cpu')
+ cpu_files = get_operator_backend_files(
+ filelist, cpu_ops_to_build, 'cpu', simd, attrs)
+
+ # Shared among ALL CPU files
+ lib_files += cpu_files.get('common', [])
+
+ # Arm® Neon™ specific files
+ lib_files += cpu_files.get('neon', [])
+
+ # SVE files only
+ lib_files_sve = cpu_files.get('sve', [])
+
+ # SVE2 files only
+ lib_files_sve2 = cpu_files.get('sve2', [])
+
+ graph_files += glob.glob('src/graph/backends/NEON/*.cpp')
+
+ # -------------------------------------
+
+ graph_files = sorted([path.replace("src/", "") for path in graph_files])
+ lib_files_sve = sorted([path.replace("src/", "") for path in lib_files_sve])
+ lib_files_sve2 = sorted([path.replace("src/", "") for path in lib_files_sve2])
+ lib_files = sorted([path.replace("src/", "") for path in lib_files])
+
+ return graph_files, lib_files_sve, lib_files_sve2, lib_files
+
+
+if "__main__" in __name__:
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--bazel", action="store_true")
+ parser.add_argument("--cmake", action="store_true")
+ args = parser.parse_args()
+
+ graph_files, lib_files_sve, lib_files_sve2, lib_files = gather_sources()
+
+ if args.bazel:
+ bazel_build_string = build_from_template_bazel(
+ graph_files, lib_files_sve, lib_files_sve2, lib_files)
+ print(bazel_build_string)
+ with open("src/BUILD.bazel", "w") as fp:
+ fp.write(bazel_build_string)
+
+ if args.cmake:
+ cmake_build_string = build_from_template_cmake(
+ graph_files, lib_files_sve, lib_files_sve2, lib_files)
+ print(cmake_build_string)
+ with open("src/CMakeLists.txt", "w") as fp:
+ fp.write(cmake_build_string)
+
+ if not args.cmake and not args.bazel:
+ print("Supply either --bazel or --cmake flag to generate build files for corresponding build")