caio/clang-p2996
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reapply "[CI] Migrate to runtimes build" (#143612)

Browse Source 
This reverts commit 6f62979a5a.

The reapplies commit 80ea5f46df.

That commit was reverted because it was causing compiler-rt test
failures due to tysan not having its dependencies set up properly within
CMake. That situation has since been rectified in
3cef099ced.

Reviewers: lnihlen, rnk, gburgessiv, cmtice

Reviewed By: rnk, cmtice

Pull Request: https://github.com/llvm/llvm-project/pull/144033
This commit is contained in:
Aiden Grossman
2025-06-20 15:07:00 -07:00
committed by GitHub
parent 94865edfa8
commit b7be8786af
4 changed files with 139 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

119
.ci/compute_projects.py
View File

@@ -49,8 +49,7 @@ DEPENDENTS_TO_TEST = {
},
"lld": {"bolt", "cross-project-tests"},
# TODO(issues/132795): LLDB should be enabled on clang changes.
"clang": {"clang-tools-extra", "compiler-rt", "cross-project-tests"},
"clang-tools-extra": {"libc"},
"clang": {"clang-tools-extra", "cross-project-tests"},
"mlir": {"flang"},
# Test everything if ci scripts are changed.
# FIXME: Figure out what is missing and add here.
@@ -64,7 +63,15 @@ DEPENDENT_RUNTIMES_TO_BUILD = {"lldb": {"libcxx", "libcxxabi", "libunwind"}}
# This mapping describes runtimes that should be tested when the key project is
# touched.
DEPENDENT_RUNTIMES_TO_TEST = {"clang": {"libcxx", "libcxxabi", "libunwind"}}
DEPENDENT_RUNTIMES_TO_TEST = {
"clang": {"compiler-rt"},
"clang-tools-extra": {"libc"},
}
DEPENDENT_RUNTIMES_TO_TEST_NEEDS_RECONFIG = {
"llvm": {"libcxx", "libcxxabi", "libunwind"},
"clang": {"libcxx", "libcxxabi", "libunwind"},
".ci": {"libcxx", "libcxxabi", "libunwind"},
}
EXCLUDE_LINUX = {
"cross-project-tests", # TODO(issues/132796): Tests are failing.
@@ -93,9 +100,6 @@ EXCLUDE_MAC = {
"cross-project-tests",
"flang",
"libc",
"libcxx",
"libcxxabi",
"libunwind",
"lldb",
"openmp",
"polly",
@@ -122,10 +126,10 @@ PROJECT_CHECK_TARGETS = {
"polly": "check-polly",
}
RUNTIMES = {"libcxx", "libcxxabi", "libunwind"}
RUNTIMES = {"libcxx", "libcxxabi", "libunwind", "compiler-rt", "libc"}
def _add_dependencies(projects: Set[str]) -> Set[str]:
def _add_dependencies(projects: Set[str], runtimes: Set[str]) -> Set[str]:
projects_with_dependents = set(projects)
current_projects_count = 0
while current_projects_count != len(projects_with_dependents):
@@ -134,9 +138,25 @@ def _add_dependencies(projects: Set[str]) -> Set[str]:
if project not in PROJECT_DEPENDENCIES:
continue
projects_with_dependents.update(PROJECT_DEPENDENCIES[project])
for runtime in runtimes:
if runtime not in PROJECT_DEPENDENCIES:
continue
projects_with_dependents.update(PROJECT_DEPENDENCIES[runtime])
return projects_with_dependents
def _exclude_projects(current_projects: Set[str], platform: str) -> Set[str]:
if platform == "Linux":
to_exclude = EXCLUDE_LINUX
elif platform == "Windows":
to_exclude = EXCLUDE_WINDOWS
elif platform == "Darwin":
to_exclude = EXCLUDE_MAC
else:
raise ValueError(f"Unexpected platform: {platform}")
return current_projects.difference(to_exclude)
def _compute_projects_to_test(modified_projects: Set[str], platform: str) -> Set[str]:
projects_to_test = set()
for modified_project in modified_projects:
@@ -154,25 +174,14 @@ def _compute_projects_to_test(modified_projects: Set[str], platform: str) -> Set
):
continue
projects_to_test.add(dependent_project)
if platform == "Linux":
for to_exclude in EXCLUDE_LINUX:
if to_exclude in projects_to_test:
projects_to_test.remove(to_exclude)
elif platform == "Windows":
for to_exclude in EXCLUDE_WINDOWS:
if to_exclude in projects_to_test:
projects_to_test.remove(to_exclude)
elif platform == "Darwin":
for to_exclude in EXCLUDE_MAC:
if to_exclude in projects_to_test:
projects_to_test.remove(to_exclude)
else:
raise ValueError("Unexpected platform.")
projects_to_test = _exclude_projects(projects_to_test, platform)
return projects_to_test
def _compute_projects_to_build(projects_to_test: Set[str]) -> Set[str]:
return _add_dependencies(projects_to_test)
def _compute_projects_to_build(
projects_to_test: Set[str], runtimes: Set[str]
) -> Set[str]:
return _add_dependencies(projects_to_test, runtimes)
def _compute_project_check_targets(projects_to_test: Set[str]) -> Set[str]:
@@ -184,24 +193,36 @@ def _compute_project_check_targets(projects_to_test: Set[str]) -> Set[str]:
return check_targets
def _compute_runtimes_to_test(projects_to_test: Set[str]) -> Set[str]:
def _compute_runtimes_to_test(modified_projects: Set[str], platform: str) -> Set[str]:
runtimes_to_test = set()
for project_to_test in projects_to_test:
if project_to_test in DEPENDENT_RUNTIMES_TO_TEST:
runtimes_to_test.update(DEPENDENT_RUNTIMES_TO_TEST[project_to_test])
if project_to_test in DEPENDENT_RUNTIMES_TO_BUILD:
runtimes_to_test.update(DEPENDENT_RUNTIMES_TO_BUILD[project_to_test])
return runtimes_to_test
def _compute_runtime_check_targets(projects_to_test: Set[str]) -> Set[str]:
check_targets = set()
for project_to_test in projects_to_test:
if project_to_test not in DEPENDENT_RUNTIMES_TO_TEST:
for modified_project in modified_projects:
if modified_project not in DEPENDENT_RUNTIMES_TO_TEST:
continue
for runtime_to_test in DEPENDENT_RUNTIMES_TO_TEST[project_to_test]:
check_targets.add(PROJECT_CHECK_TARGETS[runtime_to_test])
return check_targets
runtimes_to_test.update(DEPENDENT_RUNTIMES_TO_TEST[modified_project])
return _exclude_projects(runtimes_to_test, platform)
def _compute_runtimes_to_test_needs_reconfig(
modified_projects: Set[str], platform: str
) -> Set[str]:
runtimes_to_test = set()
for modified_project in modified_projects:
if modified_project not in DEPENDENT_RUNTIMES_TO_TEST_NEEDS_RECONFIG:
continue
runtimes_to_test.update(
DEPENDENT_RUNTIMES_TO_TEST_NEEDS_RECONFIG[modified_project]
)
return _exclude_projects(runtimes_to_test, platform)
def _compute_runtimes_to_build(
runtimes_to_test: Set[str], modified_projects: Set[str], platform: str
) -> Set[str]:
runtimes_to_build = set(runtimes_to_test)
for modified_project in modified_projects:
if modified_project in DEPENDENT_RUNTIMES_TO_BUILD:
runtimes_to_build.update(DEPENDENT_RUNTIMES_TO_BUILD[modified_project])
return _exclude_projects(runtimes_to_build, platform)
def _get_modified_projects(modified_files: list[str]) -> Set[str]:
@@ -225,10 +246,19 @@ def _get_modified_projects(modified_files: list[str]) -> Set[str]:
def get_env_variables(modified_files: list[str], platform: str) -> Set[str]:
modified_projects = _get_modified_projects(modified_files)
projects_to_test = _compute_projects_to_test(modified_projects, platform)
projects_to_build = _compute_projects_to_build(projects_to_test)
runtimes_to_test = _compute_runtimes_to_test(modified_projects, platform)
runtimes_to_test_needs_reconfig = _compute_runtimes_to_test_needs_reconfig(
modified_projects, platform
)
runtimes_to_build = _compute_runtimes_to_build(
runtimes_to_test | runtimes_to_test_needs_reconfig, modified_projects, platform
)
projects_to_build = _compute_projects_to_build(projects_to_test, runtimes_to_build)
projects_check_targets = _compute_project_check_targets(projects_to_test)
runtimes_to_build = _compute_runtimes_to_test(projects_to_test)
runtimes_check_targets = _compute_runtime_check_targets(projects_to_test)
runtimes_check_targets = _compute_project_check_targets(runtimes_to_test)
runtimes_check_targets_needs_reconfig = _compute_project_check_targets(
runtimes_to_test_needs_reconfig
)
# We use a semicolon to separate the projects/runtimes as they get passed
# to the CMake invocation and thus we need to use the CMake list separator
# (;). We use spaces to separate the check targets as they end up getting
@@ -238,6 +268,9 @@ def get_env_variables(modified_files: list[str], platform: str) -> Set[str]:
"project_check_targets": " ".join(sorted(projects_check_targets)),
"runtimes_to_build": ";".join(sorted(runtimes_to_build)),
"runtimes_check_targets": " ".join(sorted(runtimes_check_targets)),
"runtimes_check_targets_needs_reconfig": " ".join(
sorted(runtimes_check_targets_needs_reconfig)
),
}

55
.ci/compute_projects_test.py
View File

@@ -26,6 +26,10 @@ class TestComputeProjects(unittest.TestCase):
)
self.assertEqual(
env_variables["runtimes_check_targets"],
"",
)
self.assertEqual(
env_variables["runtimes_check_targets_needs_reconfig"],
"check-cxx check-cxxabi check-unwind",
)
@@ -46,6 +50,10 @@ class TestComputeProjects(unittest.TestCase):
)
self.assertEqual(
env_variables["runtimes_check_targets"],
"",
)
self.assertEqual(
env_variables["runtimes_check_targets_needs_reconfig"],
"check-cxx check-cxxabi check-unwind",
)
@@ -66,6 +74,10 @@ class TestComputeProjects(unittest.TestCase):
)
self.assertEqual(
env_variables["runtimes_check_targets"],
"",
)
self.assertEqual(
env_variables["runtimes_check_targets_needs_reconfig"],
"check-cxx check-cxxabi check-unwind",
)
@@ -75,17 +87,21 @@ class TestComputeProjects(unittest.TestCase):
)
self.assertEqual(
env_variables["projects_to_build"],
"clang;clang-tools-extra;compiler-rt;lld;llvm",
"clang;clang-tools-extra;lld;llvm",
)
self.assertEqual(
env_variables["project_check_targets"],
"check-clang check-clang-tools check-compiler-rt",
"check-clang check-clang-tools",
)
self.assertEqual(
env_variables["runtimes_to_build"], "libcxx;libcxxabi;libunwind"
env_variables["runtimes_to_build"], "compiler-rt;libcxx;libcxxabi;libunwind"
)
self.assertEqual(
env_variables["runtimes_check_targets"],
"check-compiler-rt",
)
self.assertEqual(
env_variables["runtimes_check_targets_needs_reconfig"],
"check-cxx check-cxxabi check-unwind",
)
@@ -104,6 +120,10 @@ class TestComputeProjects(unittest.TestCase):
)
self.assertEqual(
env_variables["runtimes_check_targets"],
"",
)
self.assertEqual(
env_variables["runtimes_check_targets_needs_reconfig"],
"check-cxx check-cxxabi check-unwind",
)
@@ -115,6 +135,7 @@ class TestComputeProjects(unittest.TestCase):
self.assertEqual(env_variables["project_check_targets"], "check-bolt")
self.assertEqual(env_variables["runtimes_to_build"], "")
self.assertEqual(env_variables["runtimes_check_targets"], "")
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
def test_lldb(self):
env_variables = compute_projects.get_env_variables(
@@ -124,6 +145,7 @@ class TestComputeProjects(unittest.TestCase):
self.assertEqual(env_variables["project_check_targets"], "check-lldb")
self.assertEqual(env_variables["runtimes_to_build"], "")
self.assertEqual(env_variables["runtimes_check_targets"], "")
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
def test_mlir(self):
env_variables = compute_projects.get_env_variables(
@@ -135,6 +157,7 @@ class TestComputeProjects(unittest.TestCase):
)
self.assertEqual(env_variables["runtimes_to_build"], "")
self.assertEqual(env_variables["runtimes_check_targets"], "")
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
def test_flang(self):
env_variables = compute_projects.get_env_variables(
@@ -144,6 +167,7 @@ class TestComputeProjects(unittest.TestCase):
self.assertEqual(env_variables["project_check_targets"], "check-flang")
self.assertEqual(env_variables["runtimes_to_build"], "")
self.assertEqual(env_variables["runtimes_check_targets"], "")
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
def test_invalid_subproject(self):
env_variables = compute_projects.get_env_variables(
@@ -153,6 +177,7 @@ class TestComputeProjects(unittest.TestCase):
self.assertEqual(env_variables["project_check_targets"], "")
self.assertEqual(env_variables["runtimes_to_build"], "")
self.assertEqual(env_variables["runtimes_check_targets"], "")
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
def test_top_level_file(self):
env_variables = compute_projects.get_env_variables(["README.md"], "Linux")
@@ -160,6 +185,7 @@ class TestComputeProjects(unittest.TestCase):
self.assertEqual(env_variables["project_check_targets"], "")
self.assertEqual(env_variables["runtimes_to_build"], "")
self.assertEqual(env_variables["runtimes_check_targets"], "")
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
def test_exclude_runtiems_in_projects(self):
env_variables = compute_projects.get_env_variables(
@@ -169,6 +195,7 @@ class TestComputeProjects(unittest.TestCase):
self.assertEqual(env_variables["project_check_targets"], "")
self.assertEqual(env_variables["runtimes_to_build"], "")
self.assertEqual(env_variables["runtimes_check_targets"], "")
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
def test_exclude_docs(self):
env_variables = compute_projects.get_env_variables(
@@ -178,6 +205,7 @@ class TestComputeProjects(unittest.TestCase):
self.assertEqual(env_variables["project_check_targets"], "")
self.assertEqual(env_variables["runtimes_to_build"], "")
self.assertEqual(env_variables["runtimes_check_targets"], "")
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
def test_exclude_gn(self):
env_variables = compute_projects.get_env_variables(
@@ -187,6 +215,7 @@ class TestComputeProjects(unittest.TestCase):
self.assertEqual(env_variables["project_check_targets"], "")
self.assertEqual(env_variables["runtimes_to_build"], "")
self.assertEqual(env_variables["runtimes_check_targets"], "")
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
def test_ci(self):
env_variables = compute_projects.get_env_variables(
@@ -198,10 +227,15 @@ class TestComputeProjects(unittest.TestCase):
"check-clang check-lld check-lldb check-llvm",
)
self.assertEqual(
env_variables["runtimes_to_build"], "libcxx;libcxxabi;libunwind"
env_variables["runtimes_to_build"],
"libcxx;libcxxabi;libunwind",
)
self.assertEqual(
env_variables["runtimes_check_targets"],
"",
)
self.assertEqual(
env_variables["runtimes_check_targets_needs_reconfig"],
"check-cxx check-cxxabi check-unwind",
)
@@ -215,6 +249,19 @@ class TestComputeProjects(unittest.TestCase):
env_variables["runtimes_to_build"], "libcxx;libcxxabi;libunwind"
)
self.assertEqual(env_variables["runtimes_check_targets"], "")
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
def test_clang_tools_extra(self):
env_variables = compute_projects.get_env_variables(
["clang-tools-extra/CMakeLists.txt"], "Linux"
)
self.assertEqual(
env_variables["projects_to_build"], "clang;clang-tools-extra;lld;llvm"
)
self.assertEqual(env_variables["project_check_targets"], "check-clang-tools")
self.assertEqual(env_variables["runtimes_to_build"], "libc")
self.assertEqual(env_variables["runtimes_check_targets"], "check-libc")
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
if __name__ == "__main__":

13
.ci/monolithic-linux.sh
View File

@@ -57,6 +57,7 @@ projects="${1}"
targets="${2}"
runtimes="${3}"
runtime_targets="${4}"
runtime_targets_needs_reconfig="${5}"
lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-output-file-name --timeout=1200 --time-tests"
@@ -93,9 +94,15 @@ echo "--- ninja"
# Targets are not escaped as they are passed as separate arguments.
ninja -C "${BUILD_DIR}" -k 0 ${targets}
if [[ "${runtime_targets}" != "" ]]; then
echo "--- ninja runtimes"
ninja -C "${BUILD_DIR}" ${runtime_targets}
fi
# Compiling runtimes with just-built Clang and running their tests
# as an additional testing for Clang.
if [[ "${runtimes_targets}" != "" ]]; then
if [[ "${runtime_targets_needs_reconfig}" != "" ]]; then
echo "--- cmake runtimes C++26"
cmake \
@@ -105,7 +112,7 @@ if [[ "${runtimes_targets}" != "" ]]; then
echo "--- ninja runtimes C++26"
ninja -C "${BUILD_DIR}" ${runtime_targets}
ninja -C "${BUILD_DIR}" ${runtime_targets_needs_reconfig}
echo "--- cmake runtimes clang modules"
@@ -116,5 +123,5 @@ if [[ "${runtimes_targets}" != "" ]]; then
echo "--- ninja runtimes clang modules"
ninja -C "${BUILD_DIR}" ${runtime_targets}
ninja -C "${BUILD_DIR}" ${runtime_targets_needs_reconfig}
fi

3
.github/workflows/premerge.yaml vendored
View File

@@ -56,11 +56,12 @@ jobs:
echo "Running project checks targets: ${project_check_targets}"
echo "Building runtimes: ${runtimes_to_build}"
echo "Running runtimes checks targets: ${runtimes_check_targets}"
echo "Running runtimes checks requiring reconfiguring targets: ${runtimes_check_targets_needs_reconfig}"
export CC=/opt/llvm/bin/clang
export CXX=/opt/llvm/bin/clang++
./.ci/monolithic-linux.sh "${projects_to_build}" "${project_check_targets}" "${runtimes_to_build}" "${runtimes_check_targets}"
./.ci/monolithic-linux.sh "${projects_to_build}" "${project_check_targets}" "${runtimes_to_build}" "${runtimes_check_targets}" "${runtimes_check_targets_needs_reconfig}"
- name: Upload Artifacts
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
with: