344 lines
12 KiB
Python
344 lines
12 KiB
Python
# DExTer : Debugging Experience Tester
|
|
# ~~~~~~ ~ ~~ ~ ~~
|
|
#
|
|
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
# See https://llvm.org/LICENSE.txt for license information.
|
|
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
"""Clang opt-bisect tool."""
|
|
|
|
from collections import defaultdict
|
|
import os
|
|
import csv
|
|
import re
|
|
import pickle
|
|
|
|
from dex.command.ParseCommand import get_command_infos
|
|
from dex.debugger.Debuggers import run_debugger_subprocess
|
|
from dex.debugger.DebuggerControllers.DefaultController import DefaultController
|
|
from dex.dextIR.DextIR import DextIR
|
|
from dex.heuristic import Heuristic
|
|
from dex.tools import TestToolBase
|
|
from dex.utils.Exceptions import DebuggerException, Error
|
|
from dex.utils.Exceptions import BuildScriptException, HeuristicException
|
|
from dex.utils.PrettyOutputBase import Stream
|
|
from dex.utils.ReturnCode import ReturnCode
|
|
|
|
|
|
class BisectPass(object):
|
|
def __init__(self, no, description, description_no_loc):
|
|
self.no = no
|
|
self.description = description
|
|
self.description_no_loc = description_no_loc
|
|
|
|
self.penalty = 0
|
|
self.differences = []
|
|
|
|
|
|
class Tool(TestToolBase):
|
|
"""Use the LLVM "-opt-bisect-limit=<n>" flag to get information on the
|
|
contribution of each LLVM pass to the overall DExTer score when using
|
|
clang.
|
|
|
|
Clang is run multiple times, with an increasing value of n, measuring the
|
|
debugging experience at each value.
|
|
"""
|
|
|
|
_re_running_pass = re.compile(r"^BISECT\: running pass \((\d+)\) (.+?)( \(.+\))?$")
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
super(Tool, self).__init__(*args, **kwargs)
|
|
self._all_bisect_pass_summary = defaultdict(list)
|
|
|
|
@property
|
|
def name(self):
|
|
return "DExTer clang opt bisect"
|
|
|
|
def _get_bisect_limits(self):
|
|
options = self.context.options
|
|
|
|
max_limit = 999999
|
|
limits = [max_limit for _ in options.source_files]
|
|
all_passes = [
|
|
l
|
|
for l in self._clang_opt_bisect_build(limits)[1].splitlines()
|
|
if l.startswith("BISECT: running pass (")
|
|
]
|
|
|
|
results = []
|
|
for i, pass_ in enumerate(all_passes[1:]):
|
|
if pass_.startswith("BISECT: running pass (1)"):
|
|
results.append(all_passes[i])
|
|
results.append(all_passes[-1])
|
|
|
|
assert len(results) == len(options.source_files), (
|
|
results,
|
|
options.source_files,
|
|
)
|
|
|
|
limits = [int(Tool._re_running_pass.match(r).group(1)) for r in results]
|
|
|
|
return limits
|
|
|
|
def handle_options(self, defaults):
|
|
options = self.context.options
|
|
if "clang" not in options.builder.lower():
|
|
raise Error(
|
|
"--builder %s is not supported by the clang-opt-bisect tool - only 'clang' is "
|
|
"supported " % options.builder
|
|
)
|
|
super(Tool, self).handle_options(defaults)
|
|
|
|
def _init_debugger_controller(self):
|
|
step_collection = DextIR(
|
|
executable_path=self.context.options.executable,
|
|
source_paths=self.context.options.source_files,
|
|
dexter_version=self.context.version,
|
|
)
|
|
|
|
step_collection.commands, new_source_files = get_command_infos(
|
|
self.context.options.source_files, self.context.options.source_root_dir
|
|
)
|
|
self.context.options.source_files.extend(list(new_source_files))
|
|
|
|
debugger_controller = DefaultController(self.context, step_collection)
|
|
return debugger_controller
|
|
|
|
def _run_test(self, test_name): # noqa
|
|
options = self.context.options
|
|
|
|
per_pass_score = []
|
|
current_bisect_pass_summary = defaultdict(list)
|
|
|
|
max_limits = self._get_bisect_limits()
|
|
overall_limit = sum(max_limits)
|
|
prev_score = 1.0
|
|
prev_steps_str = None
|
|
|
|
for current_limit in range(overall_limit + 1):
|
|
# Take the overall limit number and split it across buckets for
|
|
# each source file.
|
|
limit_remaining = current_limit
|
|
file_limits = [0] * len(max_limits)
|
|
for i, max_limit in enumerate(max_limits):
|
|
if limit_remaining < max_limit:
|
|
file_limits[i] += limit_remaining
|
|
break
|
|
else:
|
|
file_limits[i] = max_limit
|
|
limit_remaining -= file_limits[i]
|
|
|
|
f = [l for l in file_limits if l]
|
|
current_file_index = len(f) - 1 if f else 0
|
|
|
|
_, err, builderIR = self._clang_opt_bisect_build(file_limits)
|
|
err_lines = err.splitlines()
|
|
# Find the last line that specified a running pass.
|
|
for l in err_lines[::-1]:
|
|
match = Tool._re_running_pass.match(l)
|
|
if match:
|
|
pass_info = match.groups()
|
|
break
|
|
else:
|
|
pass_info = (0, None, None)
|
|
|
|
try:
|
|
debugger_controller = self._init_debugger_controller()
|
|
debugger_controller = run_debugger_subprocess(
|
|
debugger_controller, self.context.working_directory.path
|
|
)
|
|
steps = debugger_controller.step_collection
|
|
except DebuggerException:
|
|
steps = DextIR(
|
|
executable_path=self.context.options.executable,
|
|
source_paths=self.context.options.source_files,
|
|
dexter_version=self.context.version,
|
|
)
|
|
|
|
steps.builder = builderIR
|
|
|
|
try:
|
|
heuristic = Heuristic(self.context, steps)
|
|
except HeuristicException as e:
|
|
raise Error(e)
|
|
|
|
score_difference = heuristic.score - prev_score
|
|
prev_score = heuristic.score
|
|
|
|
isnan = heuristic.score != heuristic.score
|
|
if isnan or score_difference < 0:
|
|
color1 = "r"
|
|
color2 = "r"
|
|
elif score_difference > 0:
|
|
color1 = "g"
|
|
color2 = "g"
|
|
else:
|
|
color1 = "y"
|
|
color2 = "d"
|
|
|
|
summary = '<{}>running pass {}/{} on "{}"'.format(
|
|
color2, pass_info[0], max_limits[current_file_index], test_name
|
|
)
|
|
if len(options.source_files) > 1:
|
|
summary += " [{}/{}]".format(current_limit, overall_limit)
|
|
|
|
pass_text = "".join(p for p in pass_info[1:] if p)
|
|
summary += ": {} <{}>{:+.4f}</> <{}>{}</></>\n".format(
|
|
heuristic.summary_string, color1, score_difference, color2, pass_text
|
|
)
|
|
|
|
self.context.o.auto(summary)
|
|
|
|
heuristic_verbose_output = heuristic.verbose_output
|
|
|
|
if options.verbose:
|
|
self.context.o.auto(heuristic_verbose_output)
|
|
|
|
steps_str = str(steps)
|
|
steps_changed = steps_str != prev_steps_str
|
|
prev_steps_str = steps_str
|
|
|
|
# If a results directory has been specified and this is the first
|
|
# pass or something has changed, write a text file containing
|
|
# verbose information on the current status.
|
|
if options.results_directory and (
|
|
current_limit == 0 or score_difference or steps_changed
|
|
):
|
|
file_name = "-".join(
|
|
str(s)
|
|
for s in [
|
|
"status",
|
|
test_name,
|
|
"{{:0>{}}}".format(len(str(overall_limit))).format(
|
|
current_limit
|
|
),
|
|
"{:.4f}".format(heuristic.score).replace(".", "_"),
|
|
pass_info[1],
|
|
]
|
|
if s is not None
|
|
)
|
|
|
|
file_name = (
|
|
"".join(c for c in file_name if c.isalnum() or c in "()-_./ ")
|
|
.strip()
|
|
.replace(" ", "_")
|
|
.replace("/", "_")
|
|
)
|
|
|
|
output_text_path = os.path.join(
|
|
options.results_directory, "{}.txt".format(file_name)
|
|
)
|
|
with open(output_text_path, "w") as fp:
|
|
self.context.o.auto(summary + "\n", stream=Stream(fp))
|
|
self.context.o.auto(str(steps) + "\n", stream=Stream(fp))
|
|
self.context.o.auto(
|
|
heuristic_verbose_output + "\n", stream=Stream(fp)
|
|
)
|
|
|
|
output_dextIR_path = os.path.join(
|
|
options.results_directory, "{}.dextIR".format(file_name)
|
|
)
|
|
with open(output_dextIR_path, "wb") as fp:
|
|
pickle.dump(steps, fp, protocol=pickle.HIGHEST_PROTOCOL)
|
|
|
|
per_pass_score.append((test_name, pass_text, heuristic.score))
|
|
|
|
if pass_info[1]:
|
|
self._all_bisect_pass_summary[pass_info[1]].append(score_difference)
|
|
|
|
current_bisect_pass_summary[pass_info[1]].append(score_difference)
|
|
|
|
if options.results_directory:
|
|
per_pass_score_path = os.path.join(
|
|
options.results_directory, "{}-per_pass_score.csv".format(test_name)
|
|
)
|
|
|
|
with open(per_pass_score_path, mode="w", newline="") as fp:
|
|
writer = csv.writer(fp, delimiter=",")
|
|
writer.writerow(["Source File", "Pass", "Score"])
|
|
|
|
for path, pass_, score in per_pass_score:
|
|
writer.writerow([path, pass_, score])
|
|
self.context.o.blue('wrote "{}"\n'.format(per_pass_score_path))
|
|
|
|
pass_summary_path = os.path.join(
|
|
options.results_directory, "{}-pass-summary.csv".format(test_name)
|
|
)
|
|
|
|
self._write_pass_summary(pass_summary_path, current_bisect_pass_summary)
|
|
|
|
def _handle_results(self) -> ReturnCode:
|
|
options = self.context.options
|
|
if options.results_directory:
|
|
pass_summary_path = os.path.join(
|
|
options.results_directory, "overall-pass-summary.csv"
|
|
)
|
|
|
|
self._write_pass_summary(pass_summary_path, self._all_bisect_pass_summary)
|
|
return ReturnCode.OK
|
|
|
|
def _clang_opt_bisect_build(self, opt_bisect_limits):
|
|
options = self.context.options
|
|
compiler_options = [
|
|
"{} -mllvm -opt-bisect-limit={}".format(options.cflags, opt_bisect_limit)
|
|
for opt_bisect_limit in opt_bisect_limits
|
|
]
|
|
linker_options = options.ldflags
|
|
|
|
try:
|
|
return run_external_build_script(
|
|
self.context,
|
|
source_files=options.source_files,
|
|
compiler_options=compiler_options,
|
|
linker_options=linker_options,
|
|
script_path=self.build_script,
|
|
executable_file=options.executable,
|
|
)
|
|
except BuildScriptException as e:
|
|
raise Error(e)
|
|
|
|
def _write_pass_summary(self, path, pass_summary):
|
|
# Get a list of tuples.
|
|
pass_summary_list = list(pass_summary.items())
|
|
|
|
for i, item in enumerate(pass_summary_list):
|
|
# Add elems for the sum, min, and max of the values, as well as
|
|
# 'interestingness' which is whether any of these values are
|
|
# non-zero.
|
|
pass_summary_list[i] += (
|
|
sum(item[1]),
|
|
min(item[1]),
|
|
max(item[1]),
|
|
any(item[1]),
|
|
)
|
|
|
|
# Split the pass name into the basic name and kind.
|
|
pass_summary_list[i] += tuple(item[0].rsplit(" on ", 1))
|
|
|
|
# Sort the list by the following columns in order of precedence:
|
|
# - Is interesting (True first)
|
|
# - Sum (smallest first)
|
|
# - Number of times pass ran (largest first)
|
|
# - Kind (alphabetically)
|
|
# - Name (alphabetically)
|
|
pass_summary_list.sort(
|
|
key=lambda tup: (not tup[5], tup[2], -len(tup[1]), tup[7], tup[6])
|
|
)
|
|
|
|
with open(path, mode="w", newline="") as fp:
|
|
writer = csv.writer(fp, delimiter=",")
|
|
writer.writerow(["Pass", "Kind", "Sum", "Min", "Max", "Interesting"])
|
|
|
|
for (
|
|
_,
|
|
vals,
|
|
sum_,
|
|
min_,
|
|
max_,
|
|
interesting,
|
|
name,
|
|
kind,
|
|
) in pass_summary_list:
|
|
writer.writerow([name, kind, sum_, min_, max_, interesting] + vals)
|
|
|
|
self.context.o.blue('wrote "{}"\n'.format(path))
|