# DExTer : Debugging Experience Tester # ~~~~~~ ~ ~~ ~ ~~ # # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception """Base class for subtools that do build/run tests.""" import abc from datetime import datetime import os import sys from dex.builder import add_builder_tool_arguments from dex.builder import handle_builder_tool_options from dex.debugger.Debuggers import add_debugger_tool_arguments from dex.debugger.Debuggers import handle_debugger_tool_options from dex.heuristic.Heuristic import add_heuristic_tool_arguments from dex.tools.ToolBase import ToolBase from dex.utils import get_root_directory, warn from dex.utils.Exceptions import Error, ToolArgumentError from dex.utils.ReturnCode import ReturnCode class TestToolBase(ToolBase): def __init__(self, *args, **kwargs): super(TestToolBase, self).__init__(*args, **kwargs) self.build_script: str = None def add_tool_arguments(self, parser, defaults): parser.description = self.__doc__ add_builder_tool_arguments(parser) add_debugger_tool_arguments(parser, self.context, defaults) add_heuristic_tool_arguments(parser) parser.add_argument( 'test_path', type=str, metavar='', nargs='?', default=os.path.abspath( os.path.join(get_root_directory(), '..', 'tests')), help='directory containing test(s)') parser.add_argument( '--results-directory', type=str, metavar='', default=None, help='directory to save results (default: none)') def handle_options(self, defaults): options = self.context.options if not options.builder and (options.cflags or options.ldflags): warn(self.context, '--cflags and --ldflags will be ignored when not' ' using --builder') if options.vs_solution: options.vs_solution = os.path.abspath(options.vs_solution) if not os.path.isfile(options.vs_solution): raise Error('could not find VS solution file "{}"' .format(options.vs_solution)) elif options.binary: options.binary = os.path.abspath(options.binary) if not os.path.isfile(options.binary): raise Error('could not find binary file "{}"' .format(options.binary)) else: try: self.build_script = handle_builder_tool_options(self.context) except ToolArgumentError as e: raise Error(e) try: handle_debugger_tool_options(self.context, defaults) except ToolArgumentError as e: raise Error(e) options.test_path = os.path.abspath(options.test_path) options.test_path = os.path.normcase(options.test_path) if not os.path.isfile(options.test_path) and not os.path.isdir(options.test_path): raise Error( 'could not find test path "{}"'.format( options.test_path)) if options.results_directory: options.results_directory = os.path.abspath(options.results_directory) if not os.path.isdir(options.results_directory): try: os.makedirs(options.results_directory, exist_ok=True) except OSError as e: raise Error( 'could not create directory "{}" ({})'. format(options.results_directory, e.strerror)) def go(self) -> ReturnCode: # noqa options = self.context.options options.executable = os.path.join( self.context.working_directory.path, 'tmp.exe') # Test files contain dexter commands. options.test_files = [] # Source files are to be compiled by the builder script and may also # contains dexter commands. options.source_files = [] if os.path.isdir(options.test_path): subdirs = sorted([ r for r, _, f in os.walk(options.test_path) if 'test.cfg' in f ]) for subdir in subdirs: for f in os.listdir(subdir): # TODO: read file extensions from the test.cfg file instead so # that this isn't just limited to C and C++. file_path = os.path.normcase(os.path.join(subdir, f)) if f.endswith('.cpp'): options.source_files.append(file_path) elif f.endswith('.c'): options.source_files.append(file_path) elif f.endswith('.dex'): options.test_files.append(file_path) # Source files can contain dexter commands too. options.test_files = options.test_files + options.source_files self._run_test(self._get_test_name(subdir)) else: # We're dealing with a direct file path to a test file. If the file is non # .dex, then it must be a source file. if not options.test_path.endswith('.dex'): options.source_files = [options.test_path] options.test_files = [options.test_path] self._run_test(self._get_test_name(options.test_path)) return self._handle_results() @staticmethod def _is_current_directory(test_directory): return test_directory == '.' def _get_test_name(self, test_path): """Get the test name from either the test file, or the sub directory path it's stored in. """ # test names are distinguished by their relative path from the # specified test path. test_name = os.path.relpath(test_path, self.context.options.test_path) if self._is_current_directory(test_name): test_name = os.path.basename(test_path) return test_name @abc.abstractmethod def _run_test(self, test_dir): pass @abc.abstractmethod def _handle_results(self) -> ReturnCode: pass