[flang][driver] Extend the flang bash script to act as a driver

Until now, `f18` would:
  1. Use Flang to unparse the input files
  2. Call an external Fortran compiler to compile the unparsed source
  files (generated in step 1)

With this patch, `f18` will stop after unparsing the input source files,
i.e. step 1 above. The `flang` bash script will take care of step 2,
i.e. calling an external Fortran compiler driver to compile them. This
way:
  * the functionality of `f18` is reduced - it will only drive Flang (as
  opposed to delegating code-generation to an external tool on top of
  this)
  * we will able to switch between `f18` and `flang-new` for unparsing before
  an external Fortran compiler is called for code-generation

The updated `flang` bash script needs to specify the output file when
using the `-fdebug-unparse` action. Both `f18` and `flang-new` have been
updated accordingly.

These changes were discussed in [1] as a requirement for replacing `f18`
with `flang-new`.

[1] https://lists.llvm.org/pipermail/flang-dev/2021-April/000677.html

Differential Revision: https://reviews.llvm.org/D103177

flang/lib/Frontend/FrontendActions.cpp

@@ -257,8 +257,12 @@ void DebugUnparseAction::ExecuteAction() {
   auto &invoc = this->instance().invocation();
   auto &parseTree{instance().parsing().parseTree()};
 
+  CompilerInstance &ci = this->instance();
+  auto os{ci.CreateDefaultOutputFile(
+      /*Binary=*/false, /*InFile=*/GetCurrentFileOrBufferName())};
+
   // TODO: Options should come from CompilerInvocation
-  Unparse(llvm::outs(), *parseTree,
+  Unparse(*os, *parseTree,
       /*encoding=*/Fortran::parser::Encoding::UTF_8,
       /*capitalizeKeywords=*/true, /*backslashEscapes=*/false,
       /*preStatement=*/nullptr,

flang/tools/f18/CMakeLists.txt

@@ -62,10 +62,13 @@ add_custom_target(module_files ALL DEPENDS ${MODULE_FILES})
 
 install(TARGETS f18 DESTINATION bin)
 
+set(FLANG_DEFAULT_DRIVER "flang-new")
+if (NOT FLANG_BUILD_NEW_DRIVER)
+  set(FLANG_DEFAULT_DRIVER "f18")
+endif()
+
 # This flang shell script will only work in a POSIX shell.
 if (NOT WIN32)
-  file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/flang
-    DESTINATION ${CMAKE_BINARY_DIR}/bin
-    FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ OWNER_WRITE)
+  configure_file(${CMAKE_CURRENT_SOURCE_DIR}/flang.in ${CMAKE_BINARY_DIR}/bin/flang @ONLY)
   install(PROGRAMS ${CMAKE_BINARY_DIR}/bin/flang DESTINATION bin)
 endif()

flang/tools/f18/f18.cpp

@@ -319,7 +319,22 @@ std::string CompileFortran(std::string path, Fortran::parser::Options options,
     Fortran::parser::DumpTree(llvm::outs(), parseTree, &asFortran);
   }
   if (driver.dumpUnparse) {
-    Unparse(llvm::outs(), parseTree, driver.encoding, true /*capitalize*/,
+    // Prepare the output stream
+    std::unique_ptr<llvm::raw_fd_ostream> os;
+    std::string outputFile = "-";
+    if (!driver.outputPath.empty()) {
+      outputFile = driver.outputPath;
+    }
+
+    std::error_code EC;
+    os.reset(new llvm::raw_fd_ostream(
+        outputFile, EC, llvm::sys::fs::OF_TextWithCRLF));
+    if (EC) {
+      llvm::errs() << EC.message() << "\n";
+      std::exit(EXIT_FAILURE);
+    }
+
+    Unparse(*os, parseTree, driver.encoding, true /*capitalize*/,
         options.features.IsEnabled(
             Fortran::common::LanguageFeature::BackslashEscapes),
         nullptr /* action before each statement */,

flang/tools/f18/flang

@@ -1,16 +0,0 @@
-#!/bin/bash
-#===-- tools/f18/flang.sh -----------------------------------------*- sh -*-===#
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-#===------------------------------------------------------------------------===#
-
-wd=$(cd $(dirname "$0")/.. && pwd)
-opts="-fno-analyzed-objects-for-unparse -module-suffix .f18.mod "
-if ! $wd/bin/f18 $opts "$@"
-then status=$?
-     echo flang: in $PWD, f18 failed with exit status $status: $wd/bin/f18 $opts "$@" >&2
-     exit $status
-fi

flang/tools/f18/flang.in

@@ -0,0 +1,376 @@
+#! /usr/bin/env bash
+#===-- tools/f18/flang.sh -----------------------------------------*- sh -*-===#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===------------------------------------------------------------------------===#
+# A wrapper script for Flang's compiler driver that was developed for testing and
+# experimenting. You should be able to use it as a regular compiler driver. It
+# will:
+#   * run Flang's compiler driver to unparse the input source files
+#   * use the external compiler (defined via F18_FC environment variable) to
+#   compile the unparsed source files
+#===------------------------------------------------------------------------===#
+set -euo pipefail
+
+# Global variables to make the parsing of input arguments a bit easier
+INPUT_FILES=()
+OPTIONS=()
+OUTPUT_FILE=""
+MODULE_DIR=""
+INTRINSICS_MOD_DIR=""
+COMPILE_ONLY="False"
+PREPROCESS_ONLY="False"
+TEMP_OUTPUT="flang_temp_out_"
+
+# === parse_args ==============================================================
+#
+# Parse the input arguments passed to this script. Sets the global variables
+# declared at the top.
+#
+# INPUTS:
+#   $1 - all input arguments
+# OUTPUTS:
+#  Saved in the global variables for this script
+# =============================================================================
+parse_args()
+{
+  while [ "${1:-}" != "" ]; do
+      # CASE 1: Compiler option
+      if [[ "${1:0:1}" == "-" ]] ; then
+        # Output file - extract it into a global variable
+        if [[ "$1" == "-o" ]] ; then
+          shift
+          OUTPUT_FILE="$1"
+          shift
+          continue
+        fi
+
+        # Module directory - extract it into a global variable
+        if [[ "$1" == "-module-dir" ]]; then
+          shift
+          MODULE_DIR="$1"
+          shift
+          continue
+        fi
+
+        # Intrinsics module dir - extract it into a global var
+        if [[ "$1" == "-intrinsics-module-directory" ]]; then shift
+          INTRINSICS_MOD_DIR=$1
+          shift
+          continue
+        fi
+
+        # Module suffix cannot be modified - this script defines it before
+        # calling the driver.
+        if [[ "$1" == "-module-suffix" ]]; then
+          echo "ERROR: \'-module-suffix\' is not available when using the \'flang\' script"
+          exit 1
+        fi
+
+        # Special treatment for `J <dir>` and `-I <dir>`. We translate these
+        # into `J<dir>` and `-I<dir>` respectively.
+        if [[ "$1" == "-J" ]] || [[ "$1" == "-I" ]]; then
+          opt=$1
+          shift
+          OPTIONS+=("$opt$1")
+          shift
+          continue
+        fi
+
+        # This is a regular option - just add it to the list.
+        OPTIONS+=($1)
+        if [[ $1 == "-c" ]]; then
+          COMPILE_ONLY="True"
+        fi
+
+        if [[ $1 == "-E" ]]; then
+          PREPROCESS_ONLY="True"
+        fi
+
+        shift
+        continue
+
+      # CASE 2: A regular file (either source or a library file)
+      elif [[ -f "$1" ]]; then
+        INPUT_FILES+=($1)
+        shift
+        continue
+
+      else
+        # CASE 3: Unsupported
+        echo "ERROR: unrecognised option format $1"
+        exit 1
+      fi
+  done
+}
+
+# === categorise_files ========================================================
+#
+# Categorises input files into:
+#   * Fortran source files (to be compiled)
+#   * library files (to be linked into the final executable)
+#
+# INPUTS:
+#   $1 - all input files to be categorised (array, name reference)
+# OUTPUTS:
+#   $2 - Fortran source files extracted from $1 (array, name reference)
+#   $3 - other source files extracted from $1 (array, name reference)
+#   $4 - object files extracted from $1 (array, name reference)
+#   $4 - lib files extracted from $1 (array, name reference)
+# =============================================================================
+categorise_files()
+{
+  local -n -r all_files=$1
+  local -n fortran_sources=$2
+  local -n other_sources=$3
+  local -n libs=$4
+
+  for current_file in "${all_files[@]}"; do
+    file_ext=${current_file##*.}
+    if [[ $file_ext == "f" ]] || [[ $file_ext == "f90" ]] ||
+       [[ $file_ext == "f" ]] || [[ $file_ext == "F" ]] || [[ $file_ext == "ff" ]] ||
+       [[ $file_ext == "f90" ]] || [[ $file_ext == "F90" ]] || [[ $file_ext == "ff90" ]] ||
+       [[ $file_ext == "f95" ]] || [[ $file_ext == "F95" ]] || [[ $file_ext == "ff95" ]] ||
+       [[ $file_ext == "cuf" ]] || [[ $file_ext == "CUF" ]] || [[ $file_ext == "f18" ]] ||
+       [[ $file_ext == "F18" ]] || [[ $file_ext == "ff18" ]]; then
+      fortran_sources+=($current_file)
+    elif [[ $file_ext == "a" ]] || [[ $file_ext == "so" ]]; then
+      libs+=($current_file)
+    elif [[ $file_ext == "o" ]]; then
+      object_files+=($current_file)
+    else
+      other_sources+=($current_file)
+    fi
+  done
+}
+
+# === categorise_opts ==========================================================
+#
+# Categorises compiler options into options for:
+#   * the Flang driver (either new or the "throwaway" driver)
+#   * the external Fortran driver that will generate the code
+# Most options accepted by Flang will be claimed by it. The only exceptions are
+# `-I` and `-J`.
+#
+# INPUTS:
+#   $1 - all compiler options (array, name reference)
+# OUTPUTS:
+#   $2 - compiler options for the Flang driver (array, name reference)
+#   $3 - compiler options for the external driver (array, name reference)
+# =============================================================================
+categorise_opts()
+{
+  local -n all_opts=$1
+  local -n flang_opts=$2
+  local -n fc_opts=$3
+
+  for opt in "${all_opts[@]}"; do
+    # These options are claimed by Flang, but should've been dealt with in parse_args.
+    if  [[ $opt == "-module-dir" ]] ||
+      [[ $opt == "-o" ]] ||
+      [[ $opt == "-fintrinsic-modules-path" ]] ; then
+      echo "ERROR: $opt should've been fully processed by \`parse_args\`"
+      exit 1
+    fi
+
+    if
+      # The options claimed by Flang. This list needs to be compatible with
+      # what's supported by Flang's compiler driver (i.e. `flang-new` and f18).
+      [[ $opt == "-cpp" ]] ||
+      [[ $opt =~ ^-D.* ]] ||
+      [[ $opt == "-E" ]] ||
+      [[ $opt == "-falternative-parameter-statement" ]] ||
+      [[ $opt == "-fbackslash" ]] ||
+      [[ $opt == "-fcolor-diagnostics" ]] ||
+      [[ $opt == "-fdefault-double-8" ]] ||
+      [[ $opt == "-fdefault-integer-8" ]] ||
+      [[ $opt == "-fdefault-real-8" ]] ||
+      [[ $opt == "-ffixed-form" ]] ||
+      [[ $opt =~ ^-ffixed-line-length=.* ]] ||
+      [[ $opt == "-ffree-form" ]] ||
+      [[ $opt == "-fimplicit-none" ]] ||
+      [[ $opt =~ ^-finput-charset=.* ]] ||
+      [[ $opt == "-flarge-sizes" ]] ||
+      [[ $opt == "-flogical-abbreviations" ]] ||
+      [[ $opt == "-fno-color-diagnostics" ]] ||
+      [[ $opt == "-fopenacc" ]] ||
+      [[ $opt == "-fopenmp" ]] ||
+      [[ $opt == "-fxor-operator" ]] ||
+      [[ $opt == "-help" ]] ||
+      [[ $opt == "-nocpp" ]] ||
+      [[ $opt == "-pedantic" ]] ||
+      [[ $opt =~ ^-std=.* ]] ||
+      [[ $opt =~ ^-U.* ]] ||
+      [[ $opt == "--version" ]] ||
+      [[ $opt == "-Werror" ]]; then
+      flang_opts+=($opt)
+    elif [[ $opt =~ -I.* ]] || [[ $opt =~ -J.* ]]; then
+      # Options that are needed for both Flang and the external driver.
+      flang_opts+=($opt)
+      fc_opts+=($opt)
+    else
+      # All other options are claimed for the external driver.
+      fc_opts+=($opt)
+    fi
+  done
+}
+
+# === preprocess ==============================================================
+#
+# Runs the preprocessing. Fortran files are preprocessed using Flang. Other
+# files are preprocessed using the external Fortran compiler.
+#
+# INPUTS:
+#   $1 - Fortran source files (array, name reference)
+#   $2 - other source files (array, name reference)
+#   $3 - compiler flags (array, name reference)
+# =============================================================================
+preprocess() {
+  local -n fortran_srcs=$1
+  local -n other_srcs=$2
+  local -n opts=$3
+
+  local -r ext_fc="${F18_FC:-gfortran}"
+  local -r wd=$(cd "$(dirname "$0")/.." && pwd)
+
+  # Use the provided output file name.
+  if [[ ! -z ${OUTPUT_FILE:+x} ]]; then
+    output_definition="-o $OUTPUT_FILE"
+  fi
+
+  # Preprocess fortran sources using Flang
+  for idx in "${!fortran_srcs[@]}"; do
+    if ! "$wd/bin/@FLANG_DEFAULT_DRIVER@" -E "${opts[@]}" "${fortran_srcs[$idx]}" ${output_definition:+$output_definition}
+    then status=$?
+         echo flang: in "$PWD", @FLANG_DEFAULT_DRIVER@ failed with exit status $status: "$wd/bin/@FLANG_DEFAULT_DRIVER@" "${opts[@]}" "$@" >&2
+         exit $status
+    fi
+  done
+
+  # Preprocess other sources using Flang
+  for idx in "${!other_srcs[@]}"; do
+    if ! $ext_fc -E "${opts[@]}" "${other_srcs[$idx]}" ${output_definition:+$output_definition}
+    then status=$?
+         echo flang: in "$PWD", flang-new failed with exit status $status: "$wd/bin/flang-new" "${flang_options[@]}" "$@" >&2
+         exit $status
+    fi
+  done
+}
+
+# === main ====================================================================
+# Main entry point for this script
+# =============================================================================
+main() {
+  parse_args "$@"
+
+  fortran_source_files=()
+  other_source_files=()
+  object_files=()
+  lib_files=()
+  categorise_files INPUT_FILES fortran_source_files other_source_files object_files lib_files
+
+  if [[ $PREPROCESS_ONLY == "True" ]]; then
+    preprocess fortran_source_files other_source_files OPTIONS
+    exit 0
+  fi
+
+  # Options for the Flang driver.
+  # NOTE: We need `-fc1` to make sure that the frontend driver rather than
+  # compiler driver is used. We also need to make sure that that's the first
+  # flag that the driver will see (otherwise it assumes compiler/toolchain
+  # driver mode).`f18` will just ignore this flag when uparsing, so it's fine
+  # to add it here unconditionally.
+  flang_options=("-fc1")
+  # Options for the external Fortran Compiler
+  ext_fc_options=()
+  categorise_opts OPTIONS flang_options ext_fc_options
+
+  local -r wd=$(cd "$(dirname "$0")/.." && pwd)
+
+  # STEP 1: Unparse
+  local -r unparsed_file="flang_unparsed_source_file"
+  flang_options+=("-module-suffix")
+  flang_options+=(".f18.mod")
+  flang_options+=("-fdebug-unparse")
+  flang_options+=("-fno-analyzed-objects-for-unparse")
+
+  [[ ! -z ${MODULE_DIR} ]] && flang_options+=("-module-dir ${MODULE_DIR}")
+  [[ ! -z ${INTRINSICS_MOD_DIR} ]] && flang_options+=("-intrinsics-module-directory ${INTRINSICS_MOD_DIR}")
+  for idx in "${!fortran_source_files[@]}"; do
+    if ! "$wd/bin/@FLANG_DEFAULT_DRIVER@" "${flang_options[@]}" "${fortran_source_files[$idx]}" -o "${unparsed_file}_${idx}.f90"
+    then status=$?
+         echo flang: in "$PWD", @FLANG_DEFAULT_DRIVER@ failed with exit status $status: "$wd/bin/@FLANG_DEFAULT_DRIVER@" "${flang_options[@]}" "$@" >&2
+         exit $status
+    fi
+  done
+
+  # STEP 2: Compile Fortran Source Files
+  readonly ext_fc="${F18_FC:-gfortran}"
+  for idx in "${!fortran_source_files[@]}"; do
+    # Use the value of $OUTPUT_FILE for the output file iff `-c` was used.
+    if [[ ! -z ${OUTPUT_FILE:+x} ]] && [[ $COMPILE_ONLY == "True" ]]; then
+      output_definition="-o $OUTPUT_FILE"
+    elif [[ $COMPILE_ONLY == "False" ]]; then
+      output_definition="-o ${TEMP_OUTPUT}_${idx}"
+    fi
+
+    if ! $ext_fc -c "${ext_fc_options[@]}" "${unparsed_file}_${idx}.f90" ${output_definition:+$output_definition}
+    then status=$?
+      echo flang: in "$PWD", "$ext_fc" failed with exit status $status: "$ext_fc" "${ext_fc_options[@]}" "$@" >&2
+         exit $status
+    fi
+    object_files+=(${TEMP_OUTPUT}_${idx})
+  done
+
+  # Delete the unparsed files
+  for idx in "${!fortran_source_files[@]}"; do
+    rm "${unparsed_file}_${idx}.f90"
+  done
+
+  # STEP 3: Compile Other Source Files
+  for idx in "${!other_source_files[@]}"; do
+    # Use the value of $OUTPUT_FILE for the output file iff `-c` was used.
+    if [[ ! -z ${OUTPUT_FILE:+x} ]] && [[ $COMPILE_ONLY == "True" ]]; then
+      output_definition="-o $OUTPUT_FILE"
+    elif [[ $COMPILE_ONLY == "False" ]]; then
+      output_definition="-o ${TEMP_OUTPUT}_${idx}"
+    fi
+
+    if ! $ext_fc -c "${ext_fc_options[@]}" "${other_source_files[${idx}]}" ${output_definition:+$output_definition}
+    then status=$?
+      echo flang: in "$PWD", "$ext_fc" failed with exit status $status: "$ext_fc" "${ext_fc_options[@]}" "$@" >&2
+         exit $status
+    fi
+    object_files+=(${TEMP_OUTPUT}_${idx})
+  done
+
+  # STEP 4: Link
+  if [[ $COMPILE_ONLY == "True" ]]; then
+    exit 0;
+  fi
+
+  if [[ ${#object_files[@]} -ge 1 ]]; then
+    # If $OUTPUT_FILE was specified, use it for the output name.
+    if [[ ! -z ${OUTPUT_FILE:+x} ]]; then
+      output_definition="-o $OUTPUT_FILE"
+    else
+      output_definition=""
+    fi
+
+    if ! $ext_fc "${ext_fc_options[@]}" "${object_files[@]}" "${lib_files[@]}" ${output_definition:+$output_definition}
+    then status=$?
+         echo flang: in "$PWD", "$ext_fc" failed with exit status $status: "$ext_fc" "${ext_fc_options[@]}" "$@" >&2
+         exit $status
+    fi
+  fi
+
+  # Delete intermediate object files
+  for idx in "${!fortran_source_files[@]}"; do
+    rm "${TEMP_OUTPUT}_${idx}"
+  done
+}
+
+main "${@}"