[clang][modules] Separate parsing of modulemaps (#119740)

This separates out parsing of modulemaps from updating the
`clang::ModuleMap` information.

Currently this has no effect other than slightly changing diagnostics.
Upcoming changes will use this to allow searching for modules without
fully processing modulemaps.


This creates a new `modulemap` namespace because there are too many
things called ModuleMap* right now that mean different things. I'd like
to clean this up, but I'm not sure yet what I want to call everything.

This also drops the `SourceLocation` from `moduleMapFileRead`. This is
never used in tree, and in future patches I plan to make the modulemap
parser use a different `SourceManager` so that we can share modulemap
parsing between `CompilerInstance`s. This will make the `SourceLocation`
meaningless.

clang/include/clang/Basic/DiagnosticLexKinds.td

@@ -917,6 +917,8 @@ def warn_mmap_redundant_export_as : Warning<
   InGroup<PrivateModule>;
 def err_mmap_submodule_export_as : Error<
   "only top-level modules can be re-exported as public">;
+def err_mmap_qualified_export_as : Error<
+  "a module can only be re-exported as another top-level module">;
 
 def warn_quoted_include_in_framework_header : Warning<
   "double-quoted include \"%0\" in framework header, "

clang/include/clang/Basic/Module.h

@@ -100,6 +100,30 @@ struct ASTFileSignature : std::array<uint8_t, 20> {
   }
 };
 
+/// The set of attributes that can be attached to a module.
+struct ModuleAttributes {
+  /// Whether this is a system module.
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned IsSystem : 1;
+
+  /// Whether this is an extern "C" module.
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned IsExternC : 1;
+
+  /// Whether this is an exhaustive set of configuration macros.
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned IsExhaustive : 1;
+
+  /// Whether files in this module can only include non-modular headers
+  /// and headers from used modules.
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned NoUndeclaredIncludes : 1;
+
+  ModuleAttributes()
+      : IsSystem(false), IsExternC(false), IsExhaustive(false),
+        NoUndeclaredIncludes(false) {}
+};
+
 /// Required to construct a Module.
 ///
 /// This tag type is only constructible by ModuleMap, guaranteeing it ownership

clang/include/clang/Lex/ModuleMap.h

@@ -232,29 +232,7 @@ private:
 
   llvm::DenseMap<Module *, unsigned> ModuleScopeIDs;
 
-  /// The set of attributes that can be attached to a module.
-  struct Attributes {
-    /// Whether this is a system module.
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned IsSystem : 1;
-
-    /// Whether this is an extern "C" module.
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned IsExternC : 1;
-
-    /// Whether this is an exhaustive set of configuration macros.
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned IsExhaustive : 1;
-
-    /// Whether files in this module can only include non-modular headers
-    /// and headers from used modules.
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned NoUndeclaredIncludes : 1;
-
-    Attributes()
-        : IsSystem(false), IsExternC(false), IsExhaustive(false),
-          NoUndeclaredIncludes(false) {}
-  };
+  using Attributes = ModuleAttributes;
 
   /// A directory for which framework modules can be inferred.
   struct InferredDirectory {

clang/include/clang/Lex/ModuleMapFile.h

@@ -0,0 +1,162 @@
+//===- ModuleMapFile.h - Parsing and representation -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LEX_MODULEMAPFILE_H
+#define LLVM_CLANG_LEX_MODULEMAPFILE_H
+
+#include "clang/Basic/LLVM.h"
+// TODO: Consider moving ModuleId to another header, parsing a modulemap file is
+//   intended to not depend on anything about the clang::Module class.
+#include "clang/Basic/Module.h"
+#include "clang/Basic/SourceLocation.h"
+#include "llvm/ADT/StringRef.h"
+
+#include <optional>
+#include <variant>
+
+namespace clang {
+
+class DiagnosticsEngine;
+class SourceManager;
+
+namespace modulemap {
+
+struct ExportDecl;
+
+/// All declarations that can appear in a `module` declaration.
+using Decl =
+    std::variant<struct RequiresDecl, struct HeaderDecl, struct UmbrellaDirDecl,
+                 struct ModuleDecl, struct ExcludeDecl, struct ExportDecl,
+                 struct ExportAsDecl, struct ExternModuleDecl, struct UseDecl,
+                 struct LinkDecl, struct ConfigMacrosDecl, struct ConflictDecl>;
+
+struct RequiresFeature {
+  StringRef Feature;
+  SourceLocation Location;
+  bool RequiredState = true; /// False if preceded by '!'.
+};
+
+struct RequiresDecl {
+  SourceLocation Location;
+  std::vector<RequiresFeature> Features;
+};
+
+struct HeaderDecl {
+  StringRef Path;
+  SourceLocation Location;
+  SourceLocation PathLoc;
+  std::optional<int64_t> Size;
+  std::optional<int64_t> MTime;
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned Private : 1;
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned Textual : 1;
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned Umbrella : 1;
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned Excluded : 1;
+};
+
+struct UmbrellaDirDecl {
+  StringRef Path;
+  SourceLocation Location;
+};
+
+struct ModuleDecl {
+  ModuleId Id;
+  SourceLocation Location; /// Points to the first keyword in the decl.
+  ModuleAttributes Attrs;
+  std::vector<Decl> Decls;
+
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned Explicit : 1;
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned Framework : 1;
+};
+
+struct ExcludeDecl {
+  SourceLocation Location;
+  StringRef Module;
+};
+
+struct ExportDecl {
+  ModuleId Id;
+  SourceLocation Location;
+  bool Wildcard; /// True if the last element of the ModuleId is '*'.
+};
+
+struct ExportAsDecl {
+  SourceLocation Location;
+  ModuleId Id;
+};
+
+struct ExternModuleDecl {
+  SourceLocation Location;
+  ModuleId Id;
+  StringRef Path;
+};
+
+struct UseDecl {
+  SourceLocation Location;
+  ModuleId Id;
+};
+
+struct LinkDecl {
+  StringRef Library;
+  SourceLocation Location;
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned Framework : 1;
+};
+
+struct ConfigMacrosDecl {
+  std::vector<StringRef> Macros;
+  SourceLocation Location;
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned Exhaustive : 1;
+};
+
+struct ConflictDecl {
+  SourceLocation Location;
+  ModuleId Id;
+  StringRef Message;
+};
+
+using TopLevelDecl = std::variant<ModuleDecl, ExternModuleDecl>;
+
+/// Represents the parsed form of a module map file.
+///
+/// This holds many reference types (StringRef, SourceLocation, etc.) whose
+/// lifetimes are bound by the SourceManager and FileManager used.
+struct ModuleMapFile {
+  /// Beginning of the file, used for moduleMapFileRead callback.
+  SourceLocation Start;
+  std::vector<TopLevelDecl> Decls;
+
+  void dump(llvm::raw_ostream &out) const;
+};
+
+/// Parse a module map file into an in memory representation.
+///
+/// \param ID a valid local FileID.
+/// \param Dir the directory in which this module map was found.
+/// \param SM the SourceManager for \a ID.
+/// \param Diags where to send the diagnostics.
+/// \param IsSystem was this module map found in a system search path.
+/// \param Offset optional offset into the buffer associated with \a ID. This is
+///               used for handling `#pragma clang module build`. Set to the end
+///               of the module map on return.
+///
+/// \returns The parsed ModuleMapFile if successful, std::nullopt otherwise.
+std::optional<ModuleMapFile>
+parseModuleMap(FileID ID, clang::DirectoryEntryRef Dir, SourceManager &SM,
+               DiagnosticsEngine &Diags, bool IsSystem, unsigned *Offset);
+
+} // namespace modulemap
+} // namespace clang
+
+#endif

clang/lib/Lex/CMakeLists.txt

@@ -16,6 +16,7 @@ add_clang_library(clangLex
   MacroArgs.cpp
   MacroInfo.cpp
   ModuleMap.cpp
+  ModuleMapFile.cpp
   PPCaching.cpp
   PPCallbacks.cpp
   PPConditionalDirectiveRecord.cpp

clang/test/Modules/Inputs/export_as_test.modulemap

@@ -7,3 +7,7 @@ module PrivateFoo {
     export_as Wibble
   }
 }
+
+module B {
+  export_as C.B
+}

clang/test/Modules/diagnostics.modulemap

@@ -1,36 +1,39 @@
 // RUN: rm -rf %t
-// RUN: not %clang_cc1 -fmodules -fmodules-cache-path=%t -fmodule-map-file=%S/Inputs/diagnostics-aux.modulemap -fmodule-map-file=%s -fsyntax-only -x c++ /dev/null 2>&1 | FileCheck %s --implicit-check-not error:
+// RUN: not %clang_cc1 -fmodules -fmodules-cache-path=%t -fmodule-map-file=%S/Inputs/diagnostics-aux.modulemap -fmodule-map-file=%s -fsyntax-only -x c++ /dev/null 2>&1 | FileCheck %s
 
 // CHECK: In file included from {{.*}}diagnostics-aux.modulemap:3:
 // CHECK: diagnostics-aux-2.modulemap:2:3: error: expected
 
 // PR22299: Ensure we can produce diagnostics for duplicate modules from -fmodule-map-file=.
 //
-// CHECK: diagnostics.modulemap:[[@LINE+2]]:8: error: redefinition of module 'foo'
-// CHECK: diagnostics-aux.modulemap:1:8: note: previously defined here
+// CHECK-DAG: diagnostics.modulemap:[[@LINE+2]]:8: error: redefinition of module 'foo'
+// CHECK-DAG: diagnostics-aux.modulemap:1:8: note: previously defined here
 module foo {}
 
 //* Check that we accept BCPL comments properly, not just as an extension. */
 
 module bad_use {
-  // CHECK: diagnostics.modulemap:[[@LINE+1]]:22: error: use declarations are only allowed in top-level modules
+  // CHECK-DAG: diagnostics.modulemap:[[@LINE+1]]:22: error: use declarations are only allowed in top-level modules
   module submodule { use foo }
 }
 
 module header_attr {
-  // CHECK: diagnostics.modulemap:[[@LINE+1]]:20: error: expected a header attribute name
+  // CHECK-DAG: diagnostics.modulemap:[[@LINE+1]]:20: error: expected a header attribute name
   header "foo.h" { x }
-  // CHECK: diagnostics.modulemap:[[@LINE+1]]:27: error: header attribute 'size' specified multiple times
+  // CHECK-DAG: diagnostics.modulemap:[[@LINE+1]]:27: error: header attribute 'size' specified multiple times
   header "bar.h" { size 1 size 2 }
-  // CHECK: diagnostics.modulemap:[[@LINE+1]]:25: error: expected integer literal as value for header attribute 'size'
+  // CHECK-DAG: diagnostics.modulemap:[[@LINE+1]]:25: error: expected integer literal as value for header attribute 'size'
   header "baz.h" { size "30 kilobytes" }
 
   header "quux.h" { size 1 mtime 2 }
   header "no_attrs.h" {}
 }
 
-// CHECK: diagnostics.modulemap:[[@LINE+1]]:8: error: no module named 'unknown' found, parent module must be defined before the submodule
+// CHECK-DAG: diagnostics.modulemap:[[@LINE+1]]:8: error: no module named 'unknown' found, parent module must be defined before the submodule
 module unknown.submodule {}
 module known_top_level {}
-// CHECK: diagnostics.modulemap:[[@LINE+1]]:24: error: no module named 'unknown' in 'known_top_level', parent module must be defined before the submodule
+// CHECK-DAG: diagnostics.modulemap:[[@LINE+1]]:24: error: no module named 'unknown' in 'known_top_level', parent module must be defined before the submodule
 module known_top_level.unknown.submodule {}
+
+// Check that there were no other errors emitted.
+// CHECK: 8 errors generated

clang/test/Modules/export_as_test.c

@@ -2,8 +2,7 @@
 // RUN: not %clang_cc1 -fmodules -fmodules-cache-path=%t -fmodule-map-file=%S/Inputs/export_as_test.modulemap %s 2> %t.err
 // RUN: FileCheck %s < %t.err
 
+// CHECK: export_as_test.modulemap:7:5: error: only top-level modules can be re-exported as public
+// CHECK: export_as_test.modulemap:12:15: error: a module can only be re-exported as another top-level module
 // CHECK: export_as_test.modulemap:3:13: error: conflicting re-export of module 'PrivateFoo' as 'Foo' or 'Bar'
 // CHECK: export_as_test.modulemap:4:13: warning: module 'PrivateFoo' already re-exported as 'Bar'
-// CHECK: export_as_test.modulemap:7:15: error: only top-level modules can be re-exported as public
-
-