[libc] Add --json mode for hdrgen (#127847)

This adds a feature to hdrgen to emit JSON summaries of header
files for build system integration.  For now the summaries have
only the basic information about each header that is relevant for
build and testing purposes: the standards and includes lists.

libc/utils/hdrgen/header.py

@@ -233,3 +233,12 @@ class HeaderFile:
         content.append("__END_C_DECLS")
 
         return "\n".join(content)
+
+    def json_data(self):
+        return {
+            "name": self.name,
+            "standards": self.standards,
+            "includes": [
+                str(file) for file in sorted({COMMON_HEADER} | self.includes())
+            ],
+        }

libc/utils/hdrgen/main.py

@@ -9,6 +9,7 @@
 # ==------------------------------------------------------------------------==#
 
 import argparse
+import json
 import sys
 from pathlib import Path
 
@@ -23,7 +24,7 @@ def main():
         help="Path to the YAML file containing header specification",
         metavar="FILE",
         type=Path,
-        nargs=1,
+        nargs="+",
     )
     parser.add_argument(
         "-o",
@@ -32,6 +33,11 @@ def main():
         type=Path,
         required=True,
     )
+    parser.add_argument(
+        "--json",
+        help="Write JSON instead of a header, can use multiple YAML files",
+        action="store_true",
+    )
     parser.add_argument(
         "--depfile",
         help="Path to write a depfile",
@@ -52,6 +58,11 @@ def main():
     )
     args = parser.parse_args()
 
+    if not args.json and len(args.yaml_file) != 1:
+        print("Only one YAML file at a time without --json", file=sys.stderr)
+        parser.print_usage(sys.stderr)
+        return 2
+
     files_read = set()
 
     def write_depfile():
@@ -66,35 +77,47 @@ def main():
         files_read.add(path)
         return load_yaml_file(path, HeaderFile, args.entry_point)
 
-    merge_from_files = dict()
+    def load_header(yaml_file):
+        merge_from_files = dict()
 
-    def merge_from(paths):
-        for path in paths:
-            # Load each file exactly once, in case of redundant merges.
-            if path in merge_from_files:
-                continue
-            header = load_yaml(path)
-            merge_from_files[path] = header
-            merge_from(path.parent / f for f in header.merge_yaml_files)
+        def merge_from(paths):
+            for path in paths:
+                # Load each file exactly once, in case of redundant merges.
+                if path in merge_from_files:
+                    continue
+                header = load_yaml(path)
+                merge_from_files[path] = header
+                merge_from(path.parent / f for f in header.merge_yaml_files)
 
-    # Load the main file first.
-    [yaml_file] = args.yaml_file
-    header = load_yaml(yaml_file)
+        # Load the main file first.
+        header = load_yaml(yaml_file)
 
-    # Now load all the merge_yaml_files, and any transitive merge_yaml_files.
-    merge_from(yaml_file.parent / f for f in header.merge_yaml_files)
+        # Now load all the merge_yaml_files, and transitive merge_yaml_files.
+        merge_from(yaml_file.parent / f for f in header.merge_yaml_files)
 
-    # Merge in all those files' contents.
-    for merge_from_path, merge_from_header in merge_from_files.items():
-        if merge_from_header.name is not None:
-            print(f"{merge_from_path!s}: Merge file cannot have header field", stderr)
-            return 2
-        header.merge(merge_from_header)
+        # Merge in all those files' contents.
+        for merge_from_path, merge_from_header in merge_from_files.items():
+            if merge_from_header.name is not None:
+                print(
+                    f"{merge_from_path!s}: Merge file cannot have header field",
+                    file=sys.stderr,
+                )
+                return 2
+            header.merge(merge_from_header)
 
-    # The header_template path is relative to the containing YAML file.
-    template = header.template(yaml_file.parent, files_read)
+        return header
 
-    contents = fill_public_api(header.public_api(), template)
+    if args.json:
+        contents = json.dumps(
+            [load_header(file).json_data() for file in args.yaml_file],
+            indent=2,
+        )
+    else:
+        [yaml_file] = args.yaml_file
+        header = load_header(yaml_file)
+        # The header_template path is relative to the containing YAML file.
+        template = header.template(yaml_file.parent, files_read)
+        contents = fill_public_api(header.public_api(), template)
 
     write_depfile()
 

libc/utils/hdrgen/tests/expected_output/test_small.json

@@ -0,0 +1,14 @@
+[
+  {
+    "name": "test_small.h",
+    "standards": [],
+    "includes": [
+      "__llvm-libc-common.h",
+      "llvm-libc-macros/test_more-macros.h",
+      "llvm-libc-macros/test_small-macros.h",
+      "llvm-libc-types/float128.h",
+      "llvm-libc-types/type_a.h",
+      "llvm-libc-types/type_b.h"
+    ]
+  }
+]

libc/utils/hdrgen/tests/test_integration.py

@@ -12,14 +12,14 @@ class TestHeaderGenIntegration(unittest.TestCase):
         self.main_script = self.source_dir.parent / "main.py"
         self.maxDiff = 80 * 100
 
-    def run_script(self, yaml_file, output_file, entry_points=[]):
+    def run_script(self, yaml_file, output_file, entry_points=[], switches=[]):
         command = [
             "python3",
             str(self.main_script),
             str(yaml_file),
             "--output",
             str(output_file),
-        ]
+        ] + switches
 
         for entry_point in entry_points:
             command.extend(["--entry-point", entry_point])
@@ -59,6 +59,15 @@ class TestHeaderGenIntegration(unittest.TestCase):
         self.run_script(yaml_file, output_file)
         self.compare_files(output_file, expected_output_file)
 
+    def test_generate_json(self):
+        yaml_file = self.source_dir / "input/test_small.yaml"
+        expected_output_file = self.source_dir / "expected_output/test_small.json"
+        output_file = self.output_dir / "test_small.json"
+
+        self.run_script(yaml_file, output_file, switches=["--json"])
+
+        self.compare_files(output_file, expected_output_file)
+
 
 def main():
     parser = argparse.ArgumentParser(description="TestHeaderGenIntegration arguments")