[libc] Add merge_yaml_files feature to hdrgen (#127269)

This allows a sort of "include" mechanism in the YAML files.  A
file can have a "merge_yaml_files" list of paths (relative to the
containing file's location).  These are YAML files in the same
syntax, except they cannot have their own "header" entry.  Only
the lists (types, enums, macros, functions, objects) can appear.
The main YAML file is then processed just as if each of its lists
were the (sorted) union of each YAML file's corresponding list.

This will enable maintaining a single source of truth for each
function signature and other such details, where it is necessary
to generate the same declaration in more than one header.

libc/utils/hdrgen/enumeration.py

@@ -6,12 +6,24 @@
 #
 # ==-------------------------------------------------------------------------==#
 
+from functools import total_ordering
 
+
+@total_ordering
 class Enumeration:
     def __init__(self, name, value):
         self.name = name
         self.value = value
 
+    def __eq__(self, other):
+        return self.name == other.name
+
+    def __lt__(self, other):
+        return self.name < other.name
+
+    def __hash__(self):
+        return self.name.__hash__()
+
     def __str__(self):
         if self.value != None:
             return f"{self.name} = {self.value}"

libc/utils/hdrgen/function.py

@@ -7,6 +7,7 @@
 # ==-------------------------------------------------------------------------==#
 
 import re
+from functools import total_ordering
 from type import Type
 
 
@@ -36,6 +37,7 @@ KEYWORDS = [
 NONIDENTIFIER = re.compile("[^a-zA-Z0-9_]+")
 
 
+@total_ordering
 class Function:
     def __init__(
         self, return_type, name, arguments, standards, guard=None, attributes=[]
@@ -51,6 +53,15 @@ class Function:
         self.guard = guard
         self.attributes = attributes or []
 
+    def __eq__(self, other):
+        return self.name == other.name
+
+    def __lt__(self, other):
+        return self.name < other.name
+
+    def __hash__(self):
+        return self.name.__hash__()
+
     def signature_types(self):
         def collapse(type_string):
             assert type_string

libc/utils/hdrgen/header.py

@@ -73,6 +73,7 @@ class HeaderFile:
         self.objects = []
         self.functions = []
         self.standards = []
+        self.merge_yaml_files = []
 
     def add_macro(self, macro):
         self.macros.append(macro)
@@ -89,6 +90,13 @@ class HeaderFile:
     def add_function(self, function):
         self.functions.append(function)
 
+    def merge(self, other):
+        self.macros = sorted(set(self.macros) | set(other.macros))
+        self.types = sorted(set(self.types) | set(other.types))
+        self.enumerations = sorted(set(self.enumerations) | set(other.enumerations))
+        self.objects = sorted(set(self.objects) | set(other.objects))
+        self.functions = sorted(set(self.functions) | set(other.functions))
+
     def all_types(self):
         return reduce(
             lambda a, b: a | b,

libc/utils/hdrgen/macro.py

@@ -6,13 +6,25 @@
 #
 # ==-------------------------------------------------------------------------==#
 
+from functools import total_ordering
 
+
+@total_ordering
 class Macro:
     def __init__(self, name, value=None, header=None):
         self.name = name
         self.value = value
         self.header = header
 
+    def __eq__(self, other):
+        return self.name == other.name
+
+    def __lt__(self, other):
+        return self.name < other.name
+
+    def __hash__(self):
+        return self.name.__hash__()
+
     def __str__(self):
         if self.header != None:
             return ""

libc/utils/hdrgen/main.py

@@ -52,8 +52,7 @@ def main():
     )
     args = parser.parse_args()
 
-    [yaml_file] = args.yaml_file
-    files_read = {yaml_file}
+    files_read = set()
 
     def write_depfile():
         if not args.depfile:
@@ -63,7 +62,34 @@ def main():
         with open(args.depfile, "w") as depfile:
             depfile.write(f"{args.output}: {deps}\n")
 
-    header = load_yaml_file(yaml_file, HeaderFile, args.entry_point)
+    def load_yaml(path):
+        files_read.add(path)
+        return load_yaml_file(path, HeaderFile, args.entry_point)
+
+    merge_from_files = dict()
+
+    def merge_from(paths):
+        for path in paths:
+            # Load each file exactly once, in case of redundant merges.
+            if path in merge_from_files:
+                continue
+            header = load_yaml(path)
+            merge_from_files[path] = header
+            merge_from(path.parent / f for f in header.merge_yaml_files)
+
+    # Load the main file first.
+    [yaml_file] = args.yaml_file
+    header = load_yaml(yaml_file)
+
+    # Now load all the merge_yaml_files, and any transitive merge_yaml_files.
+    merge_from(yaml_file.parent / f for f in header.merge_yaml_files)
+
+    # Merge in all those files' contents.
+    for merge_from_path, merge_from_header in merge_from_files.items():
+        if merge_from_header.name is not None:
+            print(f"{merge_from_path!s}: Merge file cannot have header field", stderr)
+            return 2
+        header.merge(merge_from_header)
 
     # The header_template path is relative to the containing YAML file.
     template = header.template(yaml_file.parent, files_read)

libc/utils/hdrgen/object.py

@@ -6,11 +6,23 @@
 #
 # ==-------------------------------------------------------------------------==#
 
+from functools import total_ordering
 
+
+@total_ordering
 class Object:
     def __init__(self, name, type):
         self.name = name
         self.type = type
 
+    def __eq__(self, other):
+        return self.name == other.name
+
+    def __lt__(self, other):
+        return self.name < other.name
+
+    def __hash__(self):
+        return self.name.__hash__()
+
     def __str__(self):
         return f"extern {self.type} {self.name};"

libc/utils/hdrgen/tests/input/merge1.yaml

@@ -0,0 +1,19 @@
+macros:
+  - macro_name: MACRO_A
+    macro_value: 1
+types:
+  - type_name: type_a
+enums:
+  - name: enum_a
+    value: value_1
+objects:
+  - object_name: object_1
+    object_type: obj
+functions:
+  - name: func_a
+    return_type: void
+    arguments: []
+    standards:
+      - stdc
+    attributes:
+      - CONST_FUNC_A

libc/utils/hdrgen/tests/input/merge2.yaml

@@ -0,0 +1,18 @@
+macros:
+  - macro_name: MACRO_B
+    macro_value: 2
+types:
+  - type_name: type_b
+enums:
+  - name: enum_b
+    value: value_2
+objects:
+  - object_name: object_2
+    object_type: obj
+functions:
+  - name: func_b
+    return_type: float128
+    arguments: []
+    standards:
+      - stdc
+    guard: LIBC_TYPES_HAS_FLOAT128

libc/utils/hdrgen/tests/input/test_small.yaml

@@ -1,42 +1,15 @@
 header: test_small.h
 header_template: test_small.h.def
+merge_yaml_files:
+  - merge1.yaml
+  - merge2.yaml
 macros:
-  - macro_name: MACRO_A
-    macro_value: 1
-  - macro_name: MACRO_B
-    macro_value: 2
   - macro_name: MACRO_C
   - macro_name: MACRO_D
     macro_header: test_small-macros.h
   - macro_name: MACRO_E
     macro_header: test_more-macros.h
-types:
-  - type_name: type_a
-  - type_name: type_b
-enums:
-  - name: enum_a
-    value: value_1
-  - name: enum_b
-    value: value_2
-objects:
-  - object_name: object_1
-    object_type: obj
-  - object_name: object_2
-    object_type: obj
 functions:
-  - name: func_a
-    return_type: void
-    arguments: []
-    standards:
-      - stdc
-    attributes:
-      - CONST_FUNC_A
-  - name: func_b
-    return_type: float128
-    arguments: []
-    standards:
-      - stdc
-    guard: LIBC_TYPES_HAS_FLOAT128
   - name: func_c
     return_type: _Float16
     arguments:

libc/utils/hdrgen/type.py

@@ -6,7 +6,10 @@
 #
 # ==-------------------------------------------------------------------------==#
 
+from functools import total_ordering
 
+
+@total_ordering
 class Type:
     def __init__(self, type_name):
         assert type_name
@@ -15,5 +18,8 @@ class Type:
     def __eq__(self, other):
         return self.type_name == other.type_name
 
+    def __lt__(self, other):
+        return self.type_name < other.type_name
+
     def __hash__(self):
         return self.type_name.__hash__()

libc/utils/hdrgen/yaml_to_classes.py

@@ -37,6 +37,7 @@ def yaml_to_classes(yaml_data, header_class, entry_points=None):
     header = header_class(header_name)
     header.template_file = yaml_data.get("header_template")
     header.standards = yaml_data.get("standards", [])
+    header.merge_yaml_files = yaml_data.get("merge_yaml_files", [])
 
     for macro_data in yaml_data.get("macros", []):
         header.add_macro(
@@ -126,7 +127,7 @@ def load_yaml_file(yaml_file, header_class, entry_points):
     Returns:
         HeaderFile: An instance of HeaderFile populated with the data.
     """
-    with open(yaml_file, "r") as f:
+    with yaml_file.open() as f:
         yaml_data = yaml.safe_load(f)
     return yaml_to_classes(yaml_data, header_class, entry_points)
 
@@ -264,7 +265,7 @@ def main():
         add_function_to_yaml(args.yaml_file, args.add_function)
 
     header_class = GpuHeader if args.export_decls else HeaderFile
-    header = load_yaml_file(args.yaml_file, header_class, args.entry_points)
+    header = load_yaml_file(Path(args.yaml_file), header_class, args.entry_points)
 
     header_str = str(header)