[SystemZ][z/OS] yaml2obj for header and end records (#73859)

This PR implements part 1 of yaml2obj for the GOFF Object File Format.
It adds support for the header and end records.

---------

Co-authored-by: Yusra Syeda <yusra.syeda@ibm.com>

llvm/include/llvm/ObjectYAML/GOFFYAML.h

@@ -0,0 +1,49 @@
+//===- GOFFYAML.h - GOFF YAMLIO implementation ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares classes for handling the YAML representation of GOFF.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_OBJECTYAML_GOFFYAML_H
+#define LLVM_OBJECTYAML_GOFFYAML_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/BinaryFormat/GOFF.h"
+#include "llvm/ObjectYAML/YAML.h"
+#include <cstdint>
+#include <vector>
+
+namespace llvm {
+
+// The structure of the yaml files is not an exact 1:1 match to GOFF. In order
+// to use yaml::IO, we use these structures which are closer to the source.
+namespace GOFFYAML {
+
+struct FileHeader {
+  uint32_t TargetEnvironment = 0;
+  uint32_t TargetOperatingSystem = 0;
+  uint16_t CCSID = 0;
+  StringRef CharacterSetName;
+  StringRef LanguageProductIdentifier;
+  uint32_t ArchitectureLevel = 0;
+  std::optional<uint16_t> InternalCCSID;
+  std::optional<uint8_t> TargetSoftwareEnvironment;
+};
+
+struct Object {
+  FileHeader Header;
+  Object();
+};
+} // end namespace GOFFYAML
+} // end namespace llvm
+
+LLVM_YAML_DECLARE_MAPPING_TRAITS(GOFFYAML::FileHeader)
+LLVM_YAML_DECLARE_MAPPING_TRAITS(GOFFYAML::Object)
+
+#endif // LLVM_OBJECTYAML_GOFFYAML_H

llvm/include/llvm/ObjectYAML/ObjectYAML.h

@@ -13,6 +13,7 @@
 #include "llvm/ObjectYAML/COFFYAML.h"
 #include "llvm/ObjectYAML/DXContainerYAML.h"
 #include "llvm/ObjectYAML/ELFYAML.h"
+#include "llvm/ObjectYAML/GOFFYAML.h"
 #include "llvm/ObjectYAML/MachOYAML.h"
 #include "llvm/ObjectYAML/MinidumpYAML.h"
 #include "llvm/ObjectYAML/OffloadYAML.h"
@@ -30,6 +31,7 @@ struct YamlObjectFile {
   std::unique_ptr<ArchYAML::Archive> Arch;
   std::unique_ptr<ELFYAML::Object> Elf;
   std::unique_ptr<COFFYAML::Object> Coff;
+  std::unique_ptr<GOFFYAML::Object> Goff;
   std::unique_ptr<MachOYAML::Object> MachO;
   std::unique_ptr<MachOYAML::UniversalBinary> FatMachO;
   std::unique_ptr<MinidumpYAML::Object> Minidump;

llvm/include/llvm/ObjectYAML/yaml2obj.h

@@ -32,6 +32,10 @@ namespace ELFYAML {
 struct Object;
 }
 
+namespace GOFFYAML {
+struct Object;
+}
+
 namespace MinidumpYAML {
 struct Object;
 }
@@ -64,6 +68,7 @@ using ErrorHandler = llvm::function_ref<void(const Twine &Msg)>;
 
 bool yaml2archive(ArchYAML::Archive &Doc, raw_ostream &Out, ErrorHandler EH);
 bool yaml2coff(COFFYAML::Object &Doc, raw_ostream &Out, ErrorHandler EH);
+bool yaml2goff(GOFFYAML::Object &Doc, raw_ostream &Out, ErrorHandler EH);
 bool yaml2elf(ELFYAML::Object &Doc, raw_ostream &Out, ErrorHandler EH,
               uint64_t MaxSize);
 bool yaml2macho(YamlObjectFile &Doc, raw_ostream &Out, ErrorHandler EH);

llvm/lib/ObjectYAML/CMakeLists.txt

@@ -13,6 +13,8 @@ add_llvm_component_library(LLVMObjectYAML
   DXContainerYAML.cpp
   ELFEmitter.cpp
   ELFYAML.cpp
+  GOFFEmitter.cpp
+  GOFFYAML.cpp
   MachOEmitter.cpp
   MachOYAML.cpp
   ObjectYAML.cpp

llvm/lib/ObjectYAML/GOFFEmitter.cpp

@@ -0,0 +1,282 @@
+//===- yaml2goff - Convert YAML to a GOFF object file ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// The GOFF component of yaml2obj.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/IndexedMap.h"
+#include "llvm/ObjectYAML/ObjectYAML.h"
+#include "llvm/ObjectYAML/yaml2obj.h"
+#include "llvm/Support/ConvertEBCDIC.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+namespace {
+
+// Common flag values on records.
+enum {
+  // Flag: This record is continued.
+  Rec_Continued = 1,
+
+  // Flag: This record is a continuation.
+  Rec_Continuation = 1 << (8 - 6 - 1),
+};
+
+template <typename ValueType> struct BinaryBeImpl {
+  ValueType Value;
+  BinaryBeImpl(ValueType V) : Value(V) {}
+};
+
+template <typename ValueType>
+raw_ostream &operator<<(raw_ostream &OS, const BinaryBeImpl<ValueType> &BBE) {
+  char Buffer[sizeof(BBE.Value)];
+  support::endian::write<ValueType, llvm::endianness::big, support::unaligned>(
+      Buffer, BBE.Value);
+  OS.write(Buffer, sizeof(BBE.Value));
+  return OS;
+}
+
+template <typename ValueType> BinaryBeImpl<ValueType> binaryBe(ValueType V) {
+  return BinaryBeImpl<ValueType>(V);
+}
+
+struct ZerosImpl {
+  size_t NumBytes;
+};
+
+raw_ostream &operator<<(raw_ostream &OS, const ZerosImpl &Z) {
+  OS.write_zeros(Z.NumBytes);
+  return OS;
+}
+
+ZerosImpl zeros(const size_t NumBytes) { return ZerosImpl{NumBytes}; }
+
+// The GOFFOstream is responsible to write the data into the fixed physical
+// records of the format. A user of this class announces the start of a new
+// logical record and the size of its payload. While writing the payload, the
+// physical records are created for the data. Possible fill bytes at the end of
+// a physical record are written automatically.
+class GOFFOstream : public raw_ostream {
+public:
+  explicit GOFFOstream(raw_ostream &OS)
+      : OS(OS), LogicalRecords(0), RemainingSize(0), NewLogicalRecord(false) {
+    SetBufferSize(GOFF::PayloadLength);
+  }
+
+  ~GOFFOstream() { finalize(); }
+
+  void makeNewRecord(GOFF::RecordType Type, size_t Size) {
+    fillRecord();
+    CurrentType = Type;
+    RemainingSize = Size;
+    if (size_t Gap = (RemainingSize % GOFF::PayloadLength))
+      RemainingSize += GOFF::PayloadLength - Gap;
+    NewLogicalRecord = true;
+    ++LogicalRecords;
+  }
+
+  void finalize() { fillRecord(); }
+
+  uint32_t logicalRecords() { return LogicalRecords; }
+
+private:
+  // The underlying raw_ostream.
+  raw_ostream &OS;
+
+  // The number of logical records emitted so far.
+  uint32_t LogicalRecords;
+
+  // The remaining size of this logical record, including fill bytes.
+  size_t RemainingSize;
+
+  // The type of the current (logical) record.
+  GOFF::RecordType CurrentType;
+
+  // Signals start of new record.
+  bool NewLogicalRecord;
+
+  // Return the number of bytes left to write until next physical record.
+  // Please note that we maintain the total number of bytes left, not the
+  // written size.
+  size_t bytesToNextPhysicalRecord() {
+    size_t Bytes = RemainingSize % GOFF::PayloadLength;
+    return Bytes ? Bytes : GOFF::PayloadLength;
+  }
+
+  // Write the record prefix of a physical record, using the current record
+  // type.
+  static void writeRecordPrefix(raw_ostream &OS, GOFF::RecordType Type,
+                                size_t RemainingSize,
+                                uint8_t Flags = Rec_Continuation) {
+    uint8_t TypeAndFlags = Flags | (Type << 4);
+    if (RemainingSize > GOFF::RecordLength)
+      TypeAndFlags |= Rec_Continued;
+    OS << binaryBe(static_cast<unsigned char>(GOFF::PTVPrefix))
+       << binaryBe(static_cast<unsigned char>(TypeAndFlags))
+       << binaryBe(static_cast<unsigned char>(0));
+  }
+
+  // Fill the last physical record of a logical record with zero bytes.
+  void fillRecord() {
+    assert((GetNumBytesInBuffer() <= RemainingSize) &&
+           "More bytes in buffer than expected");
+    size_t Remains = RemainingSize - GetNumBytesInBuffer();
+    if (Remains) {
+      assert((Remains < GOFF::RecordLength) &&
+             "Attempting to fill more than one physical record");
+      raw_ostream::write_zeros(Remains);
+    }
+    flush();
+    assert(RemainingSize == 0 && "Not fully flushed");
+    assert(GetNumBytesInBuffer() == 0 && "Buffer not fully empty");
+  }
+
+  // See raw_ostream::write_impl.
+  void write_impl(const char *Ptr, size_t Size) override {
+    assert((RemainingSize >= Size) && "Attempt to write too much data");
+    assert(RemainingSize && "Logical record overflow");
+    if (!(RemainingSize % GOFF::PayloadLength)) {
+      writeRecordPrefix(OS, CurrentType, RemainingSize,
+                        NewLogicalRecord ? 0 : Rec_Continuation);
+      NewLogicalRecord = false;
+    }
+    assert(!NewLogicalRecord &&
+           "New logical record not on physical record boundary");
+
+    size_t Idx = 0;
+    while (Size > 0) {
+      size_t BytesToWrite = bytesToNextPhysicalRecord();
+      if (BytesToWrite > Size)
+        BytesToWrite = Size;
+      OS.write(Ptr + Idx, BytesToWrite);
+      Idx += BytesToWrite;
+      Size -= BytesToWrite;
+      RemainingSize -= BytesToWrite;
+      if (Size) {
+        writeRecordPrefix(OS, CurrentType, RemainingSize);
+      }
+    }
+  }
+
+  // Return the current position within the stream, not counting the bytes
+  // currently in the buffer.
+  uint64_t current_pos() const override { return OS.tell(); }
+};
+
+class GOFFState {
+  void writeHeader(GOFFYAML::FileHeader &FileHdr);
+  void writeEnd();
+
+  void reportError(const Twine &Msg) {
+    ErrHandler(Msg);
+    HasError = true;
+  }
+
+  GOFFState(raw_ostream &OS, GOFFYAML::Object &Doc,
+            yaml::ErrorHandler ErrHandler)
+      : GW(OS), Doc(Doc), ErrHandler(ErrHandler), HasError(false) {}
+
+  ~GOFFState() { GW.finalize(); }
+
+  bool writeObject();
+
+public:
+  static bool writeGOFF(raw_ostream &OS, GOFFYAML::Object &Doc,
+                        yaml::ErrorHandler ErrHandler);
+
+private:
+  GOFFOstream GW;
+  GOFFYAML::Object &Doc;
+  yaml::ErrorHandler ErrHandler;
+  bool HasError;
+};
+
+void GOFFState::writeHeader(GOFFYAML::FileHeader &FileHdr) {
+  SmallString<16> CCSIDName;
+  if (std::error_code EC =
+          ConverterEBCDIC::convertToEBCDIC(FileHdr.CharacterSetName, CCSIDName))
+    reportError("Conversion error on " + FileHdr.CharacterSetName);
+  if (CCSIDName.size() > 16) {
+    reportError("CharacterSetName too long");
+    CCSIDName.resize(16);
+  }
+  SmallString<16> LangProd;
+  if (std::error_code EC = ConverterEBCDIC::convertToEBCDIC(
+          FileHdr.LanguageProductIdentifier, LangProd))
+    reportError("Conversion error on " + FileHdr.LanguageProductIdentifier);
+  if (LangProd.size() > 16) {
+    reportError("LanguageProductIdentifier too long");
+    LangProd.resize(16);
+  }
+
+  GW.makeNewRecord(GOFF::RT_HDR, GOFF::PayloadLength);
+  GW << binaryBe(FileHdr.TargetEnvironment)     // TargetEnvironment
+     << binaryBe(FileHdr.TargetOperatingSystem) // TargetOperatingSystem
+     << zeros(2)                                // Reserved
+     << binaryBe(FileHdr.CCSID)                 // CCSID
+     << CCSIDName                               // CharacterSetName
+     << zeros(16 - CCSIDName.size())            // Fill bytes
+     << LangProd                                // LanguageProductIdentifier
+     << zeros(16 - LangProd.size())             // Fill bytes
+     << binaryBe(FileHdr.ArchitectureLevel);    // ArchitectureLevel
+  // The module propties are optional. Figure out if we need to write them.
+  uint16_t ModPropLen = 0;
+  if (FileHdr.TargetSoftwareEnvironment)
+    ModPropLen = 3;
+  else if (FileHdr.InternalCCSID)
+    ModPropLen = 2;
+  if (ModPropLen) {
+    GW << binaryBe(ModPropLen) << zeros(6);
+    if (ModPropLen >= 2)
+      GW << binaryBe(FileHdr.InternalCCSID ? *FileHdr.InternalCCSID : 0);
+    if (ModPropLen >= 3)
+      GW << binaryBe(FileHdr.TargetSoftwareEnvironment
+                         ? *FileHdr.TargetSoftwareEnvironment
+                         : 0);
+  }
+}
+
+void GOFFState::writeEnd() {
+  GW.makeNewRecord(GOFF::RT_END, GOFF::PayloadLength);
+  GW << binaryBe(uint8_t(0)) // No entry point
+     << binaryBe(uint8_t(0)) // No AMODE
+     << zeros(3)             // Reserved
+     << binaryBe(GW.logicalRecords());
+  // No entry point yet. Automatically fill remaining space with zero bytes.
+  GW.finalize();
+}
+
+bool GOFFState::writeObject() {
+  writeHeader(Doc.Header);
+  if (HasError)
+    return false;
+  writeEnd();
+  return true;
+}
+
+bool GOFFState::writeGOFF(raw_ostream &OS, GOFFYAML::Object &Doc,
+                          yaml::ErrorHandler ErrHandler) {
+  GOFFState State(OS, Doc, ErrHandler);
+  return State.writeObject();
+}
+} // namespace
+
+namespace llvm {
+namespace yaml {
+
+bool yaml2goff(llvm::GOFFYAML::Object &Doc, raw_ostream &Out,
+               ErrorHandler ErrHandler) {
+  return GOFFState::writeGOFF(Out, Doc, ErrHandler);
+}
+
+} // namespace yaml
+} // namespace llvm

llvm/lib/ObjectYAML/GOFFYAML.cpp

@@ -0,0 +1,46 @@
+//===-- GOFFYAML.cpp - GOFF YAMLIO implementation ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines classes for handling the YAML representation of GOFF.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ObjectYAML/GOFFYAML.h"
+#include "llvm/BinaryFormat/GOFF.h"
+#include <string.h>
+
+namespace llvm {
+namespace GOFFYAML {
+
+Object::Object() {}
+
+} // namespace GOFFYAML
+
+namespace yaml {
+
+void MappingTraits<GOFFYAML::FileHeader>::mapping(
+    IO &IO, GOFFYAML::FileHeader &FileHdr) {
+  IO.mapOptional("TargetEnvironment", FileHdr.TargetEnvironment, 0);
+  IO.mapOptional("TargetOperatingSystem", FileHdr.TargetOperatingSystem, 0);
+  IO.mapOptional("CCSID", FileHdr.CCSID, 0);
+  IO.mapOptional("CharacterSetName", FileHdr.CharacterSetName, "");
+  IO.mapOptional("LanguageProductIdentifier", FileHdr.LanguageProductIdentifier,
+                 "");
+  IO.mapOptional("ArchitectureLevel", FileHdr.ArchitectureLevel, 1);
+  IO.mapOptional("InternalCCSID", FileHdr.InternalCCSID);
+  IO.mapOptional("TargetSoftwareEnvironment",
+                 FileHdr.TargetSoftwareEnvironment);
+}
+
+void MappingTraits<GOFFYAML::Object>::mapping(IO &IO, GOFFYAML::Object &Obj) {
+  IO.mapTag("!GOFF", true);
+  IO.mapRequired("FileHeader", Obj.Header);
+}
+
+} // namespace yaml
+} // namespace llvm

llvm/lib/ObjectYAML/ObjectYAML.cpp

@@ -26,6 +26,8 @@ void MappingTraits<YamlObjectFile>::mapping(IO &IO,
       MappingTraits<ELFYAML::Object>::mapping(IO, *ObjectFile.Elf);
     if (ObjectFile.Coff)
       MappingTraits<COFFYAML::Object>::mapping(IO, *ObjectFile.Coff);
+    if (ObjectFile.Goff)
+      MappingTraits<GOFFYAML::Object>::mapping(IO, *ObjectFile.Goff);
     if (ObjectFile.MachO)
       MappingTraits<MachOYAML::Object>::mapping(IO, *ObjectFile.MachO);
     if (ObjectFile.FatMachO)
@@ -46,6 +48,9 @@ void MappingTraits<YamlObjectFile>::mapping(IO &IO,
     } else if (IO.mapTag("!COFF")) {
       ObjectFile.Coff.reset(new COFFYAML::Object());
       MappingTraits<COFFYAML::Object>::mapping(IO, *ObjectFile.Coff);
+    } else if (IO.mapTag("!GOFF")) {
+      ObjectFile.Goff.reset(new GOFFYAML::Object());
+      MappingTraits<GOFFYAML::Object>::mapping(IO, *ObjectFile.Goff);
     } else if (IO.mapTag("!mach-o")) {
       ObjectFile.MachO.reset(new MachOYAML::Object());
       MappingTraits<MachOYAML::Object>::mapping(IO, *ObjectFile.MachO);

llvm/lib/ObjectYAML/yaml2obj.cpp

@@ -38,6 +38,8 @@ bool convertYAML(yaml::Input &YIn, raw_ostream &Out, ErrorHandler ErrHandler,
       return yaml2elf(*Doc.Elf, Out, ErrHandler, MaxSize);
     if (Doc.Coff)
       return yaml2coff(*Doc.Coff, Out, ErrHandler);
+    if (Doc.Goff)
+      return yaml2goff(*Doc.Goff, Out, ErrHandler);
     if (Doc.MachO || Doc.FatMachO)
       return yaml2macho(Doc, Out, ErrHandler);
     if (Doc.Minidump)

llvm/test/tools/yaml2obj/GOFF/GOFF-header-end.yaml

@@ -0,0 +1,20 @@
+# RUN: yaml2obj %s | od -v -An -tx1 | FileCheck --ignore-case %s
+
+## Verify that GOFF Header is correct.
+# CHECK:      03 f0 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+
+## Verify GOFF Module end.
+# CHECK-NEXT: 03 40 00 00 00 00 00 00 00 00 00 02 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-EMPTY:
+
+--- !GOFF
+FileHeader:
+  ArchitectureLevel: 1

llvm/test/tools/yaml2obj/GOFF/GOFF-header-settings.yaml

@@ -0,0 +1,26 @@
+# RUN: yaml2obj %s | od -v -An -tx1 | FileCheck --ignore-case %s
+
+## Verify that GOFF Header is correct.
+# CHECK:      03 f0 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 01 00 03 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+
+## Verify GOFF Module end.
+# CHECK-NEXT: 03 40 00 00 00 00 00 00 00 00 00 02 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+
+--- !GOFF
+FileHeader:
+  TargetEnvironment: 0
+  TargetOperatingSystem: 0
+  CCSID: 0
+  CharacterSetName: ""
+  LanguageProductIdentifier: ""
+  ArchitectureLevel: 1
+  InternalCCSID: 0
+  TargetSoftwareEnvironment: 0 

llvm/test/tools/yaml2obj/GOFF/GOFF-no-header.yaml

@@ -0,0 +1,7 @@
+# RUN: not yaml2obj %s FileCheck --ignore-case %s
+
+# CHECK: yaml2obj: error: missing required key 'FileHeader'
+--- !GOFF
+## X: [] is an extra field required as workaround for 
+## 'document of unknown type' error
+X: []