caio/clang-p2996
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[ObjectYAML][DX] Support yaml2dxcontainer

Browse Source 
This patch adds a the first bits of support for a yaml representation
of dxcontainer files.

Since the YAML representation's primary purpose is testing
infrastructure, the yaml representation supports both verbose and a
more friendly format by making computable sizes and offsets optional.
If provided they are validated to be correct, otherwise they are
computed on the fly during emission.

As I expand the format I'll be able to make more size fields optional,
and I will continue to make the format easier to work with.

Depends on D124804

Reviewed By: lhames

Differential Revision: https://reviews.llvm.org/D124944
This commit is contained in:
Chris Bieneman
2022-06-01 09:10:28 -05:00
parent 8869ba3662
commit 129c056d62
13 changed files with 445 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
llvm/include/llvm/BinaryFormat/DXContainer.h
View File

@@ -13,6 +13,7 @@
#ifndef LLVM_BINARYFORMAT_DXCONTAINER_H
#define LLVM_BINARYFORMAT_DXCONTAINER_H
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/SwapByteOrder.h"
#include <stdint.h>
@@ -84,6 +85,9 @@ struct PartHeader {
uint32_t Size;
void swapBytes() { sys::swapByteOrder(Size); }
StringRef getName() const {
return StringRef(reinterpret_cast<const char *>(&Name[0]), 4);
}
// Structure is followed directly by part data: uint8_t PartData[PartSize].
};

84
llvm/include/llvm/ObjectYAML/DXContainerYAML.h Normal file
View File

@@ -0,0 +1,84 @@
//===- DXContainerYAML.h - DXContainer YAMLIO implementation ----*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// This file declares classes for handling the YAML representation
/// of DXContainer.
///
//===----------------------------------------------------------------------===//
#ifndef LLVM_OBJECTYAML_DXCONTAINERYAML_H
#define LLVM_OBJECTYAML_DXCONTAINERYAML_H
#include "llvm/ADT/StringRef.h"
#include "llvm/ObjectYAML/YAML.h"
#include "llvm/Support/YAMLTraits.h"
#include <cstdint>
#include <string>
#include <vector>
namespace llvm {
namespace DXContainerYAML {
struct VersionTuple {
uint16_t Major;
uint16_t Minor;
};
// The optional header fields are required in the binary and will be populated
// when reading from binary, but can be omitted in the YAML text because the
// emitter can calculate them.
struct FileHeader {
std::vector<llvm::yaml::Hex8> Hash;
VersionTuple Version;
Optional<uint32_t> FileSize;
uint32_t PartCount;
Optional<std::vector<uint32_t>> PartOffsets;
};
struct Part {
std::string Name;
uint32_t Size;
};
struct Object {
FileHeader Header;
std::vector<Part> Parts;
};
} // namespace DXContainerYAML
} // namespace llvm
LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::DXContainerYAML::Part)
namespace llvm {
class raw_ostream;
namespace yaml {
template <> struct MappingTraits<DXContainerYAML::VersionTuple> {
static void mapping(IO &IO, DXContainerYAML::VersionTuple &Version);
};
template <> struct MappingTraits<DXContainerYAML::FileHeader> {
static void mapping(IO &IO, DXContainerYAML::FileHeader &Header);
};
template <> struct MappingTraits<DXContainerYAML::Part> {
static void mapping(IO &IO, DXContainerYAML::Part &Version);
};
template <> struct MappingTraits<DXContainerYAML::Object> {
static void mapping(IO &IO, DXContainerYAML::Object &Obj);
};
} // namespace yaml
} // namespace llvm
#endif // LLVM_OBJECTYAML_DXCONTAINERYAML_H

2
llvm/include/llvm/ObjectYAML/ObjectYAML.h
View File

@@ -11,6 +11,7 @@
#include "llvm/ObjectYAML/ArchiveYAML.h"
#include "llvm/ObjectYAML/COFFYAML.h"
#include "llvm/ObjectYAML/DXContainerYAML.h"
#include "llvm/ObjectYAML/ELFYAML.h"
#include "llvm/ObjectYAML/MachOYAML.h"
#include "llvm/ObjectYAML/MinidumpYAML.h"
@@ -33,6 +34,7 @@ struct YamlObjectFile {
std::unique_ptr<MinidumpYAML::Object> Minidump;
std::unique_ptr<WasmYAML::Object> Wasm;
std::unique_ptr<XCOFFYAML::Object> Xcoff;
std::unique_ptr<DXContainerYAML::Object> DXContainer;
};
template <> struct MappingTraits<YamlObjectFile> {

6
llvm/include/llvm/ObjectYAML/yaml2obj.h
View File

@@ -48,6 +48,10 @@ namespace ArchYAML {
struct Archive;
}
namespace DXContainerYAML {
struct Object;
} // namespace DXContainerYAML
namespace yaml {
class Input;
struct YamlObjectFile;
@@ -63,6 +67,8 @@ bool yaml2minidump(MinidumpYAML::Object &Doc, raw_ostream &Out,
ErrorHandler EH);
bool yaml2wasm(WasmYAML::Object &Doc, raw_ostream &Out, ErrorHandler EH);
bool yaml2xcoff(XCOFFYAML::Object &Doc, raw_ostream &Out, ErrorHandler EH);
bool yaml2dxcontainer(DXContainerYAML::Object &Doc, raw_ostream &Out,
ErrorHandler EH);
bool convertYAML(Input &YIn, raw_ostream &Out, ErrorHandler ErrHandler,
unsigned DocNum = 1, uint64_t MaxSize = UINT64_MAX);

2
llvm/lib/ObjectYAML/CMakeLists.txt
View File

@@ -9,6 +9,8 @@ add_llvm_component_library(LLVMObjectYAML
COFFYAML.cpp
DWARFEmitter.cpp
DWARFYAML.cpp
DXContainerEmitter.cpp
DXContainerYAML.cpp
ELFEmitter.cpp
ELFYAML.cpp
MachOEmitter.cpp

148
llvm/lib/ObjectYAML/DXContainerEmitter.cpp Normal file
View File

@@ -0,0 +1,148 @@
//===- DXContainerEmitter.cpp - Convert YAML to a DXContainer -------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// Binary emitter for yaml to DXContainer binary
///
//===----------------------------------------------------------------------===//
#include "llvm/BinaryFormat/DXContainer.h"
#include "llvm/ObjectYAML/ObjectYAML.h"
#include "llvm/ObjectYAML/yaml2obj.h"
#include "llvm/Support/Errc.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/raw_ostream.h"
using namespace llvm;
namespace {
class DXContainerWriter {
public:
DXContainerWriter(DXContainerYAML::Object &ObjectFile)
: ObjectFile(ObjectFile) {}
Error write(raw_ostream &OS);
private:
DXContainerYAML::Object &ObjectFile;
Error computePartOffsets();
Error validatePartOffsets();
Error validateSize(uint32_t Computed);
void writeHeader(raw_ostream &OS);
void writeParts(raw_ostream &OS);
};
} // namespace
Error DXContainerWriter::validateSize(uint32_t Computed) {
if (!ObjectFile.Header.FileSize)
ObjectFile.Header.FileSize = Computed;
else if (*ObjectFile.Header.FileSize < Computed)
return createStringError(errc::result_out_of_range,
"File size specified is too small.");
return Error::success();
}
Error DXContainerWriter::validatePartOffsets() {
if (ObjectFile.Parts.size() != ObjectFile.Header.PartOffsets->size())
return createStringError(
errc::invalid_argument,
"Mismatch between number of parts and part offsets.");
uint32_t RollingOffset =
sizeof(dxbc::Header) + (ObjectFile.Header.PartCount * sizeof(uint32_t));
for (auto I : llvm::zip(ObjectFile.Parts, *ObjectFile.Header.PartOffsets)) {
if (RollingOffset > std::get<1>(I))
return createStringError(errc::invalid_argument,
"Offset mismatch, not enough space for data.");
RollingOffset =
std::get<1>(I) + sizeof(dxbc::PartHeader) + std::get<0>(I).Size;
}
if (Error Err = validateSize(RollingOffset))
return Err;
return Error::success();
}
Error DXContainerWriter::computePartOffsets() {
if (ObjectFile.Header.PartOffsets)
return validatePartOffsets();
uint32_t RollingOffset =
sizeof(dxbc::Header) + (ObjectFile.Header.PartCount * sizeof(uint32_t));
ObjectFile.Header.PartOffsets = std::vector<uint32_t>();
for (const auto &Part : ObjectFile.Parts) {
ObjectFile.Header.PartOffsets->push_back(RollingOffset);
RollingOffset += sizeof(dxbc::PartHeader) + Part.Size;
}
if (Error Err = validateSize(RollingOffset))
return Err;
return Error::success();
}
void DXContainerWriter::writeHeader(raw_ostream &OS) {
dxbc::Header Header;
memcpy(Header.Magic, "DXBC", 4);
memcpy(Header.FileHash.Digest, ObjectFile.Header.Hash.data(), 16);
Header.Version.Major = ObjectFile.Header.Version.Major;
Header.Version.Minor = ObjectFile.Header.Version.Minor;
Header.FileSize = *ObjectFile.Header.FileSize;
Header.PartCount = ObjectFile.Parts.size();
if (sys::IsBigEndianHost)
Header.swapBytes();
OS.write(reinterpret_cast<char *>(&Header), sizeof(Header));
for (auto &O : *ObjectFile.Header.PartOffsets)
if (sys::IsBigEndianHost)
sys::swapByteOrder(O);
OS.write(reinterpret_cast<char *>(ObjectFile.Header.PartOffsets->data()),
ObjectFile.Header.PartOffsets->size() * sizeof(uint32_t));
}
void DXContainerWriter::writeParts(raw_ostream &OS) {
uint32_t RollingOffset =
sizeof(dxbc::Header) + (ObjectFile.Header.PartCount * sizeof(uint32_t));
for (auto I : llvm::zip(ObjectFile.Parts, *ObjectFile.Header.PartOffsets)) {
if (RollingOffset < std::get<1>(I)) {
uint32_t PadBytes = std::get<1>(I) - RollingOffset;
std::vector<uint8_t> FillData(PadBytes, 0);
OS.write(reinterpret_cast<char *>(FillData.data()), PadBytes);
}
DXContainerYAML::Part P = std::get<0>(I);
OS.write(P.Name.c_str(), 4);
if (sys::IsBigEndianHost)
sys::swapByteOrder(P.Size);
OS.write(reinterpret_cast<const char *>(&P.Size), sizeof(uint32_t));
RollingOffset = std::get<1>(I) + sizeof(dxbc::PartHeader);
// TODO: Write Part data
}
}
Error DXContainerWriter::write(raw_ostream &OS) {
if (Error Err = computePartOffsets())
return Err;
writeHeader(OS);
writeParts(OS);
return Error::success();
}
namespace llvm {
namespace yaml {
bool yaml2dxcontainer(DXContainerYAML::Object &Doc, raw_ostream &Out,
ErrorHandler EH) {
DXContainerWriter Writer(Doc);
if (Error Err = Writer.write(Out)) {
handleAllErrors(std::move(Err),
[&](const ErrorInfoBase &Err) { EH(Err.message()); });
return false;
}
return true;
}
} // namespace yaml
} // namespace llvm

48
llvm/lib/ObjectYAML/DXContainerYAML.cpp Normal file
View File

@@ -0,0 +1,48 @@
//===- DXContainerYAML.cpp - DXContainer YAMLIO implementation ------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file defines classes for handling the YAML representation of
// DXContainerYAML.
//
//===----------------------------------------------------------------------===//
#include "llvm/ObjectYAML/DXContainerYAML.h"
namespace llvm {
namespace yaml {
void MappingTraits<DXContainerYAML::VersionTuple>::mapping(
IO &IO, DXContainerYAML::VersionTuple &Version) {
IO.mapRequired("Major", Version.Major);
IO.mapRequired("Minor", Version.Minor);
}
void MappingTraits<DXContainerYAML::FileHeader>::mapping(
IO &IO, DXContainerYAML::FileHeader &Header) {
IO.mapRequired("Hash", Header.Hash);
IO.mapRequired("Version", Header.Version);
IO.mapOptional("FileSize", Header.FileSize);
IO.mapRequired("PartCount", Header.PartCount);
IO.mapOptional("PartOffsets", Header.PartOffsets);
}
void MappingTraits<DXContainerYAML::Part>::mapping(IO &IO,
DXContainerYAML::Part &P) {
IO.mapRequired("Name", P.Name);
IO.mapRequired("Size", P.Size);
}
void MappingTraits<DXContainerYAML::Object>::mapping(
IO &IO, DXContainerYAML::Object &Obj) {
IO.mapTag("!dxcontainer", true);
IO.mapRequired("Header", Obj.Header);
IO.mapRequired("Parts", Obj.Parts);
}
} // namespace yaml
} // namespace llvm

4
llvm/lib/ObjectYAML/ObjectYAML.cpp
View File

@@ -62,6 +62,10 @@ void MappingTraits<YamlObjectFile>::mapping(IO &IO,
} else if (IO.mapTag("!XCOFF")) {
ObjectFile.Xcoff.reset(new XCOFFYAML::Object());
MappingTraits<XCOFFYAML::Object>::mapping(IO, *ObjectFile.Xcoff);
} else if (IO.mapTag("!dxcontainer")) {
ObjectFile.DXContainer.reset(new DXContainerYAML::Object());
MappingTraits<DXContainerYAML::Object>::mapping(IO,
*ObjectFile.DXContainer);
} else if (const Node *N = In.getCurrentNode()) {
if (N->getRawTag().empty())
IO.setError("YAML Object File missing document type tag!");

2
llvm/lib/ObjectYAML/yaml2obj.cpp
View File

@@ -46,6 +46,8 @@ bool convertYAML(yaml::Input &YIn, raw_ostream &Out, ErrorHandler ErrHandler,
return yaml2wasm(*Doc.Wasm, Out, ErrHandler);
if (Doc.Xcoff)
return yaml2xcoff(*Doc.Xcoff, Out, ErrHandler);
if (Doc.DXContainer)
return yaml2dxcontainer(*Doc.DXContainer, Out, ErrHandler);
ErrHandler("unknown document type");
return false;

17
llvm/test/tools/obj2yaml/DXContainer/InvalidOffset.yaml Normal file
View File

@@ -0,0 +1,17 @@
# RUN: not yaml2obj %s 2>&1 | FileCheck %s
# CHECK: yaml2obj: error: Offset mismatch, not enough space for data.
--- !dxcontainer
Header:
Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
Version:
Major: 1
Minor: 0
FileSize: 32
PartCount: 1
PartOffsets: [ 0 ]
Parts:
- Name: SFI0
Size: 8
...

18
llvm/test/tools/obj2yaml/DXContainer/InvalidSize.yaml Normal file
View File

@@ -0,0 +1,18 @@
# RUN: not yaml2obj %s 2>&1 | FileCheck %s
# CHECK: yaml2obj: error: File size specified is too small.
--- !dxcontainer
Header:
Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
Version:
Major: 1
Minor: 0
FileSize: 64
PartCount: 2
Parts:
- Name: SFI0
Size: 8
- Name: ISG1
Size: 8
...

1
llvm/unittests/ObjectYAML/CMakeLists.txt
View File

@@ -5,6 +5,7 @@ set(LLVM_LINK_COMPONENTS
add_llvm_unittest(ObjectYAMLTests
DWARFYAMLTest.cpp
DXContainerYAMLTest.cpp
ELFYAMLTest.cpp
MinidumpYAMLTest.cpp
YAML2ObjTest.cpp

109
llvm/unittests/ObjectYAML/DXContainerYAMLTest.cpp Normal file
View File

@@ -0,0 +1,109 @@
//===- DXContainerTest.cpp - Tests for DXContainerFile --------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/ObjectYAML/ObjectYAML.h"
#include "llvm/ObjectYAML/yaml2obj.h"
#include "llvm/Support/MemoryBufferRef.h"
#include "llvm/Support/YAMLTraits.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Testing/Support/Error.h"
#include "gtest/gtest.h"
using namespace llvm;
using namespace llvm::object;
static bool convert(SmallVectorImpl<char> &Output, const char *YAML) {
raw_svector_ostream OS(Output);
yaml::Input YIn(YAML);
return convertYAML(YIn, OS, [](const Twine &Err) { errs() << Err; });
}
TEST(DXCFile, ParseEmptyParts) {
SmallString<128> Storage;
// First read a fully explicit yaml with all sizes and offsets provided
ASSERT_TRUE(convert(Storage, R"(--- !dxcontainer
Header:
Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
Version:
Major: 1
Minor: 0
FileSize: 116
PartCount: 7
PartOffsets: [ 60, 68, 76, 84, 92, 100, 108 ]
Parts:
- Name: SFI0
Size: 0
- Name: ISG1
Size: 0
- Name: OSG1
Size: 0
- Name: PSV0
Size: 0
- Name: STAT
Size: 0
- Name: DXIL
Size: 0
- Name: DEAD
Size: 0
...
)"));
// Result
char Buffer[] = {
0x44, 0x58, 0x42, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
0x74, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3C, 0x00, 0x00, 0x00,
0x44, 0x00, 0x00, 0x00, 0x4C, 0x00, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00,
0x5C, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x6C, 0x00, 0x00, 0x00,
0x53, 0x46, 0x49, 0x30, 0x00, 0x00, 0x00, 0x00, 0x49, 0x53, 0x47, 0x31,
0x00, 0x00, 0x00, 0x00, 0x4F, 0x53, 0x47, 0x31, 0x00, 0x00, 0x00, 0x00,
0x50, 0x53, 0x56, 0x30, 0x00, 0x00, 0x00, 0x00, 0x53, 0x54, 0x41, 0x54,
0x00, 0x00, 0x00, 0x00, 0x44, 0x58, 0x49, 0x4C, 0x00, 0x00, 0x00, 0x00,
0x44, 0x45, 0x41, 0x44, 0x00, 0x00, 0x00, 0x00,
};
EXPECT_EQ(Storage.size(), 116u);
EXPECT_TRUE(memcmp(Buffer, Storage.data(), 116) == 0);
Storage.clear();
// Next, read the same file without the part offsets or file size. Both cases
// should result in the same final output.
ASSERT_TRUE(convert(Storage, R"(--- !dxcontainer
Header:
Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
Version:
Major: 1
Minor: 0
PartCount: 7
Parts:
- Name: SFI0
Size: 0
- Name: ISG1
Size: 0
- Name: OSG1
Size: 0
- Name: PSV0
Size: 0
- Name: STAT
Size: 0
- Name: DXIL
Size: 0
- Name: DEAD
Size: 0
...
)"));
EXPECT_EQ(Storage.size(), 116u);
EXPECT_TRUE(memcmp(Buffer, Storage.data(), 116) == 0);
}