2af2e4dbb7
This commit updates how the `SparseTensorConversion` pass handles `NewOp`. It breaks up the underlying `openSparseTensor` function into two parts (`SparseTensorReader::create` and `SparseTensorReader::readSparseTensor`) so that the pass can inject code for constructing `lvlSizes` between those two parts. Migrating the construction of `lvlSizes` out of the runtime and into the pass is a necessary first step toward fully supporting non-permutations. (The alternative would be for the pass to generate a `FuncOp` for performing the construction and then passing that to the runtime; which doesn't seem to have any benefits over the design of this commit.) And since the pass now generates the code to call these two functions, this change also removes the `Action::kFromFile` value from the enum used by `_mlir_ciface_newSparseTensor`. Reviewed By: aartbik Differential Revision: https://reviews.llvm.org/D138363
1474 lines
63 KiB
C++
1474 lines
63 KiB
C++
//===- SparseTensorConversion.cpp - Sparse tensor primitives conversion ---===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// A pass that converts sparse tensor primitives into calls into a runtime
|
|
// support library. Sparse tensor types are converted into opaque pointers
|
|
// to the underlying sparse storage schemes. The use of opaque pointers
|
|
// together with runtime support library keeps the conversion relatively
|
|
// simple, but at the expense of IR opacity, which obscures opportunities
|
|
// for subsequent optimization of the IR. An alternative is provided by
|
|
// the SparseTensorCodegen pass.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "CodegenUtils.h"
|
|
|
|
#include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
|
|
#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
|
|
#include "mlir/Dialect/Linalg/Utils/Utils.h"
|
|
#include "mlir/Dialect/MemRef/IR/MemRef.h"
|
|
#include "mlir/Dialect/SCF/IR/SCF.h"
|
|
#include "mlir/Dialect/SparseTensor/IR/Enums.h"
|
|
#include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
|
|
#include "mlir/Dialect/SparseTensor/Transforms/Passes.h"
|
|
#include "mlir/Dialect/Tensor/IR/Tensor.h"
|
|
#include "mlir/Transforms/DialectConversion.h"
|
|
|
|
using namespace mlir;
|
|
using namespace mlir::sparse_tensor;
|
|
|
|
namespace {
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Helper methods.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
/// Maps each sparse tensor type to an opaque pointer.
|
|
static Optional<Type> convertSparseTensorTypes(Type type) {
|
|
if (getSparseTensorEncoding(type) != nullptr)
|
|
return LLVM::LLVMPointerType::get(IntegerType::get(type.getContext(), 8));
|
|
return llvm::None;
|
|
}
|
|
|
|
/// Replaces the `op` with a `CallOp` to the function reference returned
|
|
/// by `getFunc()`.
|
|
static func::CallOp replaceOpWithFuncCall(RewriterBase &rewriter, Operation *op,
|
|
StringRef name, TypeRange resultType,
|
|
ValueRange operands,
|
|
EmitCInterface emitCInterface) {
|
|
auto fn = getFunc(op->getParentOfType<ModuleOp>(), name, resultType, operands,
|
|
emitCInterface);
|
|
return rewriter.replaceOpWithNewOp<func::CallOp>(op, resultType, fn,
|
|
operands);
|
|
}
|
|
|
|
/// Generates call to lookup a level-size.
|
|
static Value genLvlSizeCall(OpBuilder &builder, Location loc,
|
|
SparseTensorEncodingAttr &enc, Value src,
|
|
uint64_t lvl) {
|
|
// Generate the call.
|
|
StringRef name = "sparseLvlSize";
|
|
SmallVector<Value, 2> params{ // just two
|
|
src, constantIndex(builder, loc, toStoredDim(enc, lvl))};
|
|
Type iTp = builder.getIndexType();
|
|
return createFuncCall(builder, loc, name, iTp, params, EmitCInterface::Off)
|
|
.getResult(0);
|
|
}
|
|
|
|
/// Compute the size from type (for static sizes) or from an already-converted
|
|
/// opaque pointer source (for dynamic sizes) at the given dimension.
|
|
//
|
|
// FIXME: Need to rename this function to match `genLvlSizeCall` and hence
|
|
// match the naming convention used in the runtime library. However, it's
|
|
// not entirely clear that all callsites of this function properly make the
|
|
// "level"-vs-"dimension" distinction; so need to audit each callsite to
|
|
// ensure this still does what they mean (possibly by having two separate
|
|
// functions, one for levels and one for dimensions). That also means
|
|
// renaming `sizesFromPtr`, `sizesFromType`, etc, to make clear whether
|
|
// they mean to be referring to level-sizes vs dimension-sizes.
|
|
static Value sizeFromPtrAtDim(OpBuilder &builder, Location loc,
|
|
SparseTensorEncodingAttr &enc, ShapedType stp,
|
|
Value src, unsigned i) {
|
|
auto shape = stp.getShape();
|
|
if (shape[i] == ShapedType::kDynamic)
|
|
return genLvlSizeCall(builder, loc, enc, src, i);
|
|
return constantIndex(builder, loc, shape[i]);
|
|
}
|
|
|
|
/// Populates given sizes array from type (for static sizes) and from
|
|
/// an already-converted opaque pointer source (for dynamic sizes).
|
|
static void sizesFromPtr(OpBuilder &builder, SmallVectorImpl<Value> &sizes,
|
|
Location loc, SparseTensorEncodingAttr &enc,
|
|
ShapedType stp, Value src) {
|
|
unsigned rank = stp.getRank();
|
|
sizes.reserve(rank);
|
|
for (unsigned i = 0; i < rank; i++)
|
|
sizes.push_back(sizeFromPtrAtDim(builder, loc, enc, stp, src, i));
|
|
}
|
|
|
|
/// Populates given sizes array from type.
|
|
static void sizesFromType(OpBuilder &builder, SmallVectorImpl<Value> &sizes,
|
|
Location loc, ShapedType stp) {
|
|
auto shape = stp.getShape();
|
|
unsigned rank = stp.getRank();
|
|
sizes.reserve(rank);
|
|
for (unsigned i = 0; i < rank; i++) {
|
|
uint64_t s = shape[i] == ShapedType::kDynamic ? 0 : shape[i];
|
|
sizes.push_back(constantIndex(builder, loc, s));
|
|
}
|
|
}
|
|
|
|
/// Populates the given sizes array for concatenation from type (for static
|
|
/// sizes) and from an already-converted opaque pointer source (for dynamic
|
|
/// sizes).
|
|
static void concatSizesFromInputs(OpBuilder &builder,
|
|
SmallVectorImpl<Value> &sizes, Location loc,
|
|
ShapedType dstTp, ValueRange srcs,
|
|
unsigned dim) {
|
|
auto dstShape = dstTp.getShape();
|
|
|
|
auto srcTp = srcs[0].getType().cast<ShapedType>();
|
|
auto srcEnc = getSparseTensorEncoding(srcTp);
|
|
// We first fills the sizes from an input tensor, and then
|
|
// compute the size of the concatenation dimension if necessary.
|
|
if (srcEnc)
|
|
// Reuses sizes from an arbitrary input tensor is fine.
|
|
sizesFromPtr(builder, sizes, loc, srcEnc, srcTp, srcs[0]);
|
|
else
|
|
sizesFromSrc(builder, sizes, loc, srcs[0]);
|
|
|
|
// Sum up on the `dim` if the dimension is dynamic.
|
|
if (dstShape[dim] != ShapedType::kDynamic) {
|
|
// Faithfully take the static size.
|
|
sizes[dim] = constantIndex(builder, loc, dstShape[dim]);
|
|
} else {
|
|
// Else, compute the shape dynamically.
|
|
for (size_t i = 1, sz = srcs.size(); i < sz; i++) {
|
|
auto srcTp = srcs[i].getType().cast<ShapedType>();
|
|
auto encSrc = getSparseTensorEncoding(srcTp);
|
|
Value srcSz =
|
|
encSrc ? sizeFromPtrAtDim(builder, loc, encSrc, srcTp, srcs[i], dim)
|
|
: linalg::createOrFoldDimOp(builder, loc, srcs[i], dim);
|
|
// Sum up all the sizes.
|
|
sizes[dim] = builder.create<arith::AddIOp>(loc, sizes[dim], srcSz);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Generates an uninitialized buffer of the given size and type,
|
|
/// but returns it as type `memref<? x $tp>` (rather than as type
|
|
/// `memref<$sz x $tp>`). Unlike temporary buffers on the stack,
|
|
/// this buffer must be explicitly deallocated by client.
|
|
static Value genAlloc(RewriterBase &rewriter, Location loc, Value sz, Type tp) {
|
|
auto memTp = MemRefType::get({ShapedType::kDynamic}, tp);
|
|
return rewriter.create<memref::AllocOp>(loc, memTp, ValueRange{sz});
|
|
}
|
|
|
|
/// Generates a temporary buffer of the given type and given contents.
|
|
static Value genBuffer(OpBuilder &builder, Location loc, ValueRange values) {
|
|
unsigned sz = values.size();
|
|
assert(sz >= 1);
|
|
Value buffer = genAlloca(builder, loc, sz, values[0].getType());
|
|
for (unsigned i = 0; i < sz; i++) {
|
|
Value idx = constantIndex(builder, loc, i);
|
|
builder.create<memref::StoreOp>(loc, values[i], buffer, idx);
|
|
}
|
|
return buffer;
|
|
}
|
|
|
|
/// Generates a temporary buffer for the level-types of the given encoding.
|
|
static Value genLvlTypesBuffer(OpBuilder &builder, Location loc,
|
|
SparseTensorEncodingAttr enc) {
|
|
SmallVector<Value> lvlTypes;
|
|
auto dlts = enc.getDimLevelType();
|
|
lvlTypes.reserve(dlts.size());
|
|
for (auto dlt : dlts)
|
|
lvlTypes.push_back(constantDimLevelTypeEncoding(builder, loc, dlt));
|
|
return genBuffer(builder, loc, lvlTypes);
|
|
}
|
|
|
|
/// This class abstracts over the API of `_mlir_ciface_newSparseTensor`:
|
|
/// the "swiss army knife" method of the sparse runtime support library
|
|
/// for materializing sparse tensors into the computation. This abstraction
|
|
/// reduces the need to make modifications to client code whenever that
|
|
/// API changes.
|
|
class NewCallParams final {
|
|
public:
|
|
/// Allocates the `ValueRange` for the `func::CallOp` parameters,
|
|
/// but does not initialize them.
|
|
NewCallParams(OpBuilder &builder, Location loc)
|
|
: builder(builder), loc(loc), pTp(getOpaquePointerType(builder)) {}
|
|
|
|
/// Initializes all static parameters (i.e., those which indicate
|
|
/// type-level information such as the encoding and sizes), generating
|
|
/// MLIR buffers as needed, and returning `this` for method chaining.
|
|
/// This method does not set the action and pointer arguments, since
|
|
/// those are handled by `genNewCall` instead.
|
|
NewCallParams &genBuffers(SparseTensorEncodingAttr enc, ValueRange sizes,
|
|
ShapedType stp);
|
|
|
|
/// (Re)sets the C++ template type parameters, and returns `this`
|
|
/// for method chaining. This is already done as part of `genBuffers`,
|
|
/// but is factored out so that it can also be called independently
|
|
/// whenever subsequent `genNewCall` calls want to reuse the same
|
|
/// buffers but different type parameters.
|
|
//
|
|
// TODO: This is only ever used by sparse2sparse-viaCOO `ConvertOp`;
|
|
// is there a better way to handle that than this one-off setter method?
|
|
NewCallParams &setTemplateTypes(SparseTensorEncodingAttr enc,
|
|
ShapedType stp) {
|
|
params[kParamPtrTp] = constantPointerTypeEncoding(builder, loc, enc);
|
|
params[kParamIndTp] = constantIndexTypeEncoding(builder, loc, enc);
|
|
params[kParamValTp] =
|
|
constantPrimaryTypeEncoding(builder, loc, stp.getElementType());
|
|
return *this;
|
|
}
|
|
|
|
/// Checks whether all the static parameters have been initialized.
|
|
bool isInitialized() const {
|
|
for (unsigned i = 0; i < kNumStaticParams; ++i)
|
|
if (!params[i])
|
|
return false;
|
|
return true;
|
|
}
|
|
|
|
/// Gets the dimension-to-level mapping.
|
|
//
|
|
// TODO: This is only ever used for passing into `genAddEltCall`;
|
|
// is there a better way to encapsulate that pattern (both to avoid
|
|
// this one-off getter, and to avoid potential mixups)?
|
|
Value getDim2LvlMap() const {
|
|
assert(isInitialized() && "Must initialize before getDim2LvlMap");
|
|
return params[kParamDim2Lvl];
|
|
}
|
|
|
|
/// Generates a function call, with the current static parameters
|
|
/// and the given dynamic arguments.
|
|
Value genNewCall(Action action, Value ptr = Value()) {
|
|
assert(isInitialized() && "Must initialize before genNewCall");
|
|
StringRef name = "newSparseTensor";
|
|
params[kParamAction] = constantAction(builder, loc, action);
|
|
params[kParamPtr] = ptr ? ptr : builder.create<LLVM::NullOp>(loc, pTp);
|
|
return createFuncCall(builder, loc, name, pTp, params, EmitCInterface::On)
|
|
.getResult(0);
|
|
}
|
|
|
|
private:
|
|
static constexpr unsigned kNumStaticParams = 8;
|
|
static constexpr unsigned kNumDynamicParams = 2;
|
|
static constexpr unsigned kNumParams = kNumStaticParams + kNumDynamicParams;
|
|
static constexpr unsigned kParamDimSizes = 0;
|
|
static constexpr unsigned kParamLvlSizes = 1;
|
|
static constexpr unsigned kParamLvlTypes = 2;
|
|
static constexpr unsigned kParamLvl2Dim = 3;
|
|
static constexpr unsigned kParamDim2Lvl = 4;
|
|
static constexpr unsigned kParamPtrTp = 5;
|
|
static constexpr unsigned kParamIndTp = 6;
|
|
static constexpr unsigned kParamValTp = 7;
|
|
static constexpr unsigned kParamAction = 8;
|
|
static constexpr unsigned kParamPtr = 9;
|
|
|
|
OpBuilder &builder;
|
|
Location loc;
|
|
Type pTp;
|
|
Value params[kNumParams];
|
|
};
|
|
|
|
// TODO: see the note at `_mlir_ciface_newSparseTensor` about how
|
|
// the meaning of the various arguments (e.g., "sizes" vs "shapes")
|
|
// is inconsistent between the different actions.
|
|
NewCallParams &NewCallParams::genBuffers(SparseTensorEncodingAttr enc,
|
|
ValueRange dimSizes, ShapedType stp) {
|
|
const unsigned lvlRank = enc.getDimLevelType().size();
|
|
const unsigned dimRank = stp.getRank();
|
|
// Sparsity annotations.
|
|
params[kParamLvlTypes] = genLvlTypesBuffer(builder, loc, enc);
|
|
// Dimension-sizes array of the enveloping tensor. Useful for either
|
|
// verification of external data, or for construction of internal data.
|
|
assert(dimSizes.size() == dimRank && "Dimension-rank mismatch");
|
|
params[kParamDimSizes] = genBuffer(builder, loc, dimSizes);
|
|
// The level-sizes array must be passed as well, since for arbitrary
|
|
// dim2lvl mappings it cannot be trivially reconstructed at runtime.
|
|
// For now however, since we're still assuming permutations, we will
|
|
// initialize this parameter alongside the `dim2lvl` and `lvl2dim`
|
|
// parameters below. We preinitialize `lvlSizes` for code symmetry.
|
|
SmallVector<Value> lvlSizes(lvlRank);
|
|
// The dimension-to-level mapping and its inverse. We must preinitialize
|
|
// `dim2lvl` so that the true branch below can perform random-access
|
|
// `operator[]` assignment. We preinitialize `lvl2dim` for code symmetry.
|
|
SmallVector<Value> dim2lvl(dimRank);
|
|
SmallVector<Value> lvl2dim(lvlRank);
|
|
auto dimOrder = enc.getDimOrdering();
|
|
if (dimOrder) {
|
|
assert(dimOrder.isPermutation());
|
|
for (unsigned l = 0; l < lvlRank; l++) {
|
|
// The `d`th source variable occurs in the `l`th result position.
|
|
uint64_t d = dimOrder.getDimPosition(l);
|
|
dim2lvl[d] = constantIndex(builder, loc, l);
|
|
lvl2dim[l] = constantIndex(builder, loc, d);
|
|
lvlSizes[l] = dimSizes[d];
|
|
}
|
|
} else {
|
|
assert(dimRank == lvlRank && "Rank mismatch");
|
|
for (unsigned i = 0; i < lvlRank; i++) {
|
|
dim2lvl[i] = lvl2dim[i] = constantIndex(builder, loc, i);
|
|
lvlSizes[i] = dimSizes[i];
|
|
}
|
|
}
|
|
params[kParamLvlSizes] = genBuffer(builder, loc, lvlSizes);
|
|
params[kParamLvl2Dim] = genBuffer(builder, loc, lvl2dim);
|
|
params[kParamDim2Lvl] =
|
|
dimOrder ? genBuffer(builder, loc, dim2lvl) : params[kParamLvl2Dim];
|
|
// Secondary and primary types encoding.
|
|
setTemplateTypes(enc, stp);
|
|
// Finally, make note that initialization is complete.
|
|
assert(isInitialized() && "Initialization failed");
|
|
// And return `this` for method chaining.
|
|
return *this;
|
|
}
|
|
|
|
/// Generates a call to obtain the values array.
|
|
static Value genValuesCall(OpBuilder &builder, Location loc, ShapedType tp,
|
|
ValueRange ptr) {
|
|
SmallString<15> name{"sparseValues",
|
|
primaryTypeFunctionSuffix(tp.getElementType())};
|
|
return createFuncCall(builder, loc, name, tp, ptr, EmitCInterface::On)
|
|
.getResult(0);
|
|
}
|
|
|
|
/// Generates a call to release/delete a `SparseTensorCOO`.
|
|
static void genDelCOOCall(OpBuilder &builder, Location loc, Type elemTp,
|
|
Value coo) {
|
|
SmallString<21> name{"delSparseTensorCOO", primaryTypeFunctionSuffix(elemTp)};
|
|
createFuncCall(builder, loc, name, {}, coo, EmitCInterface::Off);
|
|
}
|
|
|
|
/// Generates a call to release/delete a `SparseTensorIterator`.
|
|
static void genDelIteratorCall(OpBuilder &builder, Location loc, Type elemTp,
|
|
Value iter) {
|
|
SmallString<26> name{"delSparseTensorIterator",
|
|
primaryTypeFunctionSuffix(elemTp)};
|
|
createFuncCall(builder, loc, name, {}, iter, EmitCInterface::Off);
|
|
}
|
|
|
|
/// Generates a call that adds one element to a coordinate scheme.
|
|
/// In particular, this generates code like the following:
|
|
/// val = a[i1,..,ik];
|
|
/// if val != 0
|
|
/// t->add(&val, [i1,..,ik], [p1,..,pk]);
|
|
static void genAddEltCall(OpBuilder &builder, Location loc, Type eltType,
|
|
Value lvlCOO, Value valPtr, Value dimInd,
|
|
Value dim2lvl) {
|
|
SmallString<9> name{"addElt", primaryTypeFunctionSuffix(eltType)};
|
|
SmallVector<Value, 4> params{lvlCOO, valPtr, dimInd, dim2lvl};
|
|
Type pTp = getOpaquePointerType(builder);
|
|
createFuncCall(builder, loc, name, pTp, params, EmitCInterface::On);
|
|
}
|
|
|
|
/// Generates a call to `iter->getNext()`. If there is a next element,
|
|
/// then it is copied into the out-parameters `ind` and `elemPtr`,
|
|
/// and the return value is true. If there isn't a next element, then
|
|
/// the return value is false.
|
|
static Value genGetNextCall(OpBuilder &builder, Location loc, Value iter,
|
|
Value ind, Value elemPtr) {
|
|
Type elemTp = elemPtr.getType().cast<ShapedType>().getElementType();
|
|
SmallString<10> name{"getNext", primaryTypeFunctionSuffix(elemTp)};
|
|
SmallVector<Value, 3> params{iter, ind, elemPtr};
|
|
Type i1 = builder.getI1Type();
|
|
return createFuncCall(builder, loc, name, i1, params, EmitCInterface::On)
|
|
.getResult(0);
|
|
}
|
|
|
|
/// Converts a pointer to COO (from calls to iter->next()) into a vector of
|
|
/// indices, apply (optional) `offset` on `offsetDim`.
|
|
static SmallVector<Value> loadIndices(OpBuilder &builder, Location loc,
|
|
unsigned rank, Value ind,
|
|
unsigned offsetDim = 0,
|
|
Value offset = Value()) {
|
|
SmallVector<Value> ivs;
|
|
ivs.reserve(rank);
|
|
for (unsigned i = 0; i < rank; i++) {
|
|
Value idx = constantIndex(builder, loc, i);
|
|
idx = builder.create<memref::LoadOp>(loc, ind, idx);
|
|
if (offsetDim == i && offset)
|
|
idx = builder.create<arith::AddIOp>(loc, idx, offset);
|
|
ivs.push_back(idx);
|
|
}
|
|
return ivs;
|
|
}
|
|
|
|
/// Converts the vector indices and store it into the memory pointed by
|
|
/// `ind`, apply (optional) `offset` on `offsetDim`.
|
|
static void storeIndices(OpBuilder &builder, Location loc, unsigned rank,
|
|
Value ind, ValueRange ivs, unsigned offsetDim = 0,
|
|
Value offset = Value()) {
|
|
for (unsigned i = 0; i < rank; i++) {
|
|
Value idx = ivs[i];
|
|
if (offsetDim == i && offset)
|
|
idx = builder.create<arith::AddIOp>(loc, idx, offset);
|
|
builder.create<memref::StoreOp>(loc, idx, ind,
|
|
constantIndex(builder, loc, i));
|
|
}
|
|
}
|
|
|
|
/// Inserts a value stored in `elemPtr` into a dense tensor created by
|
|
/// allocDenseTensor().
|
|
static void insertScalarIntoDenseTensor(OpBuilder &builder, Location loc,
|
|
Value elemPtr, Value tensor,
|
|
ValueRange ivs) {
|
|
Value elemV = builder.create<memref::LoadOp>(loc, elemPtr);
|
|
builder.create<memref::StoreOp>(loc, elemV, tensor, ivs);
|
|
}
|
|
|
|
/// Determine if the runtime library supports direct conversion to the
|
|
/// given target `dimTypes`.
|
|
static bool canUseDirectConversion(ArrayRef<DimLevelType> dimTypes) {
|
|
bool alreadyCompressed = false;
|
|
for (uint64_t rank = dimTypes.size(), r = 0; r < rank; r++) {
|
|
const DimLevelType dlt = dimTypes[r];
|
|
if (isCompressedDLT(dlt)) {
|
|
if (alreadyCompressed)
|
|
return false; // Multiple compressed dimensions not yet supported.
|
|
alreadyCompressed = true;
|
|
} else if (isDenseDLT(dlt)) {
|
|
if (alreadyCompressed)
|
|
return false; // Dense after Compressed not yet supported.
|
|
} else if (isSingletonDLT(dlt)) {
|
|
// Direct conversion doesn't have any particular problems with
|
|
// singleton after compressed.
|
|
} else { // TODO: investigate
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/// Helper method to translate indices during a reshaping operation.
|
|
/// TODO: provide as general utility to MLIR at large?
|
|
static void translateIndices(Location loc, ConversionPatternRewriter &rewriter,
|
|
ArrayRef<ReassociationIndices> reassociation,
|
|
TensorType dstTp, TensorType srcTp, Value dstIdx,
|
|
Value srcIdx, ArrayRef<Value> dstShape,
|
|
ArrayRef<Value> srcShape) {
|
|
unsigned dstRank = dstTp.getRank();
|
|
unsigned srcRank = srcTp.getRank();
|
|
|
|
SmallVector<Value> srcIndices;
|
|
for (unsigned i = 0; i < srcRank; i++) {
|
|
Value idx = rewriter.create<memref::LoadOp>(
|
|
loc, srcIdx, constantIndex(rewriter, loc, i));
|
|
srcIndices.push_back(idx);
|
|
}
|
|
|
|
SmallVector<Value> dstIndices;
|
|
translateIndicesArray(rewriter, loc, reassociation, srcIndices, srcShape,
|
|
dstShape, dstIndices);
|
|
|
|
for (unsigned i = 0; i < dstRank; i++)
|
|
rewriter.create<memref::StoreOp>(loc, dstIndices[i], dstIdx,
|
|
constantIndex(rewriter, loc, i));
|
|
}
|
|
|
|
/// Generate code for a general sparse to sparse reshaping operation.
|
|
/// Note that unlike dense reshaping (which can be done with a "cheap"
|
|
/// change of view), sparse reshaping is currently done with actual
|
|
/// data shuffling.
|
|
///
|
|
/// TODO: proportional to nnz, but still a lot of data movement
|
|
/// https://github.com/llvm/llvm-project/issues/56477
|
|
///
|
|
/// iter = src->toCOO();
|
|
/// coo = newSparseCOO()
|
|
/// while (elem = iter->getNext()) {
|
|
/// coo->add(reshape(elem.indices), elem.value)
|
|
/// }
|
|
/// s = newSparseTensor(coo)
|
|
template <typename ReshapeOp>
|
|
static LogicalResult
|
|
genSparse2SparseReshape(ReshapeOp op, typename ReshapeOp::Adaptor adaptor,
|
|
ConversionPatternRewriter &rewriter) {
|
|
Location loc = op.getLoc();
|
|
auto srcTp = op.getSrc().getType().template cast<RankedTensorType>();
|
|
auto dstTp = op.getResult().getType().template cast<RankedTensorType>();
|
|
auto encSrc = getSparseTensorEncoding(srcTp);
|
|
auto encDst = getSparseTensorEncoding(dstTp);
|
|
if (!encDst || !encSrc)
|
|
return failure();
|
|
|
|
unsigned srcRank = srcTp.getRank();
|
|
unsigned dstRank = dstTp.getRank();
|
|
Type elemTp = srcTp.getElementType();
|
|
assert(elemTp == dstTp.getElementType() &&
|
|
"reshape should not change element type");
|
|
// Start an iterator over the source tensor (in original index order).
|
|
auto noPerm = SparseTensorEncodingAttr::get(
|
|
op->getContext(), encSrc.getDimLevelType(), AffineMap(), AffineMap(),
|
|
encSrc.getPointerBitWidth(), encSrc.getIndexBitWidth());
|
|
SmallVector<Value> srcSizes;
|
|
sizesFromPtr(rewriter, srcSizes, loc, encSrc, srcTp, adaptor.getSrc());
|
|
NewCallParams params(rewriter, loc);
|
|
Value iter = params.genBuffers(noPerm, srcSizes, srcTp)
|
|
.genNewCall(Action::kToIterator, adaptor.getSrc());
|
|
// Start a new COO for the destination tensor.
|
|
SmallVector<Value> dstSizes;
|
|
if (dstTp.hasStaticShape()) {
|
|
sizesFromType(rewriter, dstSizes, loc, dstTp);
|
|
} else {
|
|
ArrayRef<int64_t> dstShape = dstTp.getShape();
|
|
genReshapeDstShape(loc, rewriter, dstSizes, srcSizes, dstShape,
|
|
op.getReassociationIndices());
|
|
}
|
|
Value coo =
|
|
params.genBuffers(encDst, dstSizes, dstTp).genNewCall(Action::kEmptyCOO);
|
|
Value dstPerm = params.getDim2LvlMap();
|
|
// Construct a while loop over the iterator.
|
|
Value srcIdx = genAlloca(rewriter, loc, srcRank, rewriter.getIndexType());
|
|
Value dstIdx = genAlloca(rewriter, loc, dstRank, rewriter.getIndexType());
|
|
Value elemPtr = genAllocaScalar(rewriter, loc, elemTp);
|
|
SmallVector<Value> noArgs;
|
|
SmallVector<Type> noTypes;
|
|
auto whileOp = rewriter.create<scf::WhileOp>(loc, noTypes, noArgs);
|
|
Block *before = rewriter.createBlock(&whileOp.getBefore(), {}, noTypes);
|
|
rewriter.setInsertionPointToEnd(before);
|
|
Value cond = genGetNextCall(rewriter, loc, iter, srcIdx, elemPtr);
|
|
rewriter.create<scf::ConditionOp>(loc, cond, before->getArguments());
|
|
// Translate indices from source to target and insert. Note that we do
|
|
// not need to store the value in elemPtr, as the value is still there.
|
|
Block *after = rewriter.createBlock(&whileOp.getAfter(), {}, noTypes);
|
|
rewriter.setInsertionPointToStart(after);
|
|
translateIndices(loc, rewriter, op.getReassociationIndices(), dstTp, srcTp,
|
|
dstIdx, srcIdx, dstSizes, srcSizes);
|
|
genAddEltCall(rewriter, loc, elemTp, coo, elemPtr, dstIdx, dstPerm);
|
|
rewriter.create<scf::YieldOp>(loc);
|
|
// Final call to construct sparse tensor storage and free temporary resources.
|
|
rewriter.setInsertionPointAfter(whileOp);
|
|
Value dst = params.genNewCall(Action::kFromCOO, coo);
|
|
genDelCOOCall(rewriter, loc, elemTp, coo);
|
|
genDelIteratorCall(rewriter, loc, elemTp, iter);
|
|
rewriter.replaceOp(op, dst);
|
|
return success();
|
|
}
|
|
|
|
// Generates a while loop that iterates over the COO list extracted
|
|
// from `t`, using `bodyBuilder` to build the loop body.
|
|
// while (elem = coo->getNext()) {
|
|
// bodyBuilder
|
|
// }
|
|
// TODO: It can be used by other operators (ReshapeOp, ConvertOP) conversion to
|
|
// reduce code repetition!
|
|
// TODO: rename to `genSparseIterationLoop`?
|
|
static void genSparseCOOIterationLoop(
|
|
ConversionPatternRewriter &rewriter, Location loc, Value t,
|
|
RankedTensorType tensorTp,
|
|
function_ref<void(OpBuilder &, Location, Value, Value)> bodyBuilder) {
|
|
auto enc = getSparseTensorEncoding(tensorTp);
|
|
assert(enc && "Generating Sparse Tensor COO Loop on a Dense Tensor!");
|
|
|
|
unsigned rank = tensorTp.getRank();
|
|
Type elemTp = tensorTp.getElementType();
|
|
|
|
// Start an iterator over the tensor (in original index order).
|
|
auto noPerm = SparseTensorEncodingAttr::get(
|
|
rewriter.getContext(), enc.getDimLevelType(), AffineMap(), AffineMap(),
|
|
enc.getPointerBitWidth(), enc.getIndexBitWidth());
|
|
SmallVector<Value> sizes;
|
|
sizesFromPtr(rewriter, sizes, loc, noPerm, tensorTp, t);
|
|
Value iter = NewCallParams(rewriter, loc)
|
|
.genBuffers(noPerm, sizes, tensorTp)
|
|
.genNewCall(Action::kToIterator, t);
|
|
|
|
// Construct a while loop over the iterator.
|
|
Value srcIdx = genAlloca(rewriter, loc, rank, rewriter.getIndexType());
|
|
Value elemPtr = genAllocaScalar(rewriter, loc, elemTp);
|
|
SmallVector<Value> noArgs;
|
|
SmallVector<Type> noTypes;
|
|
auto whileOp = rewriter.create<scf::WhileOp>(loc, noTypes, noArgs);
|
|
Block *before = rewriter.createBlock(&whileOp.getBefore(), {}, noTypes);
|
|
rewriter.setInsertionPointToEnd(before);
|
|
Value cond = genGetNextCall(rewriter, loc, iter, srcIdx, elemPtr);
|
|
rewriter.create<scf::ConditionOp>(loc, cond, before->getArguments());
|
|
Block *after = rewriter.createBlock(&whileOp.getAfter(), {}, noTypes);
|
|
rewriter.setInsertionPointToStart(after);
|
|
|
|
bool hasDenseDim = llvm::any_of(
|
|
enc.getDimLevelType(), [](DimLevelType dlt) { return isDenseDLT(dlt); });
|
|
if (hasDenseDim) {
|
|
Value elemV = rewriter.create<memref::LoadOp>(loc, elemPtr);
|
|
Value isZero = genIsNonzero(rewriter, loc, elemV);
|
|
scf::IfOp ifOp = rewriter.create<scf::IfOp>(loc, isZero, /*else*/ false);
|
|
rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
|
|
}
|
|
// Callback here to build loop body.
|
|
bodyBuilder(rewriter, loc, srcIdx, elemPtr);
|
|
|
|
// Exit the scope from the IfOp.
|
|
if (hasDenseDim)
|
|
rewriter.setInsertionPointToEnd(after);
|
|
|
|
rewriter.create<scf::YieldOp>(loc);
|
|
// Finish generating loop.
|
|
rewriter.setInsertionPointAfter(whileOp);
|
|
|
|
// Free memory for iterator.
|
|
genDelIteratorCall(rewriter, loc, elemTp, iter);
|
|
}
|
|
|
|
// Generate loop that iterates over a dense tensor.
|
|
// for i1 in dim1
|
|
// ..
|
|
// for ik in dimk
|
|
// val = a[i1,..,ik]
|
|
// if val != 0
|
|
// bodyBuilder(v, [i1, ..., ik])
|
|
// TODO: It can be used by other operators (ReshapeOp, ConvertOP) conversion to
|
|
// reduce code repetition!
|
|
static void genDenseTensorIterationLoop(
|
|
ConversionPatternRewriter &rewriter, Location loc, Value t,
|
|
RankedTensorType tensorTp,
|
|
function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuilder) {
|
|
assert(!getSparseTensorEncoding(tensorTp) &&
|
|
"Generating Dense Tensor Loop on a Sparse Tensor!");
|
|
|
|
unsigned rank = tensorTp.getRank();
|
|
Value zero = constantIndex(rewriter, loc, 0);
|
|
Value one = constantIndex(rewriter, loc, 1);
|
|
|
|
SmallVector<Value> lo;
|
|
SmallVector<Value> hi;
|
|
SmallVector<Value> st;
|
|
|
|
// Fill out loop iteration information.
|
|
for (unsigned i = 0; i < rank; i++) {
|
|
lo.push_back(zero);
|
|
hi.push_back(linalg::createOrFoldDimOp(rewriter, loc, t, i));
|
|
st.push_back(one);
|
|
}
|
|
|
|
scf::buildLoopNest(rewriter, loc, lo, hi, st, {},
|
|
[&](OpBuilder &builder, Location loc, ValueRange ivs,
|
|
ValueRange args) -> scf::ValueVector {
|
|
// Invoke callback to build the body of the loop.
|
|
bodyBuilder(builder, loc, ivs);
|
|
return {};
|
|
});
|
|
}
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Conversion rules.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
/// Sparse conversion rule for returns.
|
|
class SparseReturnConverter : public OpConversionPattern<func::ReturnOp> {
|
|
public:
|
|
using OpConversionPattern::OpConversionPattern;
|
|
LogicalResult
|
|
matchAndRewrite(func::ReturnOp op, OpAdaptor adaptor,
|
|
ConversionPatternRewriter &rewriter) const override {
|
|
rewriter.replaceOpWithNewOp<func::ReturnOp>(op, adaptor.getOperands());
|
|
return success();
|
|
}
|
|
};
|
|
|
|
/// Sparse conversion rule for dimension accesses.
|
|
class SparseTensorToDimSizeConverter
|
|
: public OpConversionPattern<tensor::DimOp> {
|
|
public:
|
|
using OpConversionPattern::OpConversionPattern;
|
|
LogicalResult
|
|
matchAndRewrite(tensor::DimOp op, OpAdaptor adaptor,
|
|
ConversionPatternRewriter &rewriter) const override {
|
|
// Only rewrite annotated DimOp with constant index.
|
|
auto enc = getSparseTensorEncoding(op.getSource().getType());
|
|
if (!enc)
|
|
return failure();
|
|
Optional<int64_t> index = op.getConstantIndex();
|
|
if (!index)
|
|
return failure();
|
|
// Generate the call.
|
|
Value src = adaptor.getOperands()[0];
|
|
int64_t idx = *index;
|
|
rewriter.replaceOp(op,
|
|
genLvlSizeCall(rewriter, op->getLoc(), enc, src, idx));
|
|
return success();
|
|
}
|
|
};
|
|
|
|
/// Sparse conversion rule for trivial tensor casts.
|
|
class SparseCastConverter : public OpConversionPattern<tensor::CastOp> {
|
|
public:
|
|
using OpConversionPattern::OpConversionPattern;
|
|
LogicalResult
|
|
matchAndRewrite(tensor::CastOp op, OpAdaptor adaptor,
|
|
ConversionPatternRewriter &rewriter) const override {
|
|
// Only rewrite identically annotated source/dest.
|
|
auto encDst = getSparseTensorEncoding(op.getType());
|
|
auto encSrc = getSparseTensorEncoding(op.getSource().getType());
|
|
if (!encDst || encDst != encSrc)
|
|
return failure();
|
|
rewriter.replaceOp(op, adaptor.getOperands());
|
|
return success();
|
|
}
|
|
};
|
|
|
|
/// Sparse conversion rule for a reshape operator.
|
|
template <typename ReshapeOp>
|
|
class SparseReshapeConverter : public OpConversionPattern<ReshapeOp> {
|
|
public:
|
|
using OpAdaptor = typename OpConversionPattern<ReshapeOp>::OpAdaptor;
|
|
using OpConversionPattern<ReshapeOp>::OpConversionPattern;
|
|
LogicalResult
|
|
matchAndRewrite(ReshapeOp op, OpAdaptor adaptor,
|
|
ConversionPatternRewriter &rewriter) const override {
|
|
return genSparse2SparseReshape(op, adaptor, rewriter);
|
|
}
|
|
};
|
|
|
|
/// Sparse conversion rule for the new operator.
|
|
class SparseTensorNewConverter : public OpConversionPattern<NewOp> {
|
|
public:
|
|
using OpConversionPattern::OpConversionPattern;
|
|
LogicalResult
|
|
matchAndRewrite(NewOp op, OpAdaptor adaptor,
|
|
ConversionPatternRewriter &rewriter) const override {
|
|
Location loc = op.getLoc();
|
|
auto stp = op.getType().cast<ShapedType>();
|
|
auto enc = getSparseTensorEncoding(stp);
|
|
if (!enc)
|
|
return failure();
|
|
const unsigned dimRank = stp.getRank();
|
|
const unsigned lvlRank = enc.getDimLevelType().size();
|
|
// Construct the dimShape.
|
|
const auto dimShape = stp.getShape();
|
|
SmallVector<Value> dimShapeValues;
|
|
sizesFromType(rewriter, dimShapeValues, loc, stp);
|
|
Value dimShapeBuffer = genBuffer(rewriter, loc, dimShapeValues);
|
|
// Allocate `SparseTensorReader` and perform all initial setup that
|
|
// does not depend on lvlSizes (nor dim2lvl, lvl2dim, etc).
|
|
Type opaqueTp = getOpaquePointerType(rewriter);
|
|
Value valTp =
|
|
constantPrimaryTypeEncoding(rewriter, loc, stp.getElementType());
|
|
Value reader =
|
|
createFuncCall(rewriter, loc, "createCheckedSparseTensorReader",
|
|
opaqueTp,
|
|
{adaptor.getOperands()[0], dimShapeBuffer, valTp},
|
|
EmitCInterface::On)
|
|
.getResult(0);
|
|
// Construct the lvlSizes. If the dimShape is static, then it's
|
|
// identical to dimSizes: so we can compute lvlSizes entirely at
|
|
// compile-time. If dimShape is dynamic, then we'll need to generate
|
|
// code for computing lvlSizes from the `reader`'s actual dimSizes.
|
|
//
|
|
// TODO: For now we're still assuming `dim2lvl` is a permutation.
|
|
// But since we're computing lvlSizes here (rather than in the runtime),
|
|
// we can easily generalize that simply by adjusting this code.
|
|
//
|
|
// FIXME: reduce redundancy vs `NewCallParams::genBuffers`.
|
|
Value dimSizesBuffer;
|
|
if (!stp.hasStaticShape()) {
|
|
Type indexTp = rewriter.getIndexType();
|
|
auto memTp = MemRefType::get({ShapedType::kDynamic}, indexTp);
|
|
dimSizesBuffer =
|
|
createFuncCall(rewriter, loc, "getSparseTensorReaderDimSizes", memTp,
|
|
reader, EmitCInterface::On)
|
|
.getResult(0);
|
|
}
|
|
Value lvlSizesBuffer;
|
|
Value lvl2dimBuffer;
|
|
Value dim2lvlBuffer;
|
|
if (auto dimOrder = enc.getDimOrdering()) {
|
|
assert(dimOrder.isPermutation() && "Got non-permutation");
|
|
// We preinitialize `dim2lvlValues` since we need random-access writing.
|
|
// And we preinitialize the others for stylistic consistency.
|
|
SmallVector<Value> lvlSizeValues(lvlRank);
|
|
SmallVector<Value> lvl2dimValues(lvlRank);
|
|
SmallVector<Value> dim2lvlValues(dimRank);
|
|
for (unsigned l = 0; l < lvlRank; l++) {
|
|
// The `d`th source variable occurs in the `l`th result position.
|
|
uint64_t d = dimOrder.getDimPosition(l);
|
|
Value lvl = constantIndex(rewriter, loc, l);
|
|
Value dim = constantIndex(rewriter, loc, d);
|
|
dim2lvlValues[d] = lvl;
|
|
lvl2dimValues[l] = dim;
|
|
lvlSizeValues[l] =
|
|
(dimShape[d] == ShapedType::kDynamic)
|
|
? rewriter.create<memref::LoadOp>(loc, dimSizesBuffer, dim)
|
|
: dimShapeValues[d];
|
|
}
|
|
lvlSizesBuffer = genBuffer(rewriter, loc, lvlSizeValues);
|
|
lvl2dimBuffer = genBuffer(rewriter, loc, lvl2dimValues);
|
|
dim2lvlBuffer = genBuffer(rewriter, loc, dim2lvlValues);
|
|
} else {
|
|
assert(dimRank == lvlRank && "Rank mismatch");
|
|
SmallVector<Value> iotaValues;
|
|
iotaValues.reserve(lvlRank);
|
|
for (unsigned i = 0; i < lvlRank; i++)
|
|
iotaValues.push_back(constantIndex(rewriter, loc, i));
|
|
lvlSizesBuffer = dimSizesBuffer ? dimSizesBuffer : dimShapeBuffer;
|
|
dim2lvlBuffer = lvl2dimBuffer = genBuffer(rewriter, loc, iotaValues);
|
|
}
|
|
// Use the `reader` to parse the file.
|
|
SmallVector<Value, 8> params{
|
|
reader,
|
|
lvlSizesBuffer,
|
|
genLvlTypesBuffer(rewriter, loc, enc),
|
|
lvl2dimBuffer,
|
|
dim2lvlBuffer,
|
|
constantPointerTypeEncoding(rewriter, loc, enc),
|
|
constantIndexTypeEncoding(rewriter, loc, enc),
|
|
valTp};
|
|
Value tensor = createFuncCall(rewriter, loc, "newSparseTensorFromReader",
|
|
opaqueTp, params, EmitCInterface::On)
|
|
.getResult(0);
|
|
// Free the memory for `reader`.
|
|
createFuncCall(rewriter, loc, "delSparseTensorReader", {}, {reader},
|
|
EmitCInterface::Off);
|
|
rewriter.replaceOp(op, tensor);
|
|
return success();
|
|
}
|
|
};
|
|
|
|
/// Sparse conversion rule for the alloc operator.
|
|
class SparseTensorAllocConverter
|
|
: public OpConversionPattern<bufferization::AllocTensorOp> {
|
|
public:
|
|
using OpConversionPattern::OpConversionPattern;
|
|
LogicalResult
|
|
matchAndRewrite(bufferization::AllocTensorOp op, OpAdaptor adaptor,
|
|
ConversionPatternRewriter &rewriter) const override {
|
|
if (op.getCopy())
|
|
return rewriter.notifyMatchFailure(op,
|
|
"sparse tensor copy not implemented");
|
|
Location loc = op.getLoc();
|
|
RankedTensorType resType = op.getType();
|
|
auto enc = getSparseTensorEncoding(resType);
|
|
if (!enc)
|
|
return failure();
|
|
// Gather all dimension sizes as SSA values.
|
|
SmallVector<Value> sizes;
|
|
unsigned int operandCtr = 0;
|
|
for (int64_t i = 0; i < resType.getRank(); ++i) {
|
|
if (resType.isDynamicDim(i)) {
|
|
sizes.push_back(adaptor.getOperands()[operandCtr++]);
|
|
} else {
|
|
sizes.push_back(
|
|
rewriter.create<arith::ConstantIndexOp>(loc, op.getStaticSize(i)));
|
|
}
|
|
}
|
|
// Generate the call to construct empty tensor. The sizes are
|
|
// explicitly defined by the arguments to the alloc operator.
|
|
rewriter.replaceOp(op,
|
|
NewCallParams(rewriter, loc)
|
|
.genBuffers(enc, sizes, resType.cast<ShapedType>())
|
|
.genNewCall(Action::kEmpty));
|
|
return success();
|
|
}
|
|
};
|
|
|
|
/// Sparse conversion rule for the convert operator.
|
|
class SparseTensorConvertConverter : public OpConversionPattern<ConvertOp> {
|
|
public:
|
|
using OpConversionPattern::OpConversionPattern;
|
|
SparseTensorConvertConverter(MLIRContext *context,
|
|
SparseTensorConversionOptions o)
|
|
: OpConversionPattern<ConvertOp>(context), options(o) {}
|
|
SparseTensorConvertConverter(TypeConverter &typeConv, MLIRContext *context,
|
|
SparseTensorConversionOptions o)
|
|
: OpConversionPattern<ConvertOp>(typeConv, context), options(o) {}
|
|
|
|
LogicalResult
|
|
matchAndRewrite(ConvertOp op, OpAdaptor adaptor,
|
|
ConversionPatternRewriter &rewriter) const override {
|
|
Location loc = op->getLoc();
|
|
Type resType = op.getType();
|
|
Type srcType = op.getSource().getType();
|
|
auto encDst = getSparseTensorEncoding(resType);
|
|
auto encSrc = getSparseTensorEncoding(srcType);
|
|
Value src = adaptor.getOperands()[0];
|
|
if (encDst && encSrc) {
|
|
// This is a sparse => sparse conversion, which is handled as follows:
|
|
// t = src->toCOO(); ; src to COO in dst order
|
|
// dst = newSparseTensor(t)
|
|
// Using the coordinate scheme as an intermediate does not always
|
|
// yield the fastest conversion but avoids the need for a full
|
|
// O(N^2) conversion matrix.
|
|
if (encDst == encSrc) {
|
|
rewriter.replaceOp(op, adaptor.getOperands()); // hidden nop cast
|
|
return success();
|
|
}
|
|
SmallVector<Value> sizes;
|
|
NewCallParams params(rewriter, loc);
|
|
ShapedType stp = srcType.cast<ShapedType>();
|
|
sizesFromPtr(rewriter, sizes, loc, encSrc, stp, src);
|
|
bool useDirectConversion;
|
|
switch (options.sparseToSparseStrategy) {
|
|
case SparseToSparseConversionStrategy::kViaCOO:
|
|
useDirectConversion = false;
|
|
break;
|
|
case SparseToSparseConversionStrategy::kDirect:
|
|
useDirectConversion = true;
|
|
assert(canUseDirectConversion(encDst.getDimLevelType()) &&
|
|
"Unsupported target for direct sparse-to-sparse conversion");
|
|
break;
|
|
case SparseToSparseConversionStrategy::kAuto:
|
|
useDirectConversion = canUseDirectConversion(encDst.getDimLevelType());
|
|
break;
|
|
}
|
|
if (useDirectConversion) {
|
|
rewriter.replaceOp(op, params.genBuffers(encDst, sizes, stp)
|
|
.genNewCall(Action::kSparseToSparse, src));
|
|
} else { // use via-COO conversion.
|
|
// Set up encoding with right mix of src and dst so that the two
|
|
// method calls can share most parameters, while still providing
|
|
// the correct sparsity information to either of them.
|
|
auto enc = SparseTensorEncodingAttr::get(
|
|
op->getContext(), encDst.getDimLevelType(), encDst.getDimOrdering(),
|
|
encDst.getHigherOrdering(), encSrc.getPointerBitWidth(),
|
|
encSrc.getIndexBitWidth());
|
|
// TODO: This is the only place where `kToCOO` (or `kToIterator`)
|
|
// is called with a non-identity permutation. Is there any clean
|
|
// way to push the permutation over to the `kFromCOO` side instead?
|
|
Value coo =
|
|
params.genBuffers(enc, sizes, stp).genNewCall(Action::kToCOO, src);
|
|
Value dst = params.setTemplateTypes(encDst, stp)
|
|
.genNewCall(Action::kFromCOO, coo);
|
|
genDelCOOCall(rewriter, loc, stp.getElementType(), coo);
|
|
rewriter.replaceOp(op, dst);
|
|
}
|
|
return success();
|
|
}
|
|
if (!encDst && encSrc) {
|
|
// This is sparse => dense conversion, which is handled as follows:
|
|
// dst = new Tensor(0);
|
|
// iter = new SparseTensorIterator(src);
|
|
// while (elem = iter->getNext()) {
|
|
// dst[elem.indices] = elem.value;
|
|
// }
|
|
// delete iter;
|
|
RankedTensorType dstTensorTp = resType.cast<RankedTensorType>();
|
|
RankedTensorType srcTensorTp = srcType.cast<RankedTensorType>();
|
|
unsigned rank = dstTensorTp.getRank();
|
|
Type elemTp = dstTensorTp.getElementType();
|
|
// Fabricate a no-permutation encoding for NewCallParams
|
|
// The pointer/index types must be those of `src`.
|
|
// The dimLevelTypes aren't actually used by Action::kToIterator.
|
|
encDst = SparseTensorEncodingAttr::get(
|
|
op->getContext(),
|
|
SmallVector<DimLevelType>(rank, DimLevelType::Dense), AffineMap(),
|
|
AffineMap(), encSrc.getPointerBitWidth(), encSrc.getIndexBitWidth());
|
|
SmallVector<Value> sizes;
|
|
sizesFromPtr(rewriter, sizes, loc, encSrc, srcTensorTp, src);
|
|
Value iter = NewCallParams(rewriter, loc)
|
|
.genBuffers(encDst, sizes, dstTensorTp)
|
|
.genNewCall(Action::kToIterator, src);
|
|
Value ind = genAlloca(rewriter, loc, rank, rewriter.getIndexType());
|
|
Value elemPtr = genAllocaScalar(rewriter, loc, elemTp);
|
|
Block *insertionBlock = rewriter.getInsertionBlock();
|
|
// TODO: Dense buffers should be allocated/deallocated via the callback
|
|
// in BufferizationOptions.
|
|
Value dst = allocDenseTensor(rewriter, loc, dstTensorTp, sizes);
|
|
SmallVector<Value> noArgs;
|
|
SmallVector<Type> noTypes;
|
|
auto whileOp = rewriter.create<scf::WhileOp>(loc, noTypes, noArgs);
|
|
Block *before = rewriter.createBlock(&whileOp.getBefore(), {}, noTypes);
|
|
rewriter.setInsertionPointToEnd(before);
|
|
Value cond = genGetNextCall(rewriter, loc, iter, ind, elemPtr);
|
|
rewriter.create<scf::ConditionOp>(loc, cond, before->getArguments());
|
|
Block *after = rewriter.createBlock(&whileOp.getAfter(), {}, noTypes);
|
|
rewriter.setInsertionPointToStart(after);
|
|
SmallVector<Value> ivs = loadIndices(rewriter, loc, rank, ind);
|
|
insertScalarIntoDenseTensor(rewriter, loc, elemPtr, dst, ivs);
|
|
rewriter.create<scf::YieldOp>(loc);
|
|
rewriter.setInsertionPointAfter(whileOp);
|
|
genDelIteratorCall(rewriter, loc, elemTp, iter);
|
|
rewriter.replaceOpWithNewOp<bufferization::ToTensorOp>(op, resType, dst);
|
|
// Deallocate the buffer.
|
|
if (bufferization::allocationDoesNotEscape(op->getOpResult(0))) {
|
|
rewriter.setInsertionPoint(insertionBlock->getTerminator());
|
|
deallocDenseTensor(rewriter, loc, dst);
|
|
}
|
|
return success();
|
|
}
|
|
if (!encDst && !encSrc) {
|
|
// dense => dense
|
|
return failure();
|
|
}
|
|
// This is a dense => sparse conversion or a sparse constant in COO =>
|
|
// sparse conversion, which is handled as follows:
|
|
// t = newSparseCOO()
|
|
// ...code to fill the COO tensor t...
|
|
// s = newSparseTensor(t)
|
|
//
|
|
// To fill the COO tensor from a dense tensor:
|
|
// for i1 in dim1
|
|
// ..
|
|
// for ik in dimk
|
|
// val = a[i1,..,ik]
|
|
// if val != 0
|
|
// t->add(val, [i1,..,ik], [p1,..,pk])
|
|
//
|
|
// To fill the COO tensor from a sparse constant in COO format:
|
|
// for i in range(NNZ)
|
|
// val = values[i]
|
|
// [i1,..,ik] = indices[i]
|
|
// t->add(val, [i1,..,ik], [p1,..,pk])
|
|
//
|
|
// Note that the dense tensor traversal code is actually implemented
|
|
// using MLIR IR to avoid having to expose too much low-level
|
|
// memref traversal details to the runtime support library.
|
|
// Also note that the code below only generates the "new" ops and
|
|
// the loop-nest per se; whereas the entire body of the innermost
|
|
// loop is generated by genAddElt().
|
|
ShapedType stp = resType.cast<ShapedType>();
|
|
unsigned rank = stp.getRank();
|
|
SmallVector<Value> sizes;
|
|
sizesFromSrc(rewriter, sizes, loc, src);
|
|
NewCallParams params(rewriter, loc);
|
|
Value coo =
|
|
params.genBuffers(encDst, sizes, stp).genNewCall(Action::kEmptyCOO);
|
|
Value ind = genAlloca(rewriter, loc, rank, rewriter.getIndexType());
|
|
Value perm = params.getDim2LvlMap();
|
|
Type eltType = stp.getElementType();
|
|
Value elemPtr = genAllocaScalar(rewriter, loc, eltType);
|
|
genDenseTensorOrSparseConstantIterLoop(
|
|
rewriter, loc, src, rank,
|
|
[&](OpBuilder &builder, Location loc, Value val, ValueRange indices) {
|
|
for (unsigned i = 0; i < rank; i++) {
|
|
Value idx = constantIndex(builder, loc, i);
|
|
builder.create<memref::StoreOp>(loc, indices[i], ind, idx);
|
|
}
|
|
builder.create<memref::StoreOp>(loc, val, elemPtr);
|
|
genAddEltCall(builder, loc, eltType, coo, elemPtr, ind, perm);
|
|
});
|
|
// Final call to construct sparse tensor storage.
|
|
Value dst = params.genNewCall(Action::kFromCOO, coo);
|
|
genDelCOOCall(rewriter, loc, eltType, coo);
|
|
rewriter.replaceOp(op, dst);
|
|
return success();
|
|
}
|
|
|
|
private:
|
|
/// Options to control sparse code generation.
|
|
SparseTensorConversionOptions options;
|
|
};
|
|
|
|
/// Sparse conversion rule for the dealloc operator.
|
|
class SparseTensorDeallocConverter
|
|
: public OpConversionPattern<bufferization::DeallocTensorOp> {
|
|
public:
|
|
using OpConversionPattern::OpConversionPattern;
|
|
LogicalResult
|
|
matchAndRewrite(bufferization::DeallocTensorOp op, OpAdaptor adaptor,
|
|
ConversionPatternRewriter &rewriter) const override {
|
|
auto enc = getSparseTensorEncoding(op.getTensor().getType());
|
|
if (!enc)
|
|
return failure();
|
|
StringRef name = "delSparseTensor";
|
|
createFuncCall(rewriter, op->getLoc(), name, {}, adaptor.getOperands(),
|
|
EmitCInterface::Off);
|
|
rewriter.eraseOp(op);
|
|
return success();
|
|
}
|
|
};
|
|
|
|
/// Sparse conversion rule for pointer accesses.
|
|
class SparseTensorToPointersConverter
|
|
: public OpConversionPattern<ToPointersOp> {
|
|
public:
|
|
using OpConversionPattern::OpConversionPattern;
|
|
LogicalResult
|
|
matchAndRewrite(ToPointersOp op, OpAdaptor adaptor,
|
|
ConversionPatternRewriter &rewriter) const override {
|
|
Type resType = op.getType();
|
|
Type ptrType = resType.cast<ShapedType>().getElementType();
|
|
SmallString<16> name{"sparsePointers", overheadTypeFunctionSuffix(ptrType)};
|
|
Value dim =
|
|
constantIndex(rewriter, op->getLoc(), op.getDimension().getZExtValue());
|
|
replaceOpWithFuncCall(rewriter, op, name, resType,
|
|
{adaptor.getTensor(), dim}, EmitCInterface::On);
|
|
return success();
|
|
}
|
|
};
|
|
|
|
/// Sparse conversion rule for index accesses.
|
|
class SparseTensorToIndicesConverter : public OpConversionPattern<ToIndicesOp> {
|
|
public:
|
|
using OpConversionPattern::OpConversionPattern;
|
|
LogicalResult
|
|
matchAndRewrite(ToIndicesOp op, OpAdaptor adaptor,
|
|
ConversionPatternRewriter &rewriter) const override {
|
|
Type resType = op.getType();
|
|
Type indType = resType.cast<ShapedType>().getElementType();
|
|
SmallString<15> name{"sparseIndices", overheadTypeFunctionSuffix(indType)};
|
|
Value dim =
|
|
constantIndex(rewriter, op->getLoc(), op.getDimension().getZExtValue());
|
|
replaceOpWithFuncCall(rewriter, op, name, resType,
|
|
{adaptor.getTensor(), dim}, EmitCInterface::On);
|
|
return success();
|
|
}
|
|
};
|
|
|
|
/// Sparse conversion rule for value accesses.
|
|
class SparseTensorToValuesConverter : public OpConversionPattern<ToValuesOp> {
|
|
public:
|
|
using OpConversionPattern::OpConversionPattern;
|
|
LogicalResult
|
|
matchAndRewrite(ToValuesOp op, OpAdaptor adaptor,
|
|
ConversionPatternRewriter &rewriter) const override {
|
|
auto resType = op.getType().cast<ShapedType>();
|
|
rewriter.replaceOp(op, genValuesCall(rewriter, op.getLoc(), resType,
|
|
adaptor.getOperands()));
|
|
return success();
|
|
}
|
|
};
|
|
|
|
/// Sparse conversion rule for number of entries operator.
|
|
class SparseNumberOfEntriesConverter
|
|
: public OpConversionPattern<NumberOfEntriesOp> {
|
|
public:
|
|
using OpConversionPattern::OpConversionPattern;
|
|
LogicalResult
|
|
matchAndRewrite(NumberOfEntriesOp op, OpAdaptor adaptor,
|
|
ConversionPatternRewriter &rewriter) const override {
|
|
Location loc = op.getLoc();
|
|
// Query values array size for the actually stored values size.
|
|
Type eltType = op.getTensor().getType().cast<ShapedType>().getElementType();
|
|
auto resTp = MemRefType::get({ShapedType::kDynamic}, eltType);
|
|
Value values = genValuesCall(rewriter, loc, resTp, adaptor.getOperands());
|
|
rewriter.replaceOpWithNewOp<memref::DimOp>(op, values,
|
|
constantIndex(rewriter, loc, 0));
|
|
return success();
|
|
}
|
|
};
|
|
|
|
/// Sparse conversion rule for tensor rematerialization.
|
|
class SparseTensorLoadConverter : public OpConversionPattern<LoadOp> {
|
|
public:
|
|
using OpConversionPattern::OpConversionPattern;
|
|
LogicalResult
|
|
matchAndRewrite(LoadOp op, OpAdaptor adaptor,
|
|
ConversionPatternRewriter &rewriter) const override {
|
|
if (op.getHasInserts()) {
|
|
// Finalize any pending insertions.
|
|
StringRef name = "endInsert";
|
|
createFuncCall(rewriter, op->getLoc(), name, {}, adaptor.getOperands(),
|
|
EmitCInterface::Off);
|
|
}
|
|
rewriter.replaceOp(op, adaptor.getOperands());
|
|
return success();
|
|
}
|
|
};
|
|
|
|
/// Sparse conversion rule for the insertion operator.
|
|
class SparseTensorInsertConverter : public OpConversionPattern<InsertOp> {
|
|
public:
|
|
using OpConversionPattern::OpConversionPattern;
|
|
LogicalResult
|
|
matchAndRewrite(InsertOp op, OpAdaptor adaptor,
|
|
ConversionPatternRewriter &rewriter) const override {
|
|
// Note that the current regime only allows for strict lexicographic
|
|
// index order. All values are passed by reference through stack
|
|
// allocated memrefs.
|
|
Location loc = op->getLoc();
|
|
auto tp = op.getTensor().getType().cast<RankedTensorType>();
|
|
auto elemTp = tp.getElementType();
|
|
unsigned rank = tp.getRank();
|
|
auto mref = genAlloca(rewriter, loc, rank, rewriter.getIndexType());
|
|
auto vref = genAllocaScalar(rewriter, loc, elemTp);
|
|
for (unsigned i = 0; i < rank; i++)
|
|
rewriter.create<memref::StoreOp>(loc, adaptor.getIndices()[i], mref,
|
|
constantIndex(rewriter, loc, i));
|
|
rewriter.create<memref::StoreOp>(loc, adaptor.getValue(), vref);
|
|
SmallString<12> name{"lexInsert", primaryTypeFunctionSuffix(elemTp)};
|
|
createFuncCall(rewriter, loc, name, {}, {adaptor.getTensor(), mref, vref},
|
|
EmitCInterface::On);
|
|
rewriter.replaceOp(op, adaptor.getTensor());
|
|
return success();
|
|
}
|
|
};
|
|
|
|
/// Sparse conversion rule for the expand operator.
|
|
class SparseTensorExpandConverter : public OpConversionPattern<ExpandOp> {
|
|
public:
|
|
using OpConversionPattern::OpConversionPattern;
|
|
LogicalResult
|
|
matchAndRewrite(ExpandOp op, OpAdaptor adaptor,
|
|
ConversionPatternRewriter &rewriter) const override {
|
|
Location loc = op->getLoc();
|
|
RankedTensorType srcType =
|
|
op.getTensor().getType().cast<RankedTensorType>();
|
|
Type eltType = srcType.getElementType();
|
|
Type boolType = rewriter.getIntegerType(1);
|
|
Type idxType = rewriter.getIndexType();
|
|
// All initialization should be done on entry of the loop nest.
|
|
rewriter.setInsertionPointAfter(op.getTensor().getDefiningOp());
|
|
// Determine the size for access expansion (always the innermost stored
|
|
// dimension size, translated back to original dimension).
|
|
auto enc = getSparseTensorEncoding(srcType);
|
|
unsigned innerDim = toOrigDim(srcType, srcType.getRank() - 1);
|
|
auto sz = sizeFromPtrAtDim(rewriter, loc, enc, srcType, adaptor.getTensor(),
|
|
innerDim);
|
|
// Allocate temporary buffers for values, filled-switch, and indices.
|
|
// We do not use stack buffers for this, since the expanded size may
|
|
// be rather large (as it envelops a single expanded dense dimension).
|
|
Value values = genAlloc(rewriter, loc, sz, eltType);
|
|
Value filled = genAlloc(rewriter, loc, sz, boolType);
|
|
Value indices = genAlloc(rewriter, loc, sz, idxType);
|
|
Value zero = constantZero(rewriter, loc, idxType);
|
|
// Reset the values/filled-switch to all-zero/false. Note that this
|
|
// introduces an O(N) operation into the computation, but this reset
|
|
// operation is amortized over the innermost loops for the access
|
|
// pattern expansion. As noted in the operation doc, we would like
|
|
// to amortize this setup cost even between kernels.
|
|
rewriter.create<linalg::FillOp>(
|
|
loc, ValueRange{constantZero(rewriter, loc, eltType)},
|
|
ValueRange{values});
|
|
rewriter.create<linalg::FillOp>(
|
|
loc, ValueRange{constantZero(rewriter, loc, boolType)},
|
|
ValueRange{filled});
|
|
// Replace expansion op with these buffers and initial index.
|
|
assert(op.getNumResults() == 4);
|
|
rewriter.replaceOp(op, {values, filled, indices, zero});
|
|
return success();
|
|
}
|
|
};
|
|
|
|
/// Sparse conversion rule for the compress operator.
|
|
class SparseTensorCompressConverter : public OpConversionPattern<CompressOp> {
|
|
public:
|
|
using OpConversionPattern::OpConversionPattern;
|
|
LogicalResult
|
|
matchAndRewrite(CompressOp op, OpAdaptor adaptor,
|
|
ConversionPatternRewriter &rewriter) const override {
|
|
Location loc = op->getLoc();
|
|
// Note that this method call resets the values/filled-switch back to
|
|
// all-zero/false by only iterating over the set elements, so the
|
|
// complexity remains proportional to the sparsity of the expanded
|
|
// access pattern.
|
|
Value values = adaptor.getValues();
|
|
Value filled = adaptor.getFilled();
|
|
Value added = adaptor.getAdded();
|
|
Value count = adaptor.getCount();
|
|
Value tensor = adaptor.getTensor();
|
|
auto tp = op.getTensor().getType().cast<RankedTensorType>();
|
|
Type elemTp = tp.getElementType();
|
|
unsigned rank = tp.getRank();
|
|
auto mref = genAlloca(rewriter, loc, rank, rewriter.getIndexType());
|
|
for (unsigned i = 0; i < rank - 1; i++)
|
|
rewriter.create<memref::StoreOp>(loc, adaptor.getIndices()[i], mref,
|
|
constantIndex(rewriter, loc, i));
|
|
SmallString<12> name{"expInsert", primaryTypeFunctionSuffix(elemTp)};
|
|
createFuncCall(rewriter, loc, name, {},
|
|
{tensor, mref, values, filled, added, count},
|
|
EmitCInterface::On);
|
|
rewriter.replaceOp(op, adaptor.getTensor());
|
|
// Deallocate the buffers on exit of the loop nest.
|
|
Operation *parent = getTop(op);
|
|
rewriter.setInsertionPointAfter(parent);
|
|
rewriter.create<memref::DeallocOp>(loc, values);
|
|
rewriter.create<memref::DeallocOp>(loc, filled);
|
|
rewriter.create<memref::DeallocOp>(loc, added);
|
|
return success();
|
|
}
|
|
};
|
|
|
|
/// Sparse conversion rule for the concatenate operator.
|
|
class SparseTensorConcatConverter : public OpConversionPattern<ConcatenateOp> {
|
|
public:
|
|
using OpConversionPattern::OpConversionPattern;
|
|
LogicalResult
|
|
matchAndRewrite(ConcatenateOp op, OpAdaptor adaptor,
|
|
ConversionPatternRewriter &rewriter) const override {
|
|
// The conversion works as follow:
|
|
// (1). When output is sparse, and mix of inputs:
|
|
// a_sparse = concat (b_dense, c_sparse, ....)
|
|
// =>
|
|
// coo_for_a = newSparseCOO(shapeOf(a))
|
|
// for i, j, k // dense input
|
|
// coo->add(adjustForOffset(i,j,k), b[i,j,k])
|
|
//
|
|
// for elem in sparse_input
|
|
// coo->add(adjustForOffset(elem.indices), elem.value)
|
|
// ...
|
|
// a = newSparseTensor(coo_for_a)
|
|
// return a
|
|
//
|
|
// (2). When output is dense, and mix of inputs:
|
|
// a_dense = concat (b_dense, c_sparse, ....)
|
|
// =>
|
|
// a = malloc(shapeOf(a))
|
|
// for i, j, k // dense input
|
|
// a[ adjustForOffset(i,j,k) ] = b[i,j,k]
|
|
//
|
|
// for elem in sparse_input
|
|
// a[ adjustForOffset(elem.indices) ] = elem.value
|
|
// return a
|
|
Location loc = op.getLoc();
|
|
auto dstTp = op.getType().cast<RankedTensorType>();
|
|
auto encDst = getSparseTensorEncoding(dstTp);
|
|
Type elemTp = dstTp.getElementType();
|
|
uint64_t concatDim = op.getDimension().getZExtValue();
|
|
unsigned rank = dstTp.getRank();
|
|
|
|
Value dst; // destination tensor
|
|
Value dstPerm; // destination tensor permutation (if sparse out)
|
|
// A pointer to the value being inserted (if dense => sparse)
|
|
Value elemPtr;
|
|
// Memory that holds the COO for destination tensor (if sparse out)
|
|
Value dstIdx;
|
|
// The offset applied to the dimenstion to be concated (starting from 0)
|
|
Value offset = constantIndex(rewriter, loc, 0);
|
|
|
|
SmallVector<Value> sizes;
|
|
NewCallParams params(rewriter, loc);
|
|
concatSizesFromInputs(rewriter, sizes, loc, dstTp, op.getInputs(),
|
|
concatDim);
|
|
|
|
if (encDst) {
|
|
// Start a new COO for the destination tensor.
|
|
dst =
|
|
params.genBuffers(encDst, sizes, dstTp).genNewCall(Action::kEmptyCOO);
|
|
dstPerm = params.getDim2LvlMap();
|
|
elemPtr = genAllocaScalar(rewriter, loc, elemTp);
|
|
dstIdx = genAlloca(rewriter, loc, rank, rewriter.getIndexType());
|
|
} else {
|
|
// TODO: Dense buffers should be allocated/deallocated via the callback
|
|
// in BufferizationOptions.
|
|
dst = allocDenseTensor(rewriter, loc, dstTp, sizes);
|
|
}
|
|
for (auto it : llvm::zip(op.getInputs(), adaptor.getInputs())) {
|
|
Value orignalOp = std::get<0>(it); // Input (with encoding) from Op
|
|
Value adaptedOp = std::get<1>(it); // Input (type converted) from adaptor
|
|
RankedTensorType srcTp = orignalOp.getType().cast<RankedTensorType>();
|
|
auto encSrc = getSparseTensorEncoding(srcTp);
|
|
if (encSrc) {
|
|
genSparseCOOIterationLoop(
|
|
rewriter, loc, adaptedOp, srcTp,
|
|
[&](OpBuilder &builder, Location loc, Value idx,
|
|
Value elemPtr) -> void {
|
|
auto indVec =
|
|
loadIndices(builder, loc, rank, idx, concatDim, offset);
|
|
if (encDst) {
|
|
// Case: sparse => sparse
|
|
storeIndices(builder, loc, rank, dstIdx, indVec);
|
|
genAddEltCall(builder, loc, elemTp, dst, elemPtr, dstIdx,
|
|
dstPerm);
|
|
} else {
|
|
// Case: sparse => dense
|
|
insertScalarIntoDenseTensor(builder, loc, elemPtr, dst, indVec);
|
|
}
|
|
});
|
|
} else {
|
|
genDenseTensorIterationLoop(
|
|
rewriter, loc, adaptedOp, srcTp,
|
|
[&](OpBuilder &builder, Location loc, ValueRange idx) -> void {
|
|
if (encDst) {
|
|
// Case: dense => sparse
|
|
storeIndices(builder, loc, rank, dstIdx, idx, concatDim,
|
|
offset);
|
|
Value val = genValueForDense(builder, loc, adaptedOp, idx);
|
|
builder.create<memref::StoreOp>(loc, val, elemPtr);
|
|
genAddEltCall(builder, loc, elemTp, dst, elemPtr, dstIdx,
|
|
dstPerm);
|
|
} else {
|
|
// Case: dense => dense
|
|
Value val = genValueForDense(builder, loc, adaptedOp, idx);
|
|
SmallVector<Value> indVec(idx);
|
|
// Apply offset.
|
|
indVec[concatDim] = builder.create<arith::AddIOp>(
|
|
loc, indVec[concatDim], offset);
|
|
builder.create<memref::StoreOp>(loc, val, dst, indVec);
|
|
}
|
|
});
|
|
}
|
|
// Accumulate offset.
|
|
// TODO: avoid calling sparseDimSize multiple times by caching the result!
|
|
Value curDim = encSrc ? sizeFromPtrAtDim(rewriter, loc, encSrc, srcTp,
|
|
adaptedOp, concatDim)
|
|
: linalg::createOrFoldDimOp(rewriter, loc,
|
|
adaptedOp, concatDim);
|
|
|
|
offset = rewriter.create<arith::AddIOp>(loc, offset, curDim);
|
|
}
|
|
if (encDst) {
|
|
// In sparse output case, the destination holds the COO.
|
|
Value coo = dst;
|
|
dst = params.genNewCall(Action::kFromCOO, coo);
|
|
// Release resources.
|
|
genDelCOOCall(rewriter, loc, elemTp, coo);
|
|
rewriter.replaceOp(op, dst);
|
|
} else {
|
|
rewriter.replaceOpWithNewOp<bufferization::ToTensorOp>(op, dstTp, dst);
|
|
}
|
|
return success();
|
|
}
|
|
};
|
|
|
|
/// Sparse conversion rule for the output operator.
|
|
class SparseTensorOutConverter : public OpConversionPattern<OutOp> {
|
|
public:
|
|
using OpConversionPattern::OpConversionPattern;
|
|
LogicalResult
|
|
matchAndRewrite(OutOp op, OpAdaptor adaptor,
|
|
ConversionPatternRewriter &rewriter) const override {
|
|
Location loc = op->getLoc();
|
|
ShapedType srcType = op.getTensor().getType().cast<ShapedType>();
|
|
// Convert to default permuted COO.
|
|
Value src = adaptor.getOperands()[0];
|
|
auto encSrc = getSparseTensorEncoding(srcType);
|
|
SmallVector<Value> sizes;
|
|
sizesFromPtr(rewriter, sizes, loc, encSrc, srcType, src);
|
|
auto enc = SparseTensorEncodingAttr::get(
|
|
op->getContext(), encSrc.getDimLevelType(), AffineMap(), AffineMap(),
|
|
encSrc.getPointerBitWidth(), encSrc.getIndexBitWidth());
|
|
Value coo = NewCallParams(rewriter, loc)
|
|
.genBuffers(enc, sizes, srcType)
|
|
.genNewCall(Action::kToCOO, src);
|
|
// Then output the tensor to external file with indices in the externally
|
|
// visible lexicographic index order. A sort is required if the source was
|
|
// not in that order yet (note that the sort can be dropped altogether if
|
|
// external format does not care about the order at all, but here we assume
|
|
// it does).
|
|
Value sort = constantI1(rewriter, loc,
|
|
encSrc.getDimOrdering() &&
|
|
!encSrc.getDimOrdering().isIdentity());
|
|
SmallVector<Value, 3> outParams{coo, adaptor.getOperands()[1], sort};
|
|
Type eltType = srcType.getElementType();
|
|
SmallString<18> name{"outSparseTensor", primaryTypeFunctionSuffix(eltType)};
|
|
createFuncCall(rewriter, loc, name, {}, outParams, EmitCInterface::Off);
|
|
genDelCOOCall(rewriter, loc, eltType, coo);
|
|
rewriter.eraseOp(op);
|
|
return success();
|
|
}
|
|
};
|
|
|
|
} // namespace
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Sparse tensor type conversion into opaque pointer.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
mlir::SparseTensorTypeToPtrConverter::SparseTensorTypeToPtrConverter() {
|
|
addConversion([](Type type) { return type; });
|
|
addConversion(convertSparseTensorTypes);
|
|
}
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Public method for populating conversion rules.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
/// Populates the given patterns list with conversion rules required for
|
|
/// the sparsification of linear algebra operations.
|
|
void mlir::populateSparseTensorConversionPatterns(
|
|
TypeConverter &typeConverter, RewritePatternSet &patterns,
|
|
const SparseTensorConversionOptions &options) {
|
|
patterns.add<SparseReturnConverter, SparseTensorToDimSizeConverter,
|
|
SparseCastConverter, SparseTensorNewConverter,
|
|
SparseReshapeConverter<tensor::ExpandShapeOp>,
|
|
SparseReshapeConverter<tensor::CollapseShapeOp>,
|
|
SparseTensorConcatConverter, SparseTensorAllocConverter,
|
|
SparseTensorDeallocConverter, SparseTensorToPointersConverter,
|
|
SparseTensorToIndicesConverter, SparseTensorToValuesConverter,
|
|
SparseNumberOfEntriesConverter, SparseTensorLoadConverter,
|
|
SparseTensorInsertConverter, SparseTensorExpandConverter,
|
|
SparseTensorCompressConverter, SparseTensorOutConverter>(
|
|
typeConverter, patterns.getContext());
|
|
|
|
patterns.add<SparseTensorConvertConverter>(typeConverter,
|
|
patterns.getContext(), options);
|
|
}
|