fac349a169
…ted. (#89998)" (#90250)
This partially reverts commit 7aedd7dc75.
This change removes calls to the deprecated member functions. It does
not mark the functions deprecated yet and does not disable the
deprecation warning in TypeSwitch. This seems to cause problems with
MSVC.
2999 lines
116 KiB
C++
2999 lines
116 KiB
C++
//===-- PPCIntrinsicCall.cpp ----------------------------------------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// Helper routines for constructing the FIR dialect of MLIR for PowerPC
|
|
// intrinsics. Extensive use of MLIR interfaces and MLIR's coding style
|
|
// (https://mlir.llvm.org/getting_started/DeveloperGuide/) is used in this
|
|
// module.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "flang/Optimizer/Builder/PPCIntrinsicCall.h"
|
|
#include "flang/Evaluate/common.h"
|
|
#include "flang/Optimizer/Builder/FIRBuilder.h"
|
|
#include "flang/Optimizer/Builder/MutableBox.h"
|
|
#include "mlir/Dialect/Vector/IR/VectorOps.h"
|
|
|
|
namespace fir {
|
|
|
|
using PI = PPCIntrinsicLibrary;
|
|
|
|
// PPC specific intrinsic handlers.
|
|
static constexpr IntrinsicHandler ppcHandlers[]{
|
|
{"__ppc_mma_assemble_acc",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::AssembleAcc, MMAHandlerOp::SubToFunc>),
|
|
{{{"acc", asAddr},
|
|
{"arg1", asValue},
|
|
{"arg2", asValue},
|
|
{"arg3", asValue},
|
|
{"arg4", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_assemble_pair",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::AssemblePair, MMAHandlerOp::SubToFunc>),
|
|
{{{"pair", asAddr}, {"arg1", asValue}, {"arg2", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_build_acc",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::AssembleAcc,
|
|
MMAHandlerOp::SubToFuncReverseArgOnLE>),
|
|
{{{"acc", asAddr},
|
|
{"arg1", asValue},
|
|
{"arg2", asValue},
|
|
{"arg3", asValue},
|
|
{"arg4", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_disassemble_acc",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::DisassembleAcc, MMAHandlerOp::SubToFunc>),
|
|
{{{"data", asAddr}, {"acc", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_disassemble_pair",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::DisassemblePair, MMAHandlerOp::SubToFunc>),
|
|
{{{"data", asAddr}, {"pair", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvbf16ger2_",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvbf16ger2, MMAHandlerOp::SubToFunc>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue},
|
|
{"pmask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvbf16ger2nn",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvbf16ger2nn,
|
|
MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue},
|
|
{"pmask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvbf16ger2np",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvbf16ger2np,
|
|
MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue},
|
|
{"pmask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvbf16ger2pn",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvbf16ger2pn,
|
|
MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue},
|
|
{"pmask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvbf16ger2pp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvbf16ger2pp,
|
|
MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue},
|
|
{"pmask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvf16ger2_",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvf16ger2, MMAHandlerOp::SubToFunc>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue},
|
|
{"pmask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvf16ger2nn",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvf16ger2nn, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue},
|
|
{"pmask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvf16ger2np",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvf16ger2np, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue},
|
|
{"pmask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvf16ger2pn",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvf16ger2pn, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue},
|
|
{"pmask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvf16ger2pp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvf16ger2pp, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue},
|
|
{"pmask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvf32ger",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvf32ger, MMAHandlerOp::SubToFunc>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvf32gernn",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvf32gernn, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvf32gernp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvf32gernp, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvf32gerpn",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvf32gerpn, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvf32gerpp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvf32gerpp, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvf64ger",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvf64ger, MMAHandlerOp::SubToFunc>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvf64gernn",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvf64gernn, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvf64gernp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvf64gernp, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvf64gerpn",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvf64gerpn, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvf64gerpp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvf64gerpp, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvi16ger2_",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvi16ger2, MMAHandlerOp::SubToFunc>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue},
|
|
{"pmask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvi16ger2pp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvi16ger2pp, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue},
|
|
{"pmask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvi16ger2s",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvi16ger2s, MMAHandlerOp::SubToFunc>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue},
|
|
{"pmask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvi16ger2spp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvi16ger2spp,
|
|
MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue},
|
|
{"pmask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvi4ger8_",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvi4ger8, MMAHandlerOp::SubToFunc>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue},
|
|
{"pmask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvi4ger8pp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvi4ger8pp, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue},
|
|
{"pmask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvi8ger4_",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvi8ger4, MMAHandlerOp::SubToFunc>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue},
|
|
{"pmask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvi8ger4pp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvi8ger4pp, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue},
|
|
{"pmask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_pmxvi8ger4spp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Pmxvi8ger4spp, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr},
|
|
{"a", asValue},
|
|
{"b", asValue},
|
|
{"xmask", asValue},
|
|
{"ymask", asValue},
|
|
{"pmask", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvbf16ger2_",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvbf16ger2, MMAHandlerOp::SubToFunc>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvbf16ger2nn",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvbf16ger2nn, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvbf16ger2np",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvbf16ger2np, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvbf16ger2pn",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvbf16ger2pn, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvbf16ger2pp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvbf16ger2pp, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvf16ger2_",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvf16ger2, MMAHandlerOp::SubToFunc>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvf16ger2nn",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvf16ger2nn, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvf16ger2np",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvf16ger2np, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvf16ger2pn",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvf16ger2pn, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvf16ger2pp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvf16ger2pp, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvf32ger",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvf32ger, MMAHandlerOp::SubToFunc>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvf32gernn",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvf32gernn, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvf32gernp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvf32gernp, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvf32gerpn",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvf32gerpn, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvf32gerpp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvf32gerpp, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvf64ger",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvf64ger, MMAHandlerOp::SubToFunc>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvf64gernn",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvf64gernn, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvf64gernp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvf64gernp, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvf64gerpn",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvf64gerpn, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvf64gerpp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvf64gerpp, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvi16ger2_",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvi16ger2, MMAHandlerOp::SubToFunc>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvi16ger2pp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvi16ger2pp, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvi16ger2s",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvi16ger2s, MMAHandlerOp::SubToFunc>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvi16ger2spp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvi16ger2spp, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvi4ger8_",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvi4ger8, MMAHandlerOp::SubToFunc>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvi4ger8pp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvi4ger8pp, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvi8ger4_",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvi8ger4, MMAHandlerOp::SubToFunc>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvi8ger4pp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvi8ger4pp, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xvi8ger4spp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xvi8ger4spp, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}, {"a", asValue}, {"b", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xxmfacc",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xxmfacc, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xxmtacc",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xxmtacc, MMAHandlerOp::FirstArgIsResult>),
|
|
{{{"acc", asAddr}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mma_xxsetaccz",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genMmaIntr<MMAOp::Xxsetaccz, MMAHandlerOp::SubToFunc>),
|
|
{{{"acc", asAddr}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_mtfsf",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(&PI::genMtfsf<false>),
|
|
{{{"mask", asValue}, {"r", asValue}}},
|
|
/*isElemental=*/false},
|
|
{"__ppc_mtfsfi",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(&PI::genMtfsf<true>),
|
|
{{{"bf", asValue}, {"i", asValue}}},
|
|
/*isElemental=*/false},
|
|
{"__ppc_vec_abs",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(&PI::genVecAbs),
|
|
{{{"arg1", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_add",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecAddAndMulSubXor<VecOp::Add>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_and",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecAddAndMulSubXor<VecOp::And>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_any_ge",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecAnyCompare<VecOp::Anyge>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_cmpge",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecCmp<VecOp::Cmpge>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_cmpgt",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecCmp<VecOp::Cmpgt>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_cmple",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecCmp<VecOp::Cmple>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_cmplt",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecCmp<VecOp::Cmplt>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_convert",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecConvert<VecOp::Convert>),
|
|
{{{"v", asValue}, {"mold", asValue}}},
|
|
/*isElemental=*/false},
|
|
{"__ppc_vec_ctf",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecConvert<VecOp::Ctf>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_cvf",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecConvert<VecOp::Cvf>),
|
|
{{{"arg1", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_extract",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(&PI::genVecExtract),
|
|
{{{"arg1", asValue}, {"arg2", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_insert",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(&PI::genVecInsert),
|
|
{{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_ld",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecLdCallGrp<VecOp::Ld>),
|
|
{{{"arg1", asValue}, {"arg2", asAddr}}},
|
|
/*isElemental=*/false},
|
|
{"__ppc_vec_lde",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecLdCallGrp<VecOp::Lde>),
|
|
{{{"arg1", asValue}, {"arg2", asAddr}}},
|
|
/*isElemental=*/false},
|
|
{"__ppc_vec_ldl",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecLdCallGrp<VecOp::Ldl>),
|
|
{{{"arg1", asValue}, {"arg2", asAddr}}},
|
|
/*isElemental=*/false},
|
|
{"__ppc_vec_lvsl",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecLvsGrp<VecOp::Lvsl>),
|
|
{{{"arg1", asValue}, {"arg2", asAddr}}},
|
|
/*isElemental=*/false},
|
|
{"__ppc_vec_lvsr",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecLvsGrp<VecOp::Lvsr>),
|
|
{{{"arg1", asValue}, {"arg2", asAddr}}},
|
|
/*isElemental=*/false},
|
|
{"__ppc_vec_lxv",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecLdNoCallGrp<VecOp::Lxv>),
|
|
{{{"arg1", asValue}, {"arg2", asAddr}}},
|
|
/*isElemental=*/false},
|
|
{"__ppc_vec_lxvp",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecLdCallGrp<VecOp::Lxvp>),
|
|
{{{"arg1", asValue}, {"arg2", asAddr}}},
|
|
/*isElemental=*/false},
|
|
{"__ppc_vec_mergeh",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecMerge<VecOp::Mergeh>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_mergel",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecMerge<VecOp::Mergel>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_msub",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecNmaddMsub<VecOp::Msub>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_mul",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecAddAndMulSubXor<VecOp::Mul>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_nmadd",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecNmaddMsub<VecOp::Nmadd>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_perm",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecPerm<VecOp::Perm>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_permi",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecPerm<VecOp::Permi>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_sel",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(&PI::genVecSel),
|
|
{{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_sl",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecShift<VecOp::Sl>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_sld",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecShift<VecOp::Sld>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_sldw",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecShift<VecOp::Sldw>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_sll",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecShift<VecOp::Sll>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_slo",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecShift<VecOp::Slo>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_splat",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecSplat<VecOp::Splat>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_splat_s32_",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecSplat<VecOp::Splat_s32>),
|
|
{{{"arg1", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_splats",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecSplat<VecOp::Splats>),
|
|
{{{"arg1", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_sr",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecShift<VecOp::Sr>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_srl",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecShift<VecOp::Srl>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_sro",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecShift<VecOp::Sro>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_st",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genVecStore<VecOp::St>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}},
|
|
/*isElemental=*/false},
|
|
{"__ppc_vec_ste",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genVecStore<VecOp::Ste>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}},
|
|
/*isElemental=*/false},
|
|
{"__ppc_vec_stxv",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genVecXStore<VecOp::Stxv>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}},
|
|
/*isElemental=*/false},
|
|
{"__ppc_vec_stxvp",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genVecStore<VecOp::Stxvp>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}},
|
|
/*isElemental=*/false},
|
|
{"__ppc_vec_sub",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecAddAndMulSubXor<VecOp::Sub>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_xl",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(&PI::genVecXlGrp),
|
|
{{{"arg1", asValue}, {"arg2", asAddr}}},
|
|
/*isElemental=*/false},
|
|
{"__ppc_vec_xl_be",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecLdNoCallGrp<VecOp::Xlbe>),
|
|
{{{"arg1", asValue}, {"arg2", asAddr}}},
|
|
/*isElemental=*/false},
|
|
{"__ppc_vec_xld2_",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecLdCallGrp<VecOp::Xld2>),
|
|
{{{"arg1", asValue}, {"arg2", asAddr}}},
|
|
/*isElemental=*/false},
|
|
{"__ppc_vec_xlds",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(&PI::genVecXlds),
|
|
{{{"arg1", asValue}, {"arg2", asAddr}}},
|
|
/*isElemental=*/false},
|
|
{"__ppc_vec_xlw4_",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecLdCallGrp<VecOp::Xlw4>),
|
|
{{{"arg1", asValue}, {"arg2", asAddr}}},
|
|
/*isElemental=*/false},
|
|
{"__ppc_vec_xor",
|
|
static_cast<IntrinsicLibrary::ExtendedGenerator>(
|
|
&PI::genVecAddAndMulSubXor<VecOp::Xor>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}}},
|
|
/*isElemental=*/true},
|
|
{"__ppc_vec_xst",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genVecXStore<VecOp::Xst>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}},
|
|
/*isElemental=*/false},
|
|
{"__ppc_vec_xst_be",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genVecXStore<VecOp::Xst_be>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}},
|
|
/*isElemental=*/false},
|
|
{"__ppc_vec_xstd2_",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genVecXStore<VecOp::Xstd2>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}},
|
|
/*isElemental=*/false},
|
|
{"__ppc_vec_xstw4_",
|
|
static_cast<IntrinsicLibrary::SubroutineGenerator>(
|
|
&PI::genVecXStore<VecOp::Xstw4>),
|
|
{{{"arg1", asValue}, {"arg2", asValue}, {"arg3", asAddr}}},
|
|
/*isElemental=*/false},
|
|
};
|
|
|
|
static constexpr MathOperation ppcMathOperations[] = {
|
|
// fcfi is just another name for fcfid, there is no llvm.ppc.fcfi.
|
|
{"__ppc_fcfi", "llvm.ppc.fcfid", genFuncType<Ty::Real<8>, Ty::Real<8>>,
|
|
genLibCall},
|
|
{"__ppc_fcfid", "llvm.ppc.fcfid", genFuncType<Ty::Real<8>, Ty::Real<8>>,
|
|
genLibCall},
|
|
{"__ppc_fcfud", "llvm.ppc.fcfud", genFuncType<Ty::Real<8>, Ty::Real<8>>,
|
|
genLibCall},
|
|
{"__ppc_fctid", "llvm.ppc.fctid", genFuncType<Ty::Real<8>, Ty::Real<8>>,
|
|
genLibCall},
|
|
{"__ppc_fctidz", "llvm.ppc.fctidz", genFuncType<Ty::Real<8>, Ty::Real<8>>,
|
|
genLibCall},
|
|
{"__ppc_fctiw", "llvm.ppc.fctiw", genFuncType<Ty::Real<8>, Ty::Real<8>>,
|
|
genLibCall},
|
|
{"__ppc_fctiwz", "llvm.ppc.fctiwz", genFuncType<Ty::Real<8>, Ty::Real<8>>,
|
|
genLibCall},
|
|
{"__ppc_fctudz", "llvm.ppc.fctudz", genFuncType<Ty::Real<8>, Ty::Real<8>>,
|
|
genLibCall},
|
|
{"__ppc_fctuwz", "llvm.ppc.fctuwz", genFuncType<Ty::Real<8>, Ty::Real<8>>,
|
|
genLibCall},
|
|
{"__ppc_fmadd", "llvm.fma.f32",
|
|
genFuncType<Ty::Real<4>, Ty::Real<4>, Ty::Real<4>, Ty::Real<4>>,
|
|
genMathOp<mlir::math::FmaOp>},
|
|
{"__ppc_fmadd", "llvm.fma.f64",
|
|
genFuncType<Ty::Real<8>, Ty::Real<8>, Ty::Real<8>, Ty::Real<8>>,
|
|
genMathOp<mlir::math::FmaOp>},
|
|
{"__ppc_fmsub", "llvm.ppc.fmsubs",
|
|
genFuncType<Ty::Real<4>, Ty::Real<4>, Ty::Real<4>, Ty::Real<4>>,
|
|
genLibCall},
|
|
{"__ppc_fmsub", "llvm.ppc.fmsub",
|
|
genFuncType<Ty::Real<8>, Ty::Real<8>, Ty::Real<8>, Ty::Real<8>>,
|
|
genLibCall},
|
|
{"__ppc_fnabs", "llvm.ppc.fnabss", genFuncType<Ty::Real<4>, Ty::Real<4>>,
|
|
genLibCall},
|
|
{"__ppc_fnabs", "llvm.ppc.fnabs", genFuncType<Ty::Real<8>, Ty::Real<8>>,
|
|
genLibCall},
|
|
{"__ppc_fnmadd", "llvm.ppc.fnmadds",
|
|
genFuncType<Ty::Real<4>, Ty::Real<4>, Ty::Real<4>, Ty::Real<4>>,
|
|
genLibCall},
|
|
{"__ppc_fnmadd", "llvm.ppc.fnmadd",
|
|
genFuncType<Ty::Real<8>, Ty::Real<8>, Ty::Real<8>, Ty::Real<8>>,
|
|
genLibCall},
|
|
{"__ppc_fnmsub", "llvm.ppc.fnmsub.f32",
|
|
genFuncType<Ty::Real<4>, Ty::Real<4>, Ty::Real<4>, Ty::Real<4>>,
|
|
genLibCall},
|
|
{"__ppc_fnmsub", "llvm.ppc.fnmsub.f64",
|
|
genFuncType<Ty::Real<8>, Ty::Real<8>, Ty::Real<8>, Ty::Real<8>>,
|
|
genLibCall},
|
|
{"__ppc_fre", "llvm.ppc.fre", genFuncType<Ty::Real<8>, Ty::Real<8>>,
|
|
genLibCall},
|
|
{"__ppc_fres", "llvm.ppc.fres", genFuncType<Ty::Real<4>, Ty::Real<4>>,
|
|
genLibCall},
|
|
{"__ppc_frsqrte", "llvm.ppc.frsqrte", genFuncType<Ty::Real<8>, Ty::Real<8>>,
|
|
genLibCall},
|
|
{"__ppc_frsqrtes", "llvm.ppc.frsqrtes",
|
|
genFuncType<Ty::Real<4>, Ty::Real<4>>, genLibCall},
|
|
{"__ppc_vec_cvbf16spn", "llvm.ppc.vsx.xvcvbf16spn",
|
|
genFuncType<Ty::UnsignedVector<1>, Ty::UnsignedVector<1>>, genLibCall},
|
|
{"__ppc_vec_cvspbf16_", "llvm.ppc.vsx.xvcvspbf16",
|
|
genFuncType<Ty::UnsignedVector<1>, Ty::UnsignedVector<1>>, genLibCall},
|
|
{"__ppc_vec_madd", "llvm.fma.v4f32",
|
|
genFuncType<Ty::RealVector<4>, Ty::RealVector<4>, Ty::RealVector<4>,
|
|
Ty::RealVector<4>>,
|
|
genLibCall},
|
|
{"__ppc_vec_madd", "llvm.fma.v2f64",
|
|
genFuncType<Ty::RealVector<8>, Ty::RealVector<8>, Ty::RealVector<8>,
|
|
Ty::RealVector<8>>,
|
|
genLibCall},
|
|
{"__ppc_vec_max", "llvm.ppc.altivec.vmaxsb",
|
|
genFuncType<Ty::IntegerVector<1>, Ty::IntegerVector<1>,
|
|
Ty::IntegerVector<1>>,
|
|
genLibCall},
|
|
{"__ppc_vec_max", "llvm.ppc.altivec.vmaxsh",
|
|
genFuncType<Ty::IntegerVector<2>, Ty::IntegerVector<2>,
|
|
Ty::IntegerVector<2>>,
|
|
genLibCall},
|
|
{"__ppc_vec_max", "llvm.ppc.altivec.vmaxsw",
|
|
genFuncType<Ty::IntegerVector<4>, Ty::IntegerVector<4>,
|
|
Ty::IntegerVector<4>>,
|
|
genLibCall},
|
|
{"__ppc_vec_max", "llvm.ppc.altivec.vmaxsd",
|
|
genFuncType<Ty::IntegerVector<8>, Ty::IntegerVector<8>,
|
|
Ty::IntegerVector<8>>,
|
|
genLibCall},
|
|
{"__ppc_vec_max", "llvm.ppc.altivec.vmaxub",
|
|
genFuncType<Ty::UnsignedVector<1>, Ty::UnsignedVector<1>,
|
|
Ty::UnsignedVector<1>>,
|
|
genLibCall},
|
|
{"__ppc_vec_max", "llvm.ppc.altivec.vmaxuh",
|
|
genFuncType<Ty::UnsignedVector<2>, Ty::UnsignedVector<2>,
|
|
Ty::UnsignedVector<2>>,
|
|
genLibCall},
|
|
{"__ppc_vec_max", "llvm.ppc.altivec.vmaxuw",
|
|
genFuncType<Ty::UnsignedVector<4>, Ty::UnsignedVector<4>,
|
|
Ty::UnsignedVector<4>>,
|
|
genLibCall},
|
|
{"__ppc_vec_max", "llvm.ppc.altivec.vmaxud",
|
|
genFuncType<Ty::UnsignedVector<8>, Ty::UnsignedVector<8>,
|
|
Ty::UnsignedVector<8>>,
|
|
genLibCall},
|
|
{"__ppc_vec_max", "llvm.ppc.vsx.xvmaxsp",
|
|
genFuncType<Ty::RealVector<4>, Ty::RealVector<4>, Ty::RealVector<4>>,
|
|
genLibCall},
|
|
{"__ppc_vec_max", "llvm.ppc.vsx.xvmaxdp",
|
|
genFuncType<Ty::RealVector<8>, Ty::RealVector<8>, Ty::RealVector<8>>,
|
|
genLibCall},
|
|
{"__ppc_vec_min", "llvm.ppc.altivec.vminsb",
|
|
genFuncType<Ty::IntegerVector<1>, Ty::IntegerVector<1>,
|
|
Ty::IntegerVector<1>>,
|
|
genLibCall},
|
|
{"__ppc_vec_min", "llvm.ppc.altivec.vminsh",
|
|
genFuncType<Ty::IntegerVector<2>, Ty::IntegerVector<2>,
|
|
Ty::IntegerVector<2>>,
|
|
genLibCall},
|
|
{"__ppc_vec_min", "llvm.ppc.altivec.vminsw",
|
|
genFuncType<Ty::IntegerVector<4>, Ty::IntegerVector<4>,
|
|
Ty::IntegerVector<4>>,
|
|
genLibCall},
|
|
{"__ppc_vec_min", "llvm.ppc.altivec.vminsd",
|
|
genFuncType<Ty::IntegerVector<8>, Ty::IntegerVector<8>,
|
|
Ty::IntegerVector<8>>,
|
|
genLibCall},
|
|
{"__ppc_vec_min", "llvm.ppc.altivec.vminub",
|
|
genFuncType<Ty::UnsignedVector<1>, Ty::UnsignedVector<1>,
|
|
Ty::UnsignedVector<1>>,
|
|
genLibCall},
|
|
{"__ppc_vec_min", "llvm.ppc.altivec.vminuh",
|
|
genFuncType<Ty::UnsignedVector<2>, Ty::UnsignedVector<2>,
|
|
Ty::UnsignedVector<2>>,
|
|
genLibCall},
|
|
{"__ppc_vec_min", "llvm.ppc.altivec.vminuw",
|
|
genFuncType<Ty::UnsignedVector<4>, Ty::UnsignedVector<4>,
|
|
Ty::UnsignedVector<4>>,
|
|
genLibCall},
|
|
{"__ppc_vec_min", "llvm.ppc.altivec.vminud",
|
|
genFuncType<Ty::UnsignedVector<8>, Ty::UnsignedVector<8>,
|
|
Ty::UnsignedVector<8>>,
|
|
genLibCall},
|
|
{"__ppc_vec_min", "llvm.ppc.vsx.xvminsp",
|
|
genFuncType<Ty::RealVector<4>, Ty::RealVector<4>, Ty::RealVector<4>>,
|
|
genLibCall},
|
|
{"__ppc_vec_min", "llvm.ppc.vsx.xvmindp",
|
|
genFuncType<Ty::RealVector<8>, Ty::RealVector<8>, Ty::RealVector<8>>,
|
|
genLibCall},
|
|
{"__ppc_vec_nmsub", "llvm.ppc.fnmsub.v4f32",
|
|
genFuncType<Ty::RealVector<4>, Ty::RealVector<4>, Ty::RealVector<4>,
|
|
Ty::RealVector<4>>,
|
|
genLibCall},
|
|
{"__ppc_vec_nmsub", "llvm.ppc.fnmsub.v2f64",
|
|
genFuncType<Ty::RealVector<8>, Ty::RealVector<8>, Ty::RealVector<8>,
|
|
Ty::RealVector<8>>,
|
|
genLibCall},
|
|
};
|
|
|
|
const IntrinsicHandler *findPPCIntrinsicHandler(llvm::StringRef name) {
|
|
auto compare = [](const IntrinsicHandler &ppcHandler, llvm::StringRef name) {
|
|
return name.compare(ppcHandler.name) > 0;
|
|
};
|
|
auto result = llvm::lower_bound(ppcHandlers, name, compare);
|
|
return result != std::end(ppcHandlers) && result->name == name ? result
|
|
: nullptr;
|
|
}
|
|
|
|
using RtMap = Fortran::common::StaticMultimapView<MathOperation>;
|
|
static constexpr RtMap ppcMathOps(ppcMathOperations);
|
|
static_assert(ppcMathOps.Verify() && "map must be sorted");
|
|
|
|
std::pair<const MathOperation *, const MathOperation *>
|
|
checkPPCMathOperationsRange(llvm::StringRef name) {
|
|
return ppcMathOps.equal_range(name);
|
|
}
|
|
|
|
// Helper functions for vector element ordering.
|
|
bool PPCIntrinsicLibrary::isBEVecElemOrderOnLE() {
|
|
const auto triple{fir::getTargetTriple(builder.getModule())};
|
|
return (triple.isLittleEndian() &&
|
|
converter->getLoweringOptions().getNoPPCNativeVecElemOrder());
|
|
}
|
|
bool PPCIntrinsicLibrary::isNativeVecElemOrderOnLE() {
|
|
const auto triple{fir::getTargetTriple(builder.getModule())};
|
|
return (triple.isLittleEndian() &&
|
|
!converter->getLoweringOptions().getNoPPCNativeVecElemOrder());
|
|
}
|
|
bool PPCIntrinsicLibrary::changeVecElemOrder() {
|
|
const auto triple{fir::getTargetTriple(builder.getModule())};
|
|
return (triple.isLittleEndian() !=
|
|
converter->getLoweringOptions().getNoPPCNativeVecElemOrder());
|
|
}
|
|
|
|
static mlir::FunctionType genMmaVpFuncType(mlir::MLIRContext *context,
|
|
int quadCnt, int pairCnt, int vecCnt,
|
|
int intCnt = 0,
|
|
int vecElemBitSize = 8,
|
|
int intBitSize = 32) {
|
|
// Constructs a function type with the following signature:
|
|
// Result type: __vector_pair
|
|
// Arguments:
|
|
// quadCnt: number of arguments that has __vector_quad type, followed by
|
|
// pairCnt: number of arguments that has __vector_pair type, followed by
|
|
// vecCnt: number of arguments that has vector(integer) type, followed by
|
|
// intCnt: number of arguments that has integer type
|
|
// vecElemBitSize: specifies the size of vector elements in bits
|
|
// intBitSize: specifies the size of integer arguments in bits
|
|
auto vType{mlir::VectorType::get(
|
|
128 / vecElemBitSize, mlir::IntegerType::get(context, vecElemBitSize))};
|
|
auto vpType{fir::VectorType::get(256, mlir::IntegerType::get(context, 1))};
|
|
auto vqType{fir::VectorType::get(512, mlir::IntegerType::get(context, 1))};
|
|
auto iType{mlir::IntegerType::get(context, intBitSize)};
|
|
llvm::SmallVector<mlir::Type> argTypes;
|
|
for (int i = 0; i < quadCnt; ++i) {
|
|
argTypes.push_back(vqType);
|
|
}
|
|
for (int i = 0; i < pairCnt; ++i) {
|
|
argTypes.push_back(vpType);
|
|
}
|
|
for (int i = 0; i < vecCnt; ++i) {
|
|
argTypes.push_back(vType);
|
|
}
|
|
for (int i = 0; i < intCnt; ++i) {
|
|
argTypes.push_back(iType);
|
|
}
|
|
|
|
return mlir::FunctionType::get(context, argTypes, {vpType});
|
|
}
|
|
|
|
static mlir::FunctionType genMmaVqFuncType(mlir::MLIRContext *context,
|
|
int quadCnt, int pairCnt, int vecCnt,
|
|
int intCnt = 0,
|
|
int vecElemBitSize = 8,
|
|
int intBitSize = 32) {
|
|
// Constructs a function type with the following signature:
|
|
// Result type: __vector_quad
|
|
// Arguments:
|
|
// quadCnt: number of arguments that has __vector_quad type, followed by
|
|
// pairCnt: number of arguments that has __vector_pair type, followed by
|
|
// vecCnt: number of arguments that has vector(integer) type, followed by
|
|
// intCnt: number of arguments that has integer type
|
|
// vecElemBitSize: specifies the size of vector elements in bits
|
|
// intBitSize: specifies the size of integer arguments in bits
|
|
auto vType{mlir::VectorType::get(
|
|
128 / vecElemBitSize, mlir::IntegerType::get(context, vecElemBitSize))};
|
|
auto vpType{fir::VectorType::get(256, mlir::IntegerType::get(context, 1))};
|
|
auto vqType{fir::VectorType::get(512, mlir::IntegerType::get(context, 1))};
|
|
auto iType{mlir::IntegerType::get(context, intBitSize)};
|
|
llvm::SmallVector<mlir::Type> argTypes;
|
|
for (int i = 0; i < quadCnt; ++i) {
|
|
argTypes.push_back(vqType);
|
|
}
|
|
for (int i = 0; i < pairCnt; ++i) {
|
|
argTypes.push_back(vpType);
|
|
}
|
|
for (int i = 0; i < vecCnt; ++i) {
|
|
argTypes.push_back(vType);
|
|
}
|
|
for (int i = 0; i < intCnt; ++i) {
|
|
argTypes.push_back(iType);
|
|
}
|
|
|
|
return mlir::FunctionType::get(context, argTypes, {vqType});
|
|
}
|
|
|
|
mlir::FunctionType genMmaDisassembleFuncType(mlir::MLIRContext *context,
|
|
MMAOp mmaOp) {
|
|
auto vType{mlir::VectorType::get(16, mlir::IntegerType::get(context, 8))};
|
|
llvm::SmallVector<mlir::Type> members;
|
|
|
|
if (mmaOp == MMAOp::DisassembleAcc) {
|
|
auto vqType{fir::VectorType::get(512, mlir::IntegerType::get(context, 1))};
|
|
members.push_back(vType);
|
|
members.push_back(vType);
|
|
members.push_back(vType);
|
|
members.push_back(vType);
|
|
auto resType{mlir::LLVM::LLVMStructType::getLiteral(context, members)};
|
|
return mlir::FunctionType::get(context, {vqType}, {resType});
|
|
} else if (mmaOp == MMAOp::DisassemblePair) {
|
|
auto vpType{fir::VectorType::get(256, mlir::IntegerType::get(context, 1))};
|
|
members.push_back(vType);
|
|
members.push_back(vType);
|
|
auto resType{mlir::LLVM::LLVMStructType::getLiteral(context, members)};
|
|
return mlir::FunctionType::get(context, {vpType}, {resType});
|
|
} else {
|
|
llvm_unreachable(
|
|
"Unsupported intrinsic code for function signature generator");
|
|
}
|
|
}
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// PowerPC specific intrinsic handlers.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
// MTFSF, MTFSFI
|
|
template <bool isImm>
|
|
void PPCIntrinsicLibrary::genMtfsf(llvm::ArrayRef<fir::ExtendedValue> args) {
|
|
assert(args.size() == 2);
|
|
llvm::SmallVector<mlir::Value> scalarArgs;
|
|
for (const fir::ExtendedValue &arg : args)
|
|
if (arg.getUnboxed())
|
|
scalarArgs.emplace_back(fir::getBase(arg));
|
|
else
|
|
mlir::emitError(loc, "nonscalar intrinsic argument");
|
|
|
|
mlir::FunctionType libFuncType;
|
|
mlir::func::FuncOp funcOp;
|
|
if (isImm) {
|
|
libFuncType = genFuncType<Ty::Void, Ty::Integer<4>, Ty::Integer<4>>(
|
|
builder.getContext(), builder);
|
|
funcOp = builder.createFunction(loc, "llvm.ppc.mtfsfi", libFuncType);
|
|
} else {
|
|
libFuncType = genFuncType<Ty::Void, Ty::Integer<4>, Ty::Real<8>>(
|
|
builder.getContext(), builder);
|
|
funcOp = builder.createFunction(loc, "llvm.ppc.mtfsf", libFuncType);
|
|
}
|
|
builder.create<fir::CallOp>(loc, funcOp, scalarArgs);
|
|
}
|
|
|
|
// VEC_ABS
|
|
fir::ExtendedValue
|
|
PPCIntrinsicLibrary::genVecAbs(mlir::Type resultType,
|
|
llvm::ArrayRef<fir::ExtendedValue> args) {
|
|
assert(args.size() == 1);
|
|
auto context{builder.getContext()};
|
|
auto argBases{getBasesForArgs(args)};
|
|
auto vTypeInfo{getVecTypeFromFir(argBases[0])};
|
|
|
|
mlir::func::FuncOp funcOp{nullptr};
|
|
mlir::FunctionType ftype;
|
|
llvm::StringRef fname{};
|
|
if (vTypeInfo.isFloat()) {
|
|
if (vTypeInfo.isFloat32()) {
|
|
fname = "llvm.fabs.v4f32";
|
|
ftype =
|
|
genFuncType<Ty::RealVector<4>, Ty::RealVector<4>>(context, builder);
|
|
} else if (vTypeInfo.isFloat64()) {
|
|
fname = "llvm.fabs.v2f64";
|
|
ftype =
|
|
genFuncType<Ty::RealVector<8>, Ty::RealVector<8>>(context, builder);
|
|
}
|
|
|
|
funcOp = builder.createFunction(loc, fname, ftype);
|
|
auto callOp{builder.create<fir::CallOp>(loc, funcOp, argBases[0])};
|
|
return callOp.getResult(0);
|
|
} else if (auto eleTy = mlir::dyn_cast<mlir::IntegerType>(vTypeInfo.eleTy)) {
|
|
// vec_abs(arg1) = max(0 - arg1, arg1)
|
|
|
|
auto newVecTy{mlir::VectorType::get(vTypeInfo.len, eleTy)};
|
|
auto varg1{builder.createConvert(loc, newVecTy, argBases[0])};
|
|
// construct vector(0,..)
|
|
auto zeroVal{builder.createIntegerConstant(loc, eleTy, 0)};
|
|
auto vZero{
|
|
builder.create<mlir::vector::BroadcastOp>(loc, newVecTy, zeroVal)};
|
|
auto zeroSubVarg1{builder.create<mlir::arith::SubIOp>(loc, vZero, varg1)};
|
|
|
|
mlir::func::FuncOp funcOp{nullptr};
|
|
switch (eleTy.getWidth()) {
|
|
case 8:
|
|
fname = "llvm.ppc.altivec.vmaxsb";
|
|
ftype = genFuncType<Ty::IntegerVector<1>, Ty::IntegerVector<1>,
|
|
Ty::IntegerVector<1>>(context, builder);
|
|
break;
|
|
case 16:
|
|
fname = "llvm.ppc.altivec.vmaxsh";
|
|
ftype = genFuncType<Ty::IntegerVector<2>, Ty::IntegerVector<2>,
|
|
Ty::IntegerVector<2>>(context, builder);
|
|
break;
|
|
case 32:
|
|
fname = "llvm.ppc.altivec.vmaxsw";
|
|
ftype = genFuncType<Ty::IntegerVector<4>, Ty::IntegerVector<4>,
|
|
Ty::IntegerVector<4>>(context, builder);
|
|
break;
|
|
case 64:
|
|
fname = "llvm.ppc.altivec.vmaxsd";
|
|
ftype = genFuncType<Ty::IntegerVector<8>, Ty::IntegerVector<8>,
|
|
Ty::IntegerVector<8>>(context, builder);
|
|
break;
|
|
default:
|
|
llvm_unreachable("invalid integer size");
|
|
}
|
|
funcOp = builder.createFunction(loc, fname, ftype);
|
|
|
|
mlir::Value args[] = {zeroSubVarg1, varg1};
|
|
auto callOp{builder.create<fir::CallOp>(loc, funcOp, args)};
|
|
return builder.createConvert(loc, argBases[0].getType(),
|
|
callOp.getResult(0));
|
|
}
|
|
|
|
llvm_unreachable("unknown vector type");
|
|
}
|
|
|
|
// VEC_ADD, VEC_AND, VEC_SUB, VEC_MUL, VEC_XOR
|
|
template <VecOp vop>
|
|
fir::ExtendedValue PPCIntrinsicLibrary::genVecAddAndMulSubXor(
|
|
mlir::Type resultType, llvm::ArrayRef<fir::ExtendedValue> args) {
|
|
assert(args.size() == 2);
|
|
auto argBases{getBasesForArgs(args)};
|
|
auto argsTy{getTypesForArgs(argBases)};
|
|
assert(mlir::isa<fir::VectorType>(argsTy[0]) &&
|
|
mlir::isa<fir::VectorType>(argsTy[1]));
|
|
|
|
auto vecTyInfo{getVecTypeFromFir(argBases[0])};
|
|
|
|
const auto isInteger{mlir::isa<mlir::IntegerType>(vecTyInfo.eleTy)};
|
|
const auto isFloat{mlir::isa<mlir::FloatType>(vecTyInfo.eleTy)};
|
|
assert((isInteger || isFloat) && "unknown vector type");
|
|
|
|
auto vargs{convertVecArgs(builder, loc, vecTyInfo, argBases)};
|
|
|
|
mlir::Value r{nullptr};
|
|
switch (vop) {
|
|
case VecOp::Add:
|
|
if (isInteger)
|
|
r = builder.create<mlir::arith::AddIOp>(loc, vargs[0], vargs[1]);
|
|
else if (isFloat)
|
|
r = builder.create<mlir::arith::AddFOp>(loc, vargs[0], vargs[1]);
|
|
break;
|
|
case VecOp::Mul:
|
|
if (isInteger)
|
|
r = builder.create<mlir::arith::MulIOp>(loc, vargs[0], vargs[1]);
|
|
else if (isFloat)
|
|
r = builder.create<mlir::arith::MulFOp>(loc, vargs[0], vargs[1]);
|
|
break;
|
|
case VecOp::Sub:
|
|
if (isInteger)
|
|
r = builder.create<mlir::arith::SubIOp>(loc, vargs[0], vargs[1]);
|
|
else if (isFloat)
|
|
r = builder.create<mlir::arith::SubFOp>(loc, vargs[0], vargs[1]);
|
|
break;
|
|
case VecOp::And:
|
|
case VecOp::Xor: {
|
|
mlir::Value arg1{nullptr};
|
|
mlir::Value arg2{nullptr};
|
|
if (isInteger) {
|
|
arg1 = vargs[0];
|
|
arg2 = vargs[1];
|
|
} else if (isFloat) {
|
|
// bitcast the arguments to integer
|
|
auto wd{mlir::dyn_cast<mlir::FloatType>(vecTyInfo.eleTy).getWidth()};
|
|
auto ftype{builder.getIntegerType(wd)};
|
|
auto bcVecTy{mlir::VectorType::get(vecTyInfo.len, ftype)};
|
|
arg1 = builder.create<mlir::vector::BitCastOp>(loc, bcVecTy, vargs[0]);
|
|
arg2 = builder.create<mlir::vector::BitCastOp>(loc, bcVecTy, vargs[1]);
|
|
}
|
|
if (vop == VecOp::And)
|
|
r = builder.create<mlir::arith::AndIOp>(loc, arg1, arg2);
|
|
else if (vop == VecOp::Xor)
|
|
r = builder.create<mlir::arith::XOrIOp>(loc, arg1, arg2);
|
|
|
|
if (isFloat)
|
|
r = builder.create<mlir::vector::BitCastOp>(loc, vargs[0].getType(), r);
|
|
|
|
break;
|
|
}
|
|
}
|
|
|
|
return builder.createConvert(loc, argsTy[0], r);
|
|
}
|
|
|
|
// VEC_ANY_GE
|
|
template <VecOp vop>
|
|
fir::ExtendedValue
|
|
PPCIntrinsicLibrary::genVecAnyCompare(mlir::Type resultType,
|
|
llvm::ArrayRef<fir::ExtendedValue> args) {
|
|
assert(args.size() == 2);
|
|
assert(vop == VecOp::Anyge && "unknown vector compare operation");
|
|
auto argBases{getBasesForArgs(args)};
|
|
VecTypeInfo vTypeInfo{getVecTypeFromFir(argBases[0])};
|
|
[[maybe_unused]] const auto isSupportedTy{
|
|
mlir::isa<mlir::Float32Type, mlir::Float64Type, mlir::IntegerType>(
|
|
vTypeInfo.eleTy)};
|
|
assert(isSupportedTy && "unsupported vector type");
|
|
|
|
// Constants for mapping CR6 bits to predicate result
|
|
enum { CR6_EQ_REV = 1, CR6_LT_REV = 3 };
|
|
|
|
auto context{builder.getContext()};
|
|
|
|
static std::map<std::pair<ParamTypeId, unsigned>,
|
|
std::pair<llvm::StringRef, mlir::FunctionType>>
|
|
uiBuiltin{
|
|
{std::make_pair(ParamTypeId::IntegerVector, 8),
|
|
std::make_pair(
|
|
"llvm.ppc.altivec.vcmpgtsb.p",
|
|
genFuncType<Ty::Integer<4>, Ty::Integer<4>, Ty::IntegerVector<1>,
|
|
Ty::IntegerVector<1>>(context, builder))},
|
|
{std::make_pair(ParamTypeId::IntegerVector, 16),
|
|
std::make_pair(
|
|
"llvm.ppc.altivec.vcmpgtsh.p",
|
|
genFuncType<Ty::Integer<4>, Ty::Integer<4>, Ty::IntegerVector<2>,
|
|
Ty::IntegerVector<2>>(context, builder))},
|
|
{std::make_pair(ParamTypeId::IntegerVector, 32),
|
|
std::make_pair(
|
|
"llvm.ppc.altivec.vcmpgtsw.p",
|
|
genFuncType<Ty::Integer<4>, Ty::Integer<4>, Ty::IntegerVector<4>,
|
|
Ty::IntegerVector<4>>(context, builder))},
|
|
{std::make_pair(ParamTypeId::IntegerVector, 64),
|
|
std::make_pair(
|
|
"llvm.ppc.altivec.vcmpgtsd.p",
|
|
genFuncType<Ty::Integer<4>, Ty::Integer<4>, Ty::IntegerVector<8>,
|
|
Ty::IntegerVector<8>>(context, builder))},
|
|
{std::make_pair(ParamTypeId::UnsignedVector, 8),
|
|
std::make_pair(
|
|
"llvm.ppc.altivec.vcmpgtub.p",
|
|
genFuncType<Ty::Integer<4>, Ty::Integer<4>,
|
|
Ty::UnsignedVector<1>, Ty::UnsignedVector<1>>(
|
|
context, builder))},
|
|
{std::make_pair(ParamTypeId::UnsignedVector, 16),
|
|
std::make_pair(
|
|
"llvm.ppc.altivec.vcmpgtuh.p",
|
|
genFuncType<Ty::Integer<4>, Ty::Integer<4>,
|
|
Ty::UnsignedVector<2>, Ty::UnsignedVector<2>>(
|
|
context, builder))},
|
|
{std::make_pair(ParamTypeId::UnsignedVector, 32),
|
|
std::make_pair(
|
|
"llvm.ppc.altivec.vcmpgtuw.p",
|
|
genFuncType<Ty::Integer<4>, Ty::Integer<4>,
|
|
Ty::UnsignedVector<4>, Ty::UnsignedVector<4>>(
|
|
context, builder))},
|
|
{std::make_pair(ParamTypeId::UnsignedVector, 64),
|
|
std::make_pair(
|
|
"llvm.ppc.altivec.vcmpgtud.p",
|
|
genFuncType<Ty::Integer<4>, Ty::Integer<4>,
|
|
Ty::UnsignedVector<8>, Ty::UnsignedVector<8>>(
|
|
context, builder))},
|
|
};
|
|
|
|
mlir::FunctionType ftype{nullptr};
|
|
llvm::StringRef fname;
|
|
const auto i32Ty{mlir::IntegerType::get(context, 32)};
|
|
llvm::SmallVector<mlir::Value> cmpArgs;
|
|
mlir::Value op{nullptr};
|
|
const auto width{vTypeInfo.eleTy.getIntOrFloatBitWidth()};
|
|
|
|
if (auto elementTy = mlir::dyn_cast<mlir::IntegerType>(vTypeInfo.eleTy)) {
|
|
std::pair<llvm::StringRef, mlir::FunctionType> bi;
|
|
bi = (elementTy.isUnsignedInteger())
|
|
? uiBuiltin[std::pair(ParamTypeId::UnsignedVector, width)]
|
|
: uiBuiltin[std::pair(ParamTypeId::IntegerVector, width)];
|
|
|
|
fname = std::get<0>(bi);
|
|
ftype = std::get<1>(bi);
|
|
|
|
op = builder.createIntegerConstant(loc, i32Ty, CR6_LT_REV);
|
|
cmpArgs.emplace_back(op);
|
|
// reverse the argument order
|
|
cmpArgs.emplace_back(argBases[1]);
|
|
cmpArgs.emplace_back(argBases[0]);
|
|
} else if (vTypeInfo.isFloat()) {
|
|
if (vTypeInfo.isFloat32()) {
|
|
fname = "llvm.ppc.vsx.xvcmpgesp.p";
|
|
ftype = genFuncType<Ty::Integer<4>, Ty::Integer<4>, Ty::RealVector<4>,
|
|
Ty::RealVector<4>>(context, builder);
|
|
} else {
|
|
fname = "llvm.ppc.vsx.xvcmpgedp.p";
|
|
ftype = genFuncType<Ty::Integer<4>, Ty::Integer<4>, Ty::RealVector<8>,
|
|
Ty::RealVector<8>>(context, builder);
|
|
}
|
|
op = builder.createIntegerConstant(loc, i32Ty, CR6_EQ_REV);
|
|
cmpArgs.emplace_back(op);
|
|
cmpArgs.emplace_back(argBases[0]);
|
|
cmpArgs.emplace_back(argBases[1]);
|
|
}
|
|
assert((!fname.empty() && ftype) && "invalid type");
|
|
|
|
mlir::func::FuncOp funcOp{builder.createFunction(loc, fname, ftype)};
|
|
auto callOp{builder.create<fir::CallOp>(loc, funcOp, cmpArgs)};
|
|
return callOp.getResult(0);
|
|
}
|
|
|
|
static std::pair<llvm::StringRef, mlir::FunctionType>
|
|
getVecCmpFuncTypeAndName(VecTypeInfo &vTypeInfo, VecOp vop,
|
|
fir::FirOpBuilder &builder) {
|
|
auto context{builder.getContext()};
|
|
static std::map<std::pair<ParamTypeId, unsigned>,
|
|
std::pair<llvm::StringRef, mlir::FunctionType>>
|
|
iuBuiltinName{
|
|
{std::make_pair(ParamTypeId::IntegerVector, 8),
|
|
std::make_pair(
|
|
"llvm.ppc.altivec.vcmpgtsb",
|
|
genFuncType<Ty::UnsignedVector<1>, Ty::IntegerVector<1>,
|
|
Ty::IntegerVector<1>>(context, builder))},
|
|
{std::make_pair(ParamTypeId::IntegerVector, 16),
|
|
std::make_pair(
|
|
"llvm.ppc.altivec.vcmpgtsh",
|
|
genFuncType<Ty::UnsignedVector<2>, Ty::IntegerVector<2>,
|
|
Ty::IntegerVector<2>>(context, builder))},
|
|
{std::make_pair(ParamTypeId::IntegerVector, 32),
|
|
std::make_pair(
|
|
"llvm.ppc.altivec.vcmpgtsw",
|
|
genFuncType<Ty::UnsignedVector<4>, Ty::IntegerVector<4>,
|
|
Ty::IntegerVector<4>>(context, builder))},
|
|
{std::make_pair(ParamTypeId::IntegerVector, 64),
|
|
std::make_pair(
|
|
"llvm.ppc.altivec.vcmpgtsd",
|
|
genFuncType<Ty::UnsignedVector<8>, Ty::IntegerVector<8>,
|
|
Ty::IntegerVector<8>>(context, builder))},
|
|
{std::make_pair(ParamTypeId::UnsignedVector, 8),
|
|
std::make_pair(
|
|
"llvm.ppc.altivec.vcmpgtub",
|
|
genFuncType<Ty::UnsignedVector<1>, Ty::UnsignedVector<1>,
|
|
Ty::UnsignedVector<1>>(context, builder))},
|
|
{std::make_pair(ParamTypeId::UnsignedVector, 16),
|
|
std::make_pair(
|
|
"llvm.ppc.altivec.vcmpgtuh",
|
|
genFuncType<Ty::UnsignedVector<2>, Ty::UnsignedVector<2>,
|
|
Ty::UnsignedVector<2>>(context, builder))},
|
|
{std::make_pair(ParamTypeId::UnsignedVector, 32),
|
|
std::make_pair(
|
|
"llvm.ppc.altivec.vcmpgtuw",
|
|
genFuncType<Ty::UnsignedVector<4>, Ty::UnsignedVector<4>,
|
|
Ty::UnsignedVector<4>>(context, builder))},
|
|
{std::make_pair(ParamTypeId::UnsignedVector, 64),
|
|
std::make_pair(
|
|
"llvm.ppc.altivec.vcmpgtud",
|
|
genFuncType<Ty::UnsignedVector<8>, Ty::UnsignedVector<8>,
|
|
Ty::UnsignedVector<8>>(context, builder))}};
|
|
|
|
// VSX only defines GE and GT builtins. Cmple and Cmplt use GE and GT with
|
|
// arguments revsered.
|
|
enum class Cmp { gtOrLt, geOrLe };
|
|
static std::map<std::pair<Cmp, int>,
|
|
std::pair<llvm::StringRef, mlir::FunctionType>>
|
|
rGBI{{std::make_pair(Cmp::geOrLe, 32),
|
|
std::make_pair("llvm.ppc.vsx.xvcmpgesp",
|
|
genFuncType<Ty::UnsignedVector<4>, Ty::RealVector<4>,
|
|
Ty::RealVector<4>>(context, builder))},
|
|
{std::make_pair(Cmp::geOrLe, 64),
|
|
std::make_pair("llvm.ppc.vsx.xvcmpgedp",
|
|
genFuncType<Ty::UnsignedVector<8>, Ty::RealVector<8>,
|
|
Ty::RealVector<8>>(context, builder))},
|
|
{std::make_pair(Cmp::gtOrLt, 32),
|
|
std::make_pair("llvm.ppc.vsx.xvcmpgtsp",
|
|
genFuncType<Ty::UnsignedVector<4>, Ty::RealVector<4>,
|
|
Ty::RealVector<4>>(context, builder))},
|
|
{std::make_pair(Cmp::gtOrLt, 64),
|
|
std::make_pair("llvm.ppc.vsx.xvcmpgtdp",
|
|
genFuncType<Ty::UnsignedVector<8>, Ty::RealVector<8>,
|
|
Ty::RealVector<8>>(context, builder))}};
|
|
|
|
const auto width{vTypeInfo.eleTy.getIntOrFloatBitWidth()};
|
|
std::pair<llvm::StringRef, mlir::FunctionType> specFunc;
|
|
if (auto elementTy = mlir::dyn_cast<mlir::IntegerType>(vTypeInfo.eleTy))
|
|
specFunc =
|
|
(elementTy.isUnsignedInteger())
|
|
? iuBuiltinName[std::make_pair(ParamTypeId::UnsignedVector, width)]
|
|
: iuBuiltinName[std::make_pair(ParamTypeId::IntegerVector, width)];
|
|
else if (vTypeInfo.isFloat())
|
|
specFunc = (vop == VecOp::Cmpge || vop == VecOp::Cmple)
|
|
? rGBI[std::make_pair(Cmp::geOrLe, width)]
|
|
: rGBI[std::make_pair(Cmp::gtOrLt, width)];
|
|
|
|
assert(!std::get<0>(specFunc).empty() && "unknown builtin name");
|
|
assert(std::get<1>(specFunc) && "unknown function type");
|
|
return specFunc;
|
|
}
|
|
|
|
// VEC_CMPGE, VEC_CMPGT, VEC_CMPLE, VEC_CMPLT
|
|
template <VecOp vop>
|
|
fir::ExtendedValue
|
|
PPCIntrinsicLibrary::genVecCmp(mlir::Type resultType,
|
|
llvm::ArrayRef<fir::ExtendedValue> args) {
|
|
assert(args.size() == 2);
|
|
auto context{builder.getContext()};
|
|
auto argBases{getBasesForArgs(args)};
|
|
VecTypeInfo vecTyInfo{getVecTypeFromFir(argBases[0])};
|
|
auto varg{convertVecArgs(builder, loc, vecTyInfo, argBases)};
|
|
|
|
std::pair<llvm::StringRef, mlir::FunctionType> funcTyNam{
|
|
getVecCmpFuncTypeAndName(vecTyInfo, vop, builder)};
|
|
|
|
mlir::func::FuncOp funcOp = builder.createFunction(
|
|
loc, std::get<0>(funcTyNam), std::get<1>(funcTyNam));
|
|
|
|
mlir::Value res{nullptr};
|
|
|
|
if (auto eTy = mlir::dyn_cast<mlir::IntegerType>(vecTyInfo.eleTy)) {
|
|
constexpr int firstArg{0};
|
|
constexpr int secondArg{1};
|
|
std::map<VecOp, std::array<int, 2>> argOrder{
|
|
{VecOp::Cmpge, {secondArg, firstArg}},
|
|
{VecOp::Cmple, {firstArg, secondArg}},
|
|
{VecOp::Cmpgt, {firstArg, secondArg}},
|
|
{VecOp::Cmplt, {secondArg, firstArg}}};
|
|
|
|
// Construct the function return type, unsigned vector, for conversion.
|
|
auto itype = mlir::IntegerType::get(context, eTy.getWidth(),
|
|
mlir::IntegerType::Unsigned);
|
|
auto returnType = fir::VectorType::get(vecTyInfo.len, itype);
|
|
|
|
switch (vop) {
|
|
case VecOp::Cmpgt:
|
|
case VecOp::Cmplt: {
|
|
// arg1 > arg2 --> vcmpgt(arg1, arg2)
|
|
// arg1 < arg2 --> vcmpgt(arg2, arg1)
|
|
mlir::Value vargs[]{argBases[argOrder[vop][0]],
|
|
argBases[argOrder[vop][1]]};
|
|
auto callOp{builder.create<fir::CallOp>(loc, funcOp, vargs)};
|
|
res = callOp.getResult(0);
|
|
break;
|
|
}
|
|
case VecOp::Cmpge:
|
|
case VecOp::Cmple: {
|
|
// arg1 >= arg2 --> vcmpge(arg2, arg1) xor vector(-1)
|
|
// arg1 <= arg2 --> vcmpge(arg1, arg2) xor vector(-1)
|
|
mlir::Value vargs[]{argBases[argOrder[vop][0]],
|
|
argBases[argOrder[vop][1]]};
|
|
|
|
// Construct a constant vector(-1)
|
|
auto negOneVal{builder.createIntegerConstant(
|
|
loc, getConvertedElementType(context, eTy), -1)};
|
|
auto vNegOne{builder.create<mlir::vector::BroadcastOp>(
|
|
loc, vecTyInfo.toMlirVectorType(context), negOneVal)};
|
|
|
|
auto callOp{builder.create<fir::CallOp>(loc, funcOp, vargs)};
|
|
mlir::Value callRes{callOp.getResult(0)};
|
|
auto vargs2{
|
|
convertVecArgs(builder, loc, vecTyInfo, mlir::ValueRange{callRes})};
|
|
auto xorRes{builder.create<mlir::arith::XOrIOp>(loc, vargs2[0], vNegOne)};
|
|
|
|
res = builder.createConvert(loc, returnType, xorRes);
|
|
break;
|
|
}
|
|
default:
|
|
llvm_unreachable("Invalid vector operation for generator");
|
|
}
|
|
} else if (vecTyInfo.isFloat()) {
|
|
mlir::Value vargs[2];
|
|
switch (vop) {
|
|
case VecOp::Cmpge:
|
|
case VecOp::Cmpgt:
|
|
vargs[0] = argBases[0];
|
|
vargs[1] = argBases[1];
|
|
break;
|
|
case VecOp::Cmple:
|
|
case VecOp::Cmplt:
|
|
// Swap the arguments as xvcmpg[et] is used
|
|
vargs[0] = argBases[1];
|
|
vargs[1] = argBases[0];
|
|
break;
|
|
default:
|
|
llvm_unreachable("Invalid vector operation for generator");
|
|
}
|
|
auto callOp{builder.create<fir::CallOp>(loc, funcOp, vargs)};
|
|
res = callOp.getResult(0);
|
|
} else
|
|
llvm_unreachable("invalid vector type");
|
|
|
|
return res;
|
|
}
|
|
|
|
static inline mlir::Value swapVectorWordPairs(fir::FirOpBuilder &builder,
|
|
mlir::Location loc,
|
|
mlir::Value arg) {
|
|
auto ty = arg.getType();
|
|
auto context{builder.getContext()};
|
|
auto vtype{mlir::VectorType::get(16, mlir::IntegerType::get(context, 8))};
|
|
|
|
if (ty != vtype)
|
|
arg = builder.create<mlir::LLVM::BitcastOp>(loc, vtype, arg).getResult();
|
|
|
|
llvm::SmallVector<int64_t, 16> mask{4, 5, 6, 7, 0, 1, 2, 3,
|
|
12, 13, 14, 15, 8, 9, 10, 11};
|
|
arg = builder.create<mlir::vector::ShuffleOp>(loc, arg, arg, mask);
|
|
if (ty != vtype)
|
|
arg = builder.create<mlir::LLVM::BitcastOp>(loc, ty, arg);
|
|
return arg;
|
|
}
|
|
|
|
// VEC_CONVERT, VEC_CTF, VEC_CVF
|
|
template <VecOp vop>
|
|
fir::ExtendedValue
|
|
PPCIntrinsicLibrary::genVecConvert(mlir::Type resultType,
|
|
llvm::ArrayRef<fir::ExtendedValue> args) {
|
|
auto context{builder.getContext()};
|
|
auto argBases{getBasesForArgs(args)};
|
|
auto vecTyInfo{getVecTypeFromFir(argBases[0])};
|
|
auto mlirTy{vecTyInfo.toMlirVectorType(context)};
|
|
auto vArg1{builder.createConvert(loc, mlirTy, argBases[0])};
|
|
const auto i32Ty{mlir::IntegerType::get(context, 32)};
|
|
|
|
switch (vop) {
|
|
case VecOp::Ctf: {
|
|
assert(args.size() == 2);
|
|
auto convArg{builder.createConvert(loc, i32Ty, argBases[1])};
|
|
auto eTy{mlir::dyn_cast<mlir::IntegerType>(vecTyInfo.eleTy)};
|
|
assert(eTy && "Unsupported vector type");
|
|
const auto isUnsigned{eTy.isUnsignedInteger()};
|
|
const auto width{eTy.getWidth()};
|
|
|
|
if (width == 32) {
|
|
auto ftype{(isUnsigned)
|
|
? genFuncType<Ty::RealVector<4>, Ty::UnsignedVector<4>,
|
|
Ty::Integer<4>>(context, builder)
|
|
: genFuncType<Ty::RealVector<4>, Ty::IntegerVector<4>,
|
|
Ty::Integer<4>>(context, builder)};
|
|
const llvm::StringRef fname{(isUnsigned) ? "llvm.ppc.altivec.vcfux"
|
|
: "llvm.ppc.altivec.vcfsx"};
|
|
auto funcOp{builder.createFunction(loc, fname, ftype)};
|
|
mlir::Value newArgs[] = {argBases[0], convArg};
|
|
auto callOp{builder.create<fir::CallOp>(loc, funcOp, newArgs)};
|
|
|
|
return callOp.getResult(0);
|
|
} else if (width == 64) {
|
|
auto fTy{mlir::FloatType::getF64(context)};
|
|
auto ty{mlir::VectorType::get(2, fTy)};
|
|
|
|
// vec_vtf(arg1, arg2) = fmul(1.0 / (1 << arg2), llvm.sitofp(arg1))
|
|
auto convOp{(isUnsigned)
|
|
? builder.create<mlir::LLVM::UIToFPOp>(loc, ty, vArg1)
|
|
: builder.create<mlir::LLVM::SIToFPOp>(loc, ty, vArg1)};
|
|
|
|
// construct vector<1./(1<<arg1), 1.0/(1<<arg1)>
|
|
auto constInt{mlir::dyn_cast_or_null<mlir::IntegerAttr>(
|
|
mlir::dyn_cast<mlir::arith::ConstantOp>(argBases[1].getDefiningOp())
|
|
.getValue())};
|
|
assert(constInt && "expected integer constant argument");
|
|
double f{1.0 / (1 << constInt.getInt())};
|
|
llvm::SmallVector<double> vals{f, f};
|
|
auto constOp{builder.create<mlir::arith::ConstantOp>(
|
|
loc, ty, builder.getF64VectorAttr(vals))};
|
|
|
|
auto mulOp{builder.create<mlir::LLVM::FMulOp>(
|
|
loc, ty, convOp->getResult(0), constOp)};
|
|
|
|
return builder.createConvert(loc, fir::VectorType::get(2, fTy), mulOp);
|
|
}
|
|
llvm_unreachable("invalid element integer kind");
|
|
}
|
|
case VecOp::Convert: {
|
|
assert(args.size() == 2);
|
|
// resultType has mold type (if scalar) or element type (if array)
|
|
auto resTyInfo{getVecTypeFromFirType(resultType)};
|
|
auto moldTy{resTyInfo.toMlirVectorType(context)};
|
|
auto firTy{resTyInfo.toFirVectorType()};
|
|
|
|
// vec_convert(v, mold) = bitcast v to "type of mold"
|
|
auto conv{builder.create<mlir::LLVM::BitcastOp>(loc, moldTy, vArg1)};
|
|
|
|
return builder.createConvert(loc, firTy, conv);
|
|
}
|
|
case VecOp::Cvf: {
|
|
assert(args.size() == 1);
|
|
|
|
mlir::Value newArgs[]{vArg1};
|
|
if (vecTyInfo.isFloat32()) {
|
|
if (changeVecElemOrder())
|
|
newArgs[0] = swapVectorWordPairs(builder, loc, newArgs[0]);
|
|
|
|
const llvm::StringRef fname{"llvm.ppc.vsx.xvcvspdp"};
|
|
auto ftype{
|
|
genFuncType<Ty::RealVector<8>, Ty::RealVector<4>>(context, builder)};
|
|
auto funcOp{builder.createFunction(loc, fname, ftype)};
|
|
auto callOp{builder.create<fir::CallOp>(loc, funcOp, newArgs)};
|
|
|
|
return callOp.getResult(0);
|
|
} else if (vecTyInfo.isFloat64()) {
|
|
const llvm::StringRef fname{"llvm.ppc.vsx.xvcvdpsp"};
|
|
auto ftype{
|
|
genFuncType<Ty::RealVector<4>, Ty::RealVector<8>>(context, builder)};
|
|
auto funcOp{builder.createFunction(loc, fname, ftype)};
|
|
newArgs[0] =
|
|
builder.create<fir::CallOp>(loc, funcOp, newArgs).getResult(0);
|
|
auto fvf32Ty{newArgs[0].getType()};
|
|
auto f32type{mlir::FloatType::getF32(context)};
|
|
auto mvf32Ty{mlir::VectorType::get(4, f32type)};
|
|
newArgs[0] = builder.createConvert(loc, mvf32Ty, newArgs[0]);
|
|
|
|
if (changeVecElemOrder())
|
|
newArgs[0] = swapVectorWordPairs(builder, loc, newArgs[0]);
|
|
|
|
return builder.createConvert(loc, fvf32Ty, newArgs[0]);
|
|
}
|
|
llvm_unreachable("invalid element integer kind");
|
|
}
|
|
default:
|
|
llvm_unreachable("Invalid vector operation for generator");
|
|
}
|
|
}
|
|
|
|
static mlir::Value convertVectorElementOrder(fir::FirOpBuilder &builder,
|
|
mlir::Location loc,
|
|
VecTypeInfo vecInfo,
|
|
mlir::Value idx) {
|
|
mlir::Value numSub1{
|
|
builder.createIntegerConstant(loc, idx.getType(), vecInfo.len - 1)};
|
|
return builder.create<mlir::LLVM::SubOp>(loc, idx.getType(), numSub1, idx);
|
|
}
|
|
|
|
// VEC_EXTRACT
|
|
fir::ExtendedValue
|
|
PPCIntrinsicLibrary::genVecExtract(mlir::Type resultType,
|
|
llvm::ArrayRef<fir::ExtendedValue> args) {
|
|
assert(args.size() == 2);
|
|
auto argBases{getBasesForArgs(args)};
|
|
auto argTypes{getTypesForArgs(argBases)};
|
|
auto vecTyInfo{getVecTypeFromFir(argBases[0])};
|
|
|
|
auto mlirTy{vecTyInfo.toMlirVectorType(builder.getContext())};
|
|
auto varg0{builder.createConvert(loc, mlirTy, argBases[0])};
|
|
|
|
// arg2 modulo the number of elements in arg1 to determine the element
|
|
// position
|
|
auto numEle{builder.createIntegerConstant(loc, argTypes[1], vecTyInfo.len)};
|
|
mlir::Value uremOp{
|
|
builder.create<mlir::LLVM::URemOp>(loc, argBases[1], numEle)};
|
|
|
|
if (!isNativeVecElemOrderOnLE())
|
|
uremOp = convertVectorElementOrder(builder, loc, vecTyInfo, uremOp);
|
|
|
|
return builder.create<mlir::vector::ExtractElementOp>(loc, varg0, uremOp);
|
|
}
|
|
|
|
// VEC_INSERT
|
|
fir::ExtendedValue
|
|
PPCIntrinsicLibrary::genVecInsert(mlir::Type resultType,
|
|
llvm::ArrayRef<fir::ExtendedValue> args) {
|
|
assert(args.size() == 3);
|
|
auto argBases{getBasesForArgs(args)};
|
|
auto argTypes{getTypesForArgs(argBases)};
|
|
auto vecTyInfo{getVecTypeFromFir(argBases[1])};
|
|
auto mlirTy{vecTyInfo.toMlirVectorType(builder.getContext())};
|
|
auto varg1{builder.createConvert(loc, mlirTy, argBases[1])};
|
|
|
|
auto numEle{builder.createIntegerConstant(loc, argTypes[2], vecTyInfo.len)};
|
|
mlir::Value uremOp{
|
|
builder.create<mlir::LLVM::URemOp>(loc, argBases[2], numEle)};
|
|
|
|
if (!isNativeVecElemOrderOnLE())
|
|
uremOp = convertVectorElementOrder(builder, loc, vecTyInfo, uremOp);
|
|
|
|
auto res{builder.create<mlir::vector::InsertElementOp>(loc, argBases[0],
|
|
varg1, uremOp)};
|
|
return builder.create<fir::ConvertOp>(loc, vecTyInfo.toFirVectorType(), res);
|
|
}
|
|
|
|
// VEC_MERGEH, VEC_MERGEL
|
|
template <VecOp vop>
|
|
fir::ExtendedValue
|
|
PPCIntrinsicLibrary::genVecMerge(mlir::Type resultType,
|
|
llvm::ArrayRef<fir::ExtendedValue> args) {
|
|
assert(args.size() == 2);
|
|
auto argBases{getBasesForArgs(args)};
|
|
auto vecTyInfo{getVecTypeFromFir(argBases[0])};
|
|
llvm::SmallVector<int64_t, 16> mMask; // native vector element order mask
|
|
llvm::SmallVector<int64_t, 16> rMask; // non-native vector element order mask
|
|
|
|
switch (vop) {
|
|
case VecOp::Mergeh: {
|
|
switch (vecTyInfo.len) {
|
|
case 2: {
|
|
enum { V1 = 0, V2 = 2 };
|
|
mMask = {V1 + 0, V2 + 0};
|
|
rMask = {V2 + 1, V1 + 1};
|
|
break;
|
|
}
|
|
case 4: {
|
|
enum { V1 = 0, V2 = 4 };
|
|
mMask = {V1 + 0, V2 + 0, V1 + 1, V2 + 1};
|
|
rMask = {V2 + 2, V1 + 2, V2 + 3, V1 + 3};
|
|
break;
|
|
}
|
|
case 8: {
|
|
enum { V1 = 0, V2 = 8 };
|
|
mMask = {V1 + 0, V2 + 0, V1 + 1, V2 + 1, V1 + 2, V2 + 2, V1 + 3, V2 + 3};
|
|
rMask = {V2 + 4, V1 + 4, V2 + 5, V1 + 5, V2 + 6, V1 + 6, V2 + 7, V1 + 7};
|
|
break;
|
|
}
|
|
case 16:
|
|
mMask = {0x00, 0x10, 0x01, 0x11, 0x02, 0x12, 0x03, 0x13,
|
|
0x04, 0x14, 0x05, 0x15, 0x06, 0x16, 0x07, 0x17};
|
|
rMask = {0x18, 0x08, 0x19, 0x09, 0x1A, 0x0A, 0x1B, 0x0B,
|
|
0x1C, 0x0C, 0x1D, 0x0D, 0x1E, 0x0E, 0x1F, 0x0F};
|
|
break;
|
|
default:
|
|
llvm_unreachable("unexpected vector length");
|
|
}
|
|
break;
|
|
}
|
|
case VecOp::Mergel: {
|
|
switch (vecTyInfo.len) {
|
|
case 2: {
|
|
enum { V1 = 0, V2 = 2 };
|
|
mMask = {V1 + 1, V2 + 1};
|
|
rMask = {V2 + 0, V1 + 0};
|
|
break;
|
|
}
|
|
case 4: {
|
|
enum { V1 = 0, V2 = 4 };
|
|
mMask = {V1 + 2, V2 + 2, V1 + 3, V2 + 3};
|
|
rMask = {V2 + 0, V1 + 0, V2 + 1, V1 + 1};
|
|
break;
|
|
}
|
|
case 8: {
|
|
enum { V1 = 0, V2 = 8 };
|
|
mMask = {V1 + 4, V2 + 4, V1 + 5, V2 + 5, V1 + 6, V2 + 6, V1 + 7, V2 + 7};
|
|
rMask = {V2 + 0, V1 + 0, V2 + 1, V1 + 1, V2 + 2, V1 + 2, V2 + 3, V1 + 3};
|
|
break;
|
|
}
|
|
case 16:
|
|
mMask = {0x08, 0x18, 0x09, 0x19, 0x0A, 0x1A, 0x0B, 0x1B,
|
|
0x0C, 0x1C, 0x0D, 0x1D, 0x0E, 0x1E, 0x0F, 0x1F};
|
|
rMask = {0x10, 0x00, 0x11, 0x01, 0x12, 0x02, 0x13, 0x03,
|
|
0x14, 0x04, 0x15, 0x05, 0x16, 0x06, 0x17, 0x07};
|
|
break;
|
|
default:
|
|
llvm_unreachable("unexpected vector length");
|
|
}
|
|
break;
|
|
}
|
|
default:
|
|
llvm_unreachable("invalid vector operation for generator");
|
|
}
|
|
|
|
auto vargs{convertVecArgs(builder, loc, vecTyInfo, argBases)};
|
|
|
|
llvm::SmallVector<int64_t, 16> &mergeMask =
|
|
(isBEVecElemOrderOnLE()) ? rMask : mMask;
|
|
|
|
auto callOp{builder.create<mlir::vector::ShuffleOp>(loc, vargs[0], vargs[1],
|
|
mergeMask)};
|
|
return builder.createConvert(loc, resultType, callOp);
|
|
}
|
|
|
|
static mlir::Value addOffsetToAddress(fir::FirOpBuilder &builder,
|
|
mlir::Location loc, mlir::Value baseAddr,
|
|
mlir::Value offset) {
|
|
auto typeExtent{fir::SequenceType::getUnknownExtent()};
|
|
// Construct an !fir.ref<!ref.array<?xi8>> type
|
|
auto arrRefTy{builder.getRefType(fir::SequenceType::get(
|
|
{typeExtent}, mlir::IntegerType::get(builder.getContext(), 8)))};
|
|
// Convert arg to !fir.ref<!ref.array<?xi8>>
|
|
auto resAddr{builder.create<fir::ConvertOp>(loc, arrRefTy, baseAddr)};
|
|
|
|
return builder.create<fir::CoordinateOp>(loc, arrRefTy, resAddr, offset);
|
|
}
|
|
|
|
static mlir::Value reverseVectorElements(fir::FirOpBuilder &builder,
|
|
mlir::Location loc, mlir::Value v,
|
|
int64_t len) {
|
|
assert(mlir::isa<mlir::VectorType>(v.getType()));
|
|
assert(len > 0);
|
|
llvm::SmallVector<int64_t, 16> mask;
|
|
for (int64_t i = 0; i < len; ++i) {
|
|
mask.push_back(len - 1 - i);
|
|
}
|
|
auto undefVec{builder.create<fir::UndefOp>(loc, v.getType())};
|
|
return builder.create<mlir::vector::ShuffleOp>(loc, v, undefVec, mask);
|
|
}
|
|
|
|
static mlir::NamedAttribute getAlignmentAttr(fir::FirOpBuilder &builder,
|
|
const int val) {
|
|
auto i64ty{mlir::IntegerType::get(builder.getContext(), 64)};
|
|
auto alignAttr{mlir::IntegerAttr::get(i64ty, val)};
|
|
return builder.getNamedAttr("alignment", alignAttr);
|
|
}
|
|
|
|
fir::ExtendedValue
|
|
PPCIntrinsicLibrary::genVecXlGrp(mlir::Type resultType,
|
|
llvm::ArrayRef<fir::ExtendedValue> args) {
|
|
VecTypeInfo vecTyInfo{getVecTypeFromFirType(resultType)};
|
|
switch (vecTyInfo.eleTy.getIntOrFloatBitWidth()) {
|
|
case 8:
|
|
// vec_xlb1
|
|
return genVecLdNoCallGrp<VecOp::Xl>(resultType, args);
|
|
case 16:
|
|
// vec_xlh8
|
|
return genVecLdNoCallGrp<VecOp::Xl>(resultType, args);
|
|
case 32:
|
|
// vec_xlw4
|
|
return genVecLdCallGrp<VecOp::Xlw4>(resultType, args);
|
|
case 64:
|
|
// vec_xld2
|
|
return genVecLdCallGrp<VecOp::Xld2>(resultType, args);
|
|
default:
|
|
llvm_unreachable("invalid kind");
|
|
}
|
|
llvm_unreachable("invalid vector operation for generator");
|
|
}
|
|
|
|
template <VecOp vop>
|
|
fir::ExtendedValue PPCIntrinsicLibrary::genVecLdNoCallGrp(
|
|
mlir::Type resultType, llvm::ArrayRef<fir::ExtendedValue> args) {
|
|
assert(args.size() == 2);
|
|
auto arg0{getBase(args[0])};
|
|
auto arg1{getBase(args[1])};
|
|
|
|
auto vecTyInfo{getVecTypeFromFirType(resultType)};
|
|
auto mlirTy{vecTyInfo.toMlirVectorType(builder.getContext())};
|
|
auto firTy{vecTyInfo.toFirVectorType()};
|
|
|
|
// Add the %val of arg0 to %addr of arg1
|
|
auto addr{addOffsetToAddress(builder, loc, arg1, arg0)};
|
|
|
|
const auto triple{fir::getTargetTriple(builder.getModule())};
|
|
// Need to get align 1.
|
|
auto result{builder.create<fir::LoadOp>(loc, mlirTy, addr,
|
|
getAlignmentAttr(builder, 1))};
|
|
if ((vop == VecOp::Xl && isBEVecElemOrderOnLE()) ||
|
|
(vop == VecOp::Xlbe && triple.isLittleEndian()))
|
|
return builder.createConvert(
|
|
loc, firTy, reverseVectorElements(builder, loc, result, vecTyInfo.len));
|
|
|
|
return builder.createConvert(loc, firTy, result);
|
|
}
|
|
|
|
// VEC_LD, VEC_LDE, VEC_LDL, VEC_LXVP, VEC_XLD2, VEC_XLW4
|
|
template <VecOp vop>
|
|
fir::ExtendedValue
|
|
PPCIntrinsicLibrary::genVecLdCallGrp(mlir::Type resultType,
|
|
llvm::ArrayRef<fir::ExtendedValue> args) {
|
|
assert(args.size() == 2);
|
|
auto context{builder.getContext()};
|
|
auto arg0{getBase(args[0])};
|
|
auto arg1{getBase(args[1])};
|
|
|
|
// Prepare the return type in FIR.
|
|
auto vecResTyInfo{getVecTypeFromFirType(resultType)};
|
|
auto mlirTy{vecResTyInfo.toMlirVectorType(context)};
|
|
auto firTy{vecResTyInfo.toFirVectorType()};
|
|
|
|
// llvm.ppc.altivec.lvx* returns <4xi32>
|
|
// Others, like "llvm.ppc.altivec.lvebx" too if arg2 is not of Integer type
|
|
const auto i32Ty{mlir::IntegerType::get(builder.getContext(), 32)};
|
|
const auto mVecI32Ty{mlir::VectorType::get(4, i32Ty)};
|
|
|
|
// For vec_ld, need to convert arg0 from i64 to i32
|
|
if (vop == VecOp::Ld && arg0.getType().getIntOrFloatBitWidth() == 64)
|
|
arg0 = builder.createConvert(loc, i32Ty, arg0);
|
|
|
|
// Add the %val of arg0 to %addr of arg1
|
|
auto addr{addOffsetToAddress(builder, loc, arg1, arg0)};
|
|
llvm::SmallVector<mlir::Value, 4> parsedArgs{addr};
|
|
|
|
mlir::Type intrinResTy{nullptr};
|
|
llvm::StringRef fname{};
|
|
switch (vop) {
|
|
case VecOp::Ld:
|
|
fname = "llvm.ppc.altivec.lvx";
|
|
intrinResTy = mVecI32Ty;
|
|
break;
|
|
case VecOp::Lde:
|
|
switch (vecResTyInfo.eleTy.getIntOrFloatBitWidth()) {
|
|
case 8:
|
|
fname = "llvm.ppc.altivec.lvebx";
|
|
intrinResTy = mlirTy;
|
|
break;
|
|
case 16:
|
|
fname = "llvm.ppc.altivec.lvehx";
|
|
intrinResTy = mlirTy;
|
|
break;
|
|
case 32:
|
|
fname = "llvm.ppc.altivec.lvewx";
|
|
if (mlir::isa<mlir::IntegerType>(vecResTyInfo.eleTy))
|
|
intrinResTy = mlirTy;
|
|
else
|
|
intrinResTy = mVecI32Ty;
|
|
break;
|
|
default:
|
|
llvm_unreachable("invalid vector for vec_lde");
|
|
}
|
|
break;
|
|
case VecOp::Ldl:
|
|
fname = "llvm.ppc.altivec.lvxl";
|
|
intrinResTy = mVecI32Ty;
|
|
break;
|
|
case VecOp::Lxvp:
|
|
fname = "llvm.ppc.vsx.lxvp";
|
|
intrinResTy = fir::VectorType::get(256, mlir::IntegerType::get(context, 1));
|
|
break;
|
|
case VecOp::Xld2: {
|
|
fname = isBEVecElemOrderOnLE() ? "llvm.ppc.vsx.lxvd2x.be"
|
|
: "llvm.ppc.vsx.lxvd2x";
|
|
// llvm.ppc.altivec.lxvd2x* returns <2 x double>
|
|
intrinResTy = mlir::VectorType::get(2, mlir::FloatType::getF64(context));
|
|
} break;
|
|
case VecOp::Xlw4:
|
|
fname = isBEVecElemOrderOnLE() ? "llvm.ppc.vsx.lxvw4x.be"
|
|
: "llvm.ppc.vsx.lxvw4x";
|
|
// llvm.ppc.altivec.lxvw4x* returns <4xi32>
|
|
intrinResTy = mVecI32Ty;
|
|
break;
|
|
default:
|
|
llvm_unreachable("invalid vector operation for generator");
|
|
}
|
|
|
|
auto funcType{
|
|
mlir::FunctionType::get(context, {addr.getType()}, {intrinResTy})};
|
|
auto funcOp{builder.createFunction(loc, fname, funcType)};
|
|
auto result{
|
|
builder.create<fir::CallOp>(loc, funcOp, parsedArgs).getResult(0)};
|
|
|
|
if (vop == VecOp::Lxvp)
|
|
return result;
|
|
|
|
if (intrinResTy != mlirTy)
|
|
result = builder.create<mlir::vector::BitCastOp>(loc, mlirTy, result);
|
|
|
|
if (vop != VecOp::Xld2 && vop != VecOp::Xlw4 && isBEVecElemOrderOnLE())
|
|
return builder.createConvert(
|
|
loc, firTy,
|
|
reverseVectorElements(builder, loc, result, vecResTyInfo.len));
|
|
|
|
return builder.createConvert(loc, firTy, result);
|
|
}
|
|
|
|
// VEC_LVSL, VEC_LVSR
|
|
template <VecOp vop>
|
|
fir::ExtendedValue
|
|
PPCIntrinsicLibrary::genVecLvsGrp(mlir::Type resultType,
|
|
llvm::ArrayRef<fir::ExtendedValue> args) {
|
|
assert(args.size() == 2);
|
|
auto context{builder.getContext()};
|
|
auto arg0{getBase(args[0])};
|
|
auto arg1{getBase(args[1])};
|
|
|
|
auto vecTyInfo{getVecTypeFromFirType(resultType)};
|
|
auto mlirTy{vecTyInfo.toMlirVectorType(context)};
|
|
auto firTy{vecTyInfo.toFirVectorType()};
|
|
|
|
// Convert arg0 to i64 type if needed
|
|
auto i64ty{mlir::IntegerType::get(context, 64)};
|
|
if (arg0.getType() != i64ty)
|
|
arg0 = builder.create<fir::ConvertOp>(loc, i64ty, arg0);
|
|
|
|
// offset is modulo 16, so shift left 56 bits and then right 56 bits to clear
|
|
// upper 56 bit while preserving sign
|
|
auto shiftVal{builder.createIntegerConstant(loc, i64ty, 56)};
|
|
auto offset{builder.create<mlir::arith::ShLIOp>(loc, arg0, shiftVal)};
|
|
auto offset2{builder.create<mlir::arith::ShRSIOp>(loc, offset, shiftVal)};
|
|
|
|
// Add the offsetArg to %addr of arg1
|
|
auto addr{addOffsetToAddress(builder, loc, arg1, offset2)};
|
|
llvm::SmallVector<mlir::Value, 4> parsedArgs{addr};
|
|
|
|
llvm::StringRef fname{};
|
|
switch (vop) {
|
|
case VecOp::Lvsl:
|
|
fname = "llvm.ppc.altivec.lvsl";
|
|
break;
|
|
case VecOp::Lvsr:
|
|
fname = "llvm.ppc.altivec.lvsr";
|
|
break;
|
|
default:
|
|
llvm_unreachable("invalid vector operation for generator");
|
|
}
|
|
auto funcType{mlir::FunctionType::get(context, {addr.getType()}, {mlirTy})};
|
|
auto funcOp{builder.createFunction(loc, fname, funcType)};
|
|
auto result{
|
|
builder.create<fir::CallOp>(loc, funcOp, parsedArgs).getResult(0)};
|
|
|
|
if (isNativeVecElemOrderOnLE())
|
|
return builder.createConvert(
|
|
loc, firTy, reverseVectorElements(builder, loc, result, vecTyInfo.len));
|
|
|
|
return builder.createConvert(loc, firTy, result);
|
|
}
|
|
|
|
// VEC_NMADD, VEC_MSUB
|
|
template <VecOp vop>
|
|
fir::ExtendedValue
|
|
PPCIntrinsicLibrary::genVecNmaddMsub(mlir::Type resultType,
|
|
llvm::ArrayRef<fir::ExtendedValue> args) {
|
|
assert(args.size() == 3);
|
|
auto context{builder.getContext()};
|
|
auto argBases{getBasesForArgs(args)};
|
|
auto vTypeInfo{getVecTypeFromFir(argBases[0])};
|
|
auto newArgs{convertVecArgs(builder, loc, vTypeInfo, argBases)};
|
|
const auto width{vTypeInfo.eleTy.getIntOrFloatBitWidth()};
|
|
|
|
static std::map<int, std::pair<llvm::StringRef, mlir::FunctionType>> fmaMap{
|
|
{32,
|
|
std::make_pair(
|
|
"llvm.fma.v4f32",
|
|
genFuncType<Ty::RealVector<4>, Ty::RealVector<4>, Ty::RealVector<4>>(
|
|
context, builder))},
|
|
{64,
|
|
std::make_pair(
|
|
"llvm.fma.v2f64",
|
|
genFuncType<Ty::RealVector<8>, Ty::RealVector<8>, Ty::RealVector<8>>(
|
|
context, builder))}};
|
|
|
|
auto funcOp{builder.createFunction(loc, std::get<0>(fmaMap[width]),
|
|
std::get<1>(fmaMap[width]))};
|
|
if (vop == VecOp::Nmadd) {
|
|
// vec_nmadd(arg1, arg2, arg3) = -fma(arg1, arg2, arg3)
|
|
auto callOp{builder.create<fir::CallOp>(loc, funcOp, newArgs)};
|
|
|
|
// We need to convert fir.vector to MLIR vector to use fneg and then back
|
|
// to fir.vector to store.
|
|
auto vCall{builder.createConvert(loc, vTypeInfo.toMlirVectorType(context),
|
|
callOp.getResult(0))};
|
|
auto neg{builder.create<mlir::arith::NegFOp>(loc, vCall)};
|
|
return builder.createConvert(loc, vTypeInfo.toFirVectorType(), neg);
|
|
} else if (vop == VecOp::Msub) {
|
|
// vec_msub(arg1, arg2, arg3) = fma(arg1, arg2, -arg3)
|
|
newArgs[2] = builder.create<mlir::arith::NegFOp>(loc, newArgs[2]);
|
|
|
|
auto callOp{builder.create<fir::CallOp>(loc, funcOp, newArgs)};
|
|
return callOp.getResult(0);
|
|
}
|
|
llvm_unreachable("Invalid vector operation for generator");
|
|
}
|
|
|
|
// VEC_PERM, VEC_PERMI
|
|
template <VecOp vop>
|
|
fir::ExtendedValue
|
|
PPCIntrinsicLibrary::genVecPerm(mlir::Type resultType,
|
|
llvm::ArrayRef<fir::ExtendedValue> args) {
|
|
assert(args.size() == 3);
|
|
auto context{builder.getContext()};
|
|
auto argBases{getBasesForArgs(args)};
|
|
auto argTypes{getTypesForArgs(argBases)};
|
|
auto vecTyInfo{getVecTypeFromFir(argBases[0])};
|
|
auto mlirTy{vecTyInfo.toMlirVectorType(context)};
|
|
|
|
auto vi32Ty{mlir::VectorType::get(4, mlir::IntegerType::get(context, 32))};
|
|
auto vf64Ty{mlir::VectorType::get(2, mlir::FloatType::getF64(context))};
|
|
|
|
auto mArg0{builder.createConvert(loc, mlirTy, argBases[0])};
|
|
auto mArg1{builder.createConvert(loc, mlirTy, argBases[1])};
|
|
|
|
switch (vop) {
|
|
case VecOp::Perm: {
|
|
VecTypeInfo maskVecTyInfo{getVecTypeFromFir(argBases[2])};
|
|
auto mlirMaskTy{maskVecTyInfo.toMlirVectorType(context)};
|
|
auto mMask{builder.createConvert(loc, mlirMaskTy, argBases[2])};
|
|
|
|
if (mlirTy != vi32Ty) {
|
|
mArg0 =
|
|
builder.create<mlir::LLVM::BitcastOp>(loc, vi32Ty, mArg0).getResult();
|
|
mArg1 =
|
|
builder.create<mlir::LLVM::BitcastOp>(loc, vi32Ty, mArg1).getResult();
|
|
}
|
|
|
|
auto funcOp{builder.createFunction(
|
|
loc, "llvm.ppc.altivec.vperm",
|
|
genFuncType<Ty::IntegerVector<4>, Ty::IntegerVector<4>,
|
|
Ty::IntegerVector<4>, Ty::IntegerVector<1>>(context,
|
|
builder))};
|
|
|
|
llvm::SmallVector<mlir::Value> newArgs;
|
|
if (isNativeVecElemOrderOnLE()) {
|
|
auto i8Ty{mlir::IntegerType::get(context, 8)};
|
|
auto v8Ty{mlir::VectorType::get(16, i8Ty)};
|
|
auto negOne{builder.createMinusOneInteger(loc, i8Ty)};
|
|
auto vNegOne{
|
|
builder.create<mlir::vector::BroadcastOp>(loc, v8Ty, negOne)};
|
|
|
|
mMask = builder.create<mlir::arith::XOrIOp>(loc, mMask, vNegOne);
|
|
newArgs = {mArg1, mArg0, mMask};
|
|
} else {
|
|
newArgs = {mArg0, mArg1, mMask};
|
|
}
|
|
|
|
auto res{builder.create<fir::CallOp>(loc, funcOp, newArgs).getResult(0)};
|
|
|
|
if (res.getType() != argTypes[0]) {
|
|
// fir.call llvm.ppc.altivec.vperm returns !fir.vector<i4:32>
|
|
// convert the result back to the original type
|
|
res = builder.createConvert(loc, vi32Ty, res);
|
|
if (mlirTy != vi32Ty)
|
|
res =
|
|
builder.create<mlir::LLVM::BitcastOp>(loc, mlirTy, res).getResult();
|
|
}
|
|
return builder.createConvert(loc, resultType, res);
|
|
}
|
|
case VecOp::Permi: {
|
|
// arg3 is a constant
|
|
auto constIntOp{mlir::dyn_cast_or_null<mlir::IntegerAttr>(
|
|
mlir::dyn_cast<mlir::arith::ConstantOp>(argBases[2].getDefiningOp())
|
|
.getValue())};
|
|
assert(constIntOp && "expected integer constant argument");
|
|
auto constInt{constIntOp.getInt()};
|
|
// arg1, arg2, and result type share same VecTypeInfo
|
|
if (vecTyInfo.isFloat()) {
|
|
mArg0 =
|
|
builder.create<mlir::LLVM::BitcastOp>(loc, vf64Ty, mArg0).getResult();
|
|
mArg1 =
|
|
builder.create<mlir::LLVM::BitcastOp>(loc, vf64Ty, mArg1).getResult();
|
|
}
|
|
|
|
llvm::SmallVector<int64_t, 2> nMask; // native vector element order mask
|
|
llvm::SmallVector<int64_t, 2> rMask; // non-native vector element order mask
|
|
enum { V1 = 0, V2 = 2 };
|
|
switch (constInt) {
|
|
case 0:
|
|
nMask = {V1 + 0, V2 + 0};
|
|
rMask = {V2 + 1, V1 + 1};
|
|
break;
|
|
case 1:
|
|
nMask = {V1 + 0, V2 + 1};
|
|
rMask = {V2 + 0, V1 + 1};
|
|
break;
|
|
case 2:
|
|
nMask = {V1 + 1, V2 + 0};
|
|
rMask = {V2 + 1, V1 + 0};
|
|
break;
|
|
case 3:
|
|
nMask = {V1 + 1, V2 + 1};
|
|
rMask = {V2 + 0, V1 + 0};
|
|
break;
|
|
default:
|
|
llvm_unreachable("unexpected arg3 value for vec_permi");
|
|
}
|
|
|
|
llvm::SmallVector<int64_t, 2> mask =
|
|
(isBEVecElemOrderOnLE()) ? rMask : nMask;
|
|
auto res{builder.create<mlir::vector::ShuffleOp>(loc, mArg0, mArg1, mask)};
|
|
if (res.getType() != mlirTy) {
|
|
auto cast{builder.create<mlir::LLVM::BitcastOp>(loc, mlirTy, res)};
|
|
return builder.createConvert(loc, resultType, cast);
|
|
}
|
|
return builder.createConvert(loc, resultType, res);
|
|
}
|
|
default:
|
|
llvm_unreachable("invalid vector operation for generator");
|
|
}
|
|
}
|
|
|
|
// VEC_SEL
|
|
fir::ExtendedValue
|
|
PPCIntrinsicLibrary::genVecSel(mlir::Type resultType,
|
|
llvm::ArrayRef<fir::ExtendedValue> args) {
|
|
assert(args.size() == 3);
|
|
auto argBases{getBasesForArgs(args)};
|
|
llvm::SmallVector<VecTypeInfo, 4> vecTyInfos;
|
|
for (size_t i = 0; i < argBases.size(); i++) {
|
|
vecTyInfos.push_back(getVecTypeFromFir(argBases[i]));
|
|
}
|
|
auto vargs{convertVecArgs(builder, loc, vecTyInfos, argBases)};
|
|
|
|
auto i8Ty{mlir::IntegerType::get(builder.getContext(), 8)};
|
|
auto negOne{builder.createMinusOneInteger(loc, i8Ty)};
|
|
|
|
// construct a constant <16 x i8> vector with value -1 for bitcast
|
|
auto bcVecTy{mlir::VectorType::get(16, i8Ty)};
|
|
auto vNegOne{builder.create<mlir::vector::BroadcastOp>(loc, bcVecTy, negOne)};
|
|
|
|
// bitcast arguments to bcVecTy
|
|
auto arg1{builder.create<mlir::vector::BitCastOp>(loc, bcVecTy, vargs[0])};
|
|
auto arg2{builder.create<mlir::vector::BitCastOp>(loc, bcVecTy, vargs[1])};
|
|
auto arg3{builder.create<mlir::vector::BitCastOp>(loc, bcVecTy, vargs[2])};
|
|
|
|
// vec_sel(arg1, arg2, arg3) =
|
|
// (arg2 and arg3) or (arg1 and (arg3 xor vector(-1,...)))
|
|
auto comp{builder.create<mlir::arith::XOrIOp>(loc, arg3, vNegOne)};
|
|
auto a1AndComp{builder.create<mlir::arith::AndIOp>(loc, arg1, comp)};
|
|
auto a1OrA2{builder.create<mlir::arith::AndIOp>(loc, arg2, arg3)};
|
|
auto res{builder.create<mlir::arith::OrIOp>(loc, a1AndComp, a1OrA2)};
|
|
|
|
auto bcRes{
|
|
builder.create<mlir::vector::BitCastOp>(loc, vargs[0].getType(), res)};
|
|
|
|
return builder.createConvert(loc, vecTyInfos[0].toFirVectorType(), bcRes);
|
|
}
|
|
|
|
// VEC_SL, VEC_SLD, VEC_SLDW, VEC_SLL, VEC_SLO, VEC_SR, VEC_SRL, VEC_SRO
|
|
template <VecOp vop>
|
|
fir::ExtendedValue
|
|
PPCIntrinsicLibrary::genVecShift(mlir::Type resultType,
|
|
llvm::ArrayRef<fir::ExtendedValue> args) {
|
|
auto context{builder.getContext()};
|
|
auto argBases{getBasesForArgs(args)};
|
|
auto argTypes{getTypesForArgs(argBases)};
|
|
|
|
llvm::SmallVector<VecTypeInfo, 2> vecTyInfoArgs;
|
|
vecTyInfoArgs.push_back(getVecTypeFromFir(argBases[0]));
|
|
vecTyInfoArgs.push_back(getVecTypeFromFir(argBases[1]));
|
|
|
|
// Convert the first two arguments to MLIR vectors
|
|
llvm::SmallVector<mlir::Type, 2> mlirTyArgs;
|
|
mlirTyArgs.push_back(vecTyInfoArgs[0].toMlirVectorType(context));
|
|
mlirTyArgs.push_back(vecTyInfoArgs[1].toMlirVectorType(context));
|
|
|
|
llvm::SmallVector<mlir::Value, 2> mlirVecArgs;
|
|
mlirVecArgs.push_back(builder.createConvert(loc, mlirTyArgs[0], argBases[0]));
|
|
mlirVecArgs.push_back(builder.createConvert(loc, mlirTyArgs[1], argBases[1]));
|
|
|
|
mlir::Value shftRes{nullptr};
|
|
|
|
if (vop == VecOp::Sl || vop == VecOp::Sr) {
|
|
assert(args.size() == 2);
|
|
// Construct the mask
|
|
auto width{
|
|
mlir::dyn_cast<mlir::IntegerType>(vecTyInfoArgs[1].eleTy).getWidth()};
|
|
auto vecVal{builder.createIntegerConstant(
|
|
loc, getConvertedElementType(context, vecTyInfoArgs[0].eleTy), width)};
|
|
auto mask{
|
|
builder.create<mlir::vector::BroadcastOp>(loc, mlirTyArgs[1], vecVal)};
|
|
auto shft{builder.create<mlir::arith::RemUIOp>(loc, mlirVecArgs[1], mask)};
|
|
|
|
mlir::Value res{nullptr};
|
|
if (vop == VecOp::Sr)
|
|
res = builder.create<mlir::arith::ShRUIOp>(loc, mlirVecArgs[0], shft);
|
|
else if (vop == VecOp::Sl)
|
|
res = builder.create<mlir::arith::ShLIOp>(loc, mlirVecArgs[0], shft);
|
|
|
|
shftRes = builder.createConvert(loc, argTypes[0], res);
|
|
} else if (vop == VecOp::Sll || vop == VecOp::Slo || vop == VecOp::Srl ||
|
|
vop == VecOp::Sro) {
|
|
assert(args.size() == 2);
|
|
|
|
// Bitcast to vector<4xi32>
|
|
auto bcVecTy{mlir::VectorType::get(4, builder.getIntegerType(32))};
|
|
if (mlirTyArgs[0] != bcVecTy)
|
|
mlirVecArgs[0] =
|
|
builder.create<mlir::vector::BitCastOp>(loc, bcVecTy, mlirVecArgs[0]);
|
|
if (mlirTyArgs[1] != bcVecTy)
|
|
mlirVecArgs[1] =
|
|
builder.create<mlir::vector::BitCastOp>(loc, bcVecTy, mlirVecArgs[1]);
|
|
|
|
llvm::StringRef funcName;
|
|
switch (vop) {
|
|
case VecOp::Srl:
|
|
funcName = "llvm.ppc.altivec.vsr";
|
|
break;
|
|
case VecOp::Sro:
|
|
funcName = "llvm.ppc.altivec.vsro";
|
|
break;
|
|
case VecOp::Sll:
|
|
funcName = "llvm.ppc.altivec.vsl";
|
|
break;
|
|
case VecOp::Slo:
|
|
funcName = "llvm.ppc.altivec.vslo";
|
|
break;
|
|
default:
|
|
llvm_unreachable("unknown vector shift operation");
|
|
}
|
|
auto funcTy{genFuncType<Ty::IntegerVector<4>, Ty::IntegerVector<4>,
|
|
Ty::IntegerVector<4>>(context, builder)};
|
|
mlir::func::FuncOp funcOp{builder.createFunction(loc, funcName, funcTy)};
|
|
auto callOp{builder.create<fir::CallOp>(loc, funcOp, mlirVecArgs)};
|
|
|
|
// If the result vector type is different from the original type, need
|
|
// to convert to mlir vector, bitcast and then convert back to fir vector.
|
|
if (callOp.getResult(0).getType() != argTypes[0]) {
|
|
auto res = builder.createConvert(loc, bcVecTy, callOp.getResult(0));
|
|
res = builder.create<mlir::vector::BitCastOp>(loc, mlirTyArgs[0], res);
|
|
shftRes = builder.createConvert(loc, argTypes[0], res);
|
|
} else {
|
|
shftRes = callOp.getResult(0);
|
|
}
|
|
} else if (vop == VecOp::Sld || vop == VecOp::Sldw) {
|
|
assert(args.size() == 3);
|
|
auto constIntOp = mlir::dyn_cast_or_null<mlir::IntegerAttr>(
|
|
mlir::dyn_cast<mlir::arith::ConstantOp>(argBases[2].getDefiningOp())
|
|
.getValue());
|
|
assert(constIntOp && "expected integer constant argument");
|
|
|
|
// Bitcast to vector<16xi8>
|
|
auto vi8Ty{mlir::VectorType::get(16, builder.getIntegerType(8))};
|
|
if (mlirTyArgs[0] != vi8Ty) {
|
|
mlirVecArgs[0] =
|
|
builder.create<mlir::LLVM::BitcastOp>(loc, vi8Ty, mlirVecArgs[0])
|
|
.getResult();
|
|
mlirVecArgs[1] =
|
|
builder.create<mlir::LLVM::BitcastOp>(loc, vi8Ty, mlirVecArgs[1])
|
|
.getResult();
|
|
}
|
|
|
|
// Construct the mask for shuffling
|
|
auto shiftVal{constIntOp.getInt()};
|
|
if (vop == VecOp::Sldw)
|
|
shiftVal = shiftVal << 2;
|
|
shiftVal &= 0xF;
|
|
llvm::SmallVector<int64_t, 16> mask;
|
|
// Shuffle with mask based on the endianness
|
|
const auto triple{fir::getTargetTriple(builder.getModule())};
|
|
if (triple.isLittleEndian()) {
|
|
for (int i = 16; i < 32; ++i)
|
|
mask.push_back(i - shiftVal);
|
|
shftRes = builder.create<mlir::vector::ShuffleOp>(loc, mlirVecArgs[1],
|
|
mlirVecArgs[0], mask);
|
|
} else {
|
|
for (int i = 0; i < 16; ++i)
|
|
mask.push_back(i + shiftVal);
|
|
shftRes = builder.create<mlir::vector::ShuffleOp>(loc, mlirVecArgs[0],
|
|
mlirVecArgs[1], mask);
|
|
}
|
|
|
|
// Bitcast to the original type
|
|
if (shftRes.getType() != mlirTyArgs[0])
|
|
shftRes =
|
|
builder.create<mlir::LLVM::BitcastOp>(loc, mlirTyArgs[0], shftRes);
|
|
|
|
return builder.createConvert(loc, resultType, shftRes);
|
|
} else
|
|
llvm_unreachable("Invalid vector operation for generator");
|
|
|
|
return shftRes;
|
|
}
|
|
|
|
// VEC_SPLAT, VEC_SPLATS, VEC_SPLAT_S32
|
|
template <VecOp vop>
|
|
fir::ExtendedValue
|
|
PPCIntrinsicLibrary::genVecSplat(mlir::Type resultType,
|
|
llvm::ArrayRef<fir::ExtendedValue> args) {
|
|
auto context{builder.getContext()};
|
|
auto argBases{getBasesForArgs(args)};
|
|
|
|
mlir::vector::SplatOp splatOp{nullptr};
|
|
mlir::Type retTy{nullptr};
|
|
switch (vop) {
|
|
case VecOp::Splat: {
|
|
assert(args.size() == 2);
|
|
auto vecTyInfo{getVecTypeFromFir(argBases[0])};
|
|
|
|
auto extractOp{genVecExtract(resultType, args)};
|
|
splatOp = builder.create<mlir::vector::SplatOp>(
|
|
loc, *(extractOp.getUnboxed()), vecTyInfo.toMlirVectorType(context));
|
|
retTy = vecTyInfo.toFirVectorType();
|
|
break;
|
|
}
|
|
case VecOp::Splats: {
|
|
assert(args.size() == 1);
|
|
auto vecTyInfo{getVecTypeFromEle(argBases[0])};
|
|
|
|
splatOp = builder.create<mlir::vector::SplatOp>(
|
|
loc, argBases[0], vecTyInfo.toMlirVectorType(context));
|
|
retTy = vecTyInfo.toFirVectorType();
|
|
break;
|
|
}
|
|
case VecOp::Splat_s32: {
|
|
assert(args.size() == 1);
|
|
auto eleTy{builder.getIntegerType(32)};
|
|
auto intOp{builder.createConvert(loc, eleTy, argBases[0])};
|
|
|
|
// the intrinsic always returns vector(integer(4))
|
|
splatOp = builder.create<mlir::vector::SplatOp>(
|
|
loc, intOp, mlir::VectorType::get(4, eleTy));
|
|
retTy = fir::VectorType::get(4, eleTy);
|
|
break;
|
|
}
|
|
default:
|
|
llvm_unreachable("invalid vector operation for generator");
|
|
}
|
|
return builder.createConvert(loc, retTy, splatOp);
|
|
}
|
|
|
|
fir::ExtendedValue
|
|
PPCIntrinsicLibrary::genVecXlds(mlir::Type resultType,
|
|
llvm::ArrayRef<fir::ExtendedValue> args) {
|
|
assert(args.size() == 2);
|
|
auto arg0{getBase(args[0])};
|
|
auto arg1{getBase(args[1])};
|
|
|
|
// Prepare the return type in FIR.
|
|
auto vecTyInfo{getVecTypeFromFirType(resultType)};
|
|
auto mlirTy{vecTyInfo.toMlirVectorType(builder.getContext())};
|
|
auto firTy{vecTyInfo.toFirVectorType()};
|
|
|
|
// Add the %val of arg0 to %addr of arg1
|
|
auto addr{addOffsetToAddress(builder, loc, arg1, arg0)};
|
|
|
|
auto i64Ty{mlir::IntegerType::get(builder.getContext(), 64)};
|
|
auto i64VecTy{mlir::VectorType::get(2, i64Ty)};
|
|
auto i64RefTy{builder.getRefType(i64Ty)};
|
|
auto addrConv{builder.create<fir::ConvertOp>(loc, i64RefTy, addr)};
|
|
|
|
auto addrVal{builder.create<fir::LoadOp>(loc, addrConv)};
|
|
auto splatRes{builder.create<mlir::vector::SplatOp>(loc, addrVal, i64VecTy)};
|
|
|
|
mlir::Value result{nullptr};
|
|
if (mlirTy != splatRes.getType()) {
|
|
result = builder.create<mlir::vector::BitCastOp>(loc, mlirTy, splatRes);
|
|
} else
|
|
result = splatRes;
|
|
|
|
return builder.createConvert(loc, firTy, result);
|
|
}
|
|
|
|
const char *getMmaIrIntrName(MMAOp mmaOp) {
|
|
switch (mmaOp) {
|
|
case MMAOp::AssembleAcc:
|
|
return "llvm.ppc.mma.assemble.acc";
|
|
case MMAOp::AssemblePair:
|
|
return "llvm.ppc.vsx.assemble.pair";
|
|
case MMAOp::DisassembleAcc:
|
|
return "llvm.ppc.mma.disassemble.acc";
|
|
case MMAOp::DisassemblePair:
|
|
return "llvm.ppc.vsx.disassemble.pair";
|
|
case MMAOp::Xxmfacc:
|
|
return "llvm.ppc.mma.xxmfacc";
|
|
case MMAOp::Xxmtacc:
|
|
return "llvm.ppc.mma.xxmtacc";
|
|
case MMAOp::Xxsetaccz:
|
|
return "llvm.ppc.mma.xxsetaccz";
|
|
case MMAOp::Pmxvbf16ger2:
|
|
return "llvm.ppc.mma.pmxvbf16ger2";
|
|
case MMAOp::Pmxvbf16ger2nn:
|
|
return "llvm.ppc.mma.pmxvbf16ger2nn";
|
|
case MMAOp::Pmxvbf16ger2np:
|
|
return "llvm.ppc.mma.pmxvbf16ger2np";
|
|
case MMAOp::Pmxvbf16ger2pn:
|
|
return "llvm.ppc.mma.pmxvbf16ger2pn";
|
|
case MMAOp::Pmxvbf16ger2pp:
|
|
return "llvm.ppc.mma.pmxvbf16ger2pp";
|
|
case MMAOp::Pmxvf16ger2:
|
|
return "llvm.ppc.mma.pmxvf16ger2";
|
|
case MMAOp::Pmxvf16ger2nn:
|
|
return "llvm.ppc.mma.pmxvf16ger2nn";
|
|
case MMAOp::Pmxvf16ger2np:
|
|
return "llvm.ppc.mma.pmxvf16ger2np";
|
|
case MMAOp::Pmxvf16ger2pn:
|
|
return "llvm.ppc.mma.pmxvf16ger2pn";
|
|
case MMAOp::Pmxvf16ger2pp:
|
|
return "llvm.ppc.mma.pmxvf16ger2pp";
|
|
case MMAOp::Pmxvf32ger:
|
|
return "llvm.ppc.mma.pmxvf32ger";
|
|
case MMAOp::Pmxvf32gernn:
|
|
return "llvm.ppc.mma.pmxvf32gernn";
|
|
case MMAOp::Pmxvf32gernp:
|
|
return "llvm.ppc.mma.pmxvf32gernp";
|
|
case MMAOp::Pmxvf32gerpn:
|
|
return "llvm.ppc.mma.pmxvf32gerpn";
|
|
case MMAOp::Pmxvf32gerpp:
|
|
return "llvm.ppc.mma.pmxvf32gerpp";
|
|
case MMAOp::Pmxvf64ger:
|
|
return "llvm.ppc.mma.pmxvf64ger";
|
|
case MMAOp::Pmxvf64gernn:
|
|
return "llvm.ppc.mma.pmxvf64gernn";
|
|
case MMAOp::Pmxvf64gernp:
|
|
return "llvm.ppc.mma.pmxvf64gernp";
|
|
case MMAOp::Pmxvf64gerpn:
|
|
return "llvm.ppc.mma.pmxvf64gerpn";
|
|
case MMAOp::Pmxvf64gerpp:
|
|
return "llvm.ppc.mma.pmxvf64gerpp";
|
|
case MMAOp::Pmxvi16ger2:
|
|
return "llvm.ppc.mma.pmxvi16ger2";
|
|
case MMAOp::Pmxvi16ger2pp:
|
|
return "llvm.ppc.mma.pmxvi16ger2pp";
|
|
case MMAOp::Pmxvi16ger2s:
|
|
return "llvm.ppc.mma.pmxvi16ger2s";
|
|
case MMAOp::Pmxvi16ger2spp:
|
|
return "llvm.ppc.mma.pmxvi16ger2spp";
|
|
case MMAOp::Pmxvi4ger8:
|
|
return "llvm.ppc.mma.pmxvi4ger8";
|
|
case MMAOp::Pmxvi4ger8pp:
|
|
return "llvm.ppc.mma.pmxvi4ger8pp";
|
|
case MMAOp::Pmxvi8ger4:
|
|
return "llvm.ppc.mma.pmxvi8ger4";
|
|
case MMAOp::Pmxvi8ger4pp:
|
|
return "llvm.ppc.mma.pmxvi8ger4pp";
|
|
case MMAOp::Pmxvi8ger4spp:
|
|
return "llvm.ppc.mma.pmxvi8ger4spp";
|
|
case MMAOp::Xvbf16ger2:
|
|
return "llvm.ppc.mma.xvbf16ger2";
|
|
case MMAOp::Xvbf16ger2nn:
|
|
return "llvm.ppc.mma.xvbf16ger2nn";
|
|
case MMAOp::Xvbf16ger2np:
|
|
return "llvm.ppc.mma.xvbf16ger2np";
|
|
case MMAOp::Xvbf16ger2pn:
|
|
return "llvm.ppc.mma.xvbf16ger2pn";
|
|
case MMAOp::Xvbf16ger2pp:
|
|
return "llvm.ppc.mma.xvbf16ger2pp";
|
|
case MMAOp::Xvf16ger2:
|
|
return "llvm.ppc.mma.xvf16ger2";
|
|
case MMAOp::Xvf16ger2nn:
|
|
return "llvm.ppc.mma.xvf16ger2nn";
|
|
case MMAOp::Xvf16ger2np:
|
|
return "llvm.ppc.mma.xvf16ger2np";
|
|
case MMAOp::Xvf16ger2pn:
|
|
return "llvm.ppc.mma.xvf16ger2pn";
|
|
case MMAOp::Xvf16ger2pp:
|
|
return "llvm.ppc.mma.xvf16ger2pp";
|
|
case MMAOp::Xvf32ger:
|
|
return "llvm.ppc.mma.xvf32ger";
|
|
case MMAOp::Xvf32gernn:
|
|
return "llvm.ppc.mma.xvf32gernn";
|
|
case MMAOp::Xvf32gernp:
|
|
return "llvm.ppc.mma.xvf32gernp";
|
|
case MMAOp::Xvf32gerpn:
|
|
return "llvm.ppc.mma.xvf32gerpn";
|
|
case MMAOp::Xvf32gerpp:
|
|
return "llvm.ppc.mma.xvf32gerpp";
|
|
case MMAOp::Xvf64ger:
|
|
return "llvm.ppc.mma.xvf64ger";
|
|
case MMAOp::Xvf64gernn:
|
|
return "llvm.ppc.mma.xvf64gernn";
|
|
case MMAOp::Xvf64gernp:
|
|
return "llvm.ppc.mma.xvf64gernp";
|
|
case MMAOp::Xvf64gerpn:
|
|
return "llvm.ppc.mma.xvf64gerpn";
|
|
case MMAOp::Xvf64gerpp:
|
|
return "llvm.ppc.mma.xvf64gerpp";
|
|
case MMAOp::Xvi16ger2:
|
|
return "llvm.ppc.mma.xvi16ger2";
|
|
case MMAOp::Xvi16ger2pp:
|
|
return "llvm.ppc.mma.xvi16ger2pp";
|
|
case MMAOp::Xvi16ger2s:
|
|
return "llvm.ppc.mma.xvi16ger2s";
|
|
case MMAOp::Xvi16ger2spp:
|
|
return "llvm.ppc.mma.xvi16ger2spp";
|
|
case MMAOp::Xvi4ger8:
|
|
return "llvm.ppc.mma.xvi4ger8";
|
|
case MMAOp::Xvi4ger8pp:
|
|
return "llvm.ppc.mma.xvi4ger8pp";
|
|
case MMAOp::Xvi8ger4:
|
|
return "llvm.ppc.mma.xvi8ger4";
|
|
case MMAOp::Xvi8ger4pp:
|
|
return "llvm.ppc.mma.xvi8ger4pp";
|
|
case MMAOp::Xvi8ger4spp:
|
|
return "llvm.ppc.mma.xvi8ger4spp";
|
|
}
|
|
llvm_unreachable("getMmaIrIntrName");
|
|
}
|
|
|
|
mlir::FunctionType getMmaIrFuncType(mlir::MLIRContext *context, MMAOp mmaOp) {
|
|
switch (mmaOp) {
|
|
case MMAOp::AssembleAcc:
|
|
return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 4);
|
|
case MMAOp::AssemblePair:
|
|
return genMmaVpFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::DisassembleAcc:
|
|
return genMmaDisassembleFuncType(context, mmaOp);
|
|
case MMAOp::DisassemblePair:
|
|
return genMmaDisassembleFuncType(context, mmaOp);
|
|
case MMAOp::Xxmfacc:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 0);
|
|
case MMAOp::Xxmtacc:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 0);
|
|
case MMAOp::Xxsetaccz:
|
|
return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 0);
|
|
case MMAOp::Pmxvbf16ger2:
|
|
return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 3);
|
|
case MMAOp::Pmxvbf16ger2nn:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 3);
|
|
case MMAOp::Pmxvbf16ger2np:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 3);
|
|
case MMAOp::Pmxvbf16ger2pn:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 3);
|
|
case MMAOp::Pmxvbf16ger2pp:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 3);
|
|
case MMAOp::Pmxvf16ger2:
|
|
return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 3);
|
|
case MMAOp::Pmxvf16ger2nn:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 3);
|
|
case MMAOp::Pmxvf16ger2np:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 3);
|
|
case MMAOp::Pmxvf16ger2pn:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 3);
|
|
case MMAOp::Pmxvf16ger2pp:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 3);
|
|
case MMAOp::Pmxvf32ger:
|
|
return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 2);
|
|
case MMAOp::Pmxvf32gernn:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 2);
|
|
case MMAOp::Pmxvf32gernp:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 2);
|
|
case MMAOp::Pmxvf32gerpn:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 2);
|
|
case MMAOp::Pmxvf32gerpp:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 2);
|
|
case MMAOp::Pmxvf64ger:
|
|
return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 1, /*Vector*/ 1,
|
|
/*Integer*/ 2);
|
|
case MMAOp::Pmxvf64gernn:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1,
|
|
/*Integer*/ 2);
|
|
case MMAOp::Pmxvf64gernp:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1,
|
|
/*Integer*/ 2);
|
|
case MMAOp::Pmxvf64gerpn:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1,
|
|
/*Integer*/ 2);
|
|
case MMAOp::Pmxvf64gerpp:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1,
|
|
/*Integer*/ 2);
|
|
case MMAOp::Pmxvi16ger2:
|
|
return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 3);
|
|
case MMAOp::Pmxvi16ger2pp:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 3);
|
|
case MMAOp::Pmxvi16ger2s:
|
|
return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 3);
|
|
case MMAOp::Pmxvi16ger2spp:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 3);
|
|
case MMAOp::Pmxvi4ger8:
|
|
return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 3);
|
|
case MMAOp::Pmxvi4ger8pp:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 3);
|
|
case MMAOp::Pmxvi8ger4:
|
|
return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 3);
|
|
case MMAOp::Pmxvi8ger4pp:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 3);
|
|
case MMAOp::Pmxvi8ger4spp:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2,
|
|
/*Integer*/ 3);
|
|
case MMAOp::Xvbf16ger2:
|
|
return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvbf16ger2nn:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvbf16ger2np:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvbf16ger2pn:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvbf16ger2pp:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvf16ger2:
|
|
return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvf16ger2nn:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvf16ger2np:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvf16ger2pn:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvf16ger2pp:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvf32ger:
|
|
return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvf32gernn:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvf32gernp:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvf32gerpn:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvf32gerpp:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvf64ger:
|
|
return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 1, /*Vector*/ 1);
|
|
case MMAOp::Xvf64gernn:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1);
|
|
case MMAOp::Xvf64gernp:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1);
|
|
case MMAOp::Xvf64gerpn:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1);
|
|
case MMAOp::Xvf64gerpp:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 1, /*Vector*/ 1);
|
|
case MMAOp::Xvi16ger2:
|
|
return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvi16ger2pp:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvi16ger2s:
|
|
return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvi16ger2spp:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvi4ger8:
|
|
return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvi4ger8pp:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvi8ger4:
|
|
return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvi8ger4pp:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);
|
|
case MMAOp::Xvi8ger4spp:
|
|
return genMmaVqFuncType(context, /*Quad*/ 1, /*Pair*/ 0, /*Vector*/ 2);
|
|
}
|
|
llvm_unreachable("getMmaIrFuncType");
|
|
}
|
|
|
|
template <MMAOp IntrId, MMAHandlerOp HandlerOp>
|
|
void PPCIntrinsicLibrary::genMmaIntr(llvm::ArrayRef<fir::ExtendedValue> args) {
|
|
auto context{builder.getContext()};
|
|
mlir::FunctionType intrFuncType{getMmaIrFuncType(context, IntrId)};
|
|
mlir::func::FuncOp funcOp{
|
|
builder.createFunction(loc, getMmaIrIntrName(IntrId), intrFuncType)};
|
|
llvm::SmallVector<mlir::Value> intrArgs;
|
|
|
|
// Depending on SubToFunc, change the subroutine call to a function call.
|
|
// First argument represents the result. Rest of the arguments
|
|
// are shifted one position to form the actual argument list.
|
|
size_t argStart{0};
|
|
size_t argStep{1};
|
|
size_t e{args.size()};
|
|
if (HandlerOp == MMAHandlerOp::SubToFunc) {
|
|
// The first argument becomes function result. Start from the second
|
|
// argument.
|
|
argStart = 1;
|
|
} else if (HandlerOp == MMAHandlerOp::SubToFuncReverseArgOnLE) {
|
|
// Reverse argument order on little-endian target only.
|
|
// The reversal does not depend on the setting of non-native-order option.
|
|
const auto triple{fir::getTargetTriple(builder.getModule())};
|
|
if (triple.isLittleEndian()) {
|
|
// Load the arguments in reverse order.
|
|
argStart = args.size() - 1;
|
|
// The first argument becomes function result. Stop at the second
|
|
// argument.
|
|
e = 0;
|
|
argStep = -1;
|
|
} else {
|
|
// Load the arguments in natural order.
|
|
// The first argument becomes function result. Start from the second
|
|
// argument.
|
|
argStart = 1;
|
|
}
|
|
}
|
|
|
|
for (size_t i = argStart, j = 0; i != e; i += argStep, ++j) {
|
|
auto v{fir::getBase(args[i])};
|
|
if (i == 0 && HandlerOp == MMAHandlerOp::FirstArgIsResult) {
|
|
// First argument is passed in as an address. We need to load
|
|
// the content to match the LLVM interface.
|
|
v = builder.create<fir::LoadOp>(loc, v);
|
|
}
|
|
auto vType{v.getType()};
|
|
mlir::Type targetType{intrFuncType.getInput(j)};
|
|
if (vType != targetType) {
|
|
if (mlir::isa<mlir::VectorType>(targetType)) {
|
|
// Perform vector type conversion for arguments passed by value.
|
|
auto eleTy{mlir::dyn_cast<fir::VectorType>(vType).getEleTy()};
|
|
auto len{mlir::dyn_cast<fir::VectorType>(vType).getLen()};
|
|
mlir::VectorType mlirType = mlir::VectorType::get(len, eleTy);
|
|
auto v0{builder.createConvert(loc, mlirType, v)};
|
|
auto v1{builder.create<mlir::vector::BitCastOp>(loc, targetType, v0)};
|
|
intrArgs.push_back(v1);
|
|
} else if (mlir::isa<mlir::IntegerType>(targetType) &&
|
|
mlir::isa<mlir::IntegerType>(vType)) {
|
|
auto v0{builder.createConvert(loc, targetType, v)};
|
|
intrArgs.push_back(v0);
|
|
} else {
|
|
llvm::errs() << "\nUnexpected type conversion requested: "
|
|
<< " from " << vType << " to " << targetType << "\n";
|
|
llvm_unreachable("Unsupported type conversion for argument to PowerPC "
|
|
"MMA intrinsic");
|
|
}
|
|
} else {
|
|
intrArgs.push_back(v);
|
|
}
|
|
}
|
|
auto callSt{builder.create<fir::CallOp>(loc, funcOp, intrArgs)};
|
|
if (HandlerOp == MMAHandlerOp::SubToFunc ||
|
|
HandlerOp == MMAHandlerOp::SubToFuncReverseArgOnLE ||
|
|
HandlerOp == MMAHandlerOp::FirstArgIsResult) {
|
|
// Convert pointer type if needed.
|
|
mlir::Value callResult{callSt.getResult(0)};
|
|
mlir::Value destPtr{fir::getBase(args[0])};
|
|
mlir::Type callResultPtrType{builder.getRefType(callResult.getType())};
|
|
if (destPtr.getType() != callResultPtrType) {
|
|
destPtr = builder.create<fir::ConvertOp>(loc, callResultPtrType, destPtr);
|
|
}
|
|
// Copy the result.
|
|
builder.create<fir::StoreOp>(loc, callResult, destPtr);
|
|
}
|
|
}
|
|
|
|
// VEC_ST, VEC_STE
|
|
template <VecOp vop>
|
|
void PPCIntrinsicLibrary::genVecStore(llvm::ArrayRef<fir::ExtendedValue> args) {
|
|
assert(args.size() == 3);
|
|
|
|
auto context{builder.getContext()};
|
|
auto argBases{getBasesForArgs(args)};
|
|
auto arg1TyInfo{getVecTypeFromFir(argBases[0])};
|
|
|
|
auto addr{addOffsetToAddress(builder, loc, argBases[2], argBases[1])};
|
|
|
|
llvm::StringRef fname{};
|
|
mlir::VectorType stTy{nullptr};
|
|
auto i32ty{mlir::IntegerType::get(context, 32)};
|
|
switch (vop) {
|
|
case VecOp::St:
|
|
stTy = mlir::VectorType::get(4, i32ty);
|
|
fname = "llvm.ppc.altivec.stvx";
|
|
break;
|
|
case VecOp::Ste: {
|
|
const auto width{arg1TyInfo.eleTy.getIntOrFloatBitWidth()};
|
|
const auto len{arg1TyInfo.len};
|
|
|
|
if (arg1TyInfo.isFloat32()) {
|
|
stTy = mlir::VectorType::get(len, i32ty);
|
|
fname = "llvm.ppc.altivec.stvewx";
|
|
} else if (mlir::isa<mlir::IntegerType>(arg1TyInfo.eleTy)) {
|
|
stTy = mlir::VectorType::get(len, mlir::IntegerType::get(context, width));
|
|
|
|
switch (width) {
|
|
case 8:
|
|
fname = "llvm.ppc.altivec.stvebx";
|
|
break;
|
|
case 16:
|
|
fname = "llvm.ppc.altivec.stvehx";
|
|
break;
|
|
case 32:
|
|
fname = "llvm.ppc.altivec.stvewx";
|
|
break;
|
|
default:
|
|
assert(false && "invalid element size");
|
|
}
|
|
} else
|
|
assert(false && "unknown type");
|
|
break;
|
|
}
|
|
case VecOp::Stxvp:
|
|
// __vector_pair type
|
|
stTy = mlir::VectorType::get(256, mlir::IntegerType::get(context, 1));
|
|
fname = "llvm.ppc.vsx.stxvp";
|
|
break;
|
|
default:
|
|
llvm_unreachable("invalid vector operation for generator");
|
|
}
|
|
|
|
auto funcType{
|
|
mlir::FunctionType::get(context, {stTy, addr.getType()}, std::nullopt)};
|
|
mlir::func::FuncOp funcOp = builder.createFunction(loc, fname, funcType);
|
|
|
|
llvm::SmallVector<mlir::Value, 4> biArgs;
|
|
|
|
if (vop == VecOp::Stxvp) {
|
|
biArgs.push_back(argBases[0]);
|
|
biArgs.push_back(addr);
|
|
builder.create<fir::CallOp>(loc, funcOp, biArgs);
|
|
return;
|
|
}
|
|
|
|
auto vecTyInfo{getVecTypeFromFirType(argBases[0].getType())};
|
|
auto cnv{builder.createConvert(loc, vecTyInfo.toMlirVectorType(context),
|
|
argBases[0])};
|
|
|
|
mlir::Value newArg1{nullptr};
|
|
if (stTy != arg1TyInfo.toMlirVectorType(context))
|
|
newArg1 = builder.create<mlir::vector::BitCastOp>(loc, stTy, cnv);
|
|
else
|
|
newArg1 = cnv;
|
|
|
|
if (isBEVecElemOrderOnLE())
|
|
newArg1 = builder.createConvert(
|
|
loc, stTy, reverseVectorElements(builder, loc, newArg1, 4));
|
|
|
|
biArgs.push_back(newArg1);
|
|
biArgs.push_back(addr);
|
|
|
|
builder.create<fir::CallOp>(loc, funcOp, biArgs);
|
|
}
|
|
|
|
// VEC_XST, VEC_XST_BE, VEC_STXV, VEC_XSTD2, VEC_XSTW4
|
|
template <VecOp vop>
|
|
void PPCIntrinsicLibrary::genVecXStore(
|
|
llvm::ArrayRef<fir::ExtendedValue> args) {
|
|
assert(args.size() == 3);
|
|
auto context{builder.getContext()};
|
|
auto argBases{getBasesForArgs(args)};
|
|
VecTypeInfo arg1TyInfo{getVecTypeFromFir(argBases[0])};
|
|
|
|
auto addr{addOffsetToAddress(builder, loc, argBases[2], argBases[1])};
|
|
|
|
mlir::Value trg{nullptr};
|
|
mlir::Value src{nullptr};
|
|
|
|
switch (vop) {
|
|
case VecOp::Xst:
|
|
case VecOp::Xst_be: {
|
|
src = argBases[0];
|
|
trg = builder.createConvert(loc, builder.getRefType(argBases[0].getType()),
|
|
addr);
|
|
|
|
if (vop == VecOp::Xst_be || isBEVecElemOrderOnLE()) {
|
|
auto cnv{builder.createConvert(loc, arg1TyInfo.toMlirVectorType(context),
|
|
argBases[0])};
|
|
auto shf{reverseVectorElements(builder, loc, cnv, arg1TyInfo.len)};
|
|
|
|
src = builder.createConvert(loc, arg1TyInfo.toFirVectorType(), shf);
|
|
}
|
|
break;
|
|
}
|
|
case VecOp::Xstd2:
|
|
case VecOp::Xstw4: {
|
|
// an 16-byte vector arg1 is treated as two 8-byte elements or
|
|
// four 4-byte elements
|
|
mlir::IntegerType elemTy;
|
|
uint64_t numElem = (vop == VecOp::Xstd2) ? 2 : 4;
|
|
elemTy = builder.getIntegerType(128 / numElem);
|
|
|
|
mlir::VectorType mlirVecTy{mlir::VectorType::get(numElem, elemTy)};
|
|
fir::VectorType firVecTy{fir::VectorType::get(numElem, elemTy)};
|
|
|
|
auto cnv{builder.createConvert(loc, arg1TyInfo.toMlirVectorType(context),
|
|
argBases[0])};
|
|
|
|
mlir::Type srcTy{nullptr};
|
|
if (numElem != arg1TyInfo.len) {
|
|
cnv = builder.create<mlir::vector::BitCastOp>(loc, mlirVecTy, cnv);
|
|
srcTy = firVecTy;
|
|
} else {
|
|
srcTy = arg1TyInfo.toFirVectorType();
|
|
}
|
|
|
|
trg = builder.createConvert(loc, builder.getRefType(srcTy), addr);
|
|
|
|
if (isBEVecElemOrderOnLE()) {
|
|
cnv = reverseVectorElements(builder, loc, cnv, numElem);
|
|
}
|
|
|
|
src = builder.createConvert(loc, srcTy, cnv);
|
|
break;
|
|
}
|
|
case VecOp::Stxv:
|
|
src = argBases[0];
|
|
trg = builder.createConvert(loc, builder.getRefType(argBases[0].getType()),
|
|
addr);
|
|
break;
|
|
default:
|
|
assert(false && "Invalid vector operation for generator");
|
|
}
|
|
builder.create<fir::StoreOp>(loc, mlir::TypeRange{},
|
|
mlir::ValueRange{src, trg},
|
|
getAlignmentAttr(builder, 1));
|
|
}
|
|
|
|
} // namespace fir
|