61510b51c3
This reverts commit 9c319d5bb4.
Some issues were discovered with the bootstrap builds, which
seem like they were caused by this commit. I'm reverting to investigate.
551 lines
17 KiB
LLVM
551 lines
17 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
|
|
; RUN: llc < %s -O0 -mtriple=aarch64-unknown-unknown | FileCheck %s --check-prefix=CHECKO0
|
|
; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+neon | FileCheck %s --check-prefixes=CHECK,NEON
|
|
; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+neon,+dotprod | FileCheck %s --check-prefixes=CHECK,DOT
|
|
; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+sve | FileCheck %s --check-prefixes=CHECK,SVE
|
|
; RUN: llc < %s -global-isel -mtriple=aarch64-unknown-unknown | FileCheck %s --check-prefix=GISEL
|
|
; RUN: llc < %s -O0 -global-isel -mtriple=aarch64-unknown-unknown | FileCheck %s --check-prefix=GISELO0
|
|
; RUN: llc < %s -global-isel -mtriple=aarch64-unknown-unknown -mattr=+neon | FileCheck %s --check-prefixes=GISEL,NEON-GISEL
|
|
; RUN: llc < %s -global-isel -mtriple=aarch64-unknown-unknown -mattr=+neon,+dotprod | FileCheck %s --check-prefixes=GISEL,DOT-GISEL
|
|
; RUN: llc < %s -global-isel -mtriple=aarch64-unknown-unknown -mattr=+sve | FileCheck %s --check-prefixes=GISEL,SVE-GISEL
|
|
|
|
|
|
; Function Attrs: nobuiltin nounwind readonly
|
|
define i8 @popcount128(ptr nocapture nonnull readonly %0) {
|
|
; CHECKO0-LABEL: popcount128:
|
|
; CHECKO0: // %bb.0: // %Entry
|
|
; CHECKO0-NEXT: ldr q0, [x0]
|
|
; CHECKO0-NEXT: cnt v0.16b, v0.16b
|
|
; CHECKO0-NEXT: uaddlv h0, v0.16b
|
|
; CHECKO0-NEXT: // kill: def $q0 killed $h0
|
|
; CHECKO0-NEXT: // kill: def $s0 killed $s0 killed $q0
|
|
; CHECKO0-NEXT: fmov w0, s0
|
|
; CHECKO0-NEXT: ret
|
|
;
|
|
; CHECK-LABEL: popcount128:
|
|
; CHECK: // %bb.0: // %Entry
|
|
; CHECK-NEXT: ldr d0, [x0]
|
|
; CHECK-NEXT: add x8, x0, #8
|
|
; CHECK-NEXT: ld1 { v0.d }[1], [x8]
|
|
; CHECK-NEXT: cnt v0.16b, v0.16b
|
|
; CHECK-NEXT: uaddlv h0, v0.16b
|
|
; CHECK-NEXT: fmov w0, s0
|
|
; CHECK-NEXT: ret
|
|
;
|
|
; GISEL-LABEL: popcount128:
|
|
; GISEL: // %bb.0: // %Entry
|
|
; GISEL-NEXT: ldr q0, [x0]
|
|
; GISEL-NEXT: cnt v0.16b, v0.16b
|
|
; GISEL-NEXT: uaddlv h0, v0.16b
|
|
; GISEL-NEXT: fmov w0, s0
|
|
; GISEL-NEXT: ret
|
|
;
|
|
; GISELO0-LABEL: popcount128:
|
|
; GISELO0: // %bb.0: // %Entry
|
|
; GISELO0-NEXT: ldr q0, [x0]
|
|
; GISELO0-NEXT: cnt v0.16b, v0.16b
|
|
; GISELO0-NEXT: uaddlv h0, v0.16b
|
|
; GISELO0-NEXT: // kill: def $q0 killed $h0
|
|
; GISELO0-NEXT: // kill: def $s0 killed $s0 killed $q0
|
|
; GISELO0-NEXT: fmov w0, s0
|
|
; GISELO0-NEXT: ret
|
|
Entry:
|
|
%1 = load i128, ptr %0, align 16
|
|
%2 = tail call i128 @llvm.ctpop.i128(i128 %1)
|
|
%3 = trunc i128 %2 to i8
|
|
ret i8 %3
|
|
}
|
|
|
|
; Function Attrs: nounwind readnone speculatable willreturn
|
|
declare i128 @llvm.ctpop.i128(i128)
|
|
|
|
; Function Attrs: nobuiltin nounwind readonly
|
|
define i16 @popcount256(ptr nocapture nonnull readonly %0) {
|
|
; CHECKO0-LABEL: popcount256:
|
|
; CHECKO0: // %bb.0: // %Entry
|
|
; CHECKO0-NEXT: ldr x11, [x0]
|
|
; CHECKO0-NEXT: ldr x10, [x0, #8]
|
|
; CHECKO0-NEXT: ldr x9, [x0, #16]
|
|
; CHECKO0-NEXT: ldr x8, [x0, #24]
|
|
; CHECKO0-NEXT: // implicit-def: $q1
|
|
; CHECKO0-NEXT: mov v1.d[0], x11
|
|
; CHECKO0-NEXT: mov v1.d[1], x10
|
|
; CHECKO0-NEXT: // implicit-def: $q0
|
|
; CHECKO0-NEXT: mov v0.d[0], x9
|
|
; CHECKO0-NEXT: mov v0.d[1], x8
|
|
; CHECKO0-NEXT: cnt v1.16b, v1.16b
|
|
; CHECKO0-NEXT: uaddlv h1, v1.16b
|
|
; CHECKO0-NEXT: // kill: def $q1 killed $h1
|
|
; CHECKO0-NEXT: // kill: def $s1 killed $s1 killed $q1
|
|
; CHECKO0-NEXT: fmov w0, s1
|
|
; CHECKO0-NEXT: mov w10, wzr
|
|
; CHECKO0-NEXT: mov w9, w0
|
|
; CHECKO0-NEXT: mov w8, w10
|
|
; CHECKO0-NEXT: bfi x9, x8, #32, #32
|
|
; CHECKO0-NEXT: cnt v0.16b, v0.16b
|
|
; CHECKO0-NEXT: uaddlv h0, v0.16b
|
|
; CHECKO0-NEXT: // kill: def $q0 killed $h0
|
|
; CHECKO0-NEXT: // kill: def $s0 killed $s0 killed $q0
|
|
; CHECKO0-NEXT: fmov w0, s0
|
|
; CHECKO0-NEXT: mov w8, w0
|
|
; CHECKO0-NEXT: // kill: def $x10 killed $w10
|
|
; CHECKO0-NEXT: bfi x8, x10, #32, #32
|
|
; CHECKO0-NEXT: adds x8, x8, x9
|
|
; CHECKO0-NEXT: mov w0, w8
|
|
; CHECKO0-NEXT: ret
|
|
;
|
|
; CHECK-LABEL: popcount256:
|
|
; CHECK: // %bb.0: // %Entry
|
|
; CHECK-NEXT: ldr d0, [x0, #16]
|
|
; CHECK-NEXT: ldr d1, [x0]
|
|
; CHECK-NEXT: add x8, x0, #8
|
|
; CHECK-NEXT: add x9, x0, #24
|
|
; CHECK-NEXT: ld1 { v0.d }[1], [x9]
|
|
; CHECK-NEXT: ld1 { v1.d }[1], [x8]
|
|
; CHECK-NEXT: cnt v0.16b, v0.16b
|
|
; CHECK-NEXT: cnt v1.16b, v1.16b
|
|
; CHECK-NEXT: uaddlv h0, v0.16b
|
|
; CHECK-NEXT: uaddlv h1, v1.16b
|
|
; CHECK-NEXT: fmov w8, s0
|
|
; CHECK-NEXT: fmov w9, s1
|
|
; CHECK-NEXT: add w0, w9, w8
|
|
; CHECK-NEXT: ret
|
|
;
|
|
; GISEL-LABEL: popcount256:
|
|
; GISEL: // %bb.0: // %Entry
|
|
; GISEL-NEXT: ldp x8, x9, [x0]
|
|
; GISEL-NEXT: mov v0.d[0], x8
|
|
; GISEL-NEXT: ldp x8, x10, [x0, #16]
|
|
; GISEL-NEXT: mov v1.d[0], x8
|
|
; GISEL-NEXT: mov v0.d[1], x9
|
|
; GISEL-NEXT: mov v1.d[1], x10
|
|
; GISEL-NEXT: cnt v0.16b, v0.16b
|
|
; GISEL-NEXT: cnt v1.16b, v1.16b
|
|
; GISEL-NEXT: uaddlv h0, v0.16b
|
|
; GISEL-NEXT: uaddlv h1, v1.16b
|
|
; GISEL-NEXT: mov w8, v0.s[0]
|
|
; GISEL-NEXT: fmov w9, s1
|
|
; GISEL-NEXT: add x0, x8, w9, uxtw
|
|
; GISEL-NEXT: // kill: def $w0 killed $w0 killed $x0
|
|
; GISEL-NEXT: ret
|
|
;
|
|
; GISELO0-LABEL: popcount256:
|
|
; GISELO0: // %bb.0: // %Entry
|
|
; GISELO0-NEXT: ldr x11, [x0]
|
|
; GISELO0-NEXT: ldr x10, [x0, #8]
|
|
; GISELO0-NEXT: ldr x9, [x0, #16]
|
|
; GISELO0-NEXT: ldr x8, [x0, #24]
|
|
; GISELO0-NEXT: // implicit-def: $q1
|
|
; GISELO0-NEXT: mov v1.d[0], x11
|
|
; GISELO0-NEXT: mov v1.d[1], x10
|
|
; GISELO0-NEXT: // implicit-def: $q0
|
|
; GISELO0-NEXT: mov v0.d[0], x9
|
|
; GISELO0-NEXT: mov v0.d[1], x8
|
|
; GISELO0-NEXT: cnt v1.16b, v1.16b
|
|
; GISELO0-NEXT: uaddlv h1, v1.16b
|
|
; GISELO0-NEXT: // kill: def $q1 killed $h1
|
|
; GISELO0-NEXT: // kill: def $s1 killed $s1 killed $q1
|
|
; GISELO0-NEXT: fmov w0, s1
|
|
; GISELO0-NEXT: mov w10, wzr
|
|
; GISELO0-NEXT: mov w9, w0
|
|
; GISELO0-NEXT: mov w8, w10
|
|
; GISELO0-NEXT: bfi x9, x8, #32, #32
|
|
; GISELO0-NEXT: cnt v0.16b, v0.16b
|
|
; GISELO0-NEXT: uaddlv h0, v0.16b
|
|
; GISELO0-NEXT: // kill: def $q0 killed $h0
|
|
; GISELO0-NEXT: // kill: def $s0 killed $s0 killed $q0
|
|
; GISELO0-NEXT: fmov w0, s0
|
|
; GISELO0-NEXT: mov w8, w0
|
|
; GISELO0-NEXT: // kill: def $x10 killed $w10
|
|
; GISELO0-NEXT: bfi x8, x10, #32, #32
|
|
; GISELO0-NEXT: adds x8, x8, x9
|
|
; GISELO0-NEXT: mov w0, w8
|
|
; GISELO0-NEXT: ret
|
|
Entry:
|
|
%1 = load i256, ptr %0, align 16
|
|
%2 = tail call i256 @llvm.ctpop.i256(i256 %1)
|
|
%3 = trunc i256 %2 to i16
|
|
ret i16 %3
|
|
}
|
|
|
|
; Function Attrs: nounwind readnone speculatable willreturn
|
|
declare i256 @llvm.ctpop.i256(i256)
|
|
|
|
define <1 x i128> @popcount1x128(<1 x i128> %0) {
|
|
; CHECKO0-LABEL: popcount1x128:
|
|
; CHECKO0: // %bb.0: // %Entry
|
|
; CHECKO0-NEXT: // implicit-def: $q0
|
|
; CHECKO0-NEXT: mov v0.d[0], x0
|
|
; CHECKO0-NEXT: mov v0.d[1], x1
|
|
; CHECKO0-NEXT: cnt v0.16b, v0.16b
|
|
; CHECKO0-NEXT: uaddlv h0, v0.16b
|
|
; CHECKO0-NEXT: // kill: def $q0 killed $h0
|
|
; CHECKO0-NEXT: mov x1, xzr
|
|
; CHECKO0-NEXT: // kill: def $s0 killed $s0 killed $q0
|
|
; CHECKO0-NEXT: fmov w0, s0
|
|
; CHECKO0-NEXT: mov w8, wzr
|
|
; CHECKO0-NEXT: // kill: def $x0 killed $w0
|
|
; CHECKO0-NEXT: // kill: def $x8 killed $w8
|
|
; CHECKO0-NEXT: bfi x0, x8, #32, #32
|
|
; CHECKO0-NEXT: ret
|
|
;
|
|
; CHECK-LABEL: popcount1x128:
|
|
; CHECK: // %bb.0: // %Entry
|
|
; CHECK-NEXT: fmov d1, x0
|
|
; CHECK-NEXT: movi v0.2d, #0000000000000000
|
|
; CHECK-NEXT: mov v1.d[1], x1
|
|
; CHECK-NEXT: cnt v1.16b, v1.16b
|
|
; CHECK-NEXT: uaddlv h1, v1.16b
|
|
; CHECK-NEXT: mov v0.s[0], v1.s[0]
|
|
; CHECK-NEXT: mov x1, v0.d[1]
|
|
; CHECK-NEXT: fmov x0, d0
|
|
; CHECK-NEXT: ret
|
|
;
|
|
; GISEL-LABEL: popcount1x128:
|
|
; GISEL: // %bb.0: // %Entry
|
|
; GISEL-NEXT: mov v0.d[0], x0
|
|
; GISEL-NEXT: mov v0.d[1], x1
|
|
; GISEL-NEXT: mov x1, xzr
|
|
; GISEL-NEXT: cnt v0.16b, v0.16b
|
|
; GISEL-NEXT: uaddlv h0, v0.16b
|
|
; GISEL-NEXT: mov w0, v0.s[0]
|
|
; GISEL-NEXT: ret
|
|
;
|
|
; GISELO0-LABEL: popcount1x128:
|
|
; GISELO0: // %bb.0: // %Entry
|
|
; GISELO0-NEXT: // implicit-def: $q0
|
|
; GISELO0-NEXT: mov v0.d[0], x0
|
|
; GISELO0-NEXT: mov v0.d[1], x1
|
|
; GISELO0-NEXT: cnt v0.16b, v0.16b
|
|
; GISELO0-NEXT: uaddlv h0, v0.16b
|
|
; GISELO0-NEXT: // kill: def $q0 killed $h0
|
|
; GISELO0-NEXT: mov x1, xzr
|
|
; GISELO0-NEXT: // kill: def $s0 killed $s0 killed $q0
|
|
; GISELO0-NEXT: fmov w0, s0
|
|
; GISELO0-NEXT: mov w8, wzr
|
|
; GISELO0-NEXT: // kill: def $x0 killed $w0
|
|
; GISELO0-NEXT: // kill: def $x8 killed $w8
|
|
; GISELO0-NEXT: bfi x0, x8, #32, #32
|
|
; GISELO0-NEXT: ret
|
|
Entry:
|
|
%1 = tail call <1 x i128> @llvm.ctpop.v1i128(<1 x i128> %0)
|
|
ret <1 x i128> %1
|
|
}
|
|
|
|
declare <1 x i128> @llvm.ctpop.v1i128(<1 x i128>)
|
|
|
|
define <2 x i64> @popcount2x64(<2 x i64> %0) {
|
|
; CHECKO0-LABEL: popcount2x64:
|
|
; CHECKO0: // %bb.0: // %Entry
|
|
; CHECKO0-NEXT: cnt v0.16b, v0.16b
|
|
; CHECKO0-NEXT: uaddlp v0.8h, v0.16b
|
|
; CHECKO0-NEXT: uaddlp v0.4s, v0.8h
|
|
; CHECKO0-NEXT: uaddlp v0.2d, v0.4s
|
|
; CHECKO0-NEXT: ret
|
|
;
|
|
; NEON-LABEL: popcount2x64:
|
|
; NEON: // %bb.0: // %Entry
|
|
; NEON-NEXT: cnt v0.16b, v0.16b
|
|
; NEON-NEXT: uaddlp v0.8h, v0.16b
|
|
; NEON-NEXT: uaddlp v0.4s, v0.8h
|
|
; NEON-NEXT: uaddlp v0.2d, v0.4s
|
|
; NEON-NEXT: ret
|
|
;
|
|
; DOT-LABEL: popcount2x64:
|
|
; DOT: // %bb.0: // %Entry
|
|
; DOT-NEXT: movi v1.16b, #1
|
|
; DOT-NEXT: cnt v0.16b, v0.16b
|
|
; DOT-NEXT: movi v2.2d, #0000000000000000
|
|
; DOT-NEXT: udot v2.4s, v1.16b, v0.16b
|
|
; DOT-NEXT: uaddlp v0.2d, v2.4s
|
|
; DOT-NEXT: ret
|
|
;
|
|
; SVE-LABEL: popcount2x64:
|
|
; SVE: // %bb.0: // %Entry
|
|
; SVE-NEXT: cnt v0.16b, v0.16b
|
|
; SVE-NEXT: uaddlp v0.8h, v0.16b
|
|
; SVE-NEXT: uaddlp v0.4s, v0.8h
|
|
; SVE-NEXT: uaddlp v0.2d, v0.4s
|
|
; SVE-NEXT: ret
|
|
;
|
|
; GISELO0-LABEL: popcount2x64:
|
|
; GISELO0: // %bb.0: // %Entry
|
|
; GISELO0-NEXT: cnt v0.16b, v0.16b
|
|
; GISELO0-NEXT: uaddlp v0.8h, v0.16b
|
|
; GISELO0-NEXT: uaddlp v0.4s, v0.8h
|
|
; GISELO0-NEXT: uaddlp v0.2d, v0.4s
|
|
; GISELO0-NEXT: ret
|
|
;
|
|
; NEON-GISEL-LABEL: popcount2x64:
|
|
; NEON-GISEL: // %bb.0: // %Entry
|
|
; NEON-GISEL-NEXT: cnt v0.16b, v0.16b
|
|
; NEON-GISEL-NEXT: uaddlp v0.8h, v0.16b
|
|
; NEON-GISEL-NEXT: uaddlp v0.4s, v0.8h
|
|
; NEON-GISEL-NEXT: uaddlp v0.2d, v0.4s
|
|
; NEON-GISEL-NEXT: ret
|
|
;
|
|
; DOT-GISEL-LABEL: popcount2x64:
|
|
; DOT-GISEL: // %bb.0: // %Entry
|
|
; DOT-GISEL-NEXT: movi v1.2d, #0000000000000000
|
|
; DOT-GISEL-NEXT: cnt v0.16b, v0.16b
|
|
; DOT-GISEL-NEXT: movi v2.16b, #1
|
|
; DOT-GISEL-NEXT: udot v1.4s, v2.16b, v0.16b
|
|
; DOT-GISEL-NEXT: uaddlp v0.2d, v1.4s
|
|
; DOT-GISEL-NEXT: ret
|
|
;
|
|
; SVE-GISEL-LABEL: popcount2x64:
|
|
; SVE-GISEL: // %bb.0: // %Entry
|
|
; SVE-GISEL-NEXT: cnt v0.16b, v0.16b
|
|
; SVE-GISEL-NEXT: uaddlp v0.8h, v0.16b
|
|
; SVE-GISEL-NEXT: uaddlp v0.4s, v0.8h
|
|
; SVE-GISEL-NEXT: uaddlp v0.2d, v0.4s
|
|
; SVE-GISEL-NEXT: ret
|
|
Entry:
|
|
%1 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
|
|
ret <2 x i64> %1
|
|
}
|
|
|
|
declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
|
|
|
|
define <1 x i64> @popcount1x64(<1 x i64> %0) {
|
|
; CHECKO0-LABEL: popcount1x64:
|
|
; CHECKO0: // %bb.0: // %Entry
|
|
; CHECKO0-NEXT: fmov x0, d0
|
|
; CHECKO0-NEXT: fmov d0, x0
|
|
; CHECKO0-NEXT: cnt v0.8b, v0.8b
|
|
; CHECKO0-NEXT: uaddlv h0, v0.8b
|
|
; CHECKO0-NEXT: // kill: def $q0 killed $h0
|
|
; CHECKO0-NEXT: mov w8, v0.s[0]
|
|
; CHECKO0-NEXT: // kill: def $x8 killed $w8
|
|
; CHECKO0-NEXT: fmov d0, x8
|
|
; CHECKO0-NEXT: ret
|
|
;
|
|
; CHECK-LABEL: popcount1x64:
|
|
; CHECK: // %bb.0: // %Entry
|
|
; CHECK-NEXT: cnt v0.8b, v0.8b
|
|
; CHECK-NEXT: uaddlp v0.4h, v0.8b
|
|
; CHECK-NEXT: uaddlp v0.2s, v0.4h
|
|
; CHECK-NEXT: uaddlp v0.1d, v0.2s
|
|
; CHECK-NEXT: ret
|
|
;
|
|
; GISEL-LABEL: popcount1x64:
|
|
; GISEL: // %bb.0: // %Entry
|
|
; GISEL-NEXT: cnt v0.8b, v0.8b
|
|
; GISEL-NEXT: uaddlv h0, v0.8b
|
|
; GISEL-NEXT: mov w8, v0.s[0]
|
|
; GISEL-NEXT: fmov d0, x8
|
|
; GISEL-NEXT: ret
|
|
;
|
|
; GISELO0-LABEL: popcount1x64:
|
|
; GISELO0: // %bb.0: // %Entry
|
|
; GISELO0-NEXT: fmov x0, d0
|
|
; GISELO0-NEXT: fmov d0, x0
|
|
; GISELO0-NEXT: cnt v0.8b, v0.8b
|
|
; GISELO0-NEXT: uaddlv h0, v0.8b
|
|
; GISELO0-NEXT: // kill: def $q0 killed $h0
|
|
; GISELO0-NEXT: mov w8, v0.s[0]
|
|
; GISELO0-NEXT: // kill: def $x8 killed $w8
|
|
; GISELO0-NEXT: fmov d0, x8
|
|
; GISELO0-NEXT: ret
|
|
Entry:
|
|
%1 = tail call <1 x i64> @llvm.ctpop.v1i64(<1 x i64> %0)
|
|
ret <1 x i64> %1
|
|
}
|
|
|
|
declare <1 x i64> @llvm.ctpop.v1i64(<1 x i64>)
|
|
|
|
define <4 x i32> @popcount4x32(<4 x i32> %0) {
|
|
; CHECKO0-LABEL: popcount4x32:
|
|
; CHECKO0: // %bb.0: // %Entry
|
|
; CHECKO0-NEXT: cnt v0.16b, v0.16b
|
|
; CHECKO0-NEXT: uaddlp v0.8h, v0.16b
|
|
; CHECKO0-NEXT: uaddlp v0.4s, v0.8h
|
|
; CHECKO0-NEXT: ret
|
|
;
|
|
; NEON-LABEL: popcount4x32:
|
|
; NEON: // %bb.0: // %Entry
|
|
; NEON-NEXT: cnt v0.16b, v0.16b
|
|
; NEON-NEXT: uaddlp v0.8h, v0.16b
|
|
; NEON-NEXT: uaddlp v0.4s, v0.8h
|
|
; NEON-NEXT: ret
|
|
;
|
|
; DOT-LABEL: popcount4x32:
|
|
; DOT: // %bb.0: // %Entry
|
|
; DOT-NEXT: movi v1.16b, #1
|
|
; DOT-NEXT: cnt v2.16b, v0.16b
|
|
; DOT-NEXT: movi v0.2d, #0000000000000000
|
|
; DOT-NEXT: udot v0.4s, v1.16b, v2.16b
|
|
; DOT-NEXT: ret
|
|
;
|
|
; SVE-LABEL: popcount4x32:
|
|
; SVE: // %bb.0: // %Entry
|
|
; SVE-NEXT: cnt v0.16b, v0.16b
|
|
; SVE-NEXT: uaddlp v0.8h, v0.16b
|
|
; SVE-NEXT: uaddlp v0.4s, v0.8h
|
|
; SVE-NEXT: ret
|
|
;
|
|
; GISELO0-LABEL: popcount4x32:
|
|
; GISELO0: // %bb.0: // %Entry
|
|
; GISELO0-NEXT: cnt v0.16b, v0.16b
|
|
; GISELO0-NEXT: uaddlp v0.8h, v0.16b
|
|
; GISELO0-NEXT: uaddlp v0.4s, v0.8h
|
|
; GISELO0-NEXT: ret
|
|
;
|
|
; NEON-GISEL-LABEL: popcount4x32:
|
|
; NEON-GISEL: // %bb.0: // %Entry
|
|
; NEON-GISEL-NEXT: cnt v0.16b, v0.16b
|
|
; NEON-GISEL-NEXT: uaddlp v0.8h, v0.16b
|
|
; NEON-GISEL-NEXT: uaddlp v0.4s, v0.8h
|
|
; NEON-GISEL-NEXT: ret
|
|
;
|
|
; DOT-GISEL-LABEL: popcount4x32:
|
|
; DOT-GISEL: // %bb.0: // %Entry
|
|
; DOT-GISEL-NEXT: movi v1.2d, #0000000000000000
|
|
; DOT-GISEL-NEXT: cnt v0.16b, v0.16b
|
|
; DOT-GISEL-NEXT: movi v2.16b, #1
|
|
; DOT-GISEL-NEXT: udot v1.4s, v2.16b, v0.16b
|
|
; DOT-GISEL-NEXT: mov v0.16b, v1.16b
|
|
; DOT-GISEL-NEXT: ret
|
|
;
|
|
; SVE-GISEL-LABEL: popcount4x32:
|
|
; SVE-GISEL: // %bb.0: // %Entry
|
|
; SVE-GISEL-NEXT: cnt v0.16b, v0.16b
|
|
; SVE-GISEL-NEXT: uaddlp v0.8h, v0.16b
|
|
; SVE-GISEL-NEXT: uaddlp v0.4s, v0.8h
|
|
; SVE-GISEL-NEXT: ret
|
|
Entry:
|
|
%1 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
|
|
ret <4 x i32> %1
|
|
}
|
|
|
|
declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>)
|
|
|
|
define <2 x i32> @popcount2x32(<2 x i32> %0) {
|
|
; CHECKO0-LABEL: popcount2x32:
|
|
; CHECKO0: // %bb.0: // %Entry
|
|
; CHECKO0-NEXT: cnt v0.8b, v0.8b
|
|
; CHECKO0-NEXT: uaddlp v0.4h, v0.8b
|
|
; CHECKO0-NEXT: uaddlp v0.2s, v0.4h
|
|
; CHECKO0-NEXT: ret
|
|
;
|
|
; NEON-LABEL: popcount2x32:
|
|
; NEON: // %bb.0: // %Entry
|
|
; NEON-NEXT: cnt v0.8b, v0.8b
|
|
; NEON-NEXT: uaddlp v0.4h, v0.8b
|
|
; NEON-NEXT: uaddlp v0.2s, v0.4h
|
|
; NEON-NEXT: ret
|
|
;
|
|
; DOT-LABEL: popcount2x32:
|
|
; DOT: // %bb.0: // %Entry
|
|
; DOT-NEXT: movi v1.2d, #0000000000000000
|
|
; DOT-NEXT: cnt v0.8b, v0.8b
|
|
; DOT-NEXT: movi v2.8b, #1
|
|
; DOT-NEXT: udot v1.2s, v2.8b, v0.8b
|
|
; DOT-NEXT: fmov d0, d1
|
|
; DOT-NEXT: ret
|
|
;
|
|
; SVE-LABEL: popcount2x32:
|
|
; SVE: // %bb.0: // %Entry
|
|
; SVE-NEXT: cnt v0.8b, v0.8b
|
|
; SVE-NEXT: uaddlp v0.4h, v0.8b
|
|
; SVE-NEXT: uaddlp v0.2s, v0.4h
|
|
; SVE-NEXT: ret
|
|
;
|
|
; GISELO0-LABEL: popcount2x32:
|
|
; GISELO0: // %bb.0: // %Entry
|
|
; GISELO0-NEXT: cnt v0.8b, v0.8b
|
|
; GISELO0-NEXT: uaddlp v0.4h, v0.8b
|
|
; GISELO0-NEXT: uaddlp v0.2s, v0.4h
|
|
; GISELO0-NEXT: ret
|
|
;
|
|
; NEON-GISEL-LABEL: popcount2x32:
|
|
; NEON-GISEL: // %bb.0: // %Entry
|
|
; NEON-GISEL-NEXT: cnt v0.8b, v0.8b
|
|
; NEON-GISEL-NEXT: uaddlp v0.4h, v0.8b
|
|
; NEON-GISEL-NEXT: uaddlp v0.2s, v0.4h
|
|
; NEON-GISEL-NEXT: ret
|
|
;
|
|
; DOT-GISEL-LABEL: popcount2x32:
|
|
; DOT-GISEL: // %bb.0: // %Entry
|
|
; DOT-GISEL-NEXT: movi v1.2d, #0000000000000000
|
|
; DOT-GISEL-NEXT: cnt v0.8b, v0.8b
|
|
; DOT-GISEL-NEXT: movi v2.8b, #1
|
|
; DOT-GISEL-NEXT: udot v1.2s, v2.8b, v0.8b
|
|
; DOT-GISEL-NEXT: fmov d0, d1
|
|
; DOT-GISEL-NEXT: ret
|
|
;
|
|
; SVE-GISEL-LABEL: popcount2x32:
|
|
; SVE-GISEL: // %bb.0: // %Entry
|
|
; SVE-GISEL-NEXT: cnt v0.8b, v0.8b
|
|
; SVE-GISEL-NEXT: uaddlp v0.4h, v0.8b
|
|
; SVE-GISEL-NEXT: uaddlp v0.2s, v0.4h
|
|
; SVE-GISEL-NEXT: ret
|
|
Entry:
|
|
%1 = tail call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %0)
|
|
ret <2 x i32> %1
|
|
}
|
|
|
|
declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>)
|
|
|
|
define <8 x i16> @popcount8x16(<8 x i16> %0) {
|
|
; CHECKO0-LABEL: popcount8x16:
|
|
; CHECKO0: // %bb.0: // %Entry
|
|
; CHECKO0-NEXT: cnt v0.16b, v0.16b
|
|
; CHECKO0-NEXT: uaddlp v0.8h, v0.16b
|
|
; CHECKO0-NEXT: ret
|
|
;
|
|
; CHECK-LABEL: popcount8x16:
|
|
; CHECK: // %bb.0: // %Entry
|
|
; CHECK-NEXT: cnt v0.16b, v0.16b
|
|
; CHECK-NEXT: uaddlp v0.8h, v0.16b
|
|
; CHECK-NEXT: ret
|
|
;
|
|
; GISEL-LABEL: popcount8x16:
|
|
; GISEL: // %bb.0: // %Entry
|
|
; GISEL-NEXT: cnt v0.16b, v0.16b
|
|
; GISEL-NEXT: uaddlp v0.8h, v0.16b
|
|
; GISEL-NEXT: ret
|
|
;
|
|
; GISELO0-LABEL: popcount8x16:
|
|
; GISELO0: // %bb.0: // %Entry
|
|
; GISELO0-NEXT: cnt v0.16b, v0.16b
|
|
; GISELO0-NEXT: uaddlp v0.8h, v0.16b
|
|
; GISELO0-NEXT: ret
|
|
Entry:
|
|
%1 = tail call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %0)
|
|
ret <8 x i16> %1
|
|
}
|
|
|
|
declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>)
|
|
|
|
define <4 x i16> @popcount4x16(<4 x i16> %0) {
|
|
; CHECKO0-LABEL: popcount4x16:
|
|
; CHECKO0: // %bb.0: // %Entry
|
|
; CHECKO0-NEXT: cnt v0.8b, v0.8b
|
|
; CHECKO0-NEXT: uaddlp v0.4h, v0.8b
|
|
; CHECKO0-NEXT: ret
|
|
;
|
|
; CHECK-LABEL: popcount4x16:
|
|
; CHECK: // %bb.0: // %Entry
|
|
; CHECK-NEXT: cnt v0.8b, v0.8b
|
|
; CHECK-NEXT: uaddlp v0.4h, v0.8b
|
|
; CHECK-NEXT: ret
|
|
;
|
|
; GISEL-LABEL: popcount4x16:
|
|
; GISEL: // %bb.0: // %Entry
|
|
; GISEL-NEXT: cnt v0.8b, v0.8b
|
|
; GISEL-NEXT: uaddlp v0.4h, v0.8b
|
|
; GISEL-NEXT: ret
|
|
;
|
|
; GISELO0-LABEL: popcount4x16:
|
|
; GISELO0: // %bb.0: // %Entry
|
|
; GISELO0-NEXT: cnt v0.8b, v0.8b
|
|
; GISELO0-NEXT: uaddlp v0.4h, v0.8b
|
|
; GISELO0-NEXT: ret
|
|
Entry:
|
|
%1 = tail call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> %0)
|
|
ret <4 x i16> %1
|
|
}
|
|
|
|
declare <4 x i16> @llvm.ctpop.v4i16(<4 x i16>)
|