EVOLUTION-MANAGER

Edit File: transform_kernels_arm_64.h

// Copyright 2016 The Gemmlowp Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#ifndef GEMMLOWP_META_TRANSFORM_KERNELS_ARM_64_H_
#define GEMMLOWP_META_TRANSFORM_KERNELS_ARM_64_H_

#ifdef GEMMLOWP_NEON_64

#include <cassert>
#include <cstdint>

namespace gemmlowp {
namespace meta {

template <>
inline void Transform1DKernel<int32_t, uint8_t, Requantize, 16, 0>::Transform(
    const int32_t* input, const Requantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Requantize<int32_t, uint8_t, Requantize, 16, 0>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Requantize::Prepare
      "dup v4.4s, %w[input_range_min]\n"
      "dup v5.4s, %w[output_range_min]\n"
      "dup v6.4s, %w[input_range_offset]\n"
      "dup v7.4s, %w[input_range_scale]\n"
      "dup v8.4s, %w[one_over_output_range_scale]\n"
      "fsub v4.4s, v4.4s, v5.4s\n"

"1:"
      "subs %x[count], %x[count], #16\n"

// Requantize::Transform
      "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[input]], #64\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "scvtf v3.4s, v3.4s\n"
      "fsub v0.4s, v0.4s, v6.4s\n"
      "fsub v1.4s, v1.4s, v6.4s\n"
      "fsub v2.4s, v2.4s, v6.4s\n"
      "fsub v3.4s, v3.4s, v6.4s\n"
      "fmul v0.4s, v0.4s, v7.4s\n"
      "fmul v1.4s, v1.4s, v7.4s\n"
      "fmul v2.4s, v2.4s, v7.4s\n"
      "fmul v3.4s, v3.4s, v7.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fadd v2.4s, v2.4s, v4.4s\n"
      "fadd v3.4s, v3.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v8.4s\n"
      "fmul v1.4s, v1.4s, v8.4s\n"
      "fmul v2.4s, v2.4s, v8.4s\n"
      "fmul v3.4s, v3.4s, v8.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"
      "fcvtzs v3.4s, v3.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtn v2.4h, v2.4s\n"
      "sqxtn2 v2.8h, v3.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun2 v0.16b, v2.8h\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [input_range_min] "r"(params.input_range_min),
        [output_range_min] "r"(params.output_range_min),
        [input_range_offset] "r"(params.input_range_offset),
        [one_over_output_range_scale] "r"(params.one_over_output_range_scale),
        [input_range_scale] "r"(params.input_range_scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "cc", "memory");
}

template <>
inline void Transform1DKernel<int32_t, uint8_t, Requantize, 16, 1>::Transform(
    const int32_t* input, const Requantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Requantize<int32_t, uint8_t, Requantize, 16, 1>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Reduce count by leftovers.
      "subs %x[count], %x[count], #1\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Requantize::Transform
      "ld1 {v0.s}[0], [%x[input]], #4\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "scvtf v0.4s, v0.4s\n"
      "fsub v0.4s, v0.4s, v6.4s\n"
      "fmul v0.4s, v0.4s, v7.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v8.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtun v0.8b, v0.8h\n"

"st1 {v0.b}[0], [%x[output]], #1\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [input_range_min] "r"(params.input_range_min),
        [output_range_min] "r"(params.output_range_min),
        [input_range_offset] "r"(params.input_range_offset),
        [one_over_output_range_scale] "r"(params.one_over_output_range_scale),
        [input_range_scale] "r"(params.input_range_scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "cc", "memory");
}

template <>
inline void Transform1DKernel<int32_t, uint8_t, Requantize, 16, 2>::Transform(
    const int32_t* input, const Requantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Requantize<int32_t, uint8_t, Requantize, 16, 2>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Reduce count by leftovers.
      "subs %x[count], %x[count], #2\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Requantize::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "scvtf v0.4s, v0.4s\n"
      "fsub v0.4s, v0.4s, v6.4s\n"
      "fmul v0.4s, v0.4s, v7.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v8.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtun v0.8b, v0.8h\n"

"st1 {v0.h}[0], [%x[output]], #2\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [input_range_min] "r"(params.input_range_min),
        [output_range_min] "r"(params.output_range_min),
        [input_range_offset] "r"(params.input_range_offset),
        [one_over_output_range_scale] "r"(params.one_over_output_range_scale),
        [input_range_scale] "r"(params.input_range_scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "cc", "memory");
}

template <>
inline void Transform1DKernel<int32_t, uint8_t, Requantize, 16, 3>::Transform(
    const int32_t* input, const Requantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Requantize<int32_t, uint8_t, Requantize, 16, 3>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Reduce count by leftovers.
      "subs %x[count], %x[count], #3\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Requantize::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.s}[2], [%x[input]], #4\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "scvtf v0.4s, v0.4s\n"
      "fsub v0.4s, v0.4s, v6.4s\n"
      "fmul v0.4s, v0.4s, v7.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v8.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtun v0.8b, v0.8h\n"

"st1 {v0.h}[0], [%x[output]], #2\n"
      "st1 {v0.b}[2], [%x[output]], #1\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [input_range_min] "r"(params.input_range_min),
        [output_range_min] "r"(params.output_range_min),
        [input_range_offset] "r"(params.input_range_offset),
        [one_over_output_range_scale] "r"(params.one_over_output_range_scale),
        [input_range_scale] "r"(params.input_range_scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "cc", "memory");
}

template <>
inline void Transform1DKernel<int32_t, uint8_t, Requantize, 16, 4>::Transform(
    const int32_t* input, const Requantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Requantize<int32_t, uint8_t, Requantize, 16, 4>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Reduce count by leftovers.
      "subs %x[count], %x[count], #4\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Requantize::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "scvtf v0.4s, v0.4s\n"
      "fsub v0.4s, v0.4s, v6.4s\n"
      "fmul v0.4s, v0.4s, v7.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v8.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtun v0.8b, v0.8h\n"

"st1 {v0.s}[0], [%x[output]], #4\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [input_range_min] "r"(params.input_range_min),
        [output_range_min] "r"(params.output_range_min),
        [input_range_offset] "r"(params.input_range_offset),
        [one_over_output_range_scale] "r"(params.one_over_output_range_scale),
        [input_range_scale] "r"(params.input_range_scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "cc", "memory");
}

template <>
inline void Transform1DKernel<int32_t, uint8_t, Requantize, 16, 5>::Transform(
    const int32_t* input, const Requantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Requantize<int32_t, uint8_t, Requantize, 16, 5>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Reduce count by leftovers.
      "subs %x[count], %x[count], #5\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Requantize::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "ld1 {v1.s}[0], [%x[input]], #4\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "fsub v0.4s, v0.4s, v6.4s\n"
      "fsub v1.4s, v1.4s, v6.4s\n"
      "fmul v0.4s, v0.4s, v7.4s\n"
      "fmul v1.4s, v1.4s, v7.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v8.4s\n"
      "fmul v1.4s, v1.4s, v8.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtun v0.8b, v0.8h\n"

"st1 {v0.s}[0], [%x[output]], #4\n"
      "st1 {v0.b}[4], [%x[output]], #1\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [input_range_min] "r"(params.input_range_min),
        [output_range_min] "r"(params.output_range_min),
        [input_range_offset] "r"(params.input_range_offset),
        [one_over_output_range_scale] "r"(params.one_over_output_range_scale),
        [input_range_scale] "r"(params.input_range_scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "cc", "memory");
}

template <>
inline void Transform1DKernel<int32_t, uint8_t, Requantize, 16, 6>::Transform(
    const int32_t* input, const Requantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Requantize<int32_t, uint8_t, Requantize, 16, 6>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Reduce count by leftovers.
      "subs %x[count], %x[count], #6\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Requantize::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "ld1 {v1.2s}, [%x[input]], #8\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "fsub v0.4s, v0.4s, v6.4s\n"
      "fsub v1.4s, v1.4s, v6.4s\n"
      "fmul v0.4s, v0.4s, v7.4s\n"
      "fmul v1.4s, v1.4s, v7.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v8.4s\n"
      "fmul v1.4s, v1.4s, v8.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtun v0.8b, v0.8h\n"

"st1 {v0.s}[0], [%x[output]], #4\n"
      "st1 {v0.h}[2], [%x[output]], #2\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [input_range_min] "r"(params.input_range_min),
        [output_range_min] "r"(params.output_range_min),
        [input_range_offset] "r"(params.input_range_offset),
        [one_over_output_range_scale] "r"(params.one_over_output_range_scale),
        [input_range_scale] "r"(params.input_range_scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "cc", "memory");
}

template <>
inline void Transform1DKernel<int32_t, uint8_t, Requantize, 16, 7>::Transform(
    const int32_t* input, const Requantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Requantize<int32_t, uint8_t, Requantize, 16, 7>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Reduce count by leftovers.
      "subs %x[count], %x[count], #7\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Requantize::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "ld1 {v1.2s}, [%x[input]], #8\n"
      "ld1 {v1.s}[2], [%x[input]], #4\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "fsub v0.4s, v0.4s, v6.4s\n"
      "fsub v1.4s, v1.4s, v6.4s\n"
      "fmul v0.4s, v0.4s, v7.4s\n"
      "fmul v1.4s, v1.4s, v7.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v8.4s\n"
      "fmul v1.4s, v1.4s, v8.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtun v0.8b, v0.8h\n"

"st1 {v0.s}[0], [%x[output]], #4\n"
      "st1 {v0.h}[2], [%x[output]], #2\n"
      "st1 {v0.b}[6], [%x[output]], #1\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [input_range_min] "r"(params.input_range_min),
        [output_range_min] "r"(params.output_range_min),
        [input_range_offset] "r"(params.input_range_offset),
        [one_over_output_range_scale] "r"(params.one_over_output_range_scale),
        [input_range_scale] "r"(params.input_range_scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "cc", "memory");
}

template <>
inline void Transform1DKernel<int32_t, uint8_t, Requantize, 16, 8>::Transform(
    const int32_t* input, const Requantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Requantize<int32_t, uint8_t, Requantize, 16, 8>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Reduce count by leftovers.
      "subs %x[count], %x[count], #8\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Requantize::Transform
      "ld1 {v0.4s, v1.4s}, [%x[input]], #32\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "fsub v0.4s, v0.4s, v6.4s\n"
      "fsub v1.4s, v1.4s, v6.4s\n"
      "fmul v0.4s, v0.4s, v7.4s\n"
      "fmul v1.4s, v1.4s, v7.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v8.4s\n"
      "fmul v1.4s, v1.4s, v8.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtun v0.8b, v0.8h\n"

"st1 {v0.2s}, [%x[output]], #8\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [input_range_min] "r"(params.input_range_min),
        [output_range_min] "r"(params.output_range_min),
        [input_range_offset] "r"(params.input_range_offset),
        [one_over_output_range_scale] "r"(params.one_over_output_range_scale),
        [input_range_scale] "r"(params.input_range_scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "cc", "memory");
}

template <>
inline void Transform1DKernel<int32_t, uint8_t, Requantize, 16, 9>::Transform(
    const int32_t* input, const Requantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Requantize<int32_t, uint8_t, Requantize, 16, 9>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Reduce count by leftovers.
      "subs %x[count], %x[count], #9\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Requantize::Transform
      "ld1 {v0.4s, v1.4s}, [%x[input]], #32\n"
      "ld1 {v2.s}[0], [%x[input]], #4\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "fsub v0.4s, v0.4s, v6.4s\n"
      "fsub v1.4s, v1.4s, v6.4s\n"
      "fsub v2.4s, v2.4s, v6.4s\n"
      "fmul v0.4s, v0.4s, v7.4s\n"
      "fmul v1.4s, v1.4s, v7.4s\n"
      "fmul v2.4s, v2.4s, v7.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fadd v2.4s, v2.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v8.4s\n"
      "fmul v1.4s, v1.4s, v8.4s\n"
      "fmul v2.4s, v2.4s, v8.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtn v2.4h, v2.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun2 v0.16b, v2.8h\n"

"st1 {v0.2s}, [%x[output]], #8\n"
      "st1 {v0.b}[8], [%x[output]], #1\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [input_range_min] "r"(params.input_range_min),
        [output_range_min] "r"(params.output_range_min),
        [input_range_offset] "r"(params.input_range_offset),
        [one_over_output_range_scale] "r"(params.one_over_output_range_scale),
        [input_range_scale] "r"(params.input_range_scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "cc", "memory");
}

template <>
inline void Transform1DKernel<int32_t, uint8_t, Requantize, 16, 10>::Transform(
    const int32_t* input, const Requantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Requantize<int32_t, uint8_t, Requantize, 16, 10>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Reduce count by leftovers.
      "subs %x[count], %x[count], #10\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Requantize::Transform
      "ld1 {v0.4s, v1.4s}, [%x[input]], #32\n"
      "ld1 {v2.2s}, [%x[input]], #8\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "fsub v0.4s, v0.4s, v6.4s\n"
      "fsub v1.4s, v1.4s, v6.4s\n"
      "fsub v2.4s, v2.4s, v6.4s\n"
      "fmul v0.4s, v0.4s, v7.4s\n"
      "fmul v1.4s, v1.4s, v7.4s\n"
      "fmul v2.4s, v2.4s, v7.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fadd v2.4s, v2.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v8.4s\n"
      "fmul v1.4s, v1.4s, v8.4s\n"
      "fmul v2.4s, v2.4s, v8.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtn v2.4h, v2.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun2 v0.16b, v2.8h\n"

"st1 {v0.2s}, [%x[output]], #8\n"
      "st1 {v0.h}[4], [%x[output]], #2\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [input_range_min] "r"(params.input_range_min),
        [output_range_min] "r"(params.output_range_min),
        [input_range_offset] "r"(params.input_range_offset),
        [one_over_output_range_scale] "r"(params.one_over_output_range_scale),
        [input_range_scale] "r"(params.input_range_scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "cc", "memory");
}

template <>
inline void Transform1DKernel<int32_t, uint8_t, Requantize, 16, 11>::Transform(
    const int32_t* input, const Requantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Requantize<int32_t, uint8_t, Requantize, 16, 11>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Reduce count by leftovers.
      "subs %x[count], %x[count], #11\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Requantize::Transform
      "ld1 {v0.4s, v1.4s}, [%x[input]], #32\n"
      "ld1 {v2.2s}, [%x[input]], #8\n"
      "ld1 {v2.s}[2], [%x[input]], #4\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "fsub v0.4s, v0.4s, v6.4s\n"
      "fsub v1.4s, v1.4s, v6.4s\n"
      "fsub v2.4s, v2.4s, v6.4s\n"
      "fmul v0.4s, v0.4s, v7.4s\n"
      "fmul v1.4s, v1.4s, v7.4s\n"
      "fmul v2.4s, v2.4s, v7.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fadd v2.4s, v2.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v8.4s\n"
      "fmul v1.4s, v1.4s, v8.4s\n"
      "fmul v2.4s, v2.4s, v8.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtn v2.4h, v2.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun2 v0.16b, v2.8h\n"

"st1 {v0.2s}, [%x[output]], #8\n"
      "st1 {v0.h}[4], [%x[output]], #2\n"
      "st1 {v0.b}[10], [%x[output]], #1\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [input_range_min] "r"(params.input_range_min),
        [output_range_min] "r"(params.output_range_min),
        [input_range_offset] "r"(params.input_range_offset),
        [one_over_output_range_scale] "r"(params.one_over_output_range_scale),
        [input_range_scale] "r"(params.input_range_scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "cc", "memory");
}

template <>
inline void Transform1DKernel<int32_t, uint8_t, Requantize, 16, 12>::Transform(
    const int32_t* input, const Requantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Requantize<int32_t, uint8_t, Requantize, 16, 12>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Reduce count by leftovers.
      "subs %x[count], %x[count], #12\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Requantize::Transform
      "ld1 {v0.4s, v1.4s, v2.4s}, [%x[input]], #48\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "fsub v0.4s, v0.4s, v6.4s\n"
      "fsub v1.4s, v1.4s, v6.4s\n"
      "fsub v2.4s, v2.4s, v6.4s\n"
      "fmul v0.4s, v0.4s, v7.4s\n"
      "fmul v1.4s, v1.4s, v7.4s\n"
      "fmul v2.4s, v2.4s, v7.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fadd v2.4s, v2.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v8.4s\n"
      "fmul v1.4s, v1.4s, v8.4s\n"
      "fmul v2.4s, v2.4s, v8.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtn v2.4h, v2.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun2 v0.16b, v2.8h\n"

"st1 {v0.2s}, [%x[output]], #8\n"
      "st1 {v0.s}[2], [%x[output]], #4\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [input_range_min] "r"(params.input_range_min),
        [output_range_min] "r"(params.output_range_min),
        [input_range_offset] "r"(params.input_range_offset),
        [one_over_output_range_scale] "r"(params.one_over_output_range_scale),
        [input_range_scale] "r"(params.input_range_scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "cc", "memory");
}

template <>
inline void Transform1DKernel<int32_t, uint8_t, Requantize, 16, 13>::Transform(
    const int32_t* input, const Requantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Requantize<int32_t, uint8_t, Requantize, 16, 13>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Reduce count by leftovers.
      "subs %x[count], %x[count], #13\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Requantize::Transform
      "ld1 {v0.4s, v1.4s, v2.4s}, [%x[input]], #48\n"
      "ld1 {v3.s}[0], [%x[input]], #4\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "scvtf v3.4s, v3.4s\n"
      "fsub v0.4s, v0.4s, v6.4s\n"
      "fsub v1.4s, v1.4s, v6.4s\n"
      "fsub v2.4s, v2.4s, v6.4s\n"
      "fsub v3.4s, v3.4s, v6.4s\n"
      "fmul v0.4s, v0.4s, v7.4s\n"
      "fmul v1.4s, v1.4s, v7.4s\n"
      "fmul v2.4s, v2.4s, v7.4s\n"
      "fmul v3.4s, v3.4s, v7.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fadd v2.4s, v2.4s, v4.4s\n"
      "fadd v3.4s, v3.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v8.4s\n"
      "fmul v1.4s, v1.4s, v8.4s\n"
      "fmul v2.4s, v2.4s, v8.4s\n"
      "fmul v3.4s, v3.4s, v8.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"
      "fcvtzs v3.4s, v3.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtn v2.4h, v2.4s\n"
      "sqxtn2 v2.8h, v3.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun2 v0.16b, v2.8h\n"

"st1 {v0.2s}, [%x[output]], #8\n"
      "st1 {v0.s}[2], [%x[output]], #4\n"
      "st1 {v0.b}[12], [%x[output]], #1\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [input_range_min] "r"(params.input_range_min),
        [output_range_min] "r"(params.output_range_min),
        [input_range_offset] "r"(params.input_range_offset),
        [one_over_output_range_scale] "r"(params.one_over_output_range_scale),
        [input_range_scale] "r"(params.input_range_scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "cc", "memory");
}

template <>
inline void Transform1DKernel<int32_t, uint8_t, Requantize, 16, 14>::Transform(
    const int32_t* input, const Requantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Requantize<int32_t, uint8_t, Requantize, 16, 14>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Reduce count by leftovers.
      "subs %x[count], %x[count], #14\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Requantize::Transform
      "ld1 {v0.4s, v1.4s, v2.4s}, [%x[input]], #48\n"
      "ld1 {v3.2s}, [%x[input]], #8\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "scvtf v3.4s, v3.4s\n"
      "fsub v0.4s, v0.4s, v6.4s\n"
      "fsub v1.4s, v1.4s, v6.4s\n"
      "fsub v2.4s, v2.4s, v6.4s\n"
      "fsub v3.4s, v3.4s, v6.4s\n"
      "fmul v0.4s, v0.4s, v7.4s\n"
      "fmul v1.4s, v1.4s, v7.4s\n"
      "fmul v2.4s, v2.4s, v7.4s\n"
      "fmul v3.4s, v3.4s, v7.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fadd v2.4s, v2.4s, v4.4s\n"
      "fadd v3.4s, v3.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v8.4s\n"
      "fmul v1.4s, v1.4s, v8.4s\n"
      "fmul v2.4s, v2.4s, v8.4s\n"
      "fmul v3.4s, v3.4s, v8.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"
      "fcvtzs v3.4s, v3.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtn v2.4h, v2.4s\n"
      "sqxtn2 v2.8h, v3.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun2 v0.16b, v2.8h\n"

"st1 {v0.2s}, [%x[output]], #8\n"
      "st1 {v0.s}[2], [%x[output]], #4\n"
      "st1 {v0.h}[6], [%x[output]], #2\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [input_range_min] "r"(params.input_range_min),
        [output_range_min] "r"(params.output_range_min),
        [input_range_offset] "r"(params.input_range_offset),
        [one_over_output_range_scale] "r"(params.one_over_output_range_scale),
        [input_range_scale] "r"(params.input_range_scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "cc", "memory");
}

template <>
inline void Transform1DKernel<int32_t, uint8_t, Requantize, 16, 15>::Transform(
    const int32_t* input, const Requantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Requantize<int32_t, uint8_t, Requantize, 16, 15>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Reduce count by leftovers.
      "subs %x[count], %x[count], #15\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Requantize::Transform
      "ld1 {v0.4s, v1.4s, v2.4s}, [%x[input]], #48\n"
      "ld1 {v3.2s}, [%x[input]], #8\n"
      "ld1 {v3.s}[2], [%x[input]], #4\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "scvtf v3.4s, v3.4s\n"
      "fsub v0.4s, v0.4s, v6.4s\n"
      "fsub v1.4s, v1.4s, v6.4s\n"
      "fsub v2.4s, v2.4s, v6.4s\n"
      "fsub v3.4s, v3.4s, v6.4s\n"
      "fmul v0.4s, v0.4s, v7.4s\n"
      "fmul v1.4s, v1.4s, v7.4s\n"
      "fmul v2.4s, v2.4s, v7.4s\n"
      "fmul v3.4s, v3.4s, v7.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fadd v2.4s, v2.4s, v4.4s\n"
      "fadd v3.4s, v3.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v8.4s\n"
      "fmul v1.4s, v1.4s, v8.4s\n"
      "fmul v2.4s, v2.4s, v8.4s\n"
      "fmul v3.4s, v3.4s, v8.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"
      "fcvtzs v3.4s, v3.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtn v2.4h, v2.4s\n"
      "sqxtn2 v2.8h, v3.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun2 v0.16b, v2.8h\n"

"st1 {v0.2s}, [%x[output]], #8\n"
      "st1 {v0.s}[2], [%x[output]], #4\n"
      "st1 {v0.h}[6], [%x[output]], #2\n"
      "st1 {v0.b}[14], [%x[output]], #1\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [input_range_min] "r"(params.input_range_min),
        [output_range_min] "r"(params.output_range_min),
        [input_range_offset] "r"(params.input_range_offset),
        [one_over_output_range_scale] "r"(params.one_over_output_range_scale),
        [input_range_scale] "r"(params.input_range_scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "cc", "memory");
}

template <>
inline void Transform1DKernel<float, uint8_t, Quantize, 16, 0>::Transform(
    const float* input, const Quantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Quantize<float, uint8_t, Quantize, 16, 0>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Quantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

"1:"
      "subs %x[count], %x[count], #16\n"

// Quantize::Transform
      "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[input]], #64\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "fsub v0.4s, v0.4s, v4.4s\n"
      "fsub v1.4s, v1.4s, v4.4s\n"
      "fsub v2.4s, v2.4s, v4.4s\n"
      "fsub v3.4s, v3.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fmul v2.4s, v2.4s, v6.4s\n"
      "fmul v3.4s, v3.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v5.4s\n"
      "fadd v1.4s, v1.4s, v5.4s\n"
      "fadd v2.4s, v2.4s, v5.4s\n"
      "fadd v3.4s, v3.4s, v5.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"
      "fcvtzs v3.4s, v3.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtn v2.4h, v2.4s\n"
      "sqxtn2 v2.8h, v3.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun2 v0.16b, v2.8h\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<float, uint8_t, Quantize, 16, 1>::Transform(
    const float* input, const Quantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Quantize<float, uint8_t, Quantize, 16, 1>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Quantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #1\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Quantize::Transform
      "ld1 {v0.s}[0], [%x[input]], #4\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "fsub v0.4s, v0.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v5.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtun v0.8b, v0.8h\n"

"st1 {v0.b}[0], [%x[output]], #1\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<float, uint8_t, Quantize, 16, 2>::Transform(
    const float* input, const Quantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Quantize<float, uint8_t, Quantize, 16, 2>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Quantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #2\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Quantize::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "fsub v0.4s, v0.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v5.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtun v0.8b, v0.8h\n"

"st1 {v0.h}[0], [%x[output]], #2\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<float, uint8_t, Quantize, 16, 3>::Transform(
    const float* input, const Quantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Quantize<float, uint8_t, Quantize, 16, 3>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Quantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #3\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Quantize::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.s}[2], [%x[input]], #4\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "fsub v0.4s, v0.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v5.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtun v0.8b, v0.8h\n"

"st1 {v0.h}[0], [%x[output]], #2\n"
      "st1 {v0.b}[2], [%x[output]], #1\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<float, uint8_t, Quantize, 16, 4>::Transform(
    const float* input, const Quantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Quantize<float, uint8_t, Quantize, 16, 4>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Quantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #4\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Quantize::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "fsub v0.4s, v0.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v5.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtun v0.8b, v0.8h\n"

"st1 {v0.s}[0], [%x[output]], #4\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<float, uint8_t, Quantize, 16, 5>::Transform(
    const float* input, const Quantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Quantize<float, uint8_t, Quantize, 16, 5>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Quantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #5\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Quantize::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "ld1 {v1.s}[0], [%x[input]], #4\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "fsub v0.4s, v0.4s, v4.4s\n"
      "fsub v1.4s, v1.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v5.4s\n"
      "fadd v1.4s, v1.4s, v5.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtun v0.8b, v0.8h\n"

"st1 {v0.s}[0], [%x[output]], #4\n"
      "st1 {v0.b}[4], [%x[output]], #1\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<float, uint8_t, Quantize, 16, 6>::Transform(
    const float* input, const Quantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Quantize<float, uint8_t, Quantize, 16, 6>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Quantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #6\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Quantize::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "ld1 {v1.2s}, [%x[input]], #8\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "fsub v0.4s, v0.4s, v4.4s\n"
      "fsub v1.4s, v1.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v5.4s\n"
      "fadd v1.4s, v1.4s, v5.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtun v0.8b, v0.8h\n"

"st1 {v0.s}[0], [%x[output]], #4\n"
      "st1 {v0.h}[2], [%x[output]], #2\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<float, uint8_t, Quantize, 16, 7>::Transform(
    const float* input, const Quantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Quantize<float, uint8_t, Quantize, 16, 7>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Quantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #7\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Quantize::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "ld1 {v1.2s}, [%x[input]], #8\n"
      "ld1 {v1.s}[2], [%x[input]], #4\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "fsub v0.4s, v0.4s, v4.4s\n"
      "fsub v1.4s, v1.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v5.4s\n"
      "fadd v1.4s, v1.4s, v5.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtun v0.8b, v0.8h\n"

"st1 {v0.s}[0], [%x[output]], #4\n"
      "st1 {v0.h}[2], [%x[output]], #2\n"
      "st1 {v0.b}[6], [%x[output]], #1\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<float, uint8_t, Quantize, 16, 8>::Transform(
    const float* input, const Quantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Quantize<float, uint8_t, Quantize, 16, 8>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Quantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #8\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Quantize::Transform
      "ld1 {v0.4s, v1.4s}, [%x[input]], #32\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "fsub v0.4s, v0.4s, v4.4s\n"
      "fsub v1.4s, v1.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v5.4s\n"
      "fadd v1.4s, v1.4s, v5.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtun v0.8b, v0.8h\n"

"st1 {v0.2s}, [%x[output]], #8\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<float, uint8_t, Quantize, 16, 9>::Transform(
    const float* input, const Quantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Quantize<float, uint8_t, Quantize, 16, 9>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Quantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #9\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Quantize::Transform
      "ld1 {v0.4s, v1.4s}, [%x[input]], #32\n"
      "ld1 {v2.s}[0], [%x[input]], #4\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "fsub v0.4s, v0.4s, v4.4s\n"
      "fsub v1.4s, v1.4s, v4.4s\n"
      "fsub v2.4s, v2.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fmul v2.4s, v2.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v5.4s\n"
      "fadd v1.4s, v1.4s, v5.4s\n"
      "fadd v2.4s, v2.4s, v5.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtn v2.4h, v2.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun2 v0.16b, v2.8h\n"

"st1 {v0.2s}, [%x[output]], #8\n"
      "st1 {v0.b}[8], [%x[output]], #1\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<float, uint8_t, Quantize, 16, 10>::Transform(
    const float* input, const Quantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Quantize<float, uint8_t, Quantize, 16, 10>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Quantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #10\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Quantize::Transform
      "ld1 {v0.4s, v1.4s}, [%x[input]], #32\n"
      "ld1 {v2.2s}, [%x[input]], #8\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "fsub v0.4s, v0.4s, v4.4s\n"
      "fsub v1.4s, v1.4s, v4.4s\n"
      "fsub v2.4s, v2.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fmul v2.4s, v2.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v5.4s\n"
      "fadd v1.4s, v1.4s, v5.4s\n"
      "fadd v2.4s, v2.4s, v5.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtn v2.4h, v2.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun2 v0.16b, v2.8h\n"

"st1 {v0.2s}, [%x[output]], #8\n"
      "st1 {v0.h}[4], [%x[output]], #2\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<float, uint8_t, Quantize, 16, 11>::Transform(
    const float* input, const Quantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Quantize<float, uint8_t, Quantize, 16, 11>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Quantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #11\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Quantize::Transform
      "ld1 {v0.4s, v1.4s}, [%x[input]], #32\n"
      "ld1 {v2.2s}, [%x[input]], #8\n"
      "ld1 {v2.s}[2], [%x[input]], #4\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "fsub v0.4s, v0.4s, v4.4s\n"
      "fsub v1.4s, v1.4s, v4.4s\n"
      "fsub v2.4s, v2.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fmul v2.4s, v2.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v5.4s\n"
      "fadd v1.4s, v1.4s, v5.4s\n"
      "fadd v2.4s, v2.4s, v5.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtn v2.4h, v2.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun2 v0.16b, v2.8h\n"

"st1 {v0.2s}, [%x[output]], #8\n"
      "st1 {v0.h}[4], [%x[output]], #2\n"
      "st1 {v0.b}[10], [%x[output]], #1\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<float, uint8_t, Quantize, 16, 12>::Transform(
    const float* input, const Quantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Quantize<float, uint8_t, Quantize, 16, 12>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Quantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #12\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Quantize::Transform
      "ld1 {v0.4s, v1.4s, v2.4s}, [%x[input]], #48\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "fsub v0.4s, v0.4s, v4.4s\n"
      "fsub v1.4s, v1.4s, v4.4s\n"
      "fsub v2.4s, v2.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fmul v2.4s, v2.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v5.4s\n"
      "fadd v1.4s, v1.4s, v5.4s\n"
      "fadd v2.4s, v2.4s, v5.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtn v2.4h, v2.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun2 v0.16b, v2.8h\n"

"st1 {v0.2s}, [%x[output]], #8\n"
      "st1 {v0.s}[2], [%x[output]], #4\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<float, uint8_t, Quantize, 16, 13>::Transform(
    const float* input, const Quantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Quantize<float, uint8_t, Quantize, 16, 13>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Quantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #13\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Quantize::Transform
      "ld1 {v0.4s, v1.4s, v2.4s}, [%x[input]], #48\n"
      "ld1 {v3.s}[0], [%x[input]], #4\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "fsub v0.4s, v0.4s, v4.4s\n"
      "fsub v1.4s, v1.4s, v4.4s\n"
      "fsub v2.4s, v2.4s, v4.4s\n"
      "fsub v3.4s, v3.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fmul v2.4s, v2.4s, v6.4s\n"
      "fmul v3.4s, v3.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v5.4s\n"
      "fadd v1.4s, v1.4s, v5.4s\n"
      "fadd v2.4s, v2.4s, v5.4s\n"
      "fadd v3.4s, v3.4s, v5.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"
      "fcvtzs v3.4s, v3.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtn v2.4h, v2.4s\n"
      "sqxtn2 v2.8h, v3.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun2 v0.16b, v2.8h\n"

"st1 {v0.2s}, [%x[output]], #8\n"
      "st1 {v0.s}[2], [%x[output]], #4\n"
      "st1 {v0.b}[12], [%x[output]], #1\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<float, uint8_t, Quantize, 16, 14>::Transform(
    const float* input, const Quantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Quantize<float, uint8_t, Quantize, 16, 14>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Quantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #14\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Quantize::Transform
      "ld1 {v0.4s, v1.4s, v2.4s}, [%x[input]], #48\n"
      "ld1 {v3.2s}, [%x[input]], #8\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "fsub v0.4s, v0.4s, v4.4s\n"
      "fsub v1.4s, v1.4s, v4.4s\n"
      "fsub v2.4s, v2.4s, v4.4s\n"
      "fsub v3.4s, v3.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fmul v2.4s, v2.4s, v6.4s\n"
      "fmul v3.4s, v3.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v5.4s\n"
      "fadd v1.4s, v1.4s, v5.4s\n"
      "fadd v2.4s, v2.4s, v5.4s\n"
      "fadd v3.4s, v3.4s, v5.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"
      "fcvtzs v3.4s, v3.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtn v2.4h, v2.4s\n"
      "sqxtn2 v2.8h, v3.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun2 v0.16b, v2.8h\n"

template <>
inline void Transform1DKernel<float, uint8_t, Quantize, 16, 15>::Transform(
    const float* input, const Quantize& params, uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Quantize<float, uint8_t, Quantize, 16, 15>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Quantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #15\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Quantize::Transform
      "ld1 {v0.4s, v1.4s, v2.4s}, [%x[input]], #48\n"
      "ld1 {v3.2s}, [%x[input]], #8\n"
      "ld1 {v3.s}[2], [%x[input]], #4\n"
      "prfm pldl1keep, [%x[input], #64]\n"
      "fsub v0.4s, v0.4s, v4.4s\n"
      "fsub v1.4s, v1.4s, v4.4s\n"
      "fsub v2.4s, v2.4s, v4.4s\n"
      "fsub v3.4s, v3.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fmul v2.4s, v2.4s, v6.4s\n"
      "fmul v3.4s, v3.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v5.4s\n"
      "fadd v1.4s, v1.4s, v5.4s\n"
      "fadd v2.4s, v2.4s, v5.4s\n"
      "fadd v3.4s, v3.4s, v5.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"
      "fcvtzs v3.4s, v3.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtn v2.4h, v2.4s\n"
      "sqxtn2 v2.8h, v3.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun2 v0.16b, v2.8h\n"

"st1 {v0.2s}, [%x[output]], #8\n"
      "st1 {v0.s}[2], [%x[output]], #4\n"
      "st1 {v0.h}[6], [%x[output]], #2\n"
      "st1 {v0.b}[14], [%x[output]], #1\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, float, Dequantize, 16, 0>::Transform(
    const uint8_t* input, const Dequantize& params, float* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Dequantize<uint8_t, float, Dequantize, 16, 0>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Dequantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

"1:"
      "subs %x[count], %x[count], #16\n"

// Dequantize::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl2 v1.8h, v0.16b\n"
      "uxtl v0.8h, v0.8b\n"
      "sxtl2 v3.4s, v1.8h\n"
      "sxtl v2.4s, v1.4h\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "scvtf v3.4s, v3.4s\n"
      "fsub v0.4s, v0.4s, v5.4s\n"
      "fsub v1.4s, v1.4s, v5.4s\n"
      "fsub v2.4s, v2.4s, v5.4s\n"
      "fsub v3.4s, v3.4s, v5.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fmul v2.4s, v2.4s, v6.4s\n"
      "fmul v3.4s, v3.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fadd v2.4s, v2.4s, v4.4s\n"
      "fadd v3.4s, v3.4s, v4.4s\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"

template <>
inline void Transform1DKernel<uint8_t, float, Dequantize, 16, 1>::Transform(
    const uint8_t* input, const Dequantize& params, float* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Dequantize<uint8_t, float, Dequantize, 16, 1>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Dequantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #1\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Dequantize::Transform
      "ld1 {v0.b}[0], [%x[input]], #1\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl v0.8h, v0.8b\n"
      "sxtl v0.4s, v0.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "fsub v0.4s, v0.4s, v5.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"

template <>
inline void Transform1DKernel<uint8_t, float, Dequantize, 16, 2>::Transform(
    const uint8_t* input, const Dequantize& params, float* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Dequantize<uint8_t, float, Dequantize, 16, 2>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Dequantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #2\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Dequantize::Transform
      "ld1 {v0.h}[0], [%x[input]], #2\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl v0.8h, v0.8b\n"
      "sxtl v0.4s, v0.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "fsub v0.4s, v0.4s, v5.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"

template <>
inline void Transform1DKernel<uint8_t, float, Dequantize, 16, 3>::Transform(
    const uint8_t* input, const Dequantize& params, float* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Dequantize<uint8_t, float, Dequantize, 16, 3>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Dequantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #3\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Dequantize::Transform
      "ld1 {v0.h}[0], [%x[input]], #2\n"
      "ld1 {v0.b}[2], [%x[input]], #1\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl v0.8h, v0.8b\n"
      "sxtl v0.4s, v0.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "fsub v0.4s, v0.4s, v5.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"

"st1 {v0.2s}, [%x[output]], #8\n"
      "st1 {v0.s}[2], [%x[output]], #4\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, float, Dequantize, 16, 4>::Transform(
    const uint8_t* input, const Dequantize& params, float* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Dequantize<uint8_t, float, Dequantize, 16, 4>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Dequantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #4\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Dequantize::Transform
      "ld1 {v0.s}[0], [%x[input]], #4\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl v0.8h, v0.8b\n"
      "sxtl v0.4s, v0.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "fsub v0.4s, v0.4s, v5.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, float, Dequantize, 16, 5>::Transform(
    const uint8_t* input, const Dequantize& params, float* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Dequantize<uint8_t, float, Dequantize, 16, 5>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Dequantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #5\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Dequantize::Transform
      "ld1 {v0.s}[0], [%x[input]], #4\n"
      "ld1 {v0.b}[4], [%x[input]], #1\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl v0.8h, v0.8b\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "fsub v0.4s, v0.4s, v5.4s\n"
      "fsub v1.4s, v1.4s, v5.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "st1 {v1.s}[0], [%x[output]], #4\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, float, Dequantize, 16, 6>::Transform(
    const uint8_t* input, const Dequantize& params, float* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Dequantize<uint8_t, float, Dequantize, 16, 6>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Dequantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #6\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Dequantize::Transform
      "ld1 {v0.s}[0], [%x[input]], #4\n"
      "ld1 {v0.h}[2], [%x[input]], #2\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl v0.8h, v0.8b\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "fsub v0.4s, v0.4s, v5.4s\n"
      "fsub v1.4s, v1.4s, v5.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "st1 {v1.2s}, [%x[output]], #8\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, float, Dequantize, 16, 7>::Transform(
    const uint8_t* input, const Dequantize& params, float* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Dequantize<uint8_t, float, Dequantize, 16, 7>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Dequantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #7\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Dequantize::Transform
      "ld1 {v0.s}[0], [%x[input]], #4\n"
      "ld1 {v0.h}[2], [%x[input]], #2\n"
      "ld1 {v0.b}[6], [%x[input]], #1\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl v0.8h, v0.8b\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "fsub v0.4s, v0.4s, v5.4s\n"
      "fsub v1.4s, v1.4s, v5.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "st1 {v1.2s}, [%x[output]], #8\n"
      "st1 {v1.s}[2], [%x[output]], #4\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, float, Dequantize, 16, 8>::Transform(
    const uint8_t* input, const Dequantize& params, float* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Dequantize<uint8_t, float, Dequantize, 16, 8>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Dequantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #8\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Dequantize::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl v0.8h, v0.8b\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "fsub v0.4s, v0.4s, v5.4s\n"
      "fsub v1.4s, v1.4s, v5.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"

"st1 {v0.4s, v1.4s}, [%x[output]], #32\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, float, Dequantize, 16, 9>::Transform(
    const uint8_t* input, const Dequantize& params, float* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Dequantize<uint8_t, float, Dequantize, 16, 9>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Dequantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #9\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Dequantize::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.b}[8], [%x[input]], #1\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl2 v1.8h, v0.16b\n"
      "uxtl v0.8h, v0.8b\n"
      "sxtl v2.4s, v1.4h\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "fsub v0.4s, v0.4s, v5.4s\n"
      "fsub v1.4s, v1.4s, v5.4s\n"
      "fsub v2.4s, v2.4s, v5.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fmul v2.4s, v2.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fadd v2.4s, v2.4s, v4.4s\n"

"st1 {v0.4s, v1.4s}, [%x[output]], #32\n"
      "st1 {v2.s}[0], [%x[output]], #4\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, float, Dequantize, 16, 10>::Transform(
    const uint8_t* input, const Dequantize& params, float* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Dequantize<uint8_t, float, Dequantize, 16, 10>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Dequantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #10\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Dequantize::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.h}[4], [%x[input]], #2\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl2 v1.8h, v0.16b\n"
      "uxtl v0.8h, v0.8b\n"
      "sxtl v2.4s, v1.4h\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "fsub v0.4s, v0.4s, v5.4s\n"
      "fsub v1.4s, v1.4s, v5.4s\n"
      "fsub v2.4s, v2.4s, v5.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fmul v2.4s, v2.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fadd v2.4s, v2.4s, v4.4s\n"

"st1 {v0.4s, v1.4s}, [%x[output]], #32\n"
      "st1 {v2.2s}, [%x[output]], #8\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, float, Dequantize, 16, 11>::Transform(
    const uint8_t* input, const Dequantize& params, float* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Dequantize<uint8_t, float, Dequantize, 16, 11>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Dequantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #11\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Dequantize::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.h}[4], [%x[input]], #2\n"
      "ld1 {v0.b}[10], [%x[input]], #1\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl2 v1.8h, v0.16b\n"
      "uxtl v0.8h, v0.8b\n"
      "sxtl v2.4s, v1.4h\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "fsub v0.4s, v0.4s, v5.4s\n"
      "fsub v1.4s, v1.4s, v5.4s\n"
      "fsub v2.4s, v2.4s, v5.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fmul v2.4s, v2.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fadd v2.4s, v2.4s, v4.4s\n"

"st1 {v0.4s, v1.4s}, [%x[output]], #32\n"
      "st1 {v2.2s}, [%x[output]], #8\n"
      "st1 {v2.s}[2], [%x[output]], #4\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, float, Dequantize, 16, 12>::Transform(
    const uint8_t* input, const Dequantize& params, float* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Dequantize<uint8_t, float, Dequantize, 16, 12>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Dequantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #12\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Dequantize::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.s}[2], [%x[input]], #4\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl2 v1.8h, v0.16b\n"
      "uxtl v0.8h, v0.8b\n"
      "sxtl v2.4s, v1.4h\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "fsub v0.4s, v0.4s, v5.4s\n"
      "fsub v1.4s, v1.4s, v5.4s\n"
      "fsub v2.4s, v2.4s, v5.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fmul v2.4s, v2.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fadd v2.4s, v2.4s, v4.4s\n"

"st1 {v0.4s, v1.4s, v2.4s}, [%x[output]], #48\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, float, Dequantize, 16, 13>::Transform(
    const uint8_t* input, const Dequantize& params, float* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Dequantize<uint8_t, float, Dequantize, 16, 13>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Dequantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #13\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Dequantize::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.s}[2], [%x[input]], #4\n"
      "ld1 {v0.b}[12], [%x[input]], #1\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl2 v1.8h, v0.16b\n"
      "uxtl v0.8h, v0.8b\n"
      "sxtl2 v3.4s, v1.8h\n"
      "sxtl v2.4s, v1.4h\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "scvtf v3.4s, v3.4s\n"
      "fsub v0.4s, v0.4s, v5.4s\n"
      "fsub v1.4s, v1.4s, v5.4s\n"
      "fsub v2.4s, v2.4s, v5.4s\n"
      "fsub v3.4s, v3.4s, v5.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fmul v2.4s, v2.4s, v6.4s\n"
      "fmul v3.4s, v3.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fadd v2.4s, v2.4s, v4.4s\n"
      "fadd v3.4s, v3.4s, v4.4s\n"

"st1 {v0.4s, v1.4s, v2.4s}, [%x[output]], #48\n"
      "st1 {v3.s}[0], [%x[output]], #4\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, float, Dequantize, 16, 14>::Transform(
    const uint8_t* input, const Dequantize& params, float* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Dequantize<uint8_t, float, Dequantize, 16, 14>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Dequantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #14\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Dequantize::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.s}[2], [%x[input]], #4\n"
      "ld1 {v0.h}[6], [%x[input]], #2\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl2 v1.8h, v0.16b\n"
      "uxtl v0.8h, v0.8b\n"
      "sxtl2 v3.4s, v1.8h\n"
      "sxtl v2.4s, v1.4h\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "scvtf v3.4s, v3.4s\n"
      "fsub v0.4s, v0.4s, v5.4s\n"
      "fsub v1.4s, v1.4s, v5.4s\n"
      "fsub v2.4s, v2.4s, v5.4s\n"
      "fsub v3.4s, v3.4s, v5.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fmul v2.4s, v2.4s, v6.4s\n"
      "fmul v3.4s, v3.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fadd v2.4s, v2.4s, v4.4s\n"
      "fadd v3.4s, v3.4s, v4.4s\n"

"st1 {v0.4s, v1.4s, v2.4s}, [%x[output]], #48\n"
      "st1 {v3.2s}, [%x[output]], #8\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, float, Dequantize, 16, 15>::Transform(
    const uint8_t* input, const Dequantize& params, float* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") Dequantize<uint8_t, float, Dequantize, 16, 15>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// Dequantize::Prepare
      "dup v4.4s, %w[range_min]\n"
      "dup v5.4s, %w[range_offset]\n"
      "dup v6.4s, %w[range_scale]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #15\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// Dequantize::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.s}[2], [%x[input]], #4\n"
      "ld1 {v0.h}[6], [%x[input]], #2\n"
      "ld1 {v0.b}[14], [%x[input]], #1\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl2 v1.8h, v0.16b\n"
      "uxtl v0.8h, v0.8b\n"
      "sxtl2 v3.4s, v1.8h\n"
      "sxtl v2.4s, v1.4h\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "scvtf v3.4s, v3.4s\n"
      "fsub v0.4s, v0.4s, v5.4s\n"
      "fsub v1.4s, v1.4s, v5.4s\n"
      "fsub v2.4s, v2.4s, v5.4s\n"
      "fsub v3.4s, v3.4s, v5.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fmul v2.4s, v2.4s, v6.4s\n"
      "fmul v3.4s, v3.4s, v6.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fadd v2.4s, v2.4s, v4.4s\n"
      "fadd v3.4s, v3.4s, v4.4s\n"

"st1 {v0.4s, v1.4s, v2.4s}, [%x[output]], #48\n"
      "st1 {v3.2s}, [%x[output]], #8\n"
      "st1 {v3.s}[2], [%x[output]], #4\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [range_offset] "r"(params.range_offset),
        [range_scale] "r"(params.range_scale), [range_min] "r"(params.range_min)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, uint8_t, MinMax<uint8_t>, 16,
                              0>::Transform(const uint8_t* input,
                                            const MinMax<uint8_t>& params,
                                            uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") MinMax<uint8_t><uint8_t, uint8_t, MinMax<uint8_t>, 16, "
               "0>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// MinMax::Prepare
      "dup v4.16b, %w[min]\n"
      "dup v5.16b, %w[max]\n"

"1:"
      "subs %x[count], %x[count], #16\n"

// MinMax::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [max] "r"(params.max), [min] "r"(params.min)
      : "v0", "v4", "v5", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, uint8_t, MinMax<uint8_t>, 16,
                              1>::Transform(const uint8_t* input,
                                            const MinMax<uint8_t>& params,
                                            uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") MinMax<uint8_t><uint8_t, uint8_t, MinMax<uint8_t>, 16, "
               "1>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// MinMax::Prepare
      "dup v4.16b, %w[min]\n"
      "dup v5.16b, %w[max]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #1\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

// MinMax::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// MinMax::Transform
      "ld1 {v0.b}[0], [%x[input]], #1\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

"st1 {v0.b}[0], [%x[output]], #1\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [max] "r"(params.max), [min] "r"(params.min)
      : "v0", "v4", "v5", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, uint8_t, MinMax<uint8_t>, 16,
                              2>::Transform(const uint8_t* input,
                                            const MinMax<uint8_t>& params,
                                            uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") MinMax<uint8_t><uint8_t, uint8_t, MinMax<uint8_t>, 16, "
               "2>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// MinMax::Prepare
      "dup v4.16b, %w[min]\n"
      "dup v5.16b, %w[max]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #2\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

// MinMax::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// MinMax::Transform
      "ld1 {v0.h}[0], [%x[input]], #2\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

"st1 {v0.h}[0], [%x[output]], #2\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [max] "r"(params.max), [min] "r"(params.min)
      : "v0", "v4", "v5", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, uint8_t, MinMax<uint8_t>, 16,
                              3>::Transform(const uint8_t* input,
                                            const MinMax<uint8_t>& params,
                                            uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") MinMax<uint8_t><uint8_t, uint8_t, MinMax<uint8_t>, 16, "
               "3>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// MinMax::Prepare
      "dup v4.16b, %w[min]\n"
      "dup v5.16b, %w[max]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #3\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

// MinMax::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// MinMax::Transform
      "ld1 {v0.h}[0], [%x[input]], #2\n"
      "ld1 {v0.b}[2], [%x[input]], #1\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

template <>
inline void Transform1DKernel<uint8_t, uint8_t, MinMax<uint8_t>, 16,
                              4>::Transform(const uint8_t* input,
                                            const MinMax<uint8_t>& params,
                                            uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") MinMax<uint8_t><uint8_t, uint8_t, MinMax<uint8_t>, 16, "
               "4>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// MinMax::Prepare
      "dup v4.16b, %w[min]\n"
      "dup v5.16b, %w[max]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #4\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

// MinMax::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// MinMax::Transform
      "ld1 {v0.s}[0], [%x[input]], #4\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

"st1 {v0.s}[0], [%x[output]], #4\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [max] "r"(params.max), [min] "r"(params.min)
      : "v0", "v4", "v5", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, uint8_t, MinMax<uint8_t>, 16,
                              5>::Transform(const uint8_t* input,
                                            const MinMax<uint8_t>& params,
                                            uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") MinMax<uint8_t><uint8_t, uint8_t, MinMax<uint8_t>, 16, "
               "5>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// MinMax::Prepare
      "dup v4.16b, %w[min]\n"
      "dup v5.16b, %w[max]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #5\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

// MinMax::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// MinMax::Transform
      "ld1 {v0.s}[0], [%x[input]], #4\n"
      "ld1 {v0.b}[4], [%x[input]], #1\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

template <>
inline void Transform1DKernel<uint8_t, uint8_t, MinMax<uint8_t>, 16,
                              6>::Transform(const uint8_t* input,
                                            const MinMax<uint8_t>& params,
                                            uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") MinMax<uint8_t><uint8_t, uint8_t, MinMax<uint8_t>, 16, "
               "6>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// MinMax::Prepare
      "dup v4.16b, %w[min]\n"
      "dup v5.16b, %w[max]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #6\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

// MinMax::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// MinMax::Transform
      "ld1 {v0.s}[0], [%x[input]], #4\n"
      "ld1 {v0.h}[2], [%x[input]], #2\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

template <>
inline void Transform1DKernel<uint8_t, uint8_t, MinMax<uint8_t>, 16,
                              7>::Transform(const uint8_t* input,
                                            const MinMax<uint8_t>& params,
                                            uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") MinMax<uint8_t><uint8_t, uint8_t, MinMax<uint8_t>, 16, "
               "7>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// MinMax::Prepare
      "dup v4.16b, %w[min]\n"
      "dup v5.16b, %w[max]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #7\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

// MinMax::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// MinMax::Transform
      "ld1 {v0.s}[0], [%x[input]], #4\n"
      "ld1 {v0.h}[2], [%x[input]], #2\n"
      "ld1 {v0.b}[6], [%x[input]], #1\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

template <>
inline void Transform1DKernel<uint8_t, uint8_t, MinMax<uint8_t>, 16,
                              8>::Transform(const uint8_t* input,
                                            const MinMax<uint8_t>& params,
                                            uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") MinMax<uint8_t><uint8_t, uint8_t, MinMax<uint8_t>, 16, "
               "8>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// MinMax::Prepare
      "dup v4.16b, %w[min]\n"
      "dup v5.16b, %w[max]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #8\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

// MinMax::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// MinMax::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

"st1 {v0.2s}, [%x[output]], #8\n"
      "prfm pldl1keep, [%x[output]]\n"
      : [count] "+r"(params_count_copy), [input] "+r"(input),
        [output] "+r"(output)
      : [max] "r"(params.max), [min] "r"(params.min)
      : "v0", "v4", "v5", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, uint8_t, MinMax<uint8_t>, 16,
                              9>::Transform(const uint8_t* input,
                                            const MinMax<uint8_t>& params,
                                            uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") MinMax<uint8_t><uint8_t, uint8_t, MinMax<uint8_t>, 16, "
               "9>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// MinMax::Prepare
      "dup v4.16b, %w[min]\n"
      "dup v5.16b, %w[max]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #9\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

// MinMax::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// MinMax::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.b}[8], [%x[input]], #1\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

template <>
inline void Transform1DKernel<uint8_t, uint8_t, MinMax<uint8_t>, 16,
                              10>::Transform(const uint8_t* input,
                                             const MinMax<uint8_t>& params,
                                             uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") MinMax<uint8_t><uint8_t, uint8_t, MinMax<uint8_t>, 16, "
               "10>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// MinMax::Prepare
      "dup v4.16b, %w[min]\n"
      "dup v5.16b, %w[max]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #10\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

// MinMax::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// MinMax::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.h}[4], [%x[input]], #2\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

template <>
inline void Transform1DKernel<uint8_t, uint8_t, MinMax<uint8_t>, 16,
                              11>::Transform(const uint8_t* input,
                                             const MinMax<uint8_t>& params,
                                             uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") MinMax<uint8_t><uint8_t, uint8_t, MinMax<uint8_t>, 16, "
               "11>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// MinMax::Prepare
      "dup v4.16b, %w[min]\n"
      "dup v5.16b, %w[max]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #11\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

// MinMax::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// MinMax::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.h}[4], [%x[input]], #2\n"
      "ld1 {v0.b}[10], [%x[input]], #1\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

template <>
inline void Transform1DKernel<uint8_t, uint8_t, MinMax<uint8_t>, 16,
                              12>::Transform(const uint8_t* input,
                                             const MinMax<uint8_t>& params,
                                             uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") MinMax<uint8_t><uint8_t, uint8_t, MinMax<uint8_t>, 16, "
               "12>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// MinMax::Prepare
      "dup v4.16b, %w[min]\n"
      "dup v5.16b, %w[max]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #12\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

// MinMax::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// MinMax::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.s}[2], [%x[input]], #4\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

template <>
inline void Transform1DKernel<uint8_t, uint8_t, MinMax<uint8_t>, 16,
                              13>::Transform(const uint8_t* input,
                                             const MinMax<uint8_t>& params,
                                             uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") MinMax<uint8_t><uint8_t, uint8_t, MinMax<uint8_t>, 16, "
               "13>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// MinMax::Prepare
      "dup v4.16b, %w[min]\n"
      "dup v5.16b, %w[max]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #13\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

// MinMax::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// MinMax::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.s}[2], [%x[input]], #4\n"
      "ld1 {v0.b}[12], [%x[input]], #1\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

template <>
inline void Transform1DKernel<uint8_t, uint8_t, MinMax<uint8_t>, 16,
                              14>::Transform(const uint8_t* input,
                                             const MinMax<uint8_t>& params,
                                             uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") MinMax<uint8_t><uint8_t, uint8_t, MinMax<uint8_t>, 16, "
               "14>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// MinMax::Prepare
      "dup v4.16b, %w[min]\n"
      "dup v5.16b, %w[max]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #14\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

// MinMax::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// MinMax::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.s}[2], [%x[input]], #4\n"
      "ld1 {v0.h}[6], [%x[input]], #2\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

template <>
inline void Transform1DKernel<uint8_t, uint8_t, MinMax<uint8_t>, 16,
                              15>::Transform(const uint8_t* input,
                                             const MinMax<uint8_t>& params,
                                             uint8_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") MinMax<uint8_t><uint8_t, uint8_t, MinMax<uint8_t>, 16, "
               "15>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_count_copy = params.count;
  asm volatile(

// MinMax::Prepare
      "dup v4.16b, %w[min]\n"
      "dup v5.16b, %w[max]\n"

// Reduce count by leftovers.
      "subs %x[count], %x[count], #15\n"
      "beq 2f\n"

"1:"
      "subs %x[count], %x[count], #16\n"

// MinMax::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"

"bne 1b\n"
      "2:"

// Handle leftovers.

// MinMax::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.s}[2], [%x[input]], #4\n"
      "ld1 {v0.h}[6], [%x[input]], #2\n"
      "ld1 {v0.b}[14], [%x[input]], #1\n"
      "prfm pldl1keep, [%x[input], #16]\n"
      "umax v0.16b, v0.16b, v4.16b\n"
      "umin v0.16b, v0.16b, v5.16b\n"

template <>
inline void Transform1DKernel<uint8_t, int32_t, BiasAdd<uint8_t>, 16,
                              0>::Transform(const uint8_t* input,
                                            const BiasAdd<uint8_t>& params,
                                            int32_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") BiasAdd<uint8_t><uint8_t, int32_t, BiasAdd<uint8_t>, 16, "
               "0>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_rows_copy = params.rows;
  asm volatile(
      "ldr w0, %[input_range_min]\n"
      "dup v8.4s, w0\n"
      "ldr w0, %[input_range_scale]\n"
      "dup v9.4s, w0\n"
      "ldr w0, %[bias_range_min]\n"
      "dup v10.4s, w0\n"
      "ldr w0, %[bias_range_scale]\n"
      "dup v11.4s, w0\n"
      "ldr w0, %[output_range_min]\n"
      "dup v12.4s, w0\n"
      "ldr w0, %[one_over_output_range_scale]\n"
      "dup v13.4s, w0\n"
      "ldr w0, %[output_range_offset]\n"
      "dup v14.4s, w0\n"
      "1:"
      "mov x0, %x[count]\n"
      "mov x1, %x[bias]\n"
      "2:"
      "subs x0, x0, #16\n"

// BiasAdd::Transform
      "ld1 {v0.4s}, [%x[input]], #16\n"
      "ld1 {v4.4s}, [x1], #16\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl2 v1.8h, v0.16b\n"
      "uxtl v0.8h, v0.8b\n"
      "uxtl2 v5.8h, v4.16b\n"
      "uxtl v4.8h, v4.8b\n"
      "sxtl2 v3.4s, v1.8h\n"
      "sxtl v2.4s, v1.4h\n"
      "sxtl2 v7.4s, v5.8h\n"
      "sxtl v6.4s, v5.4h\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "sxtl2 v5.4s, v4.8h\n"
      "sxtl v4.4s, v4.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "scvtf v3.4s, v3.4s\n"
      "scvtf v4.4s, v4.4s\n"
      "scvtf v5.4s, v5.4s\n"
      "scvtf v6.4s, v6.4s\n"
      "scvtf v7.4s, v7.4s\n"
      "fmul v0.4s, v0.4s, v9.4s\n"
      "fmul v1.4s, v1.4s, v9.4s\n"
      "fmul v2.4s, v2.4s, v9.4s\n"
      "fmul v3.4s, v3.4s, v9.4s\n"
      "fmul v4.4s, v4.4s, v11.4s\n"
      "fmul v5.4s, v5.4s, v11.4s\n"
      "fmul v6.4s, v6.4s, v11.4s\n"
      "fmul v7.4s, v7.4s, v11.4s\n"
      "fadd v0.4s, v0.4s, v8.4s\n"
      "fadd v1.4s, v1.4s, v8.4s\n"
      "fadd v2.4s, v2.4s, v8.4s\n"
      "fadd v3.4s, v3.4s, v8.4s\n"
      "fadd v4.4s, v4.4s, v10.4s\n"
      "fadd v5.4s, v5.4s, v10.4s\n"
      "fadd v6.4s, v6.4s, v10.4s\n"
      "fadd v7.4s, v7.4s, v10.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v5.4s\n"
      "fadd v2.4s, v2.4s, v6.4s\n"
      "fadd v3.4s, v3.4s, v7.4s\n"
      "fsub v0.4s, v0.4s, v12.4s\n"
      "fsub v1.4s, v1.4s, v12.4s\n"
      "fsub v2.4s, v2.4s, v12.4s\n"
      "fsub v3.4s, v3.4s, v12.4s\n"
      "fmul v0.4s, v0.4s, v13.4s\n"
      "fmul v1.4s, v1.4s, v13.4s\n"
      "fmul v2.4s, v2.4s, v13.4s\n"
      "fmul v3.4s, v3.4s, v13.4s\n"
      "fadd v0.4s, v0.4s, v14.4s\n"
      "fadd v1.4s, v1.4s, v14.4s\n"
      "fadd v2.4s, v2.4s, v14.4s\n"
      "fadd v3.4s, v3.4s, v14.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"
      "fcvtzs v3.4s, v3.4s\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"
      "bne 2b\n"
      "subs %x[rows], %x[rows], #1\n"
      "bne 1b\n"
      : [input] "+r"(input), [output] "+r"(output)
      : [count] "r"(params.count), [rows] "r"(params_rows_copy),
        [output_range_offset] "m"(params.output_range_offset),
        [input_range_scale] "m"(params.input_range_scale),
        [one_over_output_range_scale] "m"(params.one_over_output_range_scale),
        [bias_range_min] "m"(params.bias_range_min),
        [output_range_min] "m"(params.output_range_min),
        [bias_range_scale] "m"(params.bias_range_scale),
        [bias] "r"(params.bias), [input_range_min] "m"(params.input_range_min)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "v10", "v11", "v12", "v13", "v14", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, int32_t, BiasAdd<uint8_t>, 16,
                              1>::Transform(const uint8_t* input,
                                            const BiasAdd<uint8_t>& params,
                                            int32_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") BiasAdd<uint8_t><uint8_t, int32_t, BiasAdd<uint8_t>, 16, "
               "1>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_rows_copy = params.rows;
  asm volatile(
      "ldr w0, %[input_range_min]\n"
      "dup v8.4s, w0\n"
      "ldr w0, %[input_range_scale]\n"
      "dup v9.4s, w0\n"
      "ldr w0, %[bias_range_min]\n"
      "dup v10.4s, w0\n"
      "ldr w0, %[bias_range_scale]\n"
      "dup v11.4s, w0\n"
      "ldr w0, %[output_range_min]\n"
      "dup v12.4s, w0\n"
      "ldr w0, %[one_over_output_range_scale]\n"
      "dup v13.4s, w0\n"
      "ldr w0, %[output_range_offset]\n"
      "dup v14.4s, w0\n"
      "1:"
      "mov x0, %x[count]\n"
      "mov x1, %x[bias]\n"
      "subs x0, x0, #1\n"
      "beq 3f\n"
      "2:"
      "subs x0, x0, #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"
      "bne 2b\n"
      "3:"

// BiasAdd::Transform
      "ld1 {v0.b}[0], [%x[input]], #1\n"
      "ld1 {v1.b}[0], [x1], #1\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl v0.8h, v0.8b\n"
      "uxtl v1.8h, v1.8b\n"
      "sxtl v0.4s, v0.4h\n"
      "sxtl v1.4s, v1.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "fmul v0.4s, v0.4s, v9.4s\n"
      "fmul v1.4s, v1.4s, v11.4s\n"
      "fadd v0.4s, v0.4s, v8.4s\n"
      "fadd v1.4s, v1.4s, v10.4s\n"
      "fadd v0.4s, v0.4s, v1.4s\n"
      "fsub v0.4s, v0.4s, v12.4s\n"
      "fmul v0.4s, v0.4s, v13.4s\n"
      "fadd v0.4s, v0.4s, v14.4s\n"
      "fcvtzs v0.4s, v0.4s\n"

"st1 {v0.s}[0], [%x[output]], #4\n"
      "prfm pldl1keep, [%x[output]]\n"
      "subs %x[rows], %x[rows], #1\n"
      "bne 1b\n"
      : [input] "+r"(input), [output] "+r"(output)
      : [count] "r"(params.count), [rows] "r"(params_rows_copy),
        [output_range_offset] "m"(params.output_range_offset),
        [input_range_scale] "m"(params.input_range_scale),
        [one_over_output_range_scale] "m"(params.one_over_output_range_scale),
        [bias_range_min] "m"(params.bias_range_min),
        [output_range_min] "m"(params.output_range_min),
        [bias_range_scale] "m"(params.bias_range_scale),
        [bias] "r"(params.bias), [input_range_min] "m"(params.input_range_min)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "v10", "v11", "v12", "v13", "v14", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, int32_t, BiasAdd<uint8_t>, 16,
                              2>::Transform(const uint8_t* input,
                                            const BiasAdd<uint8_t>& params,
                                            int32_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") BiasAdd<uint8_t><uint8_t, int32_t, BiasAdd<uint8_t>, 16, "
               "2>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_rows_copy = params.rows;
  asm volatile(
      "ldr w0, %[input_range_min]\n"
      "dup v8.4s, w0\n"
      "ldr w0, %[input_range_scale]\n"
      "dup v9.4s, w0\n"
      "ldr w0, %[bias_range_min]\n"
      "dup v10.4s, w0\n"
      "ldr w0, %[bias_range_scale]\n"
      "dup v11.4s, w0\n"
      "ldr w0, %[output_range_min]\n"
      "dup v12.4s, w0\n"
      "ldr w0, %[one_over_output_range_scale]\n"
      "dup v13.4s, w0\n"
      "ldr w0, %[output_range_offset]\n"
      "dup v14.4s, w0\n"
      "1:"
      "mov x0, %x[count]\n"
      "mov x1, %x[bias]\n"
      "subs x0, x0, #2\n"
      "beq 3f\n"
      "2:"
      "subs x0, x0, #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"
      "bne 2b\n"
      "3:"

// BiasAdd::Transform
      "ld1 {v0.h}[0], [%x[input]], #2\n"
      "ld1 {v1.h}[0], [x1], #2\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl v0.8h, v0.8b\n"
      "uxtl v1.8h, v1.8b\n"
      "sxtl v0.4s, v0.4h\n"
      "sxtl v1.4s, v1.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "fmul v0.4s, v0.4s, v9.4s\n"
      "fmul v1.4s, v1.4s, v11.4s\n"
      "fadd v0.4s, v0.4s, v8.4s\n"
      "fadd v1.4s, v1.4s, v10.4s\n"
      "fadd v0.4s, v0.4s, v1.4s\n"
      "fsub v0.4s, v0.4s, v12.4s\n"
      "fmul v0.4s, v0.4s, v13.4s\n"
      "fadd v0.4s, v0.4s, v14.4s\n"
      "fcvtzs v0.4s, v0.4s\n"

"st1 {v0.2s}, [%x[output]], #8\n"
      "prfm pldl1keep, [%x[output]]\n"
      "subs %x[rows], %x[rows], #1\n"
      "bne 1b\n"
      : [input] "+r"(input), [output] "+r"(output)
      : [count] "r"(params.count), [rows] "r"(params_rows_copy),
        [output_range_offset] "m"(params.output_range_offset),
        [input_range_scale] "m"(params.input_range_scale),
        [one_over_output_range_scale] "m"(params.one_over_output_range_scale),
        [bias_range_min] "m"(params.bias_range_min),
        [output_range_min] "m"(params.output_range_min),
        [bias_range_scale] "m"(params.bias_range_scale),
        [bias] "r"(params.bias), [input_range_min] "m"(params.input_range_min)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "v10", "v11", "v12", "v13", "v14", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, int32_t, BiasAdd<uint8_t>, 16,
                              3>::Transform(const uint8_t* input,
                                            const BiasAdd<uint8_t>& params,
                                            int32_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") BiasAdd<uint8_t><uint8_t, int32_t, BiasAdd<uint8_t>, 16, "
               "3>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_rows_copy = params.rows;
  asm volatile(
      "ldr w0, %[input_range_min]\n"
      "dup v8.4s, w0\n"
      "ldr w0, %[input_range_scale]\n"
      "dup v9.4s, w0\n"
      "ldr w0, %[bias_range_min]\n"
      "dup v10.4s, w0\n"
      "ldr w0, %[bias_range_scale]\n"
      "dup v11.4s, w0\n"
      "ldr w0, %[output_range_min]\n"
      "dup v12.4s, w0\n"
      "ldr w0, %[one_over_output_range_scale]\n"
      "dup v13.4s, w0\n"
      "ldr w0, %[output_range_offset]\n"
      "dup v14.4s, w0\n"
      "1:"
      "mov x0, %x[count]\n"
      "mov x1, %x[bias]\n"
      "subs x0, x0, #3\n"
      "beq 3f\n"
      "2:"
      "subs x0, x0, #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"
      "bne 2b\n"
      "3:"

// BiasAdd::Transform
      "ld1 {v0.h}[0], [%x[input]], #2\n"
      "ld1 {v0.b}[2], [%x[input]], #1\n"
      "ld1 {v1.h}[0], [x1], #2\n"
      "ld1 {v1.b}[2], [x1], #1\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl v0.8h, v0.8b\n"
      "uxtl v1.8h, v1.8b\n"
      "sxtl v0.4s, v0.4h\n"
      "sxtl v1.4s, v1.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "fmul v0.4s, v0.4s, v9.4s\n"
      "fmul v1.4s, v1.4s, v11.4s\n"
      "fadd v0.4s, v0.4s, v8.4s\n"
      "fadd v1.4s, v1.4s, v10.4s\n"
      "fadd v0.4s, v0.4s, v1.4s\n"
      "fsub v0.4s, v0.4s, v12.4s\n"
      "fmul v0.4s, v0.4s, v13.4s\n"
      "fadd v0.4s, v0.4s, v14.4s\n"
      "fcvtzs v0.4s, v0.4s\n"

"st1 {v0.2s}, [%x[output]], #8\n"
      "st1 {v0.s}[2], [%x[output]], #4\n"
      "prfm pldl1keep, [%x[output]]\n"
      "subs %x[rows], %x[rows], #1\n"
      "bne 1b\n"
      : [input] "+r"(input), [output] "+r"(output)
      : [count] "r"(params.count), [rows] "r"(params_rows_copy),
        [output_range_offset] "m"(params.output_range_offset),
        [input_range_scale] "m"(params.input_range_scale),
        [one_over_output_range_scale] "m"(params.one_over_output_range_scale),
        [bias_range_min] "m"(params.bias_range_min),
        [output_range_min] "m"(params.output_range_min),
        [bias_range_scale] "m"(params.bias_range_scale),
        [bias] "r"(params.bias), [input_range_min] "m"(params.input_range_min)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "v10", "v11", "v12", "v13", "v14", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, int32_t, BiasAdd<uint8_t>, 16,
                              4>::Transform(const uint8_t* input,
                                            const BiasAdd<uint8_t>& params,
                                            int32_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") BiasAdd<uint8_t><uint8_t, int32_t, BiasAdd<uint8_t>, 16, "
               "4>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_rows_copy = params.rows;
  asm volatile(
      "ldr w0, %[input_range_min]\n"
      "dup v8.4s, w0\n"
      "ldr w0, %[input_range_scale]\n"
      "dup v9.4s, w0\n"
      "ldr w0, %[bias_range_min]\n"
      "dup v10.4s, w0\n"
      "ldr w0, %[bias_range_scale]\n"
      "dup v11.4s, w0\n"
      "ldr w0, %[output_range_min]\n"
      "dup v12.4s, w0\n"
      "ldr w0, %[one_over_output_range_scale]\n"
      "dup v13.4s, w0\n"
      "ldr w0, %[output_range_offset]\n"
      "dup v14.4s, w0\n"
      "1:"
      "mov x0, %x[count]\n"
      "mov x1, %x[bias]\n"
      "subs x0, x0, #4\n"
      "beq 3f\n"
      "2:"
      "subs x0, x0, #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"
      "bne 2b\n"
      "3:"

// BiasAdd::Transform
      "ld1 {v0.s}[0], [%x[input]], #4\n"
      "ld1 {v1.s}[0], [x1], #4\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl v0.8h, v0.8b\n"
      "uxtl v1.8h, v1.8b\n"
      "sxtl v0.4s, v0.4h\n"
      "sxtl v1.4s, v1.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "fmul v0.4s, v0.4s, v9.4s\n"
      "fmul v1.4s, v1.4s, v11.4s\n"
      "fadd v0.4s, v0.4s, v8.4s\n"
      "fadd v1.4s, v1.4s, v10.4s\n"
      "fadd v0.4s, v0.4s, v1.4s\n"
      "fsub v0.4s, v0.4s, v12.4s\n"
      "fmul v0.4s, v0.4s, v13.4s\n"
      "fadd v0.4s, v0.4s, v14.4s\n"
      "fcvtzs v0.4s, v0.4s\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "prfm pldl1keep, [%x[output]]\n"
      "subs %x[rows], %x[rows], #1\n"
      "bne 1b\n"
      : [input] "+r"(input), [output] "+r"(output)
      : [count] "r"(params.count), [rows] "r"(params_rows_copy),
        [output_range_offset] "m"(params.output_range_offset),
        [input_range_scale] "m"(params.input_range_scale),
        [one_over_output_range_scale] "m"(params.one_over_output_range_scale),
        [bias_range_min] "m"(params.bias_range_min),
        [output_range_min] "m"(params.output_range_min),
        [bias_range_scale] "m"(params.bias_range_scale),
        [bias] "r"(params.bias), [input_range_min] "m"(params.input_range_min)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "v10", "v11", "v12", "v13", "v14", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, int32_t, BiasAdd<uint8_t>, 16,
                              5>::Transform(const uint8_t* input,
                                            const BiasAdd<uint8_t>& params,
                                            int32_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") BiasAdd<uint8_t><uint8_t, int32_t, BiasAdd<uint8_t>, 16, "
               "5>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_rows_copy = params.rows;
  asm volatile(
      "ldr w0, %[input_range_min]\n"
      "dup v8.4s, w0\n"
      "ldr w0, %[input_range_scale]\n"
      "dup v9.4s, w0\n"
      "ldr w0, %[bias_range_min]\n"
      "dup v10.4s, w0\n"
      "ldr w0, %[bias_range_scale]\n"
      "dup v11.4s, w0\n"
      "ldr w0, %[output_range_min]\n"
      "dup v12.4s, w0\n"
      "ldr w0, %[one_over_output_range_scale]\n"
      "dup v13.4s, w0\n"
      "ldr w0, %[output_range_offset]\n"
      "dup v14.4s, w0\n"
      "1:"
      "mov x0, %x[count]\n"
      "mov x1, %x[bias]\n"
      "subs x0, x0, #5\n"
      "beq 3f\n"
      "2:"
      "subs x0, x0, #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"
      "bne 2b\n"
      "3:"

// BiasAdd::Transform
      "ld1 {v0.s}[0], [%x[input]], #4\n"
      "ld1 {v0.b}[4], [%x[input]], #1\n"
      "ld1 {v2.s}[0], [x1], #4\n"
      "ld1 {v2.b}[4], [x1], #1\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl v0.8h, v0.8b\n"
      "uxtl v2.8h, v2.8b\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "sxtl2 v3.4s, v2.8h\n"
      "sxtl v2.4s, v2.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "scvtf v3.4s, v3.4s\n"
      "fmul v0.4s, v0.4s, v9.4s\n"
      "fmul v1.4s, v1.4s, v9.4s\n"
      "fmul v2.4s, v2.4s, v11.4s\n"
      "fmul v3.4s, v3.4s, v11.4s\n"
      "fadd v0.4s, v0.4s, v8.4s\n"
      "fadd v1.4s, v1.4s, v8.4s\n"
      "fadd v2.4s, v2.4s, v10.4s\n"
      "fadd v3.4s, v3.4s, v10.4s\n"
      "fadd v0.4s, v0.4s, v2.4s\n"
      "fadd v1.4s, v1.4s, v3.4s\n"
      "fsub v0.4s, v0.4s, v12.4s\n"
      "fsub v1.4s, v1.4s, v12.4s\n"
      "fmul v0.4s, v0.4s, v13.4s\n"
      "fmul v1.4s, v1.4s, v13.4s\n"
      "fadd v0.4s, v0.4s, v14.4s\n"
      "fadd v1.4s, v1.4s, v14.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "st1 {v1.s}[0], [%x[output]], #4\n"
      "prfm pldl1keep, [%x[output]]\n"
      "subs %x[rows], %x[rows], #1\n"
      "bne 1b\n"
      : [input] "+r"(input), [output] "+r"(output)
      : [count] "r"(params.count), [rows] "r"(params_rows_copy),
        [output_range_offset] "m"(params.output_range_offset),
        [input_range_scale] "m"(params.input_range_scale),
        [one_over_output_range_scale] "m"(params.one_over_output_range_scale),
        [bias_range_min] "m"(params.bias_range_min),
        [output_range_min] "m"(params.output_range_min),
        [bias_range_scale] "m"(params.bias_range_scale),
        [bias] "r"(params.bias), [input_range_min] "m"(params.input_range_min)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "v10", "v11", "v12", "v13", "v14", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, int32_t, BiasAdd<uint8_t>, 16,
                              6>::Transform(const uint8_t* input,
                                            const BiasAdd<uint8_t>& params,
                                            int32_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") BiasAdd<uint8_t><uint8_t, int32_t, BiasAdd<uint8_t>, 16, "
               "6>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_rows_copy = params.rows;
  asm volatile(
      "ldr w0, %[input_range_min]\n"
      "dup v8.4s, w0\n"
      "ldr w0, %[input_range_scale]\n"
      "dup v9.4s, w0\n"
      "ldr w0, %[bias_range_min]\n"
      "dup v10.4s, w0\n"
      "ldr w0, %[bias_range_scale]\n"
      "dup v11.4s, w0\n"
      "ldr w0, %[output_range_min]\n"
      "dup v12.4s, w0\n"
      "ldr w0, %[one_over_output_range_scale]\n"
      "dup v13.4s, w0\n"
      "ldr w0, %[output_range_offset]\n"
      "dup v14.4s, w0\n"
      "1:"
      "mov x0, %x[count]\n"
      "mov x1, %x[bias]\n"
      "subs x0, x0, #6\n"
      "beq 3f\n"
      "2:"
      "subs x0, x0, #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"
      "bne 2b\n"
      "3:"

// BiasAdd::Transform
      "ld1 {v0.s}[0], [%x[input]], #4\n"
      "ld1 {v0.h}[2], [%x[input]], #2\n"
      "ld1 {v2.s}[0], [x1], #4\n"
      "ld1 {v2.h}[2], [x1], #2\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl v0.8h, v0.8b\n"
      "uxtl v2.8h, v2.8b\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "sxtl2 v3.4s, v2.8h\n"
      "sxtl v2.4s, v2.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "scvtf v3.4s, v3.4s\n"
      "fmul v0.4s, v0.4s, v9.4s\n"
      "fmul v1.4s, v1.4s, v9.4s\n"
      "fmul v2.4s, v2.4s, v11.4s\n"
      "fmul v3.4s, v3.4s, v11.4s\n"
      "fadd v0.4s, v0.4s, v8.4s\n"
      "fadd v1.4s, v1.4s, v8.4s\n"
      "fadd v2.4s, v2.4s, v10.4s\n"
      "fadd v3.4s, v3.4s, v10.4s\n"
      "fadd v0.4s, v0.4s, v2.4s\n"
      "fadd v1.4s, v1.4s, v3.4s\n"
      "fsub v0.4s, v0.4s, v12.4s\n"
      "fsub v1.4s, v1.4s, v12.4s\n"
      "fmul v0.4s, v0.4s, v13.4s\n"
      "fmul v1.4s, v1.4s, v13.4s\n"
      "fadd v0.4s, v0.4s, v14.4s\n"
      "fadd v1.4s, v1.4s, v14.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "st1 {v1.2s}, [%x[output]], #8\n"
      "prfm pldl1keep, [%x[output]]\n"
      "subs %x[rows], %x[rows], #1\n"
      "bne 1b\n"
      : [input] "+r"(input), [output] "+r"(output)
      : [count] "r"(params.count), [rows] "r"(params_rows_copy),
        [output_range_offset] "m"(params.output_range_offset),
        [input_range_scale] "m"(params.input_range_scale),
        [one_over_output_range_scale] "m"(params.one_over_output_range_scale),
        [bias_range_min] "m"(params.bias_range_min),
        [output_range_min] "m"(params.output_range_min),
        [bias_range_scale] "m"(params.bias_range_scale),
        [bias] "r"(params.bias), [input_range_min] "m"(params.input_range_min)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "v10", "v11", "v12", "v13", "v14", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, int32_t, BiasAdd<uint8_t>, 16,
                              7>::Transform(const uint8_t* input,
                                            const BiasAdd<uint8_t>& params,
                                            int32_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") BiasAdd<uint8_t><uint8_t, int32_t, BiasAdd<uint8_t>, 16, "
               "7>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_rows_copy = params.rows;
  asm volatile(
      "ldr w0, %[input_range_min]\n"
      "dup v8.4s, w0\n"
      "ldr w0, %[input_range_scale]\n"
      "dup v9.4s, w0\n"
      "ldr w0, %[bias_range_min]\n"
      "dup v10.4s, w0\n"
      "ldr w0, %[bias_range_scale]\n"
      "dup v11.4s, w0\n"
      "ldr w0, %[output_range_min]\n"
      "dup v12.4s, w0\n"
      "ldr w0, %[one_over_output_range_scale]\n"
      "dup v13.4s, w0\n"
      "ldr w0, %[output_range_offset]\n"
      "dup v14.4s, w0\n"
      "1:"
      "mov x0, %x[count]\n"
      "mov x1, %x[bias]\n"
      "subs x0, x0, #7\n"
      "beq 3f\n"
      "2:"
      "subs x0, x0, #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"
      "bne 2b\n"
      "3:"

// BiasAdd::Transform
      "ld1 {v0.s}[0], [%x[input]], #4\n"
      "ld1 {v0.h}[2], [%x[input]], #2\n"
      "ld1 {v0.b}[6], [%x[input]], #1\n"
      "ld1 {v2.s}[0], [x1], #4\n"
      "ld1 {v2.h}[2], [x1], #2\n"
      "ld1 {v2.b}[6], [x1], #1\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl v0.8h, v0.8b\n"
      "uxtl v2.8h, v2.8b\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "sxtl2 v3.4s, v2.8h\n"
      "sxtl v2.4s, v2.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "scvtf v3.4s, v3.4s\n"
      "fmul v0.4s, v0.4s, v9.4s\n"
      "fmul v1.4s, v1.4s, v9.4s\n"
      "fmul v2.4s, v2.4s, v11.4s\n"
      "fmul v3.4s, v3.4s, v11.4s\n"
      "fadd v0.4s, v0.4s, v8.4s\n"
      "fadd v1.4s, v1.4s, v8.4s\n"
      "fadd v2.4s, v2.4s, v10.4s\n"
      "fadd v3.4s, v3.4s, v10.4s\n"
      "fadd v0.4s, v0.4s, v2.4s\n"
      "fadd v1.4s, v1.4s, v3.4s\n"
      "fsub v0.4s, v0.4s, v12.4s\n"
      "fsub v1.4s, v1.4s, v12.4s\n"
      "fmul v0.4s, v0.4s, v13.4s\n"
      "fmul v1.4s, v1.4s, v13.4s\n"
      "fadd v0.4s, v0.4s, v14.4s\n"
      "fadd v1.4s, v1.4s, v14.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"

"st1 {v0.4s}, [%x[output]], #16\n"
      "st1 {v1.2s}, [%x[output]], #8\n"
      "st1 {v1.s}[2], [%x[output]], #4\n"
      "prfm pldl1keep, [%x[output]]\n"
      "subs %x[rows], %x[rows], #1\n"
      "bne 1b\n"
      : [input] "+r"(input), [output] "+r"(output)
      : [count] "r"(params.count), [rows] "r"(params_rows_copy),
        [output_range_offset] "m"(params.output_range_offset),
        [input_range_scale] "m"(params.input_range_scale),
        [one_over_output_range_scale] "m"(params.one_over_output_range_scale),
        [bias_range_min] "m"(params.bias_range_min),
        [output_range_min] "m"(params.output_range_min),
        [bias_range_scale] "m"(params.bias_range_scale),
        [bias] "r"(params.bias), [input_range_min] "m"(params.input_range_min)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "v10", "v11", "v12", "v13", "v14", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, int32_t, BiasAdd<uint8_t>, 16,
                              8>::Transform(const uint8_t* input,
                                            const BiasAdd<uint8_t>& params,
                                            int32_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") BiasAdd<uint8_t><uint8_t, int32_t, BiasAdd<uint8_t>, 16, "
               "8>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_rows_copy = params.rows;
  asm volatile(
      "ldr w0, %[input_range_min]\n"
      "dup v8.4s, w0\n"
      "ldr w0, %[input_range_scale]\n"
      "dup v9.4s, w0\n"
      "ldr w0, %[bias_range_min]\n"
      "dup v10.4s, w0\n"
      "ldr w0, %[bias_range_scale]\n"
      "dup v11.4s, w0\n"
      "ldr w0, %[output_range_min]\n"
      "dup v12.4s, w0\n"
      "ldr w0, %[one_over_output_range_scale]\n"
      "dup v13.4s, w0\n"
      "ldr w0, %[output_range_offset]\n"
      "dup v14.4s, w0\n"
      "1:"
      "mov x0, %x[count]\n"
      "mov x1, %x[bias]\n"
      "subs x0, x0, #8\n"
      "beq 3f\n"
      "2:"
      "subs x0, x0, #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"
      "bne 2b\n"
      "3:"

// BiasAdd::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v2.2s}, [x1], #8\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl v0.8h, v0.8b\n"
      "uxtl v2.8h, v2.8b\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "sxtl2 v3.4s, v2.8h\n"
      "sxtl v2.4s, v2.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "scvtf v3.4s, v3.4s\n"
      "fmul v0.4s, v0.4s, v9.4s\n"
      "fmul v1.4s, v1.4s, v9.4s\n"
      "fmul v2.4s, v2.4s, v11.4s\n"
      "fmul v3.4s, v3.4s, v11.4s\n"
      "fadd v0.4s, v0.4s, v8.4s\n"
      "fadd v1.4s, v1.4s, v8.4s\n"
      "fadd v2.4s, v2.4s, v10.4s\n"
      "fadd v3.4s, v3.4s, v10.4s\n"
      "fadd v0.4s, v0.4s, v2.4s\n"
      "fadd v1.4s, v1.4s, v3.4s\n"
      "fsub v0.4s, v0.4s, v12.4s\n"
      "fsub v1.4s, v1.4s, v12.4s\n"
      "fmul v0.4s, v0.4s, v13.4s\n"
      "fmul v1.4s, v1.4s, v13.4s\n"
      "fadd v0.4s, v0.4s, v14.4s\n"
      "fadd v1.4s, v1.4s, v14.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"

"st1 {v0.4s, v1.4s}, [%x[output]], #32\n"
      "prfm pldl1keep, [%x[output]]\n"
      "subs %x[rows], %x[rows], #1\n"
      "bne 1b\n"
      : [input] "+r"(input), [output] "+r"(output)
      : [count] "r"(params.count), [rows] "r"(params_rows_copy),
        [output_range_offset] "m"(params.output_range_offset),
        [input_range_scale] "m"(params.input_range_scale),
        [one_over_output_range_scale] "m"(params.one_over_output_range_scale),
        [bias_range_min] "m"(params.bias_range_min),
        [output_range_min] "m"(params.output_range_min),
        [bias_range_scale] "m"(params.bias_range_scale),
        [bias] "r"(params.bias), [input_range_min] "m"(params.input_range_min)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "v10", "v11", "v12", "v13", "v14", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, int32_t, BiasAdd<uint8_t>, 16,
                              9>::Transform(const uint8_t* input,
                                            const BiasAdd<uint8_t>& params,
                                            int32_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") BiasAdd<uint8_t><uint8_t, int32_t, BiasAdd<uint8_t>, 16, "
               "9>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_rows_copy = params.rows;
  asm volatile(
      "ldr w0, %[input_range_min]\n"
      "dup v8.4s, w0\n"
      "ldr w0, %[input_range_scale]\n"
      "dup v9.4s, w0\n"
      "ldr w0, %[bias_range_min]\n"
      "dup v10.4s, w0\n"
      "ldr w0, %[bias_range_scale]\n"
      "dup v11.4s, w0\n"
      "ldr w0, %[output_range_min]\n"
      "dup v12.4s, w0\n"
      "ldr w0, %[one_over_output_range_scale]\n"
      "dup v13.4s, w0\n"
      "ldr w0, %[output_range_offset]\n"
      "dup v14.4s, w0\n"
      "1:"
      "mov x0, %x[count]\n"
      "mov x1, %x[bias]\n"
      "subs x0, x0, #9\n"
      "beq 3f\n"
      "2:"
      "subs x0, x0, #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"
      "bne 2b\n"
      "3:"

// BiasAdd::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.b}[8], [%x[input]], #1\n"
      "ld1 {v3.2s}, [x1], #8\n"
      "ld1 {v3.b}[8], [x1], #1\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl2 v1.8h, v0.16b\n"
      "uxtl v0.8h, v0.8b\n"
      "uxtl2 v4.8h, v3.16b\n"
      "uxtl v3.8h, v3.8b\n"
      "sxtl v2.4s, v1.4h\n"
      "sxtl v5.4s, v4.4h\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "sxtl2 v4.4s, v3.8h\n"
      "sxtl v3.4s, v3.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "scvtf v3.4s, v3.4s\n"
      "scvtf v4.4s, v4.4s\n"
      "scvtf v5.4s, v5.4s\n"
      "fmul v0.4s, v0.4s, v9.4s\n"
      "fmul v1.4s, v1.4s, v9.4s\n"
      "fmul v2.4s, v2.4s, v9.4s\n"
      "fmul v3.4s, v3.4s, v11.4s\n"
      "fmul v4.4s, v4.4s, v11.4s\n"
      "fmul v5.4s, v5.4s, v11.4s\n"
      "fadd v0.4s, v0.4s, v8.4s\n"
      "fadd v1.4s, v1.4s, v8.4s\n"
      "fadd v2.4s, v2.4s, v8.4s\n"
      "fadd v3.4s, v3.4s, v10.4s\n"
      "fadd v4.4s, v4.4s, v10.4s\n"
      "fadd v5.4s, v5.4s, v10.4s\n"
      "fadd v0.4s, v0.4s, v3.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fadd v2.4s, v2.4s, v5.4s\n"
      "fsub v0.4s, v0.4s, v12.4s\n"
      "fsub v1.4s, v1.4s, v12.4s\n"
      "fsub v2.4s, v2.4s, v12.4s\n"
      "fmul v0.4s, v0.4s, v13.4s\n"
      "fmul v1.4s, v1.4s, v13.4s\n"
      "fmul v2.4s, v2.4s, v13.4s\n"
      "fadd v0.4s, v0.4s, v14.4s\n"
      "fadd v1.4s, v1.4s, v14.4s\n"
      "fadd v2.4s, v2.4s, v14.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"

"st1 {v0.4s, v1.4s}, [%x[output]], #32\n"
      "st1 {v2.s}[0], [%x[output]], #4\n"
      "prfm pldl1keep, [%x[output]]\n"
      "subs %x[rows], %x[rows], #1\n"
      "bne 1b\n"
      : [input] "+r"(input), [output] "+r"(output)
      : [count] "r"(params.count), [rows] "r"(params_rows_copy),
        [output_range_offset] "m"(params.output_range_offset),
        [input_range_scale] "m"(params.input_range_scale),
        [one_over_output_range_scale] "m"(params.one_over_output_range_scale),
        [bias_range_min] "m"(params.bias_range_min),
        [output_range_min] "m"(params.output_range_min),
        [bias_range_scale] "m"(params.bias_range_scale),
        [bias] "r"(params.bias), [input_range_min] "m"(params.input_range_min)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "v10", "v11", "v12", "v13", "v14", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, int32_t, BiasAdd<uint8_t>, 16,
                              10>::Transform(const uint8_t* input,
                                             const BiasAdd<uint8_t>& params,
                                             int32_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") BiasAdd<uint8_t><uint8_t, int32_t, BiasAdd<uint8_t>, 16, "
               "10>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_rows_copy = params.rows;
  asm volatile(
      "ldr w0, %[input_range_min]\n"
      "dup v8.4s, w0\n"
      "ldr w0, %[input_range_scale]\n"
      "dup v9.4s, w0\n"
      "ldr w0, %[bias_range_min]\n"
      "dup v10.4s, w0\n"
      "ldr w0, %[bias_range_scale]\n"
      "dup v11.4s, w0\n"
      "ldr w0, %[output_range_min]\n"
      "dup v12.4s, w0\n"
      "ldr w0, %[one_over_output_range_scale]\n"
      "dup v13.4s, w0\n"
      "ldr w0, %[output_range_offset]\n"
      "dup v14.4s, w0\n"
      "1:"
      "mov x0, %x[count]\n"
      "mov x1, %x[bias]\n"
      "subs x0, x0, #10\n"
      "beq 3f\n"
      "2:"
      "subs x0, x0, #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"
      "bne 2b\n"
      "3:"

// BiasAdd::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.h}[4], [%x[input]], #2\n"
      "ld1 {v3.2s}, [x1], #8\n"
      "ld1 {v3.h}[4], [x1], #2\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl2 v1.8h, v0.16b\n"
      "uxtl v0.8h, v0.8b\n"
      "uxtl2 v4.8h, v3.16b\n"
      "uxtl v3.8h, v3.8b\n"
      "sxtl v2.4s, v1.4h\n"
      "sxtl v5.4s, v4.4h\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "sxtl2 v4.4s, v3.8h\n"
      "sxtl v3.4s, v3.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "scvtf v3.4s, v3.4s\n"
      "scvtf v4.4s, v4.4s\n"
      "scvtf v5.4s, v5.4s\n"
      "fmul v0.4s, v0.4s, v9.4s\n"
      "fmul v1.4s, v1.4s, v9.4s\n"
      "fmul v2.4s, v2.4s, v9.4s\n"
      "fmul v3.4s, v3.4s, v11.4s\n"
      "fmul v4.4s, v4.4s, v11.4s\n"
      "fmul v5.4s, v5.4s, v11.4s\n"
      "fadd v0.4s, v0.4s, v8.4s\n"
      "fadd v1.4s, v1.4s, v8.4s\n"
      "fadd v2.4s, v2.4s, v8.4s\n"
      "fadd v3.4s, v3.4s, v10.4s\n"
      "fadd v4.4s, v4.4s, v10.4s\n"
      "fadd v5.4s, v5.4s, v10.4s\n"
      "fadd v0.4s, v0.4s, v3.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fadd v2.4s, v2.4s, v5.4s\n"
      "fsub v0.4s, v0.4s, v12.4s\n"
      "fsub v1.4s, v1.4s, v12.4s\n"
      "fsub v2.4s, v2.4s, v12.4s\n"
      "fmul v0.4s, v0.4s, v13.4s\n"
      "fmul v1.4s, v1.4s, v13.4s\n"
      "fmul v2.4s, v2.4s, v13.4s\n"
      "fadd v0.4s, v0.4s, v14.4s\n"
      "fadd v1.4s, v1.4s, v14.4s\n"
      "fadd v2.4s, v2.4s, v14.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"

"st1 {v0.4s, v1.4s}, [%x[output]], #32\n"
      "st1 {v2.2s}, [%x[output]], #8\n"
      "prfm pldl1keep, [%x[output]]\n"
      "subs %x[rows], %x[rows], #1\n"
      "bne 1b\n"
      : [input] "+r"(input), [output] "+r"(output)
      : [count] "r"(params.count), [rows] "r"(params_rows_copy),
        [output_range_offset] "m"(params.output_range_offset),
        [input_range_scale] "m"(params.input_range_scale),
        [one_over_output_range_scale] "m"(params.one_over_output_range_scale),
        [bias_range_min] "m"(params.bias_range_min),
        [output_range_min] "m"(params.output_range_min),
        [bias_range_scale] "m"(params.bias_range_scale),
        [bias] "r"(params.bias), [input_range_min] "m"(params.input_range_min)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "v10", "v11", "v12", "v13", "v14", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, int32_t, BiasAdd<uint8_t>, 16,
                              11>::Transform(const uint8_t* input,
                                             const BiasAdd<uint8_t>& params,
                                             int32_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") BiasAdd<uint8_t><uint8_t, int32_t, BiasAdd<uint8_t>, 16, "
               "11>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_rows_copy = params.rows;
  asm volatile(
      "ldr w0, %[input_range_min]\n"
      "dup v8.4s, w0\n"
      "ldr w0, %[input_range_scale]\n"
      "dup v9.4s, w0\n"
      "ldr w0, %[bias_range_min]\n"
      "dup v10.4s, w0\n"
      "ldr w0, %[bias_range_scale]\n"
      "dup v11.4s, w0\n"
      "ldr w0, %[output_range_min]\n"
      "dup v12.4s, w0\n"
      "ldr w0, %[one_over_output_range_scale]\n"
      "dup v13.4s, w0\n"
      "ldr w0, %[output_range_offset]\n"
      "dup v14.4s, w0\n"
      "1:"
      "mov x0, %x[count]\n"
      "mov x1, %x[bias]\n"
      "subs x0, x0, #11\n"
      "beq 3f\n"
      "2:"
      "subs x0, x0, #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"
      "bne 2b\n"
      "3:"

// BiasAdd::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.h}[4], [%x[input]], #2\n"
      "ld1 {v0.b}[10], [%x[input]], #1\n"
      "ld1 {v3.2s}, [x1], #8\n"
      "ld1 {v3.h}[4], [x1], #2\n"
      "ld1 {v3.b}[10], [x1], #1\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl2 v1.8h, v0.16b\n"
      "uxtl v0.8h, v0.8b\n"
      "uxtl2 v4.8h, v3.16b\n"
      "uxtl v3.8h, v3.8b\n"
      "sxtl v2.4s, v1.4h\n"
      "sxtl v5.4s, v4.4h\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "sxtl2 v4.4s, v3.8h\n"
      "sxtl v3.4s, v3.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "scvtf v3.4s, v3.4s\n"
      "scvtf v4.4s, v4.4s\n"
      "scvtf v5.4s, v5.4s\n"
      "fmul v0.4s, v0.4s, v9.4s\n"
      "fmul v1.4s, v1.4s, v9.4s\n"
      "fmul v2.4s, v2.4s, v9.4s\n"
      "fmul v3.4s, v3.4s, v11.4s\n"
      "fmul v4.4s, v4.4s, v11.4s\n"
      "fmul v5.4s, v5.4s, v11.4s\n"
      "fadd v0.4s, v0.4s, v8.4s\n"
      "fadd v1.4s, v1.4s, v8.4s\n"
      "fadd v2.4s, v2.4s, v8.4s\n"
      "fadd v3.4s, v3.4s, v10.4s\n"
      "fadd v4.4s, v4.4s, v10.4s\n"
      "fadd v5.4s, v5.4s, v10.4s\n"
      "fadd v0.4s, v0.4s, v3.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fadd v2.4s, v2.4s, v5.4s\n"
      "fsub v0.4s, v0.4s, v12.4s\n"
      "fsub v1.4s, v1.4s, v12.4s\n"
      "fsub v2.4s, v2.4s, v12.4s\n"
      "fmul v0.4s, v0.4s, v13.4s\n"
      "fmul v1.4s, v1.4s, v13.4s\n"
      "fmul v2.4s, v2.4s, v13.4s\n"
      "fadd v0.4s, v0.4s, v14.4s\n"
      "fadd v1.4s, v1.4s, v14.4s\n"
      "fadd v2.4s, v2.4s, v14.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"

"st1 {v0.4s, v1.4s}, [%x[output]], #32\n"
      "st1 {v2.2s}, [%x[output]], #8\n"
      "st1 {v2.s}[2], [%x[output]], #4\n"
      "prfm pldl1keep, [%x[output]]\n"
      "subs %x[rows], %x[rows], #1\n"
      "bne 1b\n"
      : [input] "+r"(input), [output] "+r"(output)
      : [count] "r"(params.count), [rows] "r"(params_rows_copy),
        [output_range_offset] "m"(params.output_range_offset),
        [input_range_scale] "m"(params.input_range_scale),
        [one_over_output_range_scale] "m"(params.one_over_output_range_scale),
        [bias_range_min] "m"(params.bias_range_min),
        [output_range_min] "m"(params.output_range_min),
        [bias_range_scale] "m"(params.bias_range_scale),
        [bias] "r"(params.bias), [input_range_min] "m"(params.input_range_min)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "v10", "v11", "v12", "v13", "v14", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, int32_t, BiasAdd<uint8_t>, 16,
                              12>::Transform(const uint8_t* input,
                                             const BiasAdd<uint8_t>& params,
                                             int32_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") BiasAdd<uint8_t><uint8_t, int32_t, BiasAdd<uint8_t>, 16, "
               "12>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_rows_copy = params.rows;
  asm volatile(
      "ldr w0, %[input_range_min]\n"
      "dup v8.4s, w0\n"
      "ldr w0, %[input_range_scale]\n"
      "dup v9.4s, w0\n"
      "ldr w0, %[bias_range_min]\n"
      "dup v10.4s, w0\n"
      "ldr w0, %[bias_range_scale]\n"
      "dup v11.4s, w0\n"
      "ldr w0, %[output_range_min]\n"
      "dup v12.4s, w0\n"
      "ldr w0, %[one_over_output_range_scale]\n"
      "dup v13.4s, w0\n"
      "ldr w0, %[output_range_offset]\n"
      "dup v14.4s, w0\n"
      "1:"
      "mov x0, %x[count]\n"
      "mov x1, %x[bias]\n"
      "subs x0, x0, #12\n"
      "beq 3f\n"
      "2:"
      "subs x0, x0, #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"
      "bne 2b\n"
      "3:"

// BiasAdd::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.s}[2], [%x[input]], #4\n"
      "ld1 {v3.2s}, [x1], #8\n"
      "ld1 {v3.s}[2], [x1], #4\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl2 v1.8h, v0.16b\n"
      "uxtl v0.8h, v0.8b\n"
      "uxtl2 v4.8h, v3.16b\n"
      "uxtl v3.8h, v3.8b\n"
      "sxtl v2.4s, v1.4h\n"
      "sxtl v5.4s, v4.4h\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "sxtl2 v4.4s, v3.8h\n"
      "sxtl v3.4s, v3.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "scvtf v3.4s, v3.4s\n"
      "scvtf v4.4s, v4.4s\n"
      "scvtf v5.4s, v5.4s\n"
      "fmul v0.4s, v0.4s, v9.4s\n"
      "fmul v1.4s, v1.4s, v9.4s\n"
      "fmul v2.4s, v2.4s, v9.4s\n"
      "fmul v3.4s, v3.4s, v11.4s\n"
      "fmul v4.4s, v4.4s, v11.4s\n"
      "fmul v5.4s, v5.4s, v11.4s\n"
      "fadd v0.4s, v0.4s, v8.4s\n"
      "fadd v1.4s, v1.4s, v8.4s\n"
      "fadd v2.4s, v2.4s, v8.4s\n"
      "fadd v3.4s, v3.4s, v10.4s\n"
      "fadd v4.4s, v4.4s, v10.4s\n"
      "fadd v5.4s, v5.4s, v10.4s\n"
      "fadd v0.4s, v0.4s, v3.4s\n"
      "fadd v1.4s, v1.4s, v4.4s\n"
      "fadd v2.4s, v2.4s, v5.4s\n"
      "fsub v0.4s, v0.4s, v12.4s\n"
      "fsub v1.4s, v1.4s, v12.4s\n"
      "fsub v2.4s, v2.4s, v12.4s\n"
      "fmul v0.4s, v0.4s, v13.4s\n"
      "fmul v1.4s, v1.4s, v13.4s\n"
      "fmul v2.4s, v2.4s, v13.4s\n"
      "fadd v0.4s, v0.4s, v14.4s\n"
      "fadd v1.4s, v1.4s, v14.4s\n"
      "fadd v2.4s, v2.4s, v14.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"

"st1 {v0.4s, v1.4s, v2.4s}, [%x[output]], #48\n"
      "prfm pldl1keep, [%x[output]]\n"
      "subs %x[rows], %x[rows], #1\n"
      "bne 1b\n"
      : [input] "+r"(input), [output] "+r"(output)
      : [count] "r"(params.count), [rows] "r"(params_rows_copy),
        [output_range_offset] "m"(params.output_range_offset),
        [input_range_scale] "m"(params.input_range_scale),
        [one_over_output_range_scale] "m"(params.one_over_output_range_scale),
        [bias_range_min] "m"(params.bias_range_min),
        [output_range_min] "m"(params.output_range_min),
        [bias_range_scale] "m"(params.bias_range_scale),
        [bias] "r"(params.bias), [input_range_min] "m"(params.input_range_min)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "v10", "v11", "v12", "v13", "v14", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, int32_t, BiasAdd<uint8_t>, 16,
                              13>::Transform(const uint8_t* input,
                                             const BiasAdd<uint8_t>& params,
                                             int32_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") BiasAdd<uint8_t><uint8_t, int32_t, BiasAdd<uint8_t>, 16, "
               "13>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_rows_copy = params.rows;
  asm volatile(
      "ldr w0, %[input_range_min]\n"
      "dup v8.4s, w0\n"
      "ldr w0, %[input_range_scale]\n"
      "dup v9.4s, w0\n"
      "ldr w0, %[bias_range_min]\n"
      "dup v10.4s, w0\n"
      "ldr w0, %[bias_range_scale]\n"
      "dup v11.4s, w0\n"
      "ldr w0, %[output_range_min]\n"
      "dup v12.4s, w0\n"
      "ldr w0, %[one_over_output_range_scale]\n"
      "dup v13.4s, w0\n"
      "ldr w0, %[output_range_offset]\n"
      "dup v14.4s, w0\n"
      "1:"
      "mov x0, %x[count]\n"
      "mov x1, %x[bias]\n"
      "subs x0, x0, #13\n"
      "beq 3f\n"
      "2:"
      "subs x0, x0, #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"
      "bne 2b\n"
      "3:"

// BiasAdd::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.s}[2], [%x[input]], #4\n"
      "ld1 {v0.b}[12], [%x[input]], #1\n"
      "ld1 {v4.2s}, [x1], #8\n"
      "ld1 {v4.s}[2], [x1], #4\n"
      "ld1 {v4.b}[12], [x1], #1\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl2 v1.8h, v0.16b\n"
      "uxtl v0.8h, v0.8b\n"
      "uxtl2 v5.8h, v4.16b\n"
      "uxtl v4.8h, v4.8b\n"
      "sxtl2 v3.4s, v1.8h\n"
      "sxtl v2.4s, v1.4h\n"
      "sxtl2 v7.4s, v5.8h\n"
      "sxtl v6.4s, v5.4h\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "sxtl2 v5.4s, v4.8h\n"
      "sxtl v4.4s, v4.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "scvtf v3.4s, v3.4s\n"
      "scvtf v4.4s, v4.4s\n"
      "scvtf v5.4s, v5.4s\n"
      "scvtf v6.4s, v6.4s\n"
      "scvtf v7.4s, v7.4s\n"
      "fmul v0.4s, v0.4s, v9.4s\n"
      "fmul v1.4s, v1.4s, v9.4s\n"
      "fmul v2.4s, v2.4s, v9.4s\n"
      "fmul v3.4s, v3.4s, v9.4s\n"
      "fmul v4.4s, v4.4s, v11.4s\n"
      "fmul v5.4s, v5.4s, v11.4s\n"
      "fmul v6.4s, v6.4s, v11.4s\n"
      "fmul v7.4s, v7.4s, v11.4s\n"
      "fadd v0.4s, v0.4s, v8.4s\n"
      "fadd v1.4s, v1.4s, v8.4s\n"
      "fadd v2.4s, v2.4s, v8.4s\n"
      "fadd v3.4s, v3.4s, v8.4s\n"
      "fadd v4.4s, v4.4s, v10.4s\n"
      "fadd v5.4s, v5.4s, v10.4s\n"
      "fadd v6.4s, v6.4s, v10.4s\n"
      "fadd v7.4s, v7.4s, v10.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v5.4s\n"
      "fadd v2.4s, v2.4s, v6.4s\n"
      "fadd v3.4s, v3.4s, v7.4s\n"
      "fsub v0.4s, v0.4s, v12.4s\n"
      "fsub v1.4s, v1.4s, v12.4s\n"
      "fsub v2.4s, v2.4s, v12.4s\n"
      "fsub v3.4s, v3.4s, v12.4s\n"
      "fmul v0.4s, v0.4s, v13.4s\n"
      "fmul v1.4s, v1.4s, v13.4s\n"
      "fmul v2.4s, v2.4s, v13.4s\n"
      "fmul v3.4s, v3.4s, v13.4s\n"
      "fadd v0.4s, v0.4s, v14.4s\n"
      "fadd v1.4s, v1.4s, v14.4s\n"
      "fadd v2.4s, v2.4s, v14.4s\n"
      "fadd v3.4s, v3.4s, v14.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"
      "fcvtzs v3.4s, v3.4s\n"

"st1 {v0.4s, v1.4s, v2.4s}, [%x[output]], #48\n"
      "st1 {v3.s}[0], [%x[output]], #4\n"
      "prfm pldl1keep, [%x[output]]\n"
      "subs %x[rows], %x[rows], #1\n"
      "bne 1b\n"
      : [input] "+r"(input), [output] "+r"(output)
      : [count] "r"(params.count), [rows] "r"(params_rows_copy),
        [output_range_offset] "m"(params.output_range_offset),
        [input_range_scale] "m"(params.input_range_scale),
        [one_over_output_range_scale] "m"(params.one_over_output_range_scale),
        [bias_range_min] "m"(params.bias_range_min),
        [output_range_min] "m"(params.output_range_min),
        [bias_range_scale] "m"(params.bias_range_scale),
        [bias] "r"(params.bias), [input_range_min] "m"(params.input_range_min)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "v10", "v11", "v12", "v13", "v14", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, int32_t, BiasAdd<uint8_t>, 16,
                              14>::Transform(const uint8_t* input,
                                             const BiasAdd<uint8_t>& params,
                                             int32_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") BiasAdd<uint8_t><uint8_t, int32_t, BiasAdd<uint8_t>, 16, "
               "14>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_rows_copy = params.rows;
  asm volatile(
      "ldr w0, %[input_range_min]\n"
      "dup v8.4s, w0\n"
      "ldr w0, %[input_range_scale]\n"
      "dup v9.4s, w0\n"
      "ldr w0, %[bias_range_min]\n"
      "dup v10.4s, w0\n"
      "ldr w0, %[bias_range_scale]\n"
      "dup v11.4s, w0\n"
      "ldr w0, %[output_range_min]\n"
      "dup v12.4s, w0\n"
      "ldr w0, %[one_over_output_range_scale]\n"
      "dup v13.4s, w0\n"
      "ldr w0, %[output_range_offset]\n"
      "dup v14.4s, w0\n"
      "1:"
      "mov x0, %x[count]\n"
      "mov x1, %x[bias]\n"
      "subs x0, x0, #14\n"
      "beq 3f\n"
      "2:"
      "subs x0, x0, #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"
      "bne 2b\n"
      "3:"

// BiasAdd::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.s}[2], [%x[input]], #4\n"
      "ld1 {v0.h}[6], [%x[input]], #2\n"
      "ld1 {v4.2s}, [x1], #8\n"
      "ld1 {v4.s}[2], [x1], #4\n"
      "ld1 {v4.h}[6], [x1], #2\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl2 v1.8h, v0.16b\n"
      "uxtl v0.8h, v0.8b\n"
      "uxtl2 v5.8h, v4.16b\n"
      "uxtl v4.8h, v4.8b\n"
      "sxtl2 v3.4s, v1.8h\n"
      "sxtl v2.4s, v1.4h\n"
      "sxtl2 v7.4s, v5.8h\n"
      "sxtl v6.4s, v5.4h\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "sxtl2 v5.4s, v4.8h\n"
      "sxtl v4.4s, v4.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "scvtf v3.4s, v3.4s\n"
      "scvtf v4.4s, v4.4s\n"
      "scvtf v5.4s, v5.4s\n"
      "scvtf v6.4s, v6.4s\n"
      "scvtf v7.4s, v7.4s\n"
      "fmul v0.4s, v0.4s, v9.4s\n"
      "fmul v1.4s, v1.4s, v9.4s\n"
      "fmul v2.4s, v2.4s, v9.4s\n"
      "fmul v3.4s, v3.4s, v9.4s\n"
      "fmul v4.4s, v4.4s, v11.4s\n"
      "fmul v5.4s, v5.4s, v11.4s\n"
      "fmul v6.4s, v6.4s, v11.4s\n"
      "fmul v7.4s, v7.4s, v11.4s\n"
      "fadd v0.4s, v0.4s, v8.4s\n"
      "fadd v1.4s, v1.4s, v8.4s\n"
      "fadd v2.4s, v2.4s, v8.4s\n"
      "fadd v3.4s, v3.4s, v8.4s\n"
      "fadd v4.4s, v4.4s, v10.4s\n"
      "fadd v5.4s, v5.4s, v10.4s\n"
      "fadd v6.4s, v6.4s, v10.4s\n"
      "fadd v7.4s, v7.4s, v10.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v5.4s\n"
      "fadd v2.4s, v2.4s, v6.4s\n"
      "fadd v3.4s, v3.4s, v7.4s\n"
      "fsub v0.4s, v0.4s, v12.4s\n"
      "fsub v1.4s, v1.4s, v12.4s\n"
      "fsub v2.4s, v2.4s, v12.4s\n"
      "fsub v3.4s, v3.4s, v12.4s\n"
      "fmul v0.4s, v0.4s, v13.4s\n"
      "fmul v1.4s, v1.4s, v13.4s\n"
      "fmul v2.4s, v2.4s, v13.4s\n"
      "fmul v3.4s, v3.4s, v13.4s\n"
      "fadd v0.4s, v0.4s, v14.4s\n"
      "fadd v1.4s, v1.4s, v14.4s\n"
      "fadd v2.4s, v2.4s, v14.4s\n"
      "fadd v3.4s, v3.4s, v14.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"
      "fcvtzs v3.4s, v3.4s\n"

"st1 {v0.4s, v1.4s, v2.4s}, [%x[output]], #48\n"
      "st1 {v3.2s}, [%x[output]], #8\n"
      "prfm pldl1keep, [%x[output]]\n"
      "subs %x[rows], %x[rows], #1\n"
      "bne 1b\n"
      : [input] "+r"(input), [output] "+r"(output)
      : [count] "r"(params.count), [rows] "r"(params_rows_copy),
        [output_range_offset] "m"(params.output_range_offset),
        [input_range_scale] "m"(params.input_range_scale),
        [one_over_output_range_scale] "m"(params.one_over_output_range_scale),
        [bias_range_min] "m"(params.bias_range_min),
        [output_range_min] "m"(params.output_range_min),
        [bias_range_scale] "m"(params.bias_range_scale),
        [bias] "r"(params.bias), [input_range_min] "m"(params.input_range_min)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "v10", "v11", "v12", "v13", "v14", "cc", "memory");
}

template <>
inline void Transform1DKernel<uint8_t, int32_t, BiasAdd<uint8_t>, 16,
                              15>::Transform(const uint8_t* input,
                                             const BiasAdd<uint8_t>& params,
                                             int32_t* output) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") BiasAdd<uint8_t><uint8_t, int32_t, BiasAdd<uint8_t>, 16, "
               "15>::Transform()"
            << std::endl
            << std::flush;
#endif
#endif
  int params_rows_copy = params.rows;
  asm volatile(
      "ldr w0, %[input_range_min]\n"
      "dup v8.4s, w0\n"
      "ldr w0, %[input_range_scale]\n"
      "dup v9.4s, w0\n"
      "ldr w0, %[bias_range_min]\n"
      "dup v10.4s, w0\n"
      "ldr w0, %[bias_range_scale]\n"
      "dup v11.4s, w0\n"
      "ldr w0, %[output_range_min]\n"
      "dup v12.4s, w0\n"
      "ldr w0, %[one_over_output_range_scale]\n"
      "dup v13.4s, w0\n"
      "ldr w0, %[output_range_offset]\n"
      "dup v14.4s, w0\n"
      "1:"
      "mov x0, %x[count]\n"
      "mov x1, %x[bias]\n"
      "subs x0, x0, #15\n"
      "beq 3f\n"
      "2:"
      "subs x0, x0, #16\n"

"st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%x[output]], #64\n"
      "prfm pldl1keep, [%x[output]]\n"
      "bne 2b\n"
      "3:"

// BiasAdd::Transform
      "ld1 {v0.2s}, [%x[input]], #8\n"
      "ld1 {v0.s}[2], [%x[input]], #4\n"
      "ld1 {v0.h}[6], [%x[input]], #2\n"
      "ld1 {v0.b}[14], [%x[input]], #1\n"
      "ld1 {v4.2s}, [x1], #8\n"
      "ld1 {v4.s}[2], [x1], #4\n"
      "ld1 {v4.h}[6], [x1], #2\n"
      "ld1 {v4.b}[14], [x1], #1\n"
      "prfm pldl1keep, [%x[input], #32]\n"
      "uxtl2 v1.8h, v0.16b\n"
      "uxtl v0.8h, v0.8b\n"
      "uxtl2 v5.8h, v4.16b\n"
      "uxtl v4.8h, v4.8b\n"
      "sxtl2 v3.4s, v1.8h\n"
      "sxtl v2.4s, v1.4h\n"
      "sxtl2 v7.4s, v5.8h\n"
      "sxtl v6.4s, v5.4h\n"
      "sxtl2 v1.4s, v0.8h\n"
      "sxtl v0.4s, v0.4h\n"
      "sxtl2 v5.4s, v4.8h\n"
      "sxtl v4.4s, v4.4h\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "scvtf v3.4s, v3.4s\n"
      "scvtf v4.4s, v4.4s\n"
      "scvtf v5.4s, v5.4s\n"
      "scvtf v6.4s, v6.4s\n"
      "scvtf v7.4s, v7.4s\n"
      "fmul v0.4s, v0.4s, v9.4s\n"
      "fmul v1.4s, v1.4s, v9.4s\n"
      "fmul v2.4s, v2.4s, v9.4s\n"
      "fmul v3.4s, v3.4s, v9.4s\n"
      "fmul v4.4s, v4.4s, v11.4s\n"
      "fmul v5.4s, v5.4s, v11.4s\n"
      "fmul v6.4s, v6.4s, v11.4s\n"
      "fmul v7.4s, v7.4s, v11.4s\n"
      "fadd v0.4s, v0.4s, v8.4s\n"
      "fadd v1.4s, v1.4s, v8.4s\n"
      "fadd v2.4s, v2.4s, v8.4s\n"
      "fadd v3.4s, v3.4s, v8.4s\n"
      "fadd v4.4s, v4.4s, v10.4s\n"
      "fadd v5.4s, v5.4s, v10.4s\n"
      "fadd v6.4s, v6.4s, v10.4s\n"
      "fadd v7.4s, v7.4s, v10.4s\n"
      "fadd v0.4s, v0.4s, v4.4s\n"
      "fadd v1.4s, v1.4s, v5.4s\n"
      "fadd v2.4s, v2.4s, v6.4s\n"
      "fadd v3.4s, v3.4s, v7.4s\n"
      "fsub v0.4s, v0.4s, v12.4s\n"
      "fsub v1.4s, v1.4s, v12.4s\n"
      "fsub v2.4s, v2.4s, v12.4s\n"
      "fsub v3.4s, v3.4s, v12.4s\n"
      "fmul v0.4s, v0.4s, v13.4s\n"
      "fmul v1.4s, v1.4s, v13.4s\n"
      "fmul v2.4s, v2.4s, v13.4s\n"
      "fmul v3.4s, v3.4s, v13.4s\n"
      "fadd v0.4s, v0.4s, v14.4s\n"
      "fadd v1.4s, v1.4s, v14.4s\n"
      "fadd v2.4s, v2.4s, v14.4s\n"
      "fadd v3.4s, v3.4s, v14.4s\n"
      "fcvtzs v0.4s, v0.4s\n"
      "fcvtzs v1.4s, v1.4s\n"
      "fcvtzs v2.4s, v2.4s\n"
      "fcvtzs v3.4s, v3.4s\n"

"st1 {v0.4s, v1.4s, v2.4s}, [%x[output]], #48\n"
      "st1 {v3.2s}, [%x[output]], #8\n"
      "st1 {v3.s}[2], [%x[output]], #4\n"
      "prfm pldl1keep, [%x[output]]\n"
      "subs %x[rows], %x[rows], #1\n"
      "bne 1b\n"
      : [input] "+r"(input), [output] "+r"(output)
      : [count] "r"(params.count), [rows] "r"(params_rows_copy),
        [output_range_offset] "m"(params.output_range_offset),
        [input_range_scale] "m"(params.input_range_scale),
        [one_over_output_range_scale] "m"(params.one_over_output_range_scale),
        [bias_range_min] "m"(params.bias_range_min),
        [output_range_min] "m"(params.output_range_min),
        [bias_range_scale] "m"(params.bias_range_scale),
        [bias] "r"(params.bias), [input_range_min] "m"(params.input_range_min)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "v10", "v11", "v12", "v13", "v14", "cc", "memory");
}

}  // namespace meta
}  // namespace gemmlowp

#else
#warning "Meta gemm for arm64 requires: GEMMLOWP_NEON_64!"
#endif

#endif  // GEMMLOWP_META_TRANSFORM_KERNELS_ARM_64_H_