EVOLUTION-MANAGER

Edit File: quantized_mul_kernels_arm_64.h

// Copyright 2016 The Gemmlowp Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#ifndef GEMMLOWP_META_QUANTIZED_MUL_KERNELS_ARM_64_H_
#define GEMMLOWP_META_QUANTIZED_MUL_KERNELS_ARM_64_H_

#ifdef GEMMLOWP_NEON_64

#include <cassert>
#include <cstdint>

namespace gemmlowp {
namespace meta {

template <>
inline void
MulKernel<uint8_t, uint8_t, QuantizedStaticPreprocessed, RowMajor, 1, 1,
          8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                       const FusedKernelParams<QuantizedStaticPreprocessed,
                                               RowMajor>& params,
                       uint8_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedRowMajor<uint8_t, uint8_t, "
               "QuantizedStaticPreprocessed, RowMajor, 1, 1, 8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

"ld1 {v1.2s}, [%x[lhs]], #8\n"
      "ld1 {v2.2s}, [%x[rhs]], #8\n"
      "prfm pldl1keep, [%x[lhs], #64]\n"
      "prfm pldl1keep, [%x[rhs], #64]\n"
      "umull v3.8h, v2.8b, v1.8b\n"
      "uadalp v0.4s, v3.8h\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantization::Prepare
      "ld1 {v4.4s}, [%x[lhs]], #16\n"
      "ld1 {v5.4s}, [%x[rhs]], #16\n"
      "dup v6.4s, %w[multiplicative_offset]\n"
      "dup v7.4s, %w[rounding_offset]\n"
      "dup v8.4s, %w[shift]\n"
      "dup v4.4s, v4.s[0]\n"

// RowMajorOutput::Prepare

// Reduce aggregators.
      "addp v0.4s, v0.4s, v0.4s\n"
      "addp v0.4s, v0.4s, v0.4s\n"

// StaticQuantization::Transform
      "add v0.4s, v0.4s, v4.4s\n"
      "add v0.4s, v0.4s, v5.4s\n"
      "mul v0.4s, v0.4s, v6.4s\n"
      "add v0.4s, v0.4s, v7.4s\n"
      "sshl v0.4s, v0.4s, v8.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtun v0.8b, v0.8h\n"

// RowMajorOutput::Output
      "st1 {v0.b}[0], [%x[result]], #1\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [multiplicative_offset] "r"(params.kernel.multiplicative_offset),
        [shift] "r"(params.kernel.shift),
        [stride] "r"(params.output_stream.stride),
        [rounding_offset] "r"(params.kernel.rounding_offset)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "cc", "memory");
}

template <>
inline void
MulKernel<uint8_t, uint8_t, QuantizedStaticPreprocessed, RowMajor, 1, 2,
          8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                       const FusedKernelParams<QuantizedStaticPreprocessed,
                                               RowMajor>& params,
                       uint8_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedRowMajor<uint8_t, uint8_t, "
               "QuantizedStaticPreprocessed, RowMajor, 1, 2, 8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

"ld1 {v2.2s}, [%x[lhs]], #8\n"
      "ld1 {v3.2s, v4.2s}, [%x[rhs]], #16\n"
      "prfm pldl1keep, [%x[lhs], #64]\n"
      "prfm pldl1keep, [%x[rhs], #64]\n"
      "umull v5.8h, v3.8b, v2.8b\n"
      "umull v6.8h, v4.8b, v2.8b\n"
      "uadalp v0.4s, v5.8h\n"
      "uadalp v1.4s, v6.8h\n"

// Loop break.
      "bgt 1b\n"

// RowMajorOutput::Prepare

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v0.4s, v0.4s, v0.4s\n"

// RowMajorOutput::Output
      "st1 {v0.h}[0], [%x[result]], #2\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [multiplicative_offset] "r"(params.kernel.multiplicative_offset),
        [shift] "r"(params.kernel.shift),
        [stride] "r"(params.output_stream.stride),
        [rounding_offset] "r"(params.kernel.rounding_offset)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "cc", "memory");
}

template <>
inline void
MulKernel<uint8_t, uint8_t, QuantizedStaticPreprocessed, RowMajor, 1, 3,
          8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                       const FusedKernelParams<QuantizedStaticPreprocessed,
                                               RowMajor>& params,
                       uint8_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedRowMajor<uint8_t, uint8_t, "
               "QuantizedStaticPreprocessed, RowMajor, 1, 3, 8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

"ld1 {v3.2s}, [%x[lhs]], #8\n"
      "ld1 {v4.2s, v5.2s, v6.2s}, [%x[rhs]], #24\n"
      "prfm pldl1keep, [%x[lhs], #64]\n"
      "prfm pldl1keep, [%x[rhs], #64]\n"
      "umull v7.8h, v4.8b, v3.8b\n"
      "umull v8.8h, v5.8b, v3.8b\n"
      "umull v9.8h, v6.8b, v3.8b\n"
      "uadalp v0.4s, v7.8h\n"
      "uadalp v1.4s, v8.8h\n"
      "uadalp v2.4s, v9.8h\n"

// Loop break.
      "bgt 1b\n"

// RowMajorOutput::Prepare

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v2.4s, v2.4s, v2.4s\n"
      "addp v0.4s, v0.4s, v2.4s\n"

// RowMajorOutput::Output
      "st1 {v0.h}[0], [%x[result]], #2\n"
      "st1 {v0.b}[2], [%x[result]], #1\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [multiplicative_offset] "r"(params.kernel.multiplicative_offset),
        [shift] "r"(params.kernel.shift),
        [stride] "r"(params.output_stream.stride),
        [rounding_offset] "r"(params.kernel.rounding_offset)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "cc",
        "memory");
}

template <>
inline void
MulKernel<uint8_t, uint8_t, QuantizedStaticPreprocessed, RowMajor, 1, 4,
          8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                       const FusedKernelParams<QuantizedStaticPreprocessed,
                                               RowMajor>& params,
                       uint8_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedRowMajor<uint8_t, uint8_t, "
               "QuantizedStaticPreprocessed, RowMajor, 1, 4, 8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"
      "mov v3.16b, v0.16b\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

"ld1 {v4.2s}, [%x[lhs]], #8\n"
      "ld1 {v5.2s, v6.2s, v7.2s, v8.2s}, [%x[rhs]], #32\n"
      "prfm pldl1keep, [%x[lhs], #64]\n"
      "prfm pldl1keep, [%x[rhs], #64]\n"
      "umull v9.8h, v5.8b, v4.8b\n"
      "umull v10.8h, v6.8b, v4.8b\n"
      "umull v11.8h, v7.8b, v4.8b\n"
      "umull v12.8h, v8.8b, v4.8b\n"
      "uadalp v0.4s, v9.8h\n"
      "uadalp v1.4s, v10.8h\n"
      "uadalp v2.4s, v11.8h\n"
      "uadalp v3.4s, v12.8h\n"

// Loop break.
      "bgt 1b\n"

// RowMajorOutput::Prepare

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v2.4s, v2.4s, v3.4s\n"
      "addp v0.4s, v0.4s, v2.4s\n"

// RowMajorOutput::Output
      "st1 {v0.s}[0], [%x[result]], #4\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [multiplicative_offset] "r"(params.kernel.multiplicative_offset),
        [shift] "r"(params.kernel.shift),
        [stride] "r"(params.output_stream.stride),
        [rounding_offset] "r"(params.kernel.rounding_offset)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "v12", "cc", "memory");
}

template <>
inline void
MulKernel<uint8_t, uint8_t, QuantizedStaticPreprocessed, RowMajor, 1, 5,
          8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                       const FusedKernelParams<QuantizedStaticPreprocessed,
                                               RowMajor>& params,
                       uint8_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedRowMajor<uint8_t, uint8_t, "
               "QuantizedStaticPreprocessed, RowMajor, 1, 5, 8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"
      "mov v3.16b, v0.16b\n"
      "mov v4.16b, v1.16b\n"

// General 1xM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

"ld1 {v5.2s, v6.2s, v7.2s, v8.2s}, [%x[rhs]], #32\n"
      "ld1 {v9.2s}, [%x[lhs]], #8\n"
      "prfm pldl1keep, [%x[lhs], #64]\n"
      "umull v10.8h, v5.8b, v9.8b\n"
      "umull v11.8h, v6.8b, v9.8b\n"
      "umull v12.8h, v7.8b, v9.8b\n"
      "umull v13.8h, v8.8b, v9.8b\n"
      "ld1 {v5.2s}, [%x[rhs]], #8\n"
      "prfm pldl1keep, [%x[rhs], #128]\n"
      "uadalp v0.4s, v10.8h\n"
      "uadalp v1.4s, v11.8h\n"
      "uadalp v2.4s, v12.8h\n"
      "uadalp v3.4s, v13.8h\n"
      "umull v10.8h, v5.8b, v9.8b\n"
      "uadalp v4.4s, v10.8h\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantization::Prepare
      "ld1 {v5.4s}, [%x[lhs]], #16\n"
      "ld1 {v6.4s, v7.4s}, [%x[rhs]], #32\n"
      "dup v8.4s, %w[multiplicative_offset]\n"
      "dup v9.4s, %w[rounding_offset]\n"
      "dup v10.4s, %w[shift]\n"
      "dup v5.4s, v5.s[0]\n"

// RowMajorOutput::Prepare

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v2.4s, v2.4s, v3.4s\n"
      "addp v4.4s, v4.4s, v4.4s\n"
      "addp v0.4s, v0.4s, v2.4s\n"
      "addp v1.4s, v4.4s, v4.4s\n"

// StaticQuantization::Transform
      "add v0.4s, v0.4s, v5.4s\n"
      "add v1.4s, v1.4s, v5.4s\n"
      "add v0.4s, v0.4s, v6.4s\n"
      "add v1.4s, v1.4s, v7.4s\n"
      "mul v0.4s, v0.4s, v8.4s\n"
      "mul v1.4s, v1.4s, v8.4s\n"
      "add v0.4s, v0.4s, v9.4s\n"
      "add v1.4s, v1.4s, v9.4s\n"
      "sshl v0.4s, v0.4s, v10.4s\n"
      "sshl v1.4s, v1.4s, v10.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtun v0.8b, v0.8h\n"

// RowMajorOutput::Output
      "st1 {v0.s}[0], [%x[result]], #4\n"
      "st1 {v0.b}[4], [%x[result]], #1\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [multiplicative_offset] "r"(params.kernel.multiplicative_offset),
        [shift] "r"(params.kernel.shift),
        [stride] "r"(params.output_stream.stride),
        [rounding_offset] "r"(params.kernel.rounding_offset)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "v12", "v13", "cc", "memory");
}

template <>
inline void
MulKernel<uint8_t, uint8_t, QuantizedStaticPreprocessed, RowMajor, 1, 6,
          8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                       const FusedKernelParams<QuantizedStaticPreprocessed,
                                               RowMajor>& params,
                       uint8_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedRowMajor<uint8_t, uint8_t, "
               "QuantizedStaticPreprocessed, RowMajor, 1, 6, 8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"
      "mov v3.16b, v0.16b\n"
      "mov v4.16b, v1.16b\n"
      "mov v5.16b, v2.16b\n"

// General 1xM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

"ld1 {v6.2s, v7.2s, v8.2s, v9.2s}, [%x[rhs]], #32\n"
      "ld1 {v10.2s}, [%x[lhs]], #8\n"
      "prfm pldl1keep, [%x[lhs], #64]\n"
      "umull v11.8h, v6.8b, v10.8b\n"
      "umull v12.8h, v7.8b, v10.8b\n"
      "umull v13.8h, v8.8b, v10.8b\n"
      "umull v14.8h, v9.8b, v10.8b\n"
      "ld1 {v6.2s, v7.2s}, [%x[rhs]], #16\n"
      "prfm pldl1keep, [%x[rhs], #128]\n"
      "uadalp v0.4s, v11.8h\n"
      "uadalp v1.4s, v12.8h\n"
      "uadalp v2.4s, v13.8h\n"
      "uadalp v3.4s, v14.8h\n"
      "umull v11.8h, v6.8b, v10.8b\n"
      "umull v12.8h, v7.8b, v10.8b\n"
      "uadalp v4.4s, v11.8h\n"
      "uadalp v5.4s, v12.8h\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantization::Prepare
      "ld1 {v6.4s}, [%x[lhs]], #16\n"
      "ld1 {v7.4s, v8.4s}, [%x[rhs]], #32\n"
      "dup v9.4s, %w[multiplicative_offset]\n"
      "dup v10.4s, %w[rounding_offset]\n"
      "dup v11.4s, %w[shift]\n"
      "dup v6.4s, v6.s[0]\n"

// RowMajorOutput::Prepare

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v2.4s, v2.4s, v3.4s\n"
      "addp v4.4s, v4.4s, v5.4s\n"
      "addp v0.4s, v0.4s, v2.4s\n"
      "addp v1.4s, v4.4s, v4.4s\n"

// StaticQuantization::Transform
      "add v0.4s, v0.4s, v6.4s\n"
      "add v1.4s, v1.4s, v6.4s\n"
      "add v0.4s, v0.4s, v7.4s\n"
      "add v1.4s, v1.4s, v8.4s\n"
      "mul v0.4s, v0.4s, v9.4s\n"
      "mul v1.4s, v1.4s, v9.4s\n"
      "add v0.4s, v0.4s, v10.4s\n"
      "add v1.4s, v1.4s, v10.4s\n"
      "sshl v0.4s, v0.4s, v11.4s\n"
      "sshl v1.4s, v1.4s, v11.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtun v0.8b, v0.8h\n"

// RowMajorOutput::Output
      "st1 {v0.s}[0], [%x[result]], #4\n"
      "st1 {v0.h}[2], [%x[result]], #2\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [multiplicative_offset] "r"(params.kernel.multiplicative_offset),
        [shift] "r"(params.kernel.shift),
        [stride] "r"(params.output_stream.stride),
        [rounding_offset] "r"(params.kernel.rounding_offset)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "v12", "v13", "v14", "cc", "memory");
}

template <>
inline void
MulKernel<uint8_t, uint8_t, QuantizedStaticPreprocessed, RowMajor, 1, 7,
          8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                       const FusedKernelParams<QuantizedStaticPreprocessed,
                                               RowMajor>& params,
                       uint8_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedRowMajor<uint8_t, uint8_t, "
               "QuantizedStaticPreprocessed, RowMajor, 1, 7, 8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"
      "mov v3.16b, v0.16b\n"
      "mov v4.16b, v1.16b\n"
      "mov v5.16b, v2.16b\n"
      "mov v6.16b, v3.16b\n"

// General 1xM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

"ld1 {v7.2s, v8.2s, v9.2s, v10.2s}, [%x[rhs]], #32\n"
      "ld1 {v11.2s}, [%x[lhs]], #8\n"
      "prfm pldl1keep, [%x[lhs], #64]\n"
      "umull v12.8h, v7.8b, v11.8b\n"
      "umull v13.8h, v8.8b, v11.8b\n"
      "umull v14.8h, v9.8b, v11.8b\n"
      "umull v15.8h, v10.8b, v11.8b\n"
      "ld1 {v7.2s, v8.2s, v9.2s}, [%x[rhs]], #24\n"
      "prfm pldl1keep, [%x[rhs], #128]\n"
      "uadalp v0.4s, v12.8h\n"
      "uadalp v1.4s, v13.8h\n"
      "uadalp v2.4s, v14.8h\n"
      "uadalp v3.4s, v15.8h\n"
      "umull v12.8h, v7.8b, v11.8b\n"
      "umull v13.8h, v8.8b, v11.8b\n"
      "umull v14.8h, v9.8b, v11.8b\n"
      "uadalp v4.4s, v12.8h\n"
      "uadalp v5.4s, v13.8h\n"
      "uadalp v6.4s, v14.8h\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantization::Prepare
      "ld1 {v7.4s}, [%x[lhs]], #16\n"
      "ld1 {v8.4s, v9.4s}, [%x[rhs]], #32\n"
      "dup v10.4s, %w[multiplicative_offset]\n"
      "dup v11.4s, %w[rounding_offset]\n"
      "dup v12.4s, %w[shift]\n"
      "dup v7.4s, v7.s[0]\n"

// RowMajorOutput::Prepare

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v2.4s, v2.4s, v3.4s\n"
      "addp v4.4s, v4.4s, v5.4s\n"
      "addp v6.4s, v6.4s, v6.4s\n"
      "addp v0.4s, v0.4s, v2.4s\n"
      "addp v1.4s, v4.4s, v6.4s\n"

// StaticQuantization::Transform
      "add v0.4s, v0.4s, v7.4s\n"
      "add v1.4s, v1.4s, v7.4s\n"
      "add v0.4s, v0.4s, v8.4s\n"
      "add v1.4s, v1.4s, v9.4s\n"
      "mul v0.4s, v0.4s, v10.4s\n"
      "mul v1.4s, v1.4s, v10.4s\n"
      "add v0.4s, v0.4s, v11.4s\n"
      "add v1.4s, v1.4s, v11.4s\n"
      "sshl v0.4s, v0.4s, v12.4s\n"
      "sshl v1.4s, v1.4s, v12.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtun v0.8b, v0.8h\n"

// RowMajorOutput::Output
      "st1 {v0.s}[0], [%x[result]], #4\n"
      "st1 {v0.h}[2], [%x[result]], #2\n"
      "st1 {v0.b}[6], [%x[result]], #1\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [multiplicative_offset] "r"(params.kernel.multiplicative_offset),
        [shift] "r"(params.kernel.shift),
        [stride] "r"(params.output_stream.stride),
        [rounding_offset] "r"(params.kernel.rounding_offset)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "v12", "v13", "v14", "v15", "cc", "memory");
}

template <>
inline void
MulKernel<uint8_t, uint8_t, QuantizedStaticPreprocessed, RowMajor, 1, 8,
          8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                       const FusedKernelParams<QuantizedStaticPreprocessed,
                                               RowMajor>& params,
                       uint8_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedRowMajor<uint8_t, uint8_t, "
               "QuantizedStaticPreprocessed, RowMajor, 1, 8, 8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"
      "mov v3.16b, v0.16b\n"
      "mov v4.16b, v1.16b\n"
      "mov v5.16b, v2.16b\n"
      "mov v6.16b, v3.16b\n"
      "mov v7.16b, v4.16b\n"

// 1x8 lanes loop.
      "1:"

"ld1 {v9.2s, v10.2s, v11.2s, v12.2s}, [%x[rhs]], #32\n"
      "ld1 {v8.2s}, [%x[lhs]], #8\n"
      "umull v13.8h, v8.8b, v9.8b\n"
      "umull v14.8h, v8.8b, v10.8b\n"
      "umull v15.8h, v8.8b, v11.8b\n"
      "umull v16.8h, v8.8b, v12.8b\n"
      "ld1 {v9.2s, v10.2s, v11.2s, v12.2s}, [%x[rhs]], #32\n"
      "uadalp v0.4s, v13.8h\n"
      "uadalp v1.4s, v14.8h\n"
      "uadalp v2.4s, v15.8h\n"
      "uadalp v3.4s, v16.8h\n"
      "prfm pldl1keep, [%x[rhs], #256]\n"
      "umull v17.8h, v8.8b, v9.8b\n"
      "umull v13.8h, v8.8b, v10.8b\n"
      "umull v14.8h, v8.8b, v11.8b\n"
      "umull v15.8h, v8.8b, v12.8b\n"
      "prfm pldl1keep, [%x[lhs], #32]\n"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

"uadalp v4.4s, v17.8h\n"
      "uadalp v5.4s, v13.8h\n"
      "uadalp v6.4s, v14.8h\n"
      "uadalp v7.4s, v15.8h\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantization::Prepare
      "ld1 {v8.4s}, [%x[lhs]], #16\n"
      "ld1 {v9.4s, v10.4s}, [%x[rhs]], #32\n"
      "dup v11.4s, %w[multiplicative_offset]\n"
      "dup v12.4s, %w[rounding_offset]\n"
      "dup v13.4s, %w[shift]\n"
      "dup v8.4s, v8.s[0]\n"

// RowMajorOutput::Prepare

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v2.4s, v2.4s, v3.4s\n"
      "addp v4.4s, v4.4s, v5.4s\n"
      "addp v6.4s, v6.4s, v7.4s\n"
      "addp v0.4s, v0.4s, v2.4s\n"
      "addp v1.4s, v4.4s, v6.4s\n"

// StaticQuantization::Transform
      "add v0.4s, v0.4s, v8.4s\n"
      "add v1.4s, v1.4s, v8.4s\n"
      "add v0.4s, v0.4s, v9.4s\n"
      "add v1.4s, v1.4s, v10.4s\n"
      "mul v0.4s, v0.4s, v11.4s\n"
      "mul v1.4s, v1.4s, v11.4s\n"
      "add v0.4s, v0.4s, v12.4s\n"
      "add v1.4s, v1.4s, v12.4s\n"
      "sshl v0.4s, v0.4s, v13.4s\n"
      "sshl v1.4s, v1.4s, v13.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn2 v0.8h, v1.4s\n"
      "sqxtun v0.8b, v0.8h\n"

// RowMajorOutput::Output
      "st1 {v0.2s}, [%x[result]], #8\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [multiplicative_offset] "r"(params.kernel.multiplicative_offset),
        [shift] "r"(params.kernel.shift),
        [stride] "r"(params.output_stream.stride),
        [rounding_offset] "r"(params.kernel.rounding_offset)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "v12", "v13", "v14", "v15", "v16", "v17", "cc", "memory");
}

template <>
inline void
MulKernel<uint8_t, uint8_t, QuantizedStaticPreprocessed, RowMajor, 2, 1,
          8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                       const FusedKernelParams<QuantizedStaticPreprocessed,
                                               RowMajor>& params,
                       uint8_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedRowMajor<uint8_t, uint8_t, "
               "QuantizedStaticPreprocessed, RowMajor, 2, 1, 8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

"ld1 {v2.2s, v3.2s}, [%x[lhs]], #16\n"
      "ld1 {v4.2s}, [%x[rhs]], #8\n"
      "prfm pldl1keep, [%x[lhs], #64]\n"
      "prfm pldl1keep, [%x[rhs], #64]\n"
      "umull v5.8h, v4.8b, v2.8b\n"
      "umull v6.8h, v4.8b, v3.8b\n"
      "uadalp v0.4s, v5.8h\n"
      "uadalp v1.4s, v6.8h\n"

// Loop break.
      "bgt 1b\n"

// RowMajorOutput::Prepare
      "add x0, %x[result], %x[stride]\n"

// Reduce aggregators.
      "addp v0.4s, v0.4s, v0.4s\n"
      "addp v0.4s, v0.4s, v0.4s\n"
      "addp v1.4s, v1.4s, v1.4s\n"
      "addp v1.4s, v1.4s, v1.4s\n"

// StaticQuantization::Transform
      "add v0.4s, v0.4s, v2.4s\n"
      "add v1.4s, v1.4s, v4.4s\n"
      "add v0.4s, v0.4s, v5.4s\n"
      "add v1.4s, v1.4s, v5.4s\n"
      "mul v0.4s, v0.4s, v6.4s\n"
      "mul v1.4s, v1.4s, v6.4s\n"
      "add v0.4s, v0.4s, v7.4s\n"
      "add v1.4s, v1.4s, v7.4s\n"
      "sshl v0.4s, v0.4s, v8.4s\n"
      "sshl v1.4s, v1.4s, v8.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn v1.4h, v1.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun v1.8b, v1.8h\n"

// RowMajorOutput::Output
      "st1 {v0.b}[0], [%x[result]], #1\n"
      "st1 {v1.b}[0], [x0], #1\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [multiplicative_offset] "r"(params.kernel.multiplicative_offset),
        [shift] "r"(params.kernel.shift),
        [stride] "r"(params.output_stream.stride),
        [rounding_offset] "r"(params.kernel.rounding_offset)
      : "x0", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "cc",
        "memory");
}

template <>
inline void
MulKernel<uint8_t, uint8_t, QuantizedStaticPreprocessed, RowMajor, 2, 2,
          8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                       const FusedKernelParams<QuantizedStaticPreprocessed,
                                               RowMajor>& params,
                       uint8_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedRowMajor<uint8_t, uint8_t, "
               "QuantizedStaticPreprocessed, RowMajor, 2, 2, 8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"
      "mov v3.16b, v0.16b\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

"ld1 {v4.2s, v5.2s}, [%x[lhs]], #16\n"
      "ld1 {v6.2s, v7.2s}, [%x[rhs]], #16\n"
      "prfm pldl1keep, [%x[lhs], #64]\n"
      "prfm pldl1keep, [%x[rhs], #64]\n"
      "umull v8.8h, v6.8b, v4.8b\n"
      "umull v9.8h, v7.8b, v4.8b\n"
      "umull v10.8h, v6.8b, v5.8b\n"
      "umull v11.8h, v7.8b, v5.8b\n"
      "uadalp v0.4s, v8.8h\n"
      "uadalp v1.4s, v9.8h\n"
      "uadalp v2.4s, v10.8h\n"
      "uadalp v3.4s, v11.8h\n"

// Loop break.
      "bgt 1b\n"

// RowMajorOutput::Prepare
      "add x0, %x[result], %x[stride]\n"

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v0.4s, v0.4s, v0.4s\n"
      "addp v2.4s, v2.4s, v3.4s\n"
      "addp v2.4s, v2.4s, v2.4s\n"

// StaticQuantization::Transform
      "add v0.4s, v0.4s, v9.4s\n"
      "add v2.4s, v2.4s, v4.4s\n"
      "add v0.4s, v0.4s, v5.4s\n"
      "add v2.4s, v2.4s, v5.4s\n"
      "mul v0.4s, v0.4s, v6.4s\n"
      "mul v2.4s, v2.4s, v6.4s\n"
      "add v0.4s, v0.4s, v7.4s\n"
      "add v2.4s, v2.4s, v7.4s\n"
      "sshl v0.4s, v0.4s, v8.4s\n"
      "sshl v2.4s, v2.4s, v8.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn v2.4h, v2.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun v2.8b, v2.8h\n"

// RowMajorOutput::Output
      "st1 {v0.h}[0], [%x[result]], #2\n"
      "st1 {v2.h}[0], [x0], #2\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [multiplicative_offset] "r"(params.kernel.multiplicative_offset),
        [shift] "r"(params.kernel.shift),
        [stride] "r"(params.output_stream.stride),
        [rounding_offset] "r"(params.kernel.rounding_offset)
      : "x0", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "cc", "memory");
}

template <>
inline void
MulKernel<uint8_t, uint8_t, QuantizedStaticPreprocessed, RowMajor, 2, 3,
          8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                       const FusedKernelParams<QuantizedStaticPreprocessed,
                                               RowMajor>& params,
                       uint8_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedRowMajor<uint8_t, uint8_t, "
               "QuantizedStaticPreprocessed, RowMajor, 2, 3, 8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"
      "mov v3.16b, v0.16b\n"
      "mov v4.16b, v1.16b\n"
      "mov v5.16b, v2.16b\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

"ld1 {v6.2s, v7.2s}, [%x[lhs]], #16\n"
      "ld1 {v8.2s, v9.2s, v10.2s}, [%x[rhs]], #24\n"
      "prfm pldl1keep, [%x[lhs], #64]\n"
      "prfm pldl1keep, [%x[rhs], #64]\n"
      "umull v11.8h, v8.8b, v6.8b\n"
      "umull v12.8h, v9.8b, v6.8b\n"
      "umull v13.8h, v10.8b, v6.8b\n"
      "umull v14.8h, v8.8b, v7.8b\n"
      "umull v15.8h, v9.8b, v7.8b\n"
      "umull v16.8h, v10.8b, v7.8b\n"
      "uadalp v0.4s, v11.8h\n"
      "uadalp v1.4s, v12.8h\n"
      "uadalp v2.4s, v13.8h\n"
      "uadalp v3.4s, v14.8h\n"
      "uadalp v4.4s, v15.8h\n"
      "uadalp v5.4s, v16.8h\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantization::Prepare
      "ld1 {v6.4s}, [%x[lhs]], #16\n"
      "ld1 {v7.4s}, [%x[rhs]], #16\n"
      "dup v8.4s, %w[multiplicative_offset]\n"
      "dup v9.4s, %w[rounding_offset]\n"
      "dup v10.4s, %w[shift]\n"
      "dup v11.4s, v6.s[0]\n"
      "dup v6.4s, v6.s[1]\n"

// RowMajorOutput::Prepare
      "add x0, %x[result], %x[stride]\n"

// StaticQuantization::Transform
      "add v0.4s, v0.4s, v11.4s\n"
      "add v3.4s, v3.4s, v6.4s\n"
      "add v0.4s, v0.4s, v7.4s\n"
      "add v3.4s, v3.4s, v7.4s\n"
      "mul v0.4s, v0.4s, v8.4s\n"
      "mul v3.4s, v3.4s, v8.4s\n"
      "add v0.4s, v0.4s, v9.4s\n"
      "add v3.4s, v3.4s, v9.4s\n"
      "sshl v0.4s, v0.4s, v10.4s\n"
      "sshl v3.4s, v3.4s, v10.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn v3.4h, v3.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun v3.8b, v3.8h\n"

// RowMajorOutput::Output
      "st1 {v0.h}[0], [%x[result]], #2\n"
      "st1 {v0.b}[2], [%x[result]], #1\n"
      "st1 {v3.h}[0], [x0], #2\n"
      "st1 {v3.b}[2], [x0], #1\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [multiplicative_offset] "r"(params.kernel.multiplicative_offset),
        [shift] "r"(params.kernel.shift),
        [stride] "r"(params.output_stream.stride),
        [rounding_offset] "r"(params.kernel.rounding_offset)
      : "x0", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "v12", "v13", "v14", "v15", "v16", "cc", "memory");
}

template <>
inline void
MulKernel<uint8_t, uint8_t, QuantizedStaticPreprocessed, RowMajor, 2, 4,
          8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                       const FusedKernelParams<QuantizedStaticPreprocessed,
                                               RowMajor>& params,
                       uint8_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedRowMajor<uint8_t, uint8_t, "
               "QuantizedStaticPreprocessed, RowMajor, 2, 4, 8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// 2x4 lanes loop.
      "1:"

"ld1 {v10.8b, v11.8b, v12.8b, v13.8b}, [%x[rhs]], #32\n"
      "ld1 {v8.8b}, [%x[lhs]], #8\n"
      "umull v14.8h, v8.8b, v10.8b\n"
      "ld1 {v9.8b}, [%x[lhs]], #8\n"
      "umull v15.8h, v8.8b, v11.8b\n"
      "prfm pldl1keep, [%x[rhs], #64]\n"
      "umull v16.8h, v8.8b, v12.8b\n"
      "prfm pldl1keep, [%x[lhs], #64]\n"
      "umull v17.8h, v8.8b, v13.8b\n"
      "umull v18.8h, v9.8b, v10.8b\n"
      "uadalp v0.4s, v14.8h\n"
      "uadalp v1.4s, v15.8h\n"
      "uadalp v2.4s, v16.8h\n"
      "umull v14.8h, v9.8b, v11.8b\n"
      "umull v15.8h, v9.8b, v12.8b\n"
      "umull v16.8h, v9.8b, v13.8b\n"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

"uadalp v3.4s, v17.8h\n"
      "uadalp v4.4s, v18.8h\n"
      "uadalp v5.4s, v14.8h\n"
      "uadalp v6.4s, v15.8h\n"
      "uadalp v7.4s, v16.8h\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantization::Prepare
      "ld1 {v8.4s}, [%x[lhs]], #16\n"
      "ld1 {v9.4s}, [%x[rhs]], #16\n"
      "dup v10.4s, %w[multiplicative_offset]\n"
      "dup v11.4s, %w[rounding_offset]\n"
      "dup v12.4s, %w[shift]\n"
      "dup v13.4s, v8.s[0]\n"
      "dup v8.4s, v8.s[1]\n"

// RowMajorOutput::Prepare
      "add x0, %x[result], %x[stride]\n"

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v2.4s, v2.4s, v3.4s\n"
      "addp v0.4s, v0.4s, v2.4s\n"
      "addp v4.4s, v4.4s, v5.4s\n"
      "addp v6.4s, v6.4s, v7.4s\n"
      "addp v4.4s, v4.4s, v6.4s\n"

// StaticQuantization::Transform
      "add v0.4s, v0.4s, v13.4s\n"
      "add v4.4s, v4.4s, v8.4s\n"
      "add v0.4s, v0.4s, v9.4s\n"
      "add v4.4s, v4.4s, v9.4s\n"
      "mul v0.4s, v0.4s, v10.4s\n"
      "mul v4.4s, v4.4s, v10.4s\n"
      "add v0.4s, v0.4s, v11.4s\n"
      "add v4.4s, v4.4s, v11.4s\n"
      "sshl v0.4s, v0.4s, v12.4s\n"
      "sshl v4.4s, v4.4s, v12.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn v4.4h, v4.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun v4.8b, v4.8h\n"

// RowMajorOutput::Output
      "st1 {v0.s}[0], [%x[result]], #4\n"
      "st1 {v4.s}[0], [x0], #4\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [multiplicative_offset] "r"(params.kernel.multiplicative_offset),
        [shift] "r"(params.kernel.shift),
        [stride] "r"(params.output_stream.stride),
        [rounding_offset] "r"(params.kernel.rounding_offset)
      : "x0", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "cc", "memory");
}

template <>
inline void
MulKernel<uint8_t, uint8_t, QuantizedStaticPreprocessed, RowMajor, 3, 1,
          8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                       const FusedKernelParams<QuantizedStaticPreprocessed,
                                               RowMajor>& params,
                       uint8_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedRowMajor<uint8_t, uint8_t, "
               "QuantizedStaticPreprocessed, RowMajor, 3, 1, 8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

"ld1 {v3.2s, v4.2s, v5.2s}, [%x[lhs]], #24\n"
      "ld1 {v6.2s}, [%x[rhs]], #8\n"
      "prfm pldl1keep, [%x[lhs], #64]\n"
      "prfm pldl1keep, [%x[rhs], #64]\n"
      "umull v7.8h, v6.8b, v3.8b\n"
      "umull v8.8h, v6.8b, v4.8b\n"
      "umull v9.8h, v6.8b, v5.8b\n"
      "uadalp v0.4s, v7.8h\n"
      "uadalp v1.4s, v8.8h\n"
      "uadalp v2.4s, v9.8h\n"

// Loop break.
      "bgt 1b\n"

// RowMajorOutput::Prepare
      "add x0, %x[result], %x[stride]\n"
      "add x1, x0, %x[stride]\n"

// Reduce aggregators.
      "addp v0.4s, v0.4s, v0.4s\n"
      "addp v0.4s, v0.4s, v0.4s\n"
      "addp v1.4s, v1.4s, v1.4s\n"
      "addp v1.4s, v1.4s, v1.4s\n"
      "addp v2.4s, v2.4s, v2.4s\n"
      "addp v2.4s, v2.4s, v2.4s\n"

// StaticQuantization::Transform
      "add v0.4s, v0.4s, v3.4s\n"
      "add v1.4s, v1.4s, v9.4s\n"
      "add v2.4s, v2.4s, v4.4s\n"
      "add v0.4s, v0.4s, v5.4s\n"
      "add v1.4s, v1.4s, v5.4s\n"
      "add v2.4s, v2.4s, v5.4s\n"
      "mul v0.4s, v0.4s, v6.4s\n"
      "mul v1.4s, v1.4s, v6.4s\n"
      "mul v2.4s, v2.4s, v6.4s\n"
      "add v0.4s, v0.4s, v7.4s\n"
      "add v1.4s, v1.4s, v7.4s\n"
      "add v2.4s, v2.4s, v7.4s\n"
      "sshl v0.4s, v0.4s, v8.4s\n"
      "sshl v1.4s, v1.4s, v8.4s\n"
      "sshl v2.4s, v2.4s, v8.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn v1.4h, v1.4s\n"
      "sqxtn v2.4h, v2.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun v1.8b, v1.8h\n"
      "sqxtun v2.8b, v2.8h\n"

// RowMajorOutput::Output
      "st1 {v0.b}[0], [%x[result]], #1\n"
      "st1 {v1.b}[0], [x0], #1\n"
      "st1 {v2.b}[0], [x1], #1\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [multiplicative_offset] "r"(params.kernel.multiplicative_offset),
        [shift] "r"(params.kernel.shift),
        [stride] "r"(params.output_stream.stride),
        [rounding_offset] "r"(params.kernel.rounding_offset)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "cc", "memory");
}

template <>
inline void
MulKernel<uint8_t, uint8_t, QuantizedStaticPreprocessed, RowMajor, 3, 2,
          8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                       const FusedKernelParams<QuantizedStaticPreprocessed,
                                               RowMajor>& params,
                       uint8_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedRowMajor<uint8_t, uint8_t, "
               "QuantizedStaticPreprocessed, RowMajor, 3, 2, 8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"
      "mov v3.16b, v0.16b\n"
      "mov v4.16b, v1.16b\n"
      "mov v5.16b, v2.16b\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

"ld1 {v6.2s, v7.2s, v8.2s}, [%x[lhs]], #24\n"
      "ld1 {v9.2s, v10.2s}, [%x[rhs]], #16\n"
      "prfm pldl1keep, [%x[lhs], #64]\n"
      "prfm pldl1keep, [%x[rhs], #64]\n"
      "umull v11.8h, v9.8b, v6.8b\n"
      "umull v12.8h, v10.8b, v6.8b\n"
      "umull v13.8h, v9.8b, v7.8b\n"
      "umull v14.8h, v10.8b, v7.8b\n"
      "umull v15.8h, v9.8b, v8.8b\n"
      "umull v16.8h, v10.8b, v8.8b\n"
      "uadalp v0.4s, v11.8h\n"
      "uadalp v1.4s, v12.8h\n"
      "uadalp v2.4s, v13.8h\n"
      "uadalp v3.4s, v14.8h\n"
      "uadalp v4.4s, v15.8h\n"
      "uadalp v5.4s, v16.8h\n"

// Loop break.
      "bgt 1b\n"

// RowMajorOutput::Prepare
      "add x0, %x[result], %x[stride]\n"
      "add x1, x0, %x[stride]\n"

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v0.4s, v0.4s, v0.4s\n"
      "addp v2.4s, v2.4s, v3.4s\n"
      "addp v2.4s, v2.4s, v2.4s\n"
      "addp v4.4s, v4.4s, v5.4s\n"
      "addp v4.4s, v4.4s, v4.4s\n"

// StaticQuantization::Transform
      "add v0.4s, v0.4s, v11.4s\n"
      "add v2.4s, v2.4s, v12.4s\n"
      "add v4.4s, v4.4s, v6.4s\n"
      "add v0.4s, v0.4s, v7.4s\n"
      "add v2.4s, v2.4s, v7.4s\n"
      "add v4.4s, v4.4s, v7.4s\n"
      "mul v0.4s, v0.4s, v8.4s\n"
      "mul v2.4s, v2.4s, v8.4s\n"
      "mul v4.4s, v4.4s, v8.4s\n"
      "add v0.4s, v0.4s, v9.4s\n"
      "add v2.4s, v2.4s, v9.4s\n"
      "add v4.4s, v4.4s, v9.4s\n"
      "sshl v0.4s, v0.4s, v10.4s\n"
      "sshl v2.4s, v2.4s, v10.4s\n"
      "sshl v4.4s, v4.4s, v10.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn v2.4h, v2.4s\n"
      "sqxtn v4.4h, v4.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun v2.8b, v2.8h\n"
      "sqxtun v4.8b, v4.8h\n"

// RowMajorOutput::Output
      "st1 {v0.h}[0], [%x[result]], #2\n"
      "st1 {v2.h}[0], [x0], #2\n"
      "st1 {v4.h}[0], [x1], #2\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [multiplicative_offset] "r"(params.kernel.multiplicative_offset),
        [shift] "r"(params.kernel.shift),
        [stride] "r"(params.output_stream.stride),
        [rounding_offset] "r"(params.kernel.rounding_offset)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "v10", "v11", "v12", "v13", "v14", "v15", "v16", "cc", "memory");
}

template <>
inline void
MulKernel<uint8_t, uint8_t, QuantizedStaticPreprocessed, RowMajor, 3, 3,
          8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                       const FusedKernelParams<QuantizedStaticPreprocessed,
                                               RowMajor>& params,
                       uint8_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedRowMajor<uint8_t, uint8_t, "
               "QuantizedStaticPreprocessed, RowMajor, 3, 3, 8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// 3x3 lanes loop.
      "1:"

"ld1 {v12.8b, v13.8b, v14.8b}, [%x[rhs]], #24\n"
      "ld1 {v9.8b}, [%x[lhs]], #8\n"
      "umull v15.8h, v9.8b, v12.8b\n"
      "ld1 {v10.8b}, [%x[lhs]], #8\n"
      "umull v16.8h, v9.8b, v13.8b\n"
      "ld1 {v11.8b}, [%x[lhs]], #8\n"
      "umull v17.8h, v9.8b, v14.8b\n"
      "prfm pldl1keep, [%x[lhs], #64]\n"
      "umull v18.8h, v10.8b, v12.8b\n"
      "prfm pldl1keep, [%x[rhs], #64]\n"
      "uadalp v0.4s, v15.8h\n"
      "uadalp v1.4s, v16.8h\n"
      "uadalp v2.4s, v17.8h\n"
      "uadalp v3.4s, v18.8h\n"
      "umull v15.8h, v10.8b, v13.8b\n"
      "umull v16.8h, v10.8b, v14.8b\n"
      "umull v17.8h, v11.8b, v12.8b\n"
      "umull v18.8h, v11.8b, v13.8b\n"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

"umull v9.8h, v11.8b, v14.8b\n"
      "uadalp v4.4s, v15.8h\n"
      "uadalp v5.4s, v16.8h\n"
      "uadalp v6.4s, v17.8h\n"
      "uadalp v7.4s, v18.8h\n"
      "uadalp v8.4s, v9.8h\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantization::Prepare
      "ld1 {v9.4s}, [%x[lhs]], #16\n"
      "ld1 {v10.4s}, [%x[rhs]], #16\n"
      "dup v11.4s, %w[multiplicative_offset]\n"
      "dup v12.4s, %w[rounding_offset]\n"
      "dup v13.4s, %w[shift]\n"
      "dup v14.4s, v9.s[0]\n"
      "dup v15.4s, v9.s[1]\n"
      "dup v9.4s, v9.s[2]\n"

// RowMajorOutput::Prepare
      "add x0, %x[result], %x[stride]\n"
      "add x1, x0, %x[stride]\n"

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v2.4s, v2.4s, v2.4s\n"
      "addp v0.4s, v0.4s, v2.4s\n"
      "addp v3.4s, v3.4s, v4.4s\n"
      "addp v5.4s, v5.4s, v5.4s\n"
      "addp v3.4s, v3.4s, v5.4s\n"
      "addp v6.4s, v6.4s, v7.4s\n"
      "addp v8.4s, v8.4s, v8.4s\n"
      "addp v6.4s, v6.4s, v8.4s\n"

// StaticQuantization::Transform
      "add v0.4s, v0.4s, v14.4s\n"
      "add v3.4s, v3.4s, v15.4s\n"
      "add v6.4s, v6.4s, v9.4s\n"
      "add v0.4s, v0.4s, v10.4s\n"
      "add v3.4s, v3.4s, v10.4s\n"
      "add v6.4s, v6.4s, v10.4s\n"
      "mul v0.4s, v0.4s, v11.4s\n"
      "mul v3.4s, v3.4s, v11.4s\n"
      "mul v6.4s, v6.4s, v11.4s\n"
      "add v0.4s, v0.4s, v12.4s\n"
      "add v3.4s, v3.4s, v12.4s\n"
      "add v6.4s, v6.4s, v12.4s\n"
      "sshl v0.4s, v0.4s, v13.4s\n"
      "sshl v3.4s, v3.4s, v13.4s\n"
      "sshl v6.4s, v6.4s, v13.4s\n"
      "sqxtn v0.4h, v0.4s\n"
      "sqxtn v3.4h, v3.4s\n"
      "sqxtn v6.4h, v6.4s\n"
      "sqxtun v0.8b, v0.8h\n"
      "sqxtun v3.8b, v3.8h\n"
      "sqxtun v6.8b, v6.8h\n"

// RowMajorOutput::Output
      "st1 {v0.h}[0], [%x[result]], #2\n"
      "st1 {v0.b}[2], [%x[result]], #1\n"
      "st1 {v3.h}[0], [x0], #2\n"
      "st1 {v3.b}[2], [x0], #1\n"
      "st1 {v6.h}[0], [x1], #2\n"
      "st1 {v6.b}[2], [x1], #1\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [multiplicative_offset] "r"(params.kernel.multiplicative_offset),
        [shift] "r"(params.kernel.shift),
        [stride] "r"(params.output_stream.stride),
        [rounding_offset] "r"(params.kernel.rounding_offset)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "cc",
        "memory");
}

template <>
inline void MulKernel<
    uint8_t, int32_t, QuantizedStaticPreprocessedAsInt32, RowMajor, 1, 1,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsInt32,
                                         RowMajor>& params,
                 int32_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsInt32RowMajor<uint8_t, int32_t, "
               "QuantizedStaticPreprocessedAsInt32, RowMajor, 1, 1, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationInt32::Prepare
      "ld1 {v4.4s}, [%x[lhs]], #16\n"
      "ld1 {v5.4s}, [%x[rhs]], #16\n"
      "dup v4.4s, v4.s[0]\n"

// RowMajorOutput::Prepare

// Reduce aggregators.
      "addp v0.4s, v0.4s, v0.4s\n"
      "addp v0.4s, v0.4s, v0.4s\n"

// StaticQuantizationInt32::Transform
      "add v0.4s, v0.4s, v4.4s\n"
      "add v0.4s, v0.4s, v5.4s\n"

// RowMajorOutput::Output
      "st1 {v0.s}[0], [%x[result]], #4\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride)
      : "v0", "v1", "v2", "v3", "v4", "v5", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, int32_t, QuantizedStaticPreprocessedAsInt32, RowMajor, 1, 2,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsInt32,
                                         RowMajor>& params,
                 int32_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsInt32RowMajor<uint8_t, int32_t, "
               "QuantizedStaticPreprocessedAsInt32, RowMajor, 1, 2, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationInt32::Prepare
      "ld1 {v4.4s}, [%x[lhs]], #16\n"
      "ld1 {v5.4s}, [%x[rhs]], #16\n"
      "dup v4.4s, v4.s[0]\n"

// RowMajorOutput::Prepare

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v0.4s, v0.4s, v0.4s\n"

// StaticQuantizationInt32::Transform
      "add v0.4s, v0.4s, v4.4s\n"
      "add v0.4s, v0.4s, v5.4s\n"

// RowMajorOutput::Output
      "st1 {v0.2s}, [%x[result]], #8\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, int32_t, QuantizedStaticPreprocessedAsInt32, RowMajor, 1, 3,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsInt32,
                                         RowMajor>& params,
                 int32_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsInt32RowMajor<uint8_t, int32_t, "
               "QuantizedStaticPreprocessedAsInt32, RowMajor, 1, 3, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationInt32::Prepare
      "ld1 {v4.4s}, [%x[lhs]], #16\n"
      "ld1 {v5.4s}, [%x[rhs]], #16\n"
      "dup v4.4s, v4.s[0]\n"

// RowMajorOutput::Prepare

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v2.4s, v2.4s, v2.4s\n"
      "addp v0.4s, v0.4s, v2.4s\n"

// StaticQuantizationInt32::Transform
      "add v0.4s, v0.4s, v4.4s\n"
      "add v0.4s, v0.4s, v5.4s\n"

// RowMajorOutput::Output
      "st1 {v0.2s}, [%x[result]], #8\n"
      "st1 {v0.s}[2], [%x[result]], #4\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "cc",
        "memory");
}

template <>
inline void MulKernel<
    uint8_t, int32_t, QuantizedStaticPreprocessedAsInt32, RowMajor, 1, 4,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsInt32,
                                         RowMajor>& params,
                 int32_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsInt32RowMajor<uint8_t, int32_t, "
               "QuantizedStaticPreprocessedAsInt32, RowMajor, 1, 4, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"
      "mov v3.16b, v0.16b\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationInt32::Prepare
      "ld1 {v4.4s}, [%x[lhs]], #16\n"
      "ld1 {v5.4s}, [%x[rhs]], #16\n"
      "dup v4.4s, v4.s[0]\n"

// RowMajorOutput::Prepare

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v2.4s, v2.4s, v3.4s\n"
      "addp v0.4s, v0.4s, v2.4s\n"

// StaticQuantizationInt32::Transform
      "add v0.4s, v0.4s, v4.4s\n"
      "add v0.4s, v0.4s, v5.4s\n"

// RowMajorOutput::Output
      "st1 {v0.4s}, [%x[result]], #16\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "v12", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, int32_t, QuantizedStaticPreprocessedAsInt32, RowMajor, 1, 5,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsInt32,
                                         RowMajor>& params,
                 int32_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsInt32RowMajor<uint8_t, int32_t, "
               "QuantizedStaticPreprocessedAsInt32, RowMajor, 1, 5, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"
      "mov v3.16b, v0.16b\n"
      "mov v4.16b, v1.16b\n"

// General 1xM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationInt32::Prepare
      "ld1 {v5.4s}, [%x[lhs]], #16\n"
      "ld1 {v6.4s, v7.4s}, [%x[rhs]], #32\n"
      "dup v5.4s, v5.s[0]\n"

// RowMajorOutput::Prepare

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v2.4s, v2.4s, v3.4s\n"
      "addp v4.4s, v4.4s, v4.4s\n"
      "addp v0.4s, v0.4s, v2.4s\n"
      "addp v1.4s, v4.4s, v4.4s\n"

// StaticQuantizationInt32::Transform
      "add v0.4s, v0.4s, v5.4s\n"
      "add v1.4s, v1.4s, v5.4s\n"
      "add v0.4s, v0.4s, v6.4s\n"
      "add v1.4s, v1.4s, v7.4s\n"

// RowMajorOutput::Output
      "st1 {v0.4s}, [%x[result]], #16\n"
      "st1 {v1.s}[0], [%x[result]], #4\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "v12", "v13", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, int32_t, QuantizedStaticPreprocessedAsInt32, RowMajor, 1, 6,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsInt32,
                                         RowMajor>& params,
                 int32_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsInt32RowMajor<uint8_t, int32_t, "
               "QuantizedStaticPreprocessedAsInt32, RowMajor, 1, 6, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"
      "mov v3.16b, v0.16b\n"
      "mov v4.16b, v1.16b\n"
      "mov v5.16b, v2.16b\n"

// General 1xM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationInt32::Prepare
      "ld1 {v6.4s}, [%x[lhs]], #16\n"
      "ld1 {v7.4s, v8.4s}, [%x[rhs]], #32\n"
      "dup v6.4s, v6.s[0]\n"

// RowMajorOutput::Prepare

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v2.4s, v2.4s, v3.4s\n"
      "addp v4.4s, v4.4s, v5.4s\n"
      "addp v0.4s, v0.4s, v2.4s\n"
      "addp v1.4s, v4.4s, v4.4s\n"

// StaticQuantizationInt32::Transform
      "add v0.4s, v0.4s, v6.4s\n"
      "add v1.4s, v1.4s, v6.4s\n"
      "add v0.4s, v0.4s, v7.4s\n"
      "add v1.4s, v1.4s, v8.4s\n"

// RowMajorOutput::Output
      "st1 {v0.4s}, [%x[result]], #16\n"
      "st1 {v1.2s}, [%x[result]], #8\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "v12", "v13", "v14", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, int32_t, QuantizedStaticPreprocessedAsInt32, RowMajor, 1, 7,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsInt32,
                                         RowMajor>& params,
                 int32_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsInt32RowMajor<uint8_t, int32_t, "
               "QuantizedStaticPreprocessedAsInt32, RowMajor, 1, 7, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// General 1xM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationInt32::Prepare
      "ld1 {v7.4s}, [%x[lhs]], #16\n"
      "ld1 {v8.4s, v9.4s}, [%x[rhs]], #32\n"
      "dup v7.4s, v7.s[0]\n"

// RowMajorOutput::Prepare

// StaticQuantizationInt32::Transform
      "add v0.4s, v0.4s, v7.4s\n"
      "add v1.4s, v1.4s, v7.4s\n"
      "add v0.4s, v0.4s, v8.4s\n"
      "add v1.4s, v1.4s, v9.4s\n"

// RowMajorOutput::Output
      "st1 {v0.4s}, [%x[result]], #16\n"
      "st1 {v1.2s}, [%x[result]], #8\n"
      "st1 {v1.s}[2], [%x[result]], #4\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "v12", "v13", "v14", "v15", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, int32_t, QuantizedStaticPreprocessedAsInt32, RowMajor, 1, 8,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsInt32,
                                         RowMajor>& params,
                 int32_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsInt32RowMajor<uint8_t, int32_t, "
               "QuantizedStaticPreprocessedAsInt32, RowMajor, 1, 8, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// 1x8 lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

"uadalp v4.4s, v17.8h\n"
      "uadalp v5.4s, v13.8h\n"
      "uadalp v6.4s, v14.8h\n"
      "uadalp v7.4s, v15.8h\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationInt32::Prepare
      "ld1 {v8.4s}, [%x[lhs]], #16\n"
      "ld1 {v9.4s, v10.4s}, [%x[rhs]], #32\n"
      "dup v8.4s, v8.s[0]\n"

// RowMajorOutput::Prepare

// StaticQuantizationInt32::Transform
      "add v0.4s, v0.4s, v8.4s\n"
      "add v1.4s, v1.4s, v8.4s\n"
      "add v0.4s, v0.4s, v9.4s\n"
      "add v1.4s, v1.4s, v10.4s\n"

// RowMajorOutput::Output
      "st1 {v0.4s, v1.4s}, [%x[result]], #32\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "v12", "v13", "v14", "v15", "v16", "v17", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, int32_t, QuantizedStaticPreprocessedAsInt32, RowMajor, 2, 1,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsInt32,
                                         RowMajor>& params,
                 int32_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsInt32RowMajor<uint8_t, int32_t, "
               "QuantizedStaticPreprocessedAsInt32, RowMajor, 2, 1, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationInt32::Prepare
      "ld1 {v4.4s}, [%x[lhs]], #16\n"
      "ld1 {v5.4s}, [%x[rhs]], #16\n"
      "dup v2.4s, v4.s[0]\n"
      "dup v4.4s, v4.s[1]\n"

// RowMajorOutput::Prepare
      "add x0, %x[result], %x[stride]\n"

// Reduce aggregators.
      "addp v0.4s, v0.4s, v0.4s\n"
      "addp v0.4s, v0.4s, v0.4s\n"
      "addp v1.4s, v1.4s, v1.4s\n"
      "addp v1.4s, v1.4s, v1.4s\n"

// StaticQuantizationInt32::Transform
      "add v0.4s, v0.4s, v2.4s\n"
      "add v1.4s, v1.4s, v4.4s\n"
      "add v0.4s, v0.4s, v5.4s\n"
      "add v1.4s, v1.4s, v5.4s\n"

// RowMajorOutput::Output
      "st1 {v0.s}[0], [%x[result]], #4\n"
      "st1 {v1.s}[0], [x0], #4\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride)
      : "x0", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, int32_t, QuantizedStaticPreprocessedAsInt32, RowMajor, 2, 2,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsInt32,
                                         RowMajor>& params,
                 int32_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsInt32RowMajor<uint8_t, int32_t, "
               "QuantizedStaticPreprocessedAsInt32, RowMajor, 2, 2, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"
      "mov v3.16b, v0.16b\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationInt32::Prepare
      "ld1 {v4.4s}, [%x[lhs]], #16\n"
      "ld1 {v5.4s}, [%x[rhs]], #16\n"
      "dup v6.4s, v4.s[0]\n"
      "dup v4.4s, v4.s[1]\n"

// RowMajorOutput::Prepare
      "add x0, %x[result], %x[stride]\n"

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v0.4s, v0.4s, v0.4s\n"
      "addp v2.4s, v2.4s, v3.4s\n"
      "addp v2.4s, v2.4s, v2.4s\n"

// StaticQuantizationInt32::Transform
      "add v0.4s, v0.4s, v6.4s\n"
      "add v2.4s, v2.4s, v4.4s\n"
      "add v0.4s, v0.4s, v5.4s\n"
      "add v2.4s, v2.4s, v5.4s\n"

// RowMajorOutput::Output
      "st1 {v0.2s}, [%x[result]], #8\n"
      "st1 {v2.2s}, [x0], #8\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride)
      : "x0", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, int32_t, QuantizedStaticPreprocessedAsInt32, RowMajor, 2, 3,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsInt32,
                                         RowMajor>& params,
                 int32_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsInt32RowMajor<uint8_t, int32_t, "
               "QuantizedStaticPreprocessedAsInt32, RowMajor, 2, 3, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"
      "mov v3.16b, v0.16b\n"
      "mov v4.16b, v1.16b\n"
      "mov v5.16b, v2.16b\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationInt32::Prepare
      "ld1 {v6.4s}, [%x[lhs]], #16\n"
      "ld1 {v7.4s}, [%x[rhs]], #16\n"
      "dup v8.4s, v6.s[0]\n"
      "dup v6.4s, v6.s[1]\n"

// RowMajorOutput::Prepare
      "add x0, %x[result], %x[stride]\n"

// StaticQuantizationInt32::Transform
      "add v0.4s, v0.4s, v8.4s\n"
      "add v3.4s, v3.4s, v6.4s\n"
      "add v0.4s, v0.4s, v7.4s\n"
      "add v3.4s, v3.4s, v7.4s\n"

// RowMajorOutput::Output
      "st1 {v0.2s}, [%x[result]], #8\n"
      "st1 {v0.s}[2], [%x[result]], #4\n"
      "st1 {v3.2s}, [x0], #8\n"
      "st1 {v3.s}[2], [x0], #4\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride)
      : "x0", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "v12", "v13", "v14", "v15", "v16", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, int32_t, QuantizedStaticPreprocessedAsInt32, RowMajor, 2, 4,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsInt32,
                                         RowMajor>& params,
                 int32_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsInt32RowMajor<uint8_t, int32_t, "
               "QuantizedStaticPreprocessedAsInt32, RowMajor, 2, 4, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// 2x4 lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

"uadalp v3.4s, v17.8h\n"
      "uadalp v4.4s, v18.8h\n"
      "uadalp v5.4s, v14.8h\n"
      "uadalp v6.4s, v15.8h\n"
      "uadalp v7.4s, v16.8h\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationInt32::Prepare
      "ld1 {v8.4s}, [%x[lhs]], #16\n"
      "ld1 {v9.4s}, [%x[rhs]], #16\n"
      "dup v10.4s, v8.s[0]\n"
      "dup v8.4s, v8.s[1]\n"

// RowMajorOutput::Prepare
      "add x0, %x[result], %x[stride]\n"

// StaticQuantizationInt32::Transform
      "add v0.4s, v0.4s, v10.4s\n"
      "add v4.4s, v4.4s, v8.4s\n"
      "add v0.4s, v0.4s, v9.4s\n"
      "add v4.4s, v4.4s, v9.4s\n"

// RowMajorOutput::Output
      "st1 {v0.4s}, [%x[result]], #16\n"
      "st1 {v4.4s}, [x0], #16\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride)
      : "x0", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, int32_t, QuantizedStaticPreprocessedAsInt32, RowMajor, 3, 1,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsInt32,
                                         RowMajor>& params,
                 int32_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsInt32RowMajor<uint8_t, int32_t, "
               "QuantizedStaticPreprocessedAsInt32, RowMajor, 3, 1, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationInt32::Prepare
      "ld1 {v4.4s}, [%x[lhs]], #16\n"
      "ld1 {v5.4s}, [%x[rhs]], #16\n"
      "dup v3.4s, v4.s[0]\n"
      "dup v6.4s, v4.s[1]\n"
      "dup v4.4s, v4.s[2]\n"

// RowMajorOutput::Prepare
      "add x0, %x[result], %x[stride]\n"
      "add x1, x0, %x[stride]\n"

// StaticQuantizationInt32::Transform
      "add v0.4s, v0.4s, v3.4s\n"
      "add v1.4s, v1.4s, v6.4s\n"
      "add v2.4s, v2.4s, v4.4s\n"
      "add v0.4s, v0.4s, v5.4s\n"
      "add v1.4s, v1.4s, v5.4s\n"
      "add v2.4s, v2.4s, v5.4s\n"

// RowMajorOutput::Output
      "st1 {v0.s}[0], [%x[result]], #4\n"
      "st1 {v1.s}[0], [x0], #4\n"
      "st1 {v2.s}[0], [x1], #4\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, int32_t, QuantizedStaticPreprocessedAsInt32, RowMajor, 3, 2,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsInt32,
                                         RowMajor>& params,
                 int32_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsInt32RowMajor<uint8_t, int32_t, "
               "QuantizedStaticPreprocessedAsInt32, RowMajor, 3, 2, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"
      "mov v3.16b, v0.16b\n"
      "mov v4.16b, v1.16b\n"
      "mov v5.16b, v2.16b\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationInt32::Prepare
      "ld1 {v6.4s}, [%x[lhs]], #16\n"
      "ld1 {v7.4s}, [%x[rhs]], #16\n"
      "dup v8.4s, v6.s[0]\n"
      "dup v9.4s, v6.s[1]\n"
      "dup v6.4s, v6.s[2]\n"

// RowMajorOutput::Prepare
      "add x0, %x[result], %x[stride]\n"
      "add x1, x0, %x[stride]\n"

// StaticQuantizationInt32::Transform
      "add v0.4s, v0.4s, v8.4s\n"
      "add v2.4s, v2.4s, v9.4s\n"
      "add v4.4s, v4.4s, v6.4s\n"
      "add v0.4s, v0.4s, v7.4s\n"
      "add v2.4s, v2.4s, v7.4s\n"
      "add v4.4s, v4.4s, v7.4s\n"

// RowMajorOutput::Output
      "st1 {v0.2s}, [%x[result]], #8\n"
      "st1 {v2.2s}, [x0], #8\n"
      "st1 {v4.2s}, [x1], #8\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "v10", "v11", "v12", "v13", "v14", "v15", "v16", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, int32_t, QuantizedStaticPreprocessedAsInt32, RowMajor, 3, 3,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsInt32,
                                         RowMajor>& params,
                 int32_t* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsInt32RowMajor<uint8_t, int32_t, "
               "QuantizedStaticPreprocessedAsInt32, RowMajor, 3, 3, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// 3x3 lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

"umull v9.8h, v11.8b, v14.8b\n"
      "uadalp v4.4s, v15.8h\n"
      "uadalp v5.4s, v16.8h\n"
      "uadalp v6.4s, v17.8h\n"
      "uadalp v7.4s, v18.8h\n"
      "uadalp v8.4s, v9.8h\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationInt32::Prepare
      "ld1 {v9.4s}, [%x[lhs]], #16\n"
      "ld1 {v10.4s}, [%x[rhs]], #16\n"
      "dup v11.4s, v9.s[0]\n"
      "dup v12.4s, v9.s[1]\n"
      "dup v9.4s, v9.s[2]\n"

// RowMajorOutput::Prepare
      "add x0, %x[result], %x[stride]\n"
      "add x1, x0, %x[stride]\n"

// StaticQuantizationInt32::Transform
      "add v0.4s, v0.4s, v11.4s\n"
      "add v3.4s, v3.4s, v12.4s\n"
      "add v6.4s, v6.4s, v9.4s\n"
      "add v0.4s, v0.4s, v10.4s\n"
      "add v3.4s, v3.4s, v10.4s\n"
      "add v6.4s, v6.4s, v10.4s\n"

// RowMajorOutput::Output
      "st1 {v0.2s}, [%x[result]], #8\n"
      "st1 {v0.s}[2], [%x[result]], #4\n"
      "st1 {v3.2s}, [x0], #8\n"
      "st1 {v3.s}[2], [x0], #4\n"
      "st1 {v6.2s}, [x1], #8\n"
      "st1 {v6.s}[2], [x1], #4\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "cc",
        "memory");
}

template <>
inline void MulKernel<
    uint8_t, float, QuantizedStaticPreprocessedAsFloat, RowMajor, 1, 1,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsFloat,
                                         RowMajor>& params,
                 float* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsFloatRowMajor<uint8_t, float, "
               "QuantizedStaticPreprocessedAsFloat, RowMajor, 1, 1, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationFloat::Prepare
      "ld1 {v4.4s}, [%x[lhs]], #16\n"
      "ld1 {v5.4s}, [%x[rhs]], #16\n"
      "dup v6.4s, %w[scale]\n"
      "dup v4.4s, v4.s[0]\n"

// RowMajorOutput::Prepare

// Reduce aggregators.
      "addp v0.4s, v0.4s, v0.4s\n"
      "addp v0.4s, v0.4s, v0.4s\n"

// StaticQuantizationFloat::Transform
      "add v0.4s, v0.4s, v4.4s\n"
      "add v0.4s, v0.4s, v5.4s\n"
      "scvtf v0.4s, v0.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"

// RowMajorOutput::Output
      "st1 {v0.s}[0], [%x[result]], #4\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride),
        [scale] "r"(params.kernel.scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, float, QuantizedStaticPreprocessedAsFloat, RowMajor, 1, 2,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsFloat,
                                         RowMajor>& params,
                 float* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsFloatRowMajor<uint8_t, float, "
               "QuantizedStaticPreprocessedAsFloat, RowMajor, 1, 2, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationFloat::Prepare
      "ld1 {v4.4s}, [%x[lhs]], #16\n"
      "ld1 {v5.4s}, [%x[rhs]], #16\n"
      "dup v6.4s, %w[scale]\n"
      "dup v4.4s, v4.s[0]\n"

// RowMajorOutput::Prepare

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v0.4s, v0.4s, v0.4s\n"

// StaticQuantizationFloat::Transform
      "add v0.4s, v0.4s, v4.4s\n"
      "add v0.4s, v0.4s, v5.4s\n"
      "scvtf v0.4s, v0.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"

// RowMajorOutput::Output
      "st1 {v0.2s}, [%x[result]], #8\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride),
        [scale] "r"(params.kernel.scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, float, QuantizedStaticPreprocessedAsFloat, RowMajor, 1, 3,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsFloat,
                                         RowMajor>& params,
                 float* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsFloatRowMajor<uint8_t, float, "
               "QuantizedStaticPreprocessedAsFloat, RowMajor, 1, 3, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationFloat::Prepare
      "ld1 {v4.4s}, [%x[lhs]], #16\n"
      "ld1 {v5.4s}, [%x[rhs]], #16\n"
      "dup v6.4s, %w[scale]\n"
      "dup v4.4s, v4.s[0]\n"

// RowMajorOutput::Prepare

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v2.4s, v2.4s, v2.4s\n"
      "addp v0.4s, v0.4s, v2.4s\n"

// StaticQuantizationFloat::Transform
      "add v0.4s, v0.4s, v4.4s\n"
      "add v0.4s, v0.4s, v5.4s\n"
      "scvtf v0.4s, v0.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"

// RowMajorOutput::Output
      "st1 {v0.2s}, [%x[result]], #8\n"
      "st1 {v0.s}[2], [%x[result]], #4\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride),
        [scale] "r"(params.kernel.scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "cc",
        "memory");
}

template <>
inline void MulKernel<
    uint8_t, float, QuantizedStaticPreprocessedAsFloat, RowMajor, 1, 4,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsFloat,
                                         RowMajor>& params,
                 float* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsFloatRowMajor<uint8_t, float, "
               "QuantizedStaticPreprocessedAsFloat, RowMajor, 1, 4, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"
      "mov v3.16b, v0.16b\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationFloat::Prepare
      "ld1 {v4.4s}, [%x[lhs]], #16\n"
      "ld1 {v5.4s}, [%x[rhs]], #16\n"
      "dup v6.4s, %w[scale]\n"
      "dup v4.4s, v4.s[0]\n"

// RowMajorOutput::Prepare

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v2.4s, v2.4s, v3.4s\n"
      "addp v0.4s, v0.4s, v2.4s\n"

// StaticQuantizationFloat::Transform
      "add v0.4s, v0.4s, v4.4s\n"
      "add v0.4s, v0.4s, v5.4s\n"
      "scvtf v0.4s, v0.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"

// RowMajorOutput::Output
      "st1 {v0.4s}, [%x[result]], #16\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride),
        [scale] "r"(params.kernel.scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "v12", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, float, QuantizedStaticPreprocessedAsFloat, RowMajor, 1, 5,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsFloat,
                                         RowMajor>& params,
                 float* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsFloatRowMajor<uint8_t, float, "
               "QuantizedStaticPreprocessedAsFloat, RowMajor, 1, 5, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"
      "mov v3.16b, v0.16b\n"
      "mov v4.16b, v1.16b\n"

// General 1xM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationFloat::Prepare
      "ld1 {v5.4s}, [%x[lhs]], #16\n"
      "ld1 {v6.4s, v7.4s}, [%x[rhs]], #32\n"
      "dup v8.4s, %w[scale]\n"
      "dup v5.4s, v5.s[0]\n"

// RowMajorOutput::Prepare

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v2.4s, v2.4s, v3.4s\n"
      "addp v4.4s, v4.4s, v4.4s\n"
      "addp v0.4s, v0.4s, v2.4s\n"
      "addp v1.4s, v4.4s, v4.4s\n"

// StaticQuantizationFloat::Transform
      "add v0.4s, v0.4s, v5.4s\n"
      "add v1.4s, v1.4s, v5.4s\n"
      "add v0.4s, v0.4s, v6.4s\n"
      "add v1.4s, v1.4s, v7.4s\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "fmul v0.4s, v0.4s, v8.4s\n"
      "fmul v1.4s, v1.4s, v8.4s\n"

// RowMajorOutput::Output
      "st1 {v0.4s}, [%x[result]], #16\n"
      "st1 {v1.s}[0], [%x[result]], #4\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride),
        [scale] "r"(params.kernel.scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "v12", "v13", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, float, QuantizedStaticPreprocessedAsFloat, RowMajor, 1, 6,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsFloat,
                                         RowMajor>& params,
                 float* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsFloatRowMajor<uint8_t, float, "
               "QuantizedStaticPreprocessedAsFloat, RowMajor, 1, 6, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"
      "mov v3.16b, v0.16b\n"
      "mov v4.16b, v1.16b\n"
      "mov v5.16b, v2.16b\n"

// General 1xM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationFloat::Prepare
      "ld1 {v6.4s}, [%x[lhs]], #16\n"
      "ld1 {v7.4s, v8.4s}, [%x[rhs]], #32\n"
      "dup v9.4s, %w[scale]\n"
      "dup v6.4s, v6.s[0]\n"

// RowMajorOutput::Prepare

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v2.4s, v2.4s, v3.4s\n"
      "addp v4.4s, v4.4s, v5.4s\n"
      "addp v0.4s, v0.4s, v2.4s\n"
      "addp v1.4s, v4.4s, v4.4s\n"

// StaticQuantizationFloat::Transform
      "add v0.4s, v0.4s, v6.4s\n"
      "add v1.4s, v1.4s, v6.4s\n"
      "add v0.4s, v0.4s, v7.4s\n"
      "add v1.4s, v1.4s, v8.4s\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "fmul v0.4s, v0.4s, v9.4s\n"
      "fmul v1.4s, v1.4s, v9.4s\n"

// RowMajorOutput::Output
      "st1 {v0.4s}, [%x[result]], #16\n"
      "st1 {v1.2s}, [%x[result]], #8\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride),
        [scale] "r"(params.kernel.scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "v12", "v13", "v14", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, float, QuantizedStaticPreprocessedAsFloat, RowMajor, 1, 7,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsFloat,
                                         RowMajor>& params,
                 float* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsFloatRowMajor<uint8_t, float, "
               "QuantizedStaticPreprocessedAsFloat, RowMajor, 1, 7, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// General 1xM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationFloat::Prepare
      "ld1 {v7.4s}, [%x[lhs]], #16\n"
      "ld1 {v8.4s, v9.4s}, [%x[rhs]], #32\n"
      "dup v10.4s, %w[scale]\n"
      "dup v7.4s, v7.s[0]\n"

// RowMajorOutput::Prepare

// StaticQuantizationFloat::Transform
      "add v0.4s, v0.4s, v7.4s\n"
      "add v1.4s, v1.4s, v7.4s\n"
      "add v0.4s, v0.4s, v8.4s\n"
      "add v1.4s, v1.4s, v9.4s\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "fmul v0.4s, v0.4s, v10.4s\n"
      "fmul v1.4s, v1.4s, v10.4s\n"

// RowMajorOutput::Output
      "st1 {v0.4s}, [%x[result]], #16\n"
      "st1 {v1.2s}, [%x[result]], #8\n"
      "st1 {v1.s}[2], [%x[result]], #4\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride),
        [scale] "r"(params.kernel.scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "v12", "v13", "v14", "v15", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, float, QuantizedStaticPreprocessedAsFloat, RowMajor, 1, 8,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsFloat,
                                         RowMajor>& params,
                 float* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsFloatRowMajor<uint8_t, float, "
               "QuantizedStaticPreprocessedAsFloat, RowMajor, 1, 8, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// 1x8 lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

"uadalp v4.4s, v17.8h\n"
      "uadalp v5.4s, v13.8h\n"
      "uadalp v6.4s, v14.8h\n"
      "uadalp v7.4s, v15.8h\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationFloat::Prepare
      "ld1 {v8.4s}, [%x[lhs]], #16\n"
      "ld1 {v9.4s, v10.4s}, [%x[rhs]], #32\n"
      "dup v11.4s, %w[scale]\n"
      "dup v8.4s, v8.s[0]\n"

// RowMajorOutput::Prepare

// StaticQuantizationFloat::Transform
      "add v0.4s, v0.4s, v8.4s\n"
      "add v1.4s, v1.4s, v8.4s\n"
      "add v0.4s, v0.4s, v9.4s\n"
      "add v1.4s, v1.4s, v10.4s\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "fmul v0.4s, v0.4s, v11.4s\n"
      "fmul v1.4s, v1.4s, v11.4s\n"

// RowMajorOutput::Output
      "st1 {v0.4s, v1.4s}, [%x[result]], #32\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride),
        [scale] "r"(params.kernel.scale)
      : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "v12", "v13", "v14", "v15", "v16", "v17", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, float, QuantizedStaticPreprocessedAsFloat, RowMajor, 2, 1,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsFloat,
                                         RowMajor>& params,
                 float* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsFloatRowMajor<uint8_t, float, "
               "QuantizedStaticPreprocessedAsFloat, RowMajor, 2, 1, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationFloat::Prepare
      "ld1 {v4.4s}, [%x[lhs]], #16\n"
      "ld1 {v5.4s}, [%x[rhs]], #16\n"
      "dup v6.4s, %w[scale]\n"
      "dup v2.4s, v4.s[0]\n"
      "dup v4.4s, v4.s[1]\n"

// RowMajorOutput::Prepare
      "add x0, %x[result], %x[stride]\n"

// Reduce aggregators.
      "addp v0.4s, v0.4s, v0.4s\n"
      "addp v0.4s, v0.4s, v0.4s\n"
      "addp v1.4s, v1.4s, v1.4s\n"
      "addp v1.4s, v1.4s, v1.4s\n"

// StaticQuantizationFloat::Transform
      "add v0.4s, v0.4s, v2.4s\n"
      "add v1.4s, v1.4s, v4.4s\n"
      "add v0.4s, v0.4s, v5.4s\n"
      "add v1.4s, v1.4s, v5.4s\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"

// RowMajorOutput::Output
      "st1 {v0.s}[0], [%x[result]], #4\n"
      "st1 {v1.s}[0], [x0], #4\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride),
        [scale] "r"(params.kernel.scale)
      : "x0", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, float, QuantizedStaticPreprocessedAsFloat, RowMajor, 2, 2,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsFloat,
                                         RowMajor>& params,
                 float* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsFloatRowMajor<uint8_t, float, "
               "QuantizedStaticPreprocessedAsFloat, RowMajor, 2, 2, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"
      "mov v3.16b, v0.16b\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationFloat::Prepare
      "ld1 {v4.4s}, [%x[lhs]], #16\n"
      "ld1 {v5.4s}, [%x[rhs]], #16\n"
      "dup v6.4s, %w[scale]\n"
      "dup v7.4s, v4.s[0]\n"
      "dup v4.4s, v4.s[1]\n"

// RowMajorOutput::Prepare
      "add x0, %x[result], %x[stride]\n"

// Reduce aggregators.
      "addp v0.4s, v0.4s, v1.4s\n"
      "addp v0.4s, v0.4s, v0.4s\n"
      "addp v2.4s, v2.4s, v3.4s\n"
      "addp v2.4s, v2.4s, v2.4s\n"

// StaticQuantizationFloat::Transform
      "add v0.4s, v0.4s, v7.4s\n"
      "add v2.4s, v2.4s, v4.4s\n"
      "add v0.4s, v0.4s, v5.4s\n"
      "add v2.4s, v2.4s, v5.4s\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v2.4s, v2.4s, v6.4s\n"

// RowMajorOutput::Output
      "st1 {v0.2s}, [%x[result]], #8\n"
      "st1 {v2.2s}, [x0], #8\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride),
        [scale] "r"(params.kernel.scale)
      : "x0", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, float, QuantizedStaticPreprocessedAsFloat, RowMajor, 2, 3,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsFloat,
                                         RowMajor>& params,
                 float* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsFloatRowMajor<uint8_t, float, "
               "QuantizedStaticPreprocessedAsFloat, RowMajor, 2, 3, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"
      "mov v3.16b, v0.16b\n"
      "mov v4.16b, v1.16b\n"
      "mov v5.16b, v2.16b\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationFloat::Prepare
      "ld1 {v6.4s}, [%x[lhs]], #16\n"
      "ld1 {v7.4s}, [%x[rhs]], #16\n"
      "dup v8.4s, %w[scale]\n"
      "dup v9.4s, v6.s[0]\n"
      "dup v6.4s, v6.s[1]\n"

// RowMajorOutput::Prepare
      "add x0, %x[result], %x[stride]\n"

// StaticQuantizationFloat::Transform
      "add v0.4s, v0.4s, v9.4s\n"
      "add v3.4s, v3.4s, v6.4s\n"
      "add v0.4s, v0.4s, v7.4s\n"
      "add v3.4s, v3.4s, v7.4s\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v3.4s, v3.4s\n"
      "fmul v0.4s, v0.4s, v8.4s\n"
      "fmul v3.4s, v3.4s, v8.4s\n"

// RowMajorOutput::Output
      "st1 {v0.2s}, [%x[result]], #8\n"
      "st1 {v0.s}[2], [%x[result]], #4\n"
      "st1 {v3.2s}, [x0], #8\n"
      "st1 {v3.s}[2], [x0], #4\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride),
        [scale] "r"(params.kernel.scale)
      : "x0", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "v12", "v13", "v14", "v15", "v16", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, float, QuantizedStaticPreprocessedAsFloat, RowMajor, 2, 4,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsFloat,
                                         RowMajor>& params,
                 float* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsFloatRowMajor<uint8_t, float, "
               "QuantizedStaticPreprocessedAsFloat, RowMajor, 2, 4, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// 2x4 lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

"uadalp v3.4s, v17.8h\n"
      "uadalp v4.4s, v18.8h\n"
      "uadalp v5.4s, v14.8h\n"
      "uadalp v6.4s, v15.8h\n"
      "uadalp v7.4s, v16.8h\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationFloat::Prepare
      "ld1 {v8.4s}, [%x[lhs]], #16\n"
      "ld1 {v9.4s}, [%x[rhs]], #16\n"
      "dup v10.4s, %w[scale]\n"
      "dup v11.4s, v8.s[0]\n"
      "dup v8.4s, v8.s[1]\n"

// RowMajorOutput::Prepare
      "add x0, %x[result], %x[stride]\n"

// StaticQuantizationFloat::Transform
      "add v0.4s, v0.4s, v11.4s\n"
      "add v4.4s, v4.4s, v8.4s\n"
      "add v0.4s, v0.4s, v9.4s\n"
      "add v4.4s, v4.4s, v9.4s\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v4.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v10.4s\n"
      "fmul v4.4s, v4.4s, v10.4s\n"

// RowMajorOutput::Output
      "st1 {v0.4s}, [%x[result]], #16\n"
      "st1 {v4.4s}, [x0], #16\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride),
        [scale] "r"(params.kernel.scale)
      : "x0", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
        "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, float, QuantizedStaticPreprocessedAsFloat, RowMajor, 3, 1,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsFloat,
                                         RowMajor>& params,
                 float* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsFloatRowMajor<uint8_t, float, "
               "QuantizedStaticPreprocessedAsFloat, RowMajor, 3, 1, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationFloat::Prepare
      "ld1 {v4.4s}, [%x[lhs]], #16\n"
      "ld1 {v5.4s}, [%x[rhs]], #16\n"
      "dup v6.4s, %w[scale]\n"
      "dup v3.4s, v4.s[0]\n"
      "dup v7.4s, v4.s[1]\n"
      "dup v4.4s, v4.s[2]\n"

// RowMajorOutput::Prepare
      "add x0, %x[result], %x[stride]\n"
      "add x1, x0, %x[stride]\n"

// StaticQuantizationFloat::Transform
      "add v0.4s, v0.4s, v3.4s\n"
      "add v1.4s, v1.4s, v7.4s\n"
      "add v2.4s, v2.4s, v4.4s\n"
      "add v0.4s, v0.4s, v5.4s\n"
      "add v1.4s, v1.4s, v5.4s\n"
      "add v2.4s, v2.4s, v5.4s\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v1.4s, v1.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "fmul v0.4s, v0.4s, v6.4s\n"
      "fmul v1.4s, v1.4s, v6.4s\n"
      "fmul v2.4s, v2.4s, v6.4s\n"

// RowMajorOutput::Output
      "st1 {v0.s}[0], [%x[result]], #4\n"
      "st1 {v1.s}[0], [x0], #4\n"
      "st1 {v2.s}[0], [x1], #4\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride),
        [scale] "r"(params.kernel.scale)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, float, QuantizedStaticPreprocessedAsFloat, RowMajor, 3, 2,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsFloat,
                                         RowMajor>& params,
                 float* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsFloatRowMajor<uint8_t, float, "
               "QuantizedStaticPreprocessedAsFloat, RowMajor, 3, 2, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// Clear aggregators.
      "movi v0.4s, #0\n"
      "movi v1.4s, #0\n"
      "movi v2.4s, #0\n"
      "mov v3.16b, v0.16b\n"
      "mov v4.16b, v1.16b\n"
      "mov v5.16b, v2.16b\n"

// General NxM lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationFloat::Prepare
      "ld1 {v6.4s}, [%x[lhs]], #16\n"
      "ld1 {v7.4s}, [%x[rhs]], #16\n"
      "dup v8.4s, %w[scale]\n"
      "dup v9.4s, v6.s[0]\n"
      "dup v10.4s, v6.s[1]\n"
      "dup v6.4s, v6.s[2]\n"

// RowMajorOutput::Prepare
      "add x0, %x[result], %x[stride]\n"
      "add x1, x0, %x[stride]\n"

// StaticQuantizationFloat::Transform
      "add v0.4s, v0.4s, v9.4s\n"
      "add v2.4s, v2.4s, v10.4s\n"
      "add v4.4s, v4.4s, v6.4s\n"
      "add v0.4s, v0.4s, v7.4s\n"
      "add v2.4s, v2.4s, v7.4s\n"
      "add v4.4s, v4.4s, v7.4s\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v2.4s, v2.4s\n"
      "scvtf v4.4s, v4.4s\n"
      "fmul v0.4s, v0.4s, v8.4s\n"
      "fmul v2.4s, v2.4s, v8.4s\n"
      "fmul v4.4s, v4.4s, v8.4s\n"

// RowMajorOutput::Output
      "st1 {v0.2s}, [%x[result]], #8\n"
      "st1 {v2.2s}, [x0], #8\n"
      "st1 {v4.2s}, [x1], #8\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride),
        [scale] "r"(params.kernel.scale)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "v10", "v11", "v12", "v13", "v14", "v15", "v16", "cc", "memory");
}

template <>
inline void MulKernel<
    uint8_t, float, QuantizedStaticPreprocessedAsFloat, RowMajor, 3, 3,
    8>::Multiply(const uint8_t* lhs, const uint8_t* rhs,
                 const FusedKernelParams<QuantizedStaticPreprocessedAsFloat,
                                         RowMajor>& params,
                 float* result) {
#ifdef DEBUG
#ifdef DEBUG_METAGEMM_VERBOSE
  std::cout << __FILE__ << "(" << __LINE__
            << ") QuantizedStaticPreprocessedAsFloatRowMajor<uint8_t, float, "
               "QuantizedStaticPreprocessedAsFloat, RowMajor, 3, 3, "
               "8>::Multiply()"
            << std::endl
            << std::flush;
#endif
#endif
  asm volatile(
      "prfm pldl1keep, [%x[lhs]]\n"
      "prfm pldl1keep, [%x[rhs]]\n"

// 3x3 lanes loop.
      "1:"

// Subtract counter.
      "subs %x[count], %x[count], #8\n"

"umull v9.8h, v11.8b, v14.8b\n"
      "uadalp v4.4s, v15.8h\n"
      "uadalp v5.4s, v16.8h\n"
      "uadalp v6.4s, v17.8h\n"
      "uadalp v7.4s, v18.8h\n"
      "uadalp v8.4s, v9.8h\n"

// Loop break.
      "bgt 1b\n"

// StaticQuantizationFloat::Prepare
      "ld1 {v9.4s}, [%x[lhs]], #16\n"
      "ld1 {v10.4s}, [%x[rhs]], #16\n"
      "dup v11.4s, %w[scale]\n"
      "dup v12.4s, v9.s[0]\n"
      "dup v13.4s, v9.s[1]\n"
      "dup v9.4s, v9.s[2]\n"

// RowMajorOutput::Prepare
      "add x0, %x[result], %x[stride]\n"
      "add x1, x0, %x[stride]\n"

// StaticQuantizationFloat::Transform
      "add v0.4s, v0.4s, v12.4s\n"
      "add v3.4s, v3.4s, v13.4s\n"
      "add v6.4s, v6.4s, v9.4s\n"
      "add v0.4s, v0.4s, v10.4s\n"
      "add v3.4s, v3.4s, v10.4s\n"
      "add v6.4s, v6.4s, v10.4s\n"
      "scvtf v0.4s, v0.4s\n"
      "scvtf v3.4s, v3.4s\n"
      "scvtf v6.4s, v6.4s\n"
      "fmul v0.4s, v0.4s, v11.4s\n"
      "fmul v3.4s, v3.4s, v11.4s\n"
      "fmul v6.4s, v6.4s, v11.4s\n"

// RowMajorOutput::Output
      "st1 {v0.2s}, [%x[result]], #8\n"
      "st1 {v0.s}[2], [%x[result]], #4\n"
      "st1 {v3.2s}, [x0], #8\n"
      "st1 {v3.s}[2], [x0], #4\n"
      "st1 {v6.2s}, [x1], #8\n"
      "st1 {v6.s}[2], [x1], #4\n"
      : [rhs] "+r"(rhs), [lhs] "+r"(lhs), [result] "+r"(result)
      : [count] "r"(params.kernel.count),
        [stride] "r"(params.output_stream.stride),
        [scale] "r"(params.kernel.scale)
      : "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
        "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "cc",
        "memory");
}

}  // namespace meta
}  // namespace gemmlowp

#else
#warning "Meta gemm for arm64 requires: GEMMLOWP_NEON_64!"
#endif

#endif  // GEMMLOWP_META_QUANTIZED_MUL_KERNELS_ARM_64_H_