// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "test.h"
#include <unistd.h>
#include <cstdint>
#include <cstdlib>
#include <ctime>
#include <iostream>
#include <memory>
#include <string>
#include <vector>
#ifdef __APPLE__
#include <TargetConditionals.h>
#endif
#include "../eight_bit_int_gemm/eight_bit_int_gemm.h"
#include "../internal/kernel_reference.h"
#include "test_data.h"
namespace gemmlowp {
void ReferenceEightBitIntGemm(bool transpose_a, bool transpose_b,
bool transpose_c, int m, int n, int k,
const uint8_t* a, int32_t a_offset, int lda,
const uint8_t* b, int32_t b_offset, int ldb,
uint8_t* c, int32_t c_offset, int32_t c_mult_int,
int32_t c_shift, int ldc) {
assert((c_shift >= 0) && (c_shift <= 32));
assert(a != nullptr);
assert(b != nullptr);
assert(c != nullptr);
int a_i_stride;
int a_l_stride;
if (transpose_a) {
a_i_stride = lda;
a_l_stride = 1;
} else {
a_i_stride = 1;
a_l_stride = lda;
}
int b_j_stride;
int b_l_stride;
if (transpose_b) {
b_j_stride = 1;
b_l_stride = ldb;
} else {
b_j_stride = ldb;
b_l_stride = 1;
}
int c_i_stride;
int c_j_stride;
if (transpose_c) {
c_i_stride = ldc;
c_j_stride = 1;
} else {
c_i_stride = 1;
c_j_stride = ldc;
}
int i, j, l;
const std::int32_t kRoundingTerm = (c_shift < 1) ? 0 : (1 << (c_shift - 1));
for (j = 0; j < n; j++) {
for (i = 0; i < m; i++) {
int32_t total = 0;
for (l = 0; l < k; l++) {
const int a_index = i * a_i_stride + l * a_l_stride;
const uint8_t a_as_byte = a[a_index];
const int32_t a_as_int = static_cast<int32_t>(a_as_byte) + a_offset;
const int b_index = j * b_j_stride + l * b_l_stride;
const uint8_t b_as_byte = b[b_index];
const int32_t b_as_int = static_cast<int32_t>(b_as_byte) + b_offset;
const int32_t mult_as_int = a_as_int * b_as_int;
total += mult_as_int;
}
int32_t output =
(((total + c_offset) * c_mult_int) + kRoundingTerm) >> c_shift;
if (output > 255) {
output = 255;
}
if (output < 0) {
output = 0;
}
const int c_index = i * c_i_stride + j * c_j_stride;
c[c_index] = static_cast<uint8_t>(output);
}
}
}
// *GemmWrapper's allow to wrap various Gemm functions in a uniform
// interface, so we can use the same testing code to test all of them
template <typename Kernel, typename Scalar, typename tBitDepthParams>
struct SingleThreadGemmWrapper {
typedef tBitDepthParams BitDepthParams;
static const char* Name() {
static char buf[256];
snprintf(buf, sizeof(buf), "SingleThreadGemm, Kernel: %s", Kernel().Name());
return buf;
}
typedef SingleThreadGemmContext Context;
template <MapOrder LhsOrder, MapOrder RhsOrder, MapOrder ResultOrder>
static void Gemm(Context* context,
const MatrixMap<const Scalar, LhsOrder>& lhs,
const MatrixMap<const Scalar, RhsOrder>& rhs,
MatrixMap<Scalar, ResultOrder>* result, int lhs_offset,
int rhs_offset, int result_offset, int result_mult_int,
int result_shift) {
const OffsetColDup lhs_offset_vector(lhs_offset, lhs.rows());
const OffsetRowDup rhs_offset_vector(rhs_offset, rhs.cols());
SingleThreadGemm<typename Kernel::Format, Scalar, Scalar, BitDepthParams,
LhsOrder, RhsOrder, ResultOrder,
OffsetColDup, OffsetRowDup>(
context, Kernel(), lhs, rhs, result, lhs_offset_vector,
rhs_offset_vector,
MakeStandardOutputPipeline(result_offset, result_mult_int,
result_shift));
}
};
template <typename Kernel, typename Scalar, typename tBitDepthParams>
struct MultiThreadGemmWrapper {
typedef tBitDepthParams BitDepthParams;
static const char* Name() {
static char buf[256];
snprintf(buf, sizeof(buf), "MultiThreadGemm, Kernel: %s", Kernel().Name());
return buf;
}
typedef MultiThreadGemmContext Context;
template <MapOrder LhsOrder, MapOrder RhsOrder, MapOrder ResultOrder>
static void Gemm(Context* context,
const MatrixMap<const Scalar, LhsOrder>& lhs,
const MatrixMap<const Scalar, RhsOrder>& rhs,
MatrixMap<Scalar, ResultOrder>* result, int lhs_offset,
int rhs_offset, int result_offset, int result_mult_int,
int result_shift) {
const OffsetColDup lhs_offset_vector(lhs_offset, lhs.rows());
const OffsetRowDup rhs_offset_vector(rhs_offset, rhs.cols());
MultiThreadGemm<typename Kernel::Format, Scalar, Scalar, BitDepthParams,
LhsOrder, RhsOrder, ResultOrder,
OffsetColDup, OffsetRowDup>(
context, Kernel(), lhs, rhs, result, lhs_offset_vector,
rhs_offset_vector,
MakeStandardOutputPipeline(result_offset, result_mult_int,
result_shift));
}
};
template <typename Scalar, typename tBitDepthParams>
struct PublicGemmWrapper {
typedef tBitDepthParams BitDepthParams;
static const char* Name() { return "public Gemm"; }
typedef GemmContext Context;
template <MapOrder LhsOrder, MapOrder RhsOrder, MapOrder ResultOrder>
static void Gemm(Context* context,
const MatrixMap<const Scalar, LhsOrder>& lhs,
const MatrixMap<const Scalar, RhsOrder>& rhs,
MatrixMap<Scalar, ResultOrder>* result, int lhs_offset,
int rhs_offset, int result_offset, int result_mult_int,
int result_shift) {
gemmlowp::Gemm<uint8_t, BitDepthParams, LhsOrder, RhsOrder, ResultOrder>(
context, lhs, rhs, result, lhs_offset, rhs_offset, result_offset,
result_mult_int, result_shift);
}
};
template <eight_bit_int_gemm::BitDepthSetting BitDepth>
struct BitDepthParamsForSettings {};
template <>
struct BitDepthParamsForSettings<eight_bit_int_gemm::BitDepthSetting::A8B8>
: DefaultL8R8BitDepthParams {};
template <>
struct BitDepthParamsForSettings<eight_bit_int_gemm::BitDepthSetting::A5B7>
: DefaultL7R5BitDepthParams {};
template <typename Scalar, eight_bit_int_gemm::BitDepthSetting BitDepth>
struct EightBitIntGemmWrapper {
typedef BitDepthParamsForSettings<BitDepth> BitDepthParams;
static const char* Name() { return "EightBitIntGemm"; }
typedef void Context;
template <MapOrder LhsOrder, MapOrder RhsOrder, MapOrder ResultOrder>
static void Gemm(Context*, const MatrixMap<const Scalar, LhsOrder>& lhs,
const MatrixMap<const Scalar, RhsOrder>& rhs,
MatrixMap<Scalar, ResultOrder>* result, int lhs_offset,
int rhs_offset, int result_offset, int result_mult_int,
int result_shift) {
const bool transpose_c = ResultOrder == MapOrder::RowMajor;
const bool transpose_a = LhsOrder == MapOrder::RowMajor;
const bool transpose_b = RhsOrder == MapOrder::RowMajor;
eight_bit_int_gemm::EightBitIntGemm(
transpose_a, transpose_b, transpose_c, lhs.rows(), rhs.cols(),
lhs.cols(), lhs.data(), lhs_offset, lhs.stride(), rhs.data(),
rhs_offset, rhs.stride(), result->data(), result_offset,
result_mult_int, result_shift, result->stride(), BitDepth);
}
};
template <typename Scalar>
struct ReferenceEightBitIntGemmWrapper {
typedef DefaultL8R8BitDepthParams BitDepthParams;
static const char* Name() { return "ReferenceEightBitIntGemm"; }
template <MapOrder LhsOrder, MapOrder RhsOrder, MapOrder ResultOrder>
static void Gemm(bool transpose_a, bool transpose_b, bool transpose_c,
const MatrixMap<const Scalar, LhsOrder>& lhs,
const MatrixMap<const Scalar, RhsOrder>& rhs,
MatrixMap<Scalar, ResultOrder>* result, int lhs_offset,
int rhs_offset, int result_offset, int result_mult_int,
int result_shift) {
ReferenceEightBitIntGemm(transpose_a, transpose_b, transpose_c, lhs.rows(),
rhs.cols(), lhs.cols(), lhs.data(), lhs_offset,
lhs.stride(), rhs.data(), rhs_offset, rhs.stride(),
result->data(), result_offset, result_mult_int,
result_shift, result->stride());
}
};
const char* OrderName(MapOrder order) {
return order == MapOrder::ColMajor ? "ColMajor" : "RowMajor";
}
struct ResultStats {
ResultStats()
: count(0),
med_val(0),
mean_signed_diff(0),
med_signed_diff(0),
med_unsigned_diff(0),
max_unsigned_diff(0) {}
int count;
int med_val;
float mean_signed_diff;
int med_signed_diff;
int med_unsigned_diff;
int max_unsigned_diff;
std::vector<int> count_diff_by_pot_slice;
};
void GetResultStats(const uint8_t* actual, const uint8_t* expected,
size_t count, ResultStats* stats) {
std::vector<uint8_t> results;
std::vector<int16_t> signed_diffs;
std::vector<uint8_t> unsigned_diffs;
int64_t signed_diffs_sum = 0;
for (size_t i = 0; i < count; i++) {
results.push_back(actual[i]);
int16_t signed_diff = actual[i] - expected[i];
signed_diffs.push_back(signed_diff);
unsigned_diffs.push_back(std::abs(signed_diff));
signed_diffs_sum += signed_diff;
}
std::sort(results.begin(), results.end());
std::sort(signed_diffs.begin(), signed_diffs.end());
std::sort(unsigned_diffs.begin(), unsigned_diffs.end());
const size_t middle = count / 2;
stats->count = count;
stats->med_val = results[middle];
stats->mean_signed_diff = float(signed_diffs_sum) / count;
stats->med_signed_diff = signed_diffs[middle];
stats->med_unsigned_diff = unsigned_diffs[middle];
stats->max_unsigned_diff = unsigned_diffs.back();
// Size 9 for 9 different POT values: 2^0, ..., 2^8
stats->count_diff_by_pot_slice.resize(9);
auto cur = unsigned_diffs.begin();
size_t checksum = 0;
for (int exponent = 0; exponent < 9; exponent++) {
int pot = 1 << exponent;
auto next = std::lower_bound(cur, unsigned_diffs.end(), pot);
checksum += stats->count_diff_by_pot_slice[exponent] = next - cur;
cur = next;
}
assert(checksum == count);
}
struct ResultStatsBounds {
ResultStatsBounds()
: mean_signed_diff(0),
med_signed_diff(0),
med_unsigned_diff(0),
max_unsigned_diff(0) {}
float mean_signed_diff;
int med_signed_diff;
int med_unsigned_diff;
int max_unsigned_diff;
};
bool CheckResultStatsBounds(const ResultStats& stats,
const ResultStatsBounds& bounds) {
return stats.max_unsigned_diff <= bounds.max_unsigned_diff &&
stats.med_unsigned_diff <= bounds.med_unsigned_diff &&
std::abs(stats.med_signed_diff) <= bounds.med_signed_diff &&
std::abs(stats.mean_signed_diff) <= bounds.mean_signed_diff;
}
void ReportResultStats(const ResultStats& stats,
const ResultStatsBounds& bounds) {
printf(" number of matrix entries: %d\n", stats.count);
printf(" median value: %d\n", stats.med_val);
printf(" median unsigned diff: %d (tolerating %d)\n",
stats.med_unsigned_diff, bounds.med_unsigned_diff);
printf(" max unsigned diff: %d (tolerating %d)\n", stats.max_unsigned_diff,
bounds.max_unsigned_diff);
printf(" median signed diff: %d (tolerating %d)\n", stats.med_signed_diff,
bounds.med_signed_diff);
printf(" mean signed diff: %.3g (tolerating %.3g)\n",
stats.mean_signed_diff, bounds.mean_signed_diff);
printf("No error: %.2f %% of entries\n",
100.f * stats.count_diff_by_pot_slice[0] / stats.count);
for (int exponent = 1; exponent < 9; exponent++) {
printf("Error in %d..%d range: %.2f %% of entries\n", 1 << (exponent - 1),
(1 << exponent) - 1,
100.f * stats.count_diff_by_pot_slice[exponent] / stats.count);
}
}
// Our approach to choosing result_shift values for testing, is bisection.
// This function takes an interval, [result_shift_min .. result_shift_max].
// If too much saturation occurred in either direction, it bisects accordingly,
// recursing until the interval contains only one value.
// The primary reason why we prefer this over computing optimal shift values,
// is that we actually want to exercise some saturation, as there is nontrivial
// code handling that in gemmlowp.
// Secondarily, this is faster than computing optimal shifts, since in 90% of
// cases the first-tried shift value 16 turns out to be good enough.
template <typename GemmWrapper, typename LhsType, typename RhsType,
typename ResultType>
void test_gemm_impl(typename GemmWrapper::Context* context, const LhsType& lhs,
const RhsType& rhs, ResultType* result, int lhs_offset,
int rhs_offset, int result_offset, int result_mult_int,
int result_shift_min, int result_shift_max) {
const int rows = lhs.rows();
const int cols = rhs.cols();
Check(lhs.cols() == rhs.rows());
const int depth = lhs.cols();
const int result_shift = (result_shift_min + result_shift_max) / 2;
GemmWrapper::Gemm(context, lhs.const_map(), rhs.const_map(), &result->map(),
lhs_offset, rhs_offset, result_offset, result_mult_int,
result_shift);
typedef typename ResultType::Scalar Scalar;
static const MapOrder kLhsOrder = LhsType::kOrder;
static const MapOrder kRhsOrder = RhsType::kOrder;
static const MapOrder kResultOrder = ResultType::kOrder;
ResultType ref_result(rows, cols);
const bool transpose_c = kResultOrder == MapOrder::RowMajor;
const bool transpose_a = kLhsOrder == MapOrder::RowMajor;
const bool transpose_b = kRhsOrder == MapOrder::RowMajor;
ReferenceEightBitIntGemmWrapper<Scalar>::Gemm(
transpose_a, transpose_b, transpose_c, lhs.const_map(), rhs.const_map(),
&ref_result.map(), lhs_offset, rhs_offset, result_offset, result_mult_int,
result_shift);
typedef typename GemmWrapper::BitDepthParams BitDepthParams;
ResultStats stats;
GetResultStats(result->data(), ref_result.data(), rows * cols, &stats);
// Adjust shifts until we get meaningful results
int new_result_shift_min = result_shift_min;
int new_result_shift_max = result_shift_max;
bool retry = false;
if (stats.med_val < 32) {
new_result_shift_max = (result_shift_min + result_shift_max) / 2;
retry = true;
}
if (stats.med_val > 224) {
new_result_shift_min = (result_shift_min + result_shift_max) / 2;
retry = true;
}
if (retry) {
if (result_shift_min != result_shift_max) {
test_gemm_impl<GemmWrapper>(context, lhs, rhs, result, lhs_offset,
rhs_offset, result_offset, result_mult_int,
new_result_shift_min, new_result_shift_max);
}
return;
}
ResultStatsBounds bounds;
if (BitDepthParams::LhsBitDepth::kBits < 8 ||
BitDepthParams::RhsBitDepth::kBits < 8) {
// We have very lax requirements on unsigned diff.
// We have tighter requirements on signed diff (bias), but only
// if the matrix is large enough for things to average out.
// For very small sizes, we... basically don't test anything.
// The problem is that this test uses unrealistic combinations of
// result_mult_int
// and result_shift, resulting in potentially wild requantization artifacts
// on small GEMMs.
int adjust_for_small_sizes = 1000 / (rows * cols);
bounds.max_unsigned_diff =
std::max(stats.med_val / 2, adjust_for_small_sizes);
bounds.med_unsigned_diff =
std::max(stats.med_val / 8, adjust_for_small_sizes);
bounds.med_signed_diff = std::max(2, adjust_for_small_sizes);
bounds.mean_signed_diff = std::max(2, adjust_for_small_sizes);
}
// Check results
const bool good = CheckResultStatsBounds(stats, bounds);
printf(
"%s: %dx%dx%d %s x %s -> %s, %s, offsets %d/%d/%d, mult %d, shift %d\n",
good ? "PASS" : "FAIL", rows, depth, cols, OrderName(kLhsOrder),
OrderName(kRhsOrder), OrderName(kResultOrder), GemmWrapper::Name(),
lhs_offset, rhs_offset, result_offset, result_mult_int, result_shift);
if (!good) {
ReportResultStats(stats, bounds);
int bad_coeffs_printed = 0;
for (int c = 0; c < result->cols() && bad_coeffs_printed < 20; c++) {
for (int r = 0; r < result->rows() && bad_coeffs_printed < 20; r++) {
if (ref_result(r, c) != (*result)(r, c)) {
printf("bad coeff: at (%d, %d), expected %d, got %d\n", r, c,
ref_result(r, c), (*result)(r, c));
bad_coeffs_printed++;
}
}
}
}
Check(good);
}
template <typename GemmWrapper, typename LhsType, typename RhsType,
typename ResultType>
void test_gemm(typename GemmWrapper::Context* context, const LhsType& lhs,
const RhsType& rhs, ResultType* result, int lhs_offset,
int rhs_offset, int result_offset, int result_mult_int) {
test_gemm_impl<GemmWrapper>(context, lhs, rhs, result, lhs_offset, rhs_offset,
result_offset, result_mult_int, 0, 32);
}
enum class WhatParamsToTest {
All,
OnlyGenericCase,
};
template <typename GemmWrapper, MapOrder LhsOrder, MapOrder RhsOrder,
MapOrder ResultOrder>
void test_gemm(typename GemmWrapper::Context* context, int rows, int depth,
int cols, WhatParamsToTest params_to_test) {
typedef std::uint8_t Scalar;
typedef Matrix<Scalar, LhsOrder> LhsType;
LhsType lhs(rows, depth);
MakeRandom(&lhs, 8);
typedef Matrix<Scalar, RhsOrder> RhsType;
RhsType rhs(depth, cols);
MakeRandom(&rhs, 8);
typedef Matrix<Scalar, ResultOrder> ResultType;
ResultType result(rows, cols);
MakeZero(&result);
if (params_to_test == WhatParamsToTest::All) {
test_gemm<GemmWrapper>(context, lhs, rhs, &result, 0, 0, 0, 1);
test_gemm<GemmWrapper>(context, lhs, rhs, &result, 10, 0, 0, 1);
test_gemm<GemmWrapper>(context, lhs, rhs, &result, 0, 10, 0, 1);
test_gemm<GemmWrapper>(context, lhs, rhs, &result, 0, 0, 10, 1);
test_gemm<GemmWrapper>(context, lhs, rhs, &result, 0, 0, 0, 10);
test_gemm<GemmWrapper>(context, lhs, rhs, &result, 10, 10, 10, 10);
test_gemm<GemmWrapper>(context, lhs, rhs, &result, 256, 1, 17, 4);
}
test_gemm<GemmWrapper>(context, lhs, rhs, &result, -75, -91, 74980, 123);
}
enum class WhatOrdersToTest { All, OnlyRCC };
template <typename GemmWrapper>
void test_gemm(typename GemmWrapper::Context* context, int rows, int depth,
int cols, WhatParamsToTest params_to_test,
WhatOrdersToTest orders_to_test) {
#define GEMMLOWP_ONE_TEST(LhsOrder, RhsOrder, ResultOrder) \
do { \
test_gemm<GemmWrapper, MapOrder::LhsOrder, MapOrder::RhsOrder, \
MapOrder::ResultOrder>(context, rows, depth, cols, \
params_to_test); \
} while (false)
if (orders_to_test == WhatOrdersToTest::All) {
GEMMLOWP_ONE_TEST(ColMajor, ColMajor, ColMajor);
GEMMLOWP_ONE_TEST(RowMajor, ColMajor, ColMajor);
GEMMLOWP_ONE_TEST(ColMajor, RowMajor, ColMajor);
GEMMLOWP_ONE_TEST(RowMajor, RowMajor, ColMajor);
GEMMLOWP_ONE_TEST(ColMajor, ColMajor, RowMajor);
GEMMLOWP_ONE_TEST(RowMajor, ColMajor, RowMajor);
GEMMLOWP_ONE_TEST(ColMajor, RowMajor, RowMajor);
GEMMLOWP_ONE_TEST(RowMajor, RowMajor, RowMajor);
} else {
GEMMLOWP_ONE_TEST(RowMajor, ColMajor, ColMajor);
}
#undef GEMMLOWP_ONE_TEST
}
template <typename Kernel>
void test_gemm_kernel(MultiThreadGemmContext* context) {
typedef MultiThreadGemmWrapper<Kernel, std::uint8_t,
DefaultL8R8BitDepthParams>
GemmWrapper;
test_gemm<GemmWrapper>(context, 1, 1, 1, WhatParamsToTest::OnlyGenericCase,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 2, 2, 2, WhatParamsToTest::OnlyGenericCase,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 3, 3, 3, WhatParamsToTest::OnlyGenericCase,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 4, 4, 4, WhatParamsToTest::OnlyGenericCase,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 5, 5, 5, WhatParamsToTest::OnlyGenericCase,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 9, 11, 13, WhatParamsToTest::OnlyGenericCase,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 50, 50, 50, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 200, 200, 200,
WhatParamsToTest::OnlyGenericCase,
WhatOrdersToTest::All);
test_gemm<GemmWrapper>(context, 50, 5000, 50,
WhatParamsToTest::OnlyGenericCase,
WhatOrdersToTest::OnlyRCC);
}
template <typename GemmWrapper>
void test_gemm(typename GemmWrapper::Context* context) {
test_gemm<GemmWrapper>(context, 1, 1, 1, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 2, 1, 1, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 1, 2, 1, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 1, 1, 2, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 2, 2, 2, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 3, 3, 3, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 4, 4, 4, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 5, 5, 5, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 6, 6, 6, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 3, 5, 7, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 7, 3, 5, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 5, 7, 3, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 8, 8, 8, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 16, 16, 16, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 32, 32, 32, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 64, 64, 64, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 128, 128, 128, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 16, 17, 16, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 37, 55, 73, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 57, 87, 117, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 93, 83, 73, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 109, 89, 99, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 78, 101, 82, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 512, 512, 512,
WhatParamsToTest::OnlyGenericCase,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 1024, 1024, 1024,
WhatParamsToTest::OnlyGenericCase,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 567, 2345, 123,
WhatParamsToTest::OnlyGenericCase,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 100, 5000, 100,
WhatParamsToTest::OnlyGenericCase,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 1, 1, 1000, WhatParamsToTest::OnlyGenericCase,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 1000, 1, 1, WhatParamsToTest::OnlyGenericCase,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 1, 1000, 1, WhatParamsToTest::OnlyGenericCase,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 1, 1000, 1000,
WhatParamsToTest::OnlyGenericCase,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 1000, 1, 1000,
WhatParamsToTest::OnlyGenericCase,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 1000, 1000, 1,
WhatParamsToTest::OnlyGenericCase,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 777, 3456, 1,
WhatParamsToTest::OnlyGenericCase,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 4567, 555, 1,
WhatParamsToTest::OnlyGenericCase,
WhatOrdersToTest::OnlyRCC);
// Test all storage orders
test_gemm<GemmWrapper>(context, 70, 90, 110, WhatParamsToTest::All,
WhatOrdersToTest::All);
test_gemm<GemmWrapper>(context, 300, 400, 500,
WhatParamsToTest::OnlyGenericCase,
WhatOrdersToTest::All);
}
template <typename GemmWrapper>
void test_gemv(typename GemmWrapper::Context* context) {
test_gemm<GemmWrapper>(context, 2, 2, 1, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 3, 3, 1, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 4, 4, 1, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 5, 5, 1, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 6, 6, 1, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 3, 5, 1, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 7, 3, 1, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 5, 7, 1, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 8, 8, 1, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 32, 32, 1, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 128, 128, 1, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
test_gemm<GemmWrapper>(context, 321, 123, 1, WhatParamsToTest::All,
WhatOrdersToTest::OnlyRCC);
// Test all storage orders
test_gemm<GemmWrapper>(context, 70, 90, 1, WhatParamsToTest::All,
WhatOrdersToTest::All);
test_gemm<GemmWrapper>(context, 300, 400, 1,
WhatParamsToTest::OnlyGenericCase,
WhatOrdersToTest::All);
}
const char* GetBitDepthName(eight_bit_int_gemm::BitDepthSetting b) {
switch (b) {
case eight_bit_int_gemm::BitDepthSetting::A8B8:
return "Lhs: 8 bit, Rhs: 8 bit";
case eight_bit_int_gemm::BitDepthSetting::A5B7:
return "Lhs: 7 bit, Rhs: 5 bit";
default:
abort();
return nullptr;
}
}
// Runs a small set of hand-picked data for per-channel quantized data.
// This test case comes from a set of 2 2x2 convolution filters run over a 3x3
// image.
void TestWithSmallDataPerChannelQuantization() {
const int m = 2;
const int n = 9;
const int k = 12;
// 12 x 2, columnwise.
const uint8_t a_data[] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255,
64, 64, 64, 64, 64, 64, 0, 0, 0, 255, 255, 255
};
const int lda = k;
int a_offset[] = {0, -64};
MatrixMap<const std::uint8_t, MapOrder::RowMajor> lhs(a_data, m, k, lda);
const OffsetColMap lhs_offset(a_offset, m);
// 12 x 9, columnwise.
const uint8_t b_data[] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255,
0, 0, 0, 0, 0, 0, 255, 255, 255, 0, 0, 0,
0, 0, 0, 127, 127, 127, 0, 0, 0, 127, 127, 127,
0, 0, 0, 255, 255, 255, 0, 0, 0, 0, 0, 0,
255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 127, 127, 127, 0, 0, 0, 127, 127, 127,
0, 0, 0, 0, 0, 0, 127, 127, 127, 127, 127, 127,
0, 0, 0, 0, 0, 0, 127, 127, 127, 127, 127, 127,
0, 0, 0, 127, 127, 127, 127, 127, 127, 127, 127, 127
};
const int ldb = k;
int b_offset = -127;
MatrixMap<const std::uint8_t, MapOrder::ColMajor> rhs(b_data, k, n, ldb);
const OffsetRowDup rhs_offset(b_offset, rhs.cols());
// 2 x 9, columnwise.
const uint8_t expected_c_data[] = {
255, 255,
0, 0,
127, 159,
0, 64,
0, 64,
127, 159,
127, 127,
127, 127,
127, 127
};
const int ldc = m;
int c_offset[] = {97155, 97346};
int c_mult_int[] = {2741, 2741};
const int c_shift = 21;
const int c_count = m * n;
std::unique_ptr<uint8_t[]> output_data(new uint8_t[c_count]);
MatrixMap<std::uint8_t, MapOrder::ColMajor> result(output_data.get(), m, n,
ldc);
const OffsetColMap result_offset(c_offset, m);
const OffsetColMap result_mult_int(c_mult_int, m);
const int result_shift = c_shift;
GemmContext gemm_context;
auto output_pipeline = MakeStandardOutputPipeline<VectorShape::Col>(
result_offset, result_mult_int, result_shift);
GemmWithOutputPipelinePC<uint8_t, uint8_t, DefaultL8R8BitDepthParams>(
&gemm_context, lhs, rhs, &result, lhs_offset, rhs_offset,
output_pipeline);
ResultStats stats;
GetResultStats(output_data.get(), expected_c_data, c_count, &stats);
ResultStatsBounds bounds;
const bool good = CheckResultStatsBounds(stats, bounds);
printf("TestWithSmallDataPerChannelQuantization: %s\n",
good ? "PASS" : "FAIL");
ReportResultStats(stats, bounds);
Check(good);
}
// Runs a larger set of hand-picked data for per-channel quantized data.
// This test case comes from a set of 22 3x3 convolution filters run over a 5x5
// image. Right now, I have 7 different filters and 15 copies of the first
// filter to make sure NEON code path that processes 16 rows at a time is
// covered.
void TestWithLargeDataPerChannelQuantization() {
const int m = 22;
const int n = 25;
const int k = 27;
// 27 x 22, column-wise.
const uint8_t a_data[] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 127, 127, 255, 255, 255,
127, 127, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 127, 127, 127, 0, 0, 0, 0, 0, 0, 255, 255, 255,
0, 0, 0, 0, 0, 0, 127, 127, 127, 0, 0, 0,
51, 51, 51, 51, 51, 51, 51, 51, 51, 0, 0, 0, 255, 255, 255,
0, 0, 0, 51, 51, 51, 51, 51, 51, 51, 51, 51,
51, 51, 51, 0, 0, 0, 51, 51, 51, 51, 51, 51, 255, 255, 255,
51, 51, 51, 51, 51, 51, 0, 0, 0, 51, 51, 51,
0, 0, 0, 64, 64, 64, 0, 0, 0, 64, 64, 64, 255, 255, 255,
64, 64, 64, 0, 0, 0, 64, 64, 64, 0, 0, 0,
36, 36, 36, 0, 0, 0, 36, 36, 36, 0, 0, 0, 255, 255, 255,
0, 0, 0, 36, 36, 36, 0, 0, 0, 36, 36, 36,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
};
const int lda = k;
int a_offset[] = {
0, 0, 0, -51, -51, 0, -36, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0
};
MatrixMap<const std::uint8_t, MapOrder::RowMajor> lhs(a_data, m, k, lda);
const OffsetColMap lhs_offset(a_offset, m);
// 27 x 25, column-wise.
const uint8_t b_data[] = {
127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 119, 119, 119,
119, 119, 119, 127, 127, 127, 119, 119, 119, 119, 119, 119,
127, 127, 127, 127, 127, 127, 127, 127, 127, 119, 119, 119, 119, 119, 119,
119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
127, 127, 127, 127, 127, 127, 127, 127, 127, 119, 119, 119, 119, 119, 119,
119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
127, 127, 127, 127, 127, 127, 127, 127, 127, 119, 119, 119, 119, 119, 119,
119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
127, 127, 127, 127, 127, 127, 127, 127, 127, 119, 119, 119, 119, 119, 119,
127, 127, 127, 119, 119, 119, 119, 119, 119, 127, 127, 127,
127, 127, 127, 119, 119, 119, 119, 119, 119, 127, 127, 127, 119, 119, 119,
119, 119, 119, 127, 127, 127, 119, 119, 119, 119, 119, 119,
119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
119, 119, 119, 119, 119, 119, 119, 119, 119, 136, 136, 136,
119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
119, 119, 119, 119, 119, 119, 136, 136, 136, 119, 119, 119,
119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
119, 119, 119, 136, 136, 136, 119, 119, 119, 119, 119, 119,
119, 119, 119, 119, 119, 119, 127, 127, 127, 119, 119, 119, 119, 119, 119,
127, 127, 127, 119, 119, 119, 119, 119, 119, 127, 127, 127,
127, 127, 127, 119, 119, 119, 119, 119, 119, 127, 127, 127, 119, 119, 119,
119, 119, 119, 127, 127, 127, 119, 119, 119, 119, 119, 119,
119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
136, 136, 136, 119, 119, 119, 119, 119, 119, 119, 119, 119,
119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 136, 136, 136,
119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
119, 119, 119, 119, 119, 119, 119, 119, 119, 136, 136, 136, 119, 119, 119,
119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
119, 119, 119, 119, 119, 119, 127, 127, 127, 119, 119, 119, 119, 119, 119,
127, 127, 127, 119, 119, 119, 119, 119, 119, 127, 127, 127,
127, 127, 127, 119, 119, 119, 119, 119, 119, 127, 127, 127, 119, 119, 119,
119, 119, 119, 127, 127, 127, 119, 119, 119, 119, 119, 119,
119, 119, 119, 119, 119, 119, 136, 136, 136, 119, 119, 119, 119, 119, 119,
119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
119, 119, 119, 136, 136, 136, 119, 119, 119, 119, 119, 119, 119, 119, 119,
119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
136, 136, 136, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
119, 119, 119, 119, 119, 119, 127, 127, 127, 119, 119, 119, 119, 119, 119,
127, 127, 127, 119, 119, 119, 119, 119, 119, 127, 127, 127,
127, 127, 127, 119, 119, 119, 119, 119, 119, 127, 127, 127, 119, 119, 119,
119, 119, 119, 127, 127, 127, 127, 127, 127, 127, 127, 127,
119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
119, 119, 119, 127, 127, 127, 127, 127, 127, 127, 127, 127,
119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
119, 119, 119, 127, 127, 127, 127, 127, 127, 127, 127, 127,
119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
119, 119, 119, 127, 127, 127, 127, 127, 127, 127, 127, 127,
119, 119, 119, 119, 119, 119, 127, 127, 127, 119, 119, 119, 119, 119, 119,
127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127
};
const int ldb = k;
int b_offset = -127;
MatrixMap<const std::uint8_t, MapOrder::ColMajor> rhs(b_data, k, n, ldb);
const OffsetRowDup rhs_offset(b_offset, rhs.cols());
// 22 x 25, column-wise.
const uint8_t expected_c_data[] = {
7, 37, 37, 67, 67, 39, 79, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
7, 7, 37, 87, 67, 23, 91, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
7, 7, 37, 87, 67, 23, 91, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
7, 7, 37, 87, 67, 23, 91, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
7, 37, 37, 67, 67, 39, 79, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
7, 37, 7, 67, 87, 23, 91, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 87, 87, 7, 103, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
7, 7, 71, 87, 45, 41, 77, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 87, 87, 7, 103, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
7, 37, 7, 67, 87, 23, 91, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
7, 37, 7, 67, 87, 23, 91, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
7, 71, 7, 45, 87, 41, 77, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
255, 135, 135, 255, 255, 143, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255,
7, 71, 7, 45, 87, 41, 77, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
7, 37, 7, 67, 87, 23, 91, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
7, 37, 7, 67, 87, 23, 91, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 87, 87, 7, 103, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
7, 7, 71, 87, 45, 41, 77, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 87, 87, 7, 103, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
7, 37, 7, 67, 87, 23, 91, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
7, 37, 37, 67, 67, 39, 79, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
7, 7, 37, 87, 67, 23, 91, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
7, 7, 37, 87, 67, 23, 91, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
7, 7, 37, 87, 67, 23, 91, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
7, 37, 37, 67, 67, 39, 79, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7,
99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99,
99, 99, 99, 99, 99, 99, 99,
111, 111, 111, 111, 111, 111, 111, 111, 111, 111, 111, 111, 111, 111, 111,
111, 111, 111, 111, 111, 111, 111,
};
const int ldc = m;
int c_offset[] = {
6477, 12954, 12954, 7793, 7793, 12954, 9282, 6477, 6477, 6477,
6477, 6477, 6477, 6477, 6477, 6477, 6477, 6477, 6477, 6477,
6477, 6477,
};
int c_mult_int[] = {
41121, 20560, 20560, 34267, 34267, 21937, 28784, 41121, 41121, 41121,
41121, 41121, 41121, 41121, 41121, 41121, 41121, 41121, 41121, 41121,
41121, 41121,
};
const int c_shift = 21;
const int c_count = m * n;
std::unique_ptr<uint8_t[]> output_data(new uint8_t[c_count]);
MatrixMap<std::uint8_t, MapOrder::ColMajor> result(output_data.get(), m, n,
ldc);
const OffsetColMap result_offset(c_offset, m);
const OffsetColMap result_mult_int(c_mult_int, m);
const int result_shift = c_shift;
GemmContext gemm_context;
auto output_pipeline = MakeStandardOutputPipeline<VectorShape::Col>(
result_offset, result_mult_int, result_shift);
GemmWithOutputPipelinePC<uint8_t, uint8_t, DefaultL8R8BitDepthParams>(
&gemm_context, lhs, rhs, &result, lhs_offset, rhs_offset,
output_pipeline);
ResultStats stats;
GetResultStats(output_data.get(), expected_c_data, c_count, &stats);
ResultStatsBounds bounds;
const bool good = CheckResultStatsBounds(stats, bounds);
printf("TestWithLargeDataPerChannelQuantization: %s\n",
good ? "PASS" : "FAIL");
ReportResultStats(stats, bounds);
Check(good);
}
// Runs a small set of hand-calculated data through the implementation.
void TestWithSmallData() {
const int m = 4;
const int n = 2;
const int k = 3;
// Matrix A (LHS) is:
// | 7 | 10 | 13 | 16 |
// | 8 | 11 | 14 | 17 |
// | 9 | 12 | 15 | 18 |
const uint8_t a_data[] = {7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18};
// Matrix B (RHS) is:
// | 1 | 3 | 5 |
// | 2 | 4 | 6 |
const uint8_t b_data[] = {1, 2, 3, 4, 5, 6};
// Here are the results we expect, from hand calculations:
// (1 * 7) + (3 * 8) + (5 * 9) = 76
// (2 * 7) + (4 * 8) + (6 * 9) = 100
// (1 * 10) + (3 * 11) + (5 * 12) = 103
// (2 * 10) + (4 * 11) + (6 * 12) = 136
// (1 * 13) + (3 * 14) + (5 * 15) = 130
// (2 * 13) + (4 * 14) + (6 * 15) = 172
// (1 * 16) + (3 * 17) + (5 * 18) = 157
// (2 * 16) + (4 * 17) + (6 * 18) = 208
// That means matrix C should be:
// | 76 | 103 | 130 | 157 |
// | 100 | 136 | 172 | 208 |
const uint8_t expected_data[] = {76, 100, 103, 136, 130, 172, 157, 208};
const int c_count = m * n;
std::unique_ptr<uint8_t[]> output_data(new uint8_t[c_count]);
const bool is_a_transposed = true;
const bool is_b_transposed = true;
const bool is_c_transposed = true;
const int lda = k;
const int ldb = n;
const int ldc = n;
const int a_offset = 0;
const int b_offset = 0;
const int c_offset = 0;
const int c_mult = 1;
const int c_shift = 0;
gemmlowp::eight_bit_int_gemm::EightBitIntGemm(
is_a_transposed, is_b_transposed, is_c_transposed, m, n, k, a_data,
a_offset, lda, b_data, b_offset, ldb, output_data.get(), c_offset, c_mult,
c_shift, ldc, eight_bit_int_gemm::BitDepthSetting::A8B8);
ResultStats stats;
GetResultStats(output_data.get(), expected_data, c_count, &stats);
ResultStatsBounds bounds;
const bool good = CheckResultStatsBounds(stats, bounds);
printf("TestWithSmallData: %s\n", good ? "PASS" : "FAIL");
ReportResultStats(stats, bounds);
Check(good);
}
// This is the most realistic test of how we'll be using the low-precision GEMM
// function in applications. It takes in large input matrices that have been
// captured from an actual neural network run.
void TestWithRealData(eight_bit_int_gemm::BitDepthSetting BitDepth,
int tolerance_median, int tolerance_max) {
std::unique_ptr<uint8_t[]> output_data(new uint8_t[test_data::c_count]);
gemmlowp::eight_bit_int_gemm::EightBitIntGemm(
test_data::is_a_transposed, test_data::is_b_transposed,
test_data::is_c_transposed, test_data::m, test_data::n, test_data::k,
test_data::a_data, test_data::a_offset, test_data::k, test_data::b_data,
test_data::b_offset, test_data::k, output_data.get(), test_data::c_offset,
test_data::c_mult_int, test_data::c_shift, test_data::m, BitDepth);
ResultStats stats;
GetResultStats(output_data.get(), test_data::expected_c_data,
test_data::c_count, &stats);
ResultStatsBounds bounds;
if (BitDepth == eight_bit_int_gemm::BitDepthSetting::A5B7) {
bounds.med_unsigned_diff = tolerance_median;
bounds.max_unsigned_diff = tolerance_max;
bounds.med_signed_diff = 0;
bounds.mean_signed_diff = 0.2f;
}
const bool good = CheckResultStatsBounds(stats, bounds);
printf("TestWithRealData: %s with %s\n", good ? "PASS" : "FAIL",
GetBitDepthName(BitDepth));
ReportResultStats(stats, bounds);
Check(good);
}
template <MapOrder ResultOrder>
void TestOutputStages(int rows, int depth, int cols, int result_offset,
int result_mult_int, int result_shift) {
Matrix<std::uint8_t, MapOrder::RowMajor> lhs(rows, depth);
Matrix<std::uint8_t, MapOrder::ColMajor> rhs(depth, cols);
Matrix<std::int32_t, ResultOrder> result_raw_int32(rows, cols);
MakeRandom(&lhs, 8);
MakeRandom(&rhs, 8);
const int lhs_offset = 12;
const int rhs_offset = -34;
// Test an empty pipeline, i.e. returning raw int32 accumulators.
auto empty_pipeline = std::make_tuple();
GemmContext context;
GemmWithOutputPipeline<std::uint8_t, std::int32_t, DefaultL8R8BitDepthParams>(
&context, lhs.const_map(), rhs.const_map(), &result_raw_int32, lhs_offset,
rhs_offset, empty_pipeline);
for (int r = 0; r < rows; r++) {
for (int c = 0; c < cols; c++) {
std::int32_t expected = 0;
for (int d = 0; d < depth; d++) {
std::int32_t lhs_val =
static_cast<std::int32_t>(lhs(r, d)) + lhs_offset;
std::int32_t rhs_val =
static_cast<std::int32_t>(rhs(d, c)) + rhs_offset;
expected += lhs_val * rhs_val;
}
Check(expected == result_raw_int32(r, c));
}
}
// Test a pipeline with only the quantize-down stage, still returning
// unclamped (but scaled) int32's
OutputStageQuantizeDownInt32ToUint8Scale quantize_down_stage;
quantize_down_stage.result_offset = result_offset;
quantize_down_stage.result_mult_int = result_mult_int;
quantize_down_stage.result_shift = result_shift;
auto quantize_down_pipeline = std::make_tuple(quantize_down_stage);
Matrix<std::int32_t, ResultOrder> result_quantized_down_int32(rows, cols);
GemmWithOutputPipeline<std::uint8_t, std::int32_t, DefaultL8R8BitDepthParams>(
&context, lhs.const_map(), rhs.const_map(), &result_quantized_down_int32,
lhs_offset, rhs_offset, quantize_down_pipeline);
std::uint64_t sum = 0;
for (int r = 0; r < rows; r++) {
for (int c = 0; c < cols; c++) {
std::int32_t raw = result_raw_int32(r, c);
const std::int32_t rounding =
(result_shift < 1) ? 0 : (1 << (result_shift - 1));
std::int32_t expected =
((raw + result_offset) * result_mult_int + rounding) >> result_shift;
Check(expected == result_quantized_down_int32(r, c));
sum += expected;
}
}
std::uint64_t avg = sum / (rows * cols);
// Test that the average quantized-down value falls reasonably in the
// middle of the [0..255] range. Otherwise, the multiplier / shift need to be
// adjusted.
Check(avg >= 64 && avg <= 192);
// Test the familiar default pipeline consisting of quantize-down and
// clamp-and-cast-to-uint8.
OutputStageSaturatingCastToUint8 saturating_cast_stage;
auto quantize_down_and_saturating_cast_pipeline =
std::make_tuple(quantize_down_stage, saturating_cast_stage);
Matrix<std::uint8_t, ResultOrder> result_quantized_down_saturated_uint8(rows,
cols);
GemmWithOutputPipeline<std::uint8_t, std::uint8_t, DefaultL8R8BitDepthParams>(
&context, lhs.const_map(), rhs.const_map(),
&result_quantized_down_saturated_uint8, lhs_offset, rhs_offset,
quantize_down_and_saturating_cast_pipeline);
for (int r = 0; r < rows; r++) {
for (int c = 0; c < cols; c++) {
std::int32_t quantized = result_quantized_down_int32(r, c);
std::uint8_t expected = std::min(std::max(quantized, 0), 255);
Check(expected == result_quantized_down_saturated_uint8(r, c));
}
}
// Test a bias-addition with row-vector
std::vector<std::int32_t> row_vector_data(cols);
for (int i = 0; i < cols; i++) {
row_vector_data[i] = (Random() % 1000) - 500;
}
typedef VectorMap<std::int32_t, VectorShape::Row> RowVectorMap;
RowVectorMap row_vector_map(row_vector_data.data(), cols);
OutputStageBiasAddition<RowVectorMap> row_bias_addition_stage;
row_bias_addition_stage.bias_vector = row_vector_map;
auto row_bias_addition_pipeline = std::make_tuple(row_bias_addition_stage);
Matrix<std::int32_t, ResultOrder> result_of_row_bias_addition(rows, cols);
GemmWithOutputPipeline<std::uint8_t, std::int32_t, DefaultL8R8BitDepthParams>(
&context, lhs.const_map(), rhs.const_map(), &result_of_row_bias_addition,
lhs_offset, rhs_offset, row_bias_addition_pipeline);
for (int r = 0; r < rows; r++) {
for (int c = 0; c < cols; c++) {
std::int32_t expected = result_raw_int32(r, c) + row_vector_data[c];
Check(expected == result_of_row_bias_addition(r, c));
}
}
// Test a bias-addition with column-vector
std::vector<std::int32_t> col_vector_data(rows);
for (int i = 0; i < rows; i++) {
col_vector_data[i] = (Random() % 1000) - 500;
}
typedef VectorMap<std::int32_t, VectorShape::Col> ColVectorMap;
ColVectorMap col_vector_map(col_vector_data.data(), rows);
OutputStageBiasAddition<ColVectorMap> col_bias_addition_stage;
col_bias_addition_stage.bias_vector = col_vector_map;
auto col_bias_addition_pipeline = std::make_tuple(col_bias_addition_stage);
Matrix<std::int32_t, ResultOrder> result_of_col_bias_addition(rows, cols);
GemmWithOutputPipeline<std::uint8_t, std::int32_t, DefaultL8R8BitDepthParams>(
&context, lhs.const_map(), rhs.const_map(), &result_of_col_bias_addition,
lhs_offset, rhs_offset, col_bias_addition_pipeline);
for (int r = 0; r < rows; r++) {
for (int c = 0; c < cols; c++) {
std::int32_t expected = result_raw_int32(r, c) + col_vector_data[r];
Check(expected == result_of_col_bias_addition(r, c));
}
}
// Test a clamp
OutputStageClamp clamp_stage;
// Determine min and max of raw int32 accumulators
std::int32_t raw_min = std::numeric_limits<std::int32_t>::max();
std::int32_t raw_max = std::numeric_limits<std::int32_t>::min();
for (int r = 0; r < rows; r++) {
for (int c = 0; c < cols; c++) {
raw_min = std::min(raw_min, result_raw_int32(r, c));
raw_max = std::max(raw_max, result_raw_int32(r, c));
}
}
// Pick some interesting clamp min/max bounds
clamp_stage.min = static_cast<std::int32_t>(raw_min * 0.7 + raw_max * 0.3);
clamp_stage.max = static_cast<std::int32_t>(raw_min * 0.3 + raw_max * 0.7);
assert(raw_min <= clamp_stage.min && clamp_stage.min <= clamp_stage.max &&
clamp_stage.max <= raw_max);
auto clamp_pipeline = std::make_tuple(clamp_stage);
Matrix<std::int32_t, ResultOrder> result_clamped(rows, cols);
GemmWithOutputPipeline<std::uint8_t, std::int32_t, DefaultL8R8BitDepthParams>(
&context, lhs.const_map(), rhs.const_map(), &result_clamped, lhs_offset,
rhs_offset, clamp_pipeline);
for (int r = 0; r < rows; r++) {
for (int c = 0; c < cols; c++) {
std::int32_t raw = result_raw_int32(r, c);
std::int32_t expected =
std::min(std::max(raw, clamp_stage.min), clamp_stage.max);
Check(expected == result_clamped(r, c));
}
}
// Test tanh
OutputStageTanh tanh_stage;
const std::int32_t real_zero_as_int32 = (raw_max + raw_min) / 2;
const std::int32_t real_amplitude_as_int32 = (raw_max - raw_min) / 16;
tanh_stage.real_zero_as_int32 = real_zero_as_int32;
tanh_stage.real_amplitude_as_int32 = real_amplitude_as_int32;
auto tanh_pipeline = std::make_tuple(tanh_stage);
Matrix<std::int32_t, ResultOrder> result_tanh(rows, cols);
GemmWithOutputPipeline<std::uint8_t, std::int32_t, DefaultL8R8BitDepthParams>(
&context, lhs.const_map(), rhs.const_map(), &result_tanh, lhs_offset,
rhs_offset, tanh_pipeline);
for (int r = 0; r < rows; r++) {
for (int c = 0; c < cols; c++) {
std::int32_t raw = result_raw_int32(r, c);
double real_input =
double(raw - real_zero_as_int32) / real_amplitude_as_int32;
double expected = std::tanh(real_input);
std::int32_t actual_int32 = result_tanh(r, c);
double actual =
double(actual_int32 - real_zero_as_int32) / real_amplitude_as_int32;
Check(std::abs(expected - actual) < 2e-4);
}
}
// Test a pipeline with bias and clamp
auto bias_clamp_pipeline =
std::make_tuple(col_bias_addition_stage, clamp_stage);
Matrix<std::int32_t, ResultOrder> result_biased_clamped(rows, cols);
GemmWithOutputPipeline<std::uint8_t, std::int32_t, DefaultL8R8BitDepthParams>(
&context, lhs.const_map(), rhs.const_map(), &result_biased_clamped,
lhs_offset, rhs_offset, bias_clamp_pipeline);
for (int r = 0; r < rows; r++) {
for (int c = 0; c < cols; c++) {
std::int32_t raw = result_raw_int32(r, c);
std::int32_t biased = raw + col_vector_data[r];
std::int32_t expected =
std::min(std::max(biased, clamp_stage.min), clamp_stage.max);
Check(expected == result_biased_clamped(r, c));
}
}
// Test a full pipeline with bias and clamp and quantization down to 8bit
// result
auto bias_clamp_quantize_cast_pipeline =
std::make_tuple(col_bias_addition_stage, clamp_stage, quantize_down_stage,
saturating_cast_stage);
Matrix<std::uint8_t, ResultOrder> result_biased_clamped_quantized_casted(
rows, cols);
GemmWithOutputPipeline<std::uint8_t, std::uint8_t, DefaultL8R8BitDepthParams>(
&context, lhs.const_map(), rhs.const_map(),
&result_biased_clamped_quantized_casted, lhs_offset, rhs_offset,
bias_clamp_quantize_cast_pipeline);
for (int r = 0; r < rows; r++) {
for (int c = 0; c < cols; c++) {
const std::int32_t rounding =
(result_shift < 1) ? 0 : (1 << (result_shift - 1));
std::int32_t quantized =
((result_biased_clamped(r, c) + result_offset) * result_mult_int +
rounding) >>
result_shift;
std::uint8_t expected = std::min(std::max(quantized, 0), 255);
Check(expected == result_biased_clamped_quantized_casted(r, c));
}
}
printf("TestOutputStages: PASS with ResultOrder=%s\n",
OrderName(ResultOrder));
}
#ifndef GEMMLOWP_SKIP_EXHAUSTIVE_TESTS
void TestExhaustively() {
GemmContext context;
// Test the internal GEMM interfaces
test_gemm<SingleThreadGemmWrapper<
DefaultKernel<KernelFamily::Gemm, DefaultL8R8BitDepthParams>,
std::uint8_t, DefaultL8R8BitDepthParams>>(&context);
test_gemm<MultiThreadGemmWrapper<
DefaultKernel<KernelFamily::Gemm, DefaultL8R8BitDepthParams>,
std::uint8_t, DefaultL8R8BitDepthParams>>(&context);
// Test the public GEMM interfaces
test_gemm<PublicGemmWrapper<uint8_t, DefaultL8R8BitDepthParams>>(&context);
test_gemm<EightBitIntGemmWrapper<uint8_t,
eight_bit_int_gemm::BitDepthSetting::A8B8>>(
&context);
// Test GEMV cases (internal interfaces)
test_gemv<SingleThreadGemmWrapper<
DefaultKernel<KernelFamily::Gemv, DefaultL8R8BitDepthParams>,
std::uint8_t, DefaultL8R8BitDepthParams>>(&context);
test_gemv<MultiThreadGemmWrapper<
DefaultKernel<KernelFamily::Gemv, DefaultL8R8BitDepthParams>,
std::uint8_t, DefaultL8R8BitDepthParams>>(&context);
// Test GEMV cases (public interfaces)
test_gemv<PublicGemmWrapper<uint8_t, DefaultL8R8BitDepthParams>>(&context);
test_gemv<EightBitIntGemmWrapper<uint8_t,
eight_bit_int_gemm::BitDepthSetting::A8B8>>(
&context);
// Test other bit depths
// L7R5
test_gemm<SingleThreadGemmWrapper<
DefaultKernel<KernelFamily::Gemm, DefaultL7R5BitDepthParams>,
std::uint8_t, DefaultL7R5BitDepthParams>>(&context);
test_gemv<SingleThreadGemmWrapper<
DefaultKernel<KernelFamily::Gemv, DefaultL7R5BitDepthParams>,
std::uint8_t, DefaultL7R5BitDepthParams>>(&context);
test_gemm<EightBitIntGemmWrapper<std::uint8_t,
eight_bit_int_gemm::BitDepthSetting::A5B7>>(
&context);
// Test specific kernels with various different formats,
// to exercises corner cases especially in the packing code.
test_gemm_kernel<
ReferenceKernel<KernelFormat<KernelSideFormat<CellFormat<1, 1>, 1>,
KernelSideFormat<CellFormat<1, 1>, 1>>>>(
&context);
test_gemm_kernel<
ReferenceKernel<KernelFormat<KernelSideFormat<CellFormat<4, 2>, 1>,
KernelSideFormat<CellFormat<4, 2>, 2>>>>(
&context);
test_gemm_kernel<
ReferenceKernel<KernelFormat<KernelSideFormat<CellFormat<4, 2>, 4>,
KernelSideFormat<CellFormat<4, 2>, 5>>>>(
&context);
test_gemm_kernel<ReferenceKernel<KernelFormat<
KernelSideFormat<CellFormat<3, 4, CellOrder::DepthMajor>, 2>,
KernelSideFormat<CellFormat<5, 4, CellOrder::DepthMajor>, 3>>>>(&context);
test_gemm_kernel<ReferenceKernel<KernelFormat<
KernelSideFormat<CellFormat<3, 4, CellOrder::WidthMajor>, 2>,
KernelSideFormat<CellFormat<5, 4, CellOrder::WidthMajor>, 3>>>>(&context);
test_gemm_kernel<ReferenceKernel<KernelFormat<
KernelSideFormat<CellFormat<5, 2, CellOrder::WidthMajor>, 3>,
KernelSideFormat<CellFormat<4, 2, CellOrder::DepthMajor>, 2>>>>(&context);
test_gemm_kernel<ReferenceKernel<KernelFormat<
KernelSideFormat<CellFormat<5, 2, CellOrder::DepthMajor>, 3>,
KernelSideFormat<CellFormat<4, 2, CellOrder::WidthMajor>, 2>>>>(&context);
test_gemm_kernel<ReferenceKernel<KernelFormat<
KernelSideFormat<CellFormat<8, 8, CellOrder::Diagonal>, 2>,
KernelSideFormat<CellFormat<3, 8, CellOrder::WidthMajor>, 1>>>>(&context);
test_gemm_kernel<ReferenceKernel<KernelFormat<
KernelSideFormat<CellFormat<1, 4, CellOrder::DepthMajor>, 1>,
KernelSideFormat<CellFormat<4, 4, CellOrder::Diagonal>, 1>>>>(&context);
}
#endif // not GEMMLOWP_SKIP_EXHAUSTIVE_TESTS
void test() {
#ifdef GEMMLOWP_TEST_PROFILE
RegisterCurrentThreadForProfiling();
StartProfiling();
#endif
// Run a first quick test against hand-calculated data.
TestWithSmallData();
#ifndef GEMMLOWP_SKIP_EXHAUSTIVE_TESTS
TestExhaustively();
#endif
// Run against actual data from a network evaluation.
TestWithRealData(eight_bit_int_gemm::BitDepthSetting::A8B8, 0, 0);
TestWithRealData(eight_bit_int_gemm::BitDepthSetting::A5B7, 2, 10);
// Test non-default output pipelines with various combinations of
// output stages.
TestOutputStages<MapOrder::RowMajor>(63, 10, 127, 5, 17, 14);
TestOutputStages<MapOrder::ColMajor>(63, 10, 127, 5, 17, 14);
// Test per channel quantization.
TestWithSmallDataPerChannelQuantization();
TestWithLargeDataPerChannelQuantization();
#ifdef GEMMLOWP_TEST_PROFILE
FinishProfiling();
#endif
std::cerr << "All tests passed." << std::endl;
// We have been testing the eight_bit_int_gemm, so we should free its
// persistent
// resources now to avoid having leak-checking tools report leaks.
eight_bit_int_gemm::FreePersistentResources();
}
} // end namespace gemmlowp
// For iOS, we need to define our own main(), so skip it here.
#if !(defined(__APPLE__) && (TARGET_OS_IPHONE || TARGET_IPHONE_SIMULATOR))
int main() { gemmlowp::test(); }
#endif