/** @file ***************************************************************************** Implementation of interfaces for multi-exponentiation routines. See multiexp.hpp . ***************************************************************************** * @author This file is part of libsnark, developed by SCIPR Lab * and contributors (see AUTHORS). * @copyright MIT license (see LICENSE file) *****************************************************************************/ #ifndef MULTIEXP_TCC_ #define MULTIEXP_TCC_ #include "algebra/fields/fp_aux.tcc" #include #include #include #include "common/profiling.hpp" #include "common/utils.hpp" #include "common/assert_except.hpp" #include "algebra/scalar_multiplication/wnaf.hpp" namespace libsnark { template class ordered_exponent { // to use std::push_heap and friends later public: size_t idx; bigint r; ordered_exponent(const size_t idx, const bigint &r) : idx(idx), r(r) {}; bool operator<(const ordered_exponent &other) const { #if defined(__x86_64__) && defined(USE_ASM) if (n == 3) { int64_t res; __asm__ ("// check for overflow \n\t" "mov $0, %[res] \n\t" ADD_CMP(16) ADD_CMP(8) ADD_CMP(0) "jmp done%= \n\t" "subtract%=: \n\t" "mov $1, %[res] \n\t" "done%=: \n\t" : [res] "=&r" (res) : [A] "r" (other.r.data), [mod] "r" (this->r.data) : "cc", "%rax"); return res; } else if (n == 4) { int64_t res; __asm__ ("// check for overflow \n\t" "mov $0, %[res] \n\t" ADD_CMP(24) ADD_CMP(16) ADD_CMP(8) ADD_CMP(0) "jmp done%= \n\t" "subtract%=: \n\t" "mov $1, %[res] \n\t" "done%=: \n\t" : [res] "=&r" (res) : [A] "r" (other.r.data), [mod] "r" (this->r.data) : "cc", "%rax"); return res; } else if (n == 5) { int64_t res; __asm__ ("// check for overflow \n\t" "mov $0, %[res] \n\t" ADD_CMP(32) ADD_CMP(24) ADD_CMP(16) ADD_CMP(8) ADD_CMP(0) "jmp done%= \n\t" "subtract%=: \n\t" "mov $1, %[res] \n\t" "done%=: \n\t" : [res] "=&r" (res) : [A] "r" (other.r.data), [mod] "r" (this->r.data) : "cc", "%rax"); return res; } else #endif { return (mpn_cmp(this->r.data, other.r.data, n) < 0); } } }; template T naive_exp(typename std::vector::const_iterator vec_start, typename std::vector::const_iterator vec_end, typename std::vector::const_iterator scalar_start, typename std::vector::const_iterator scalar_end) { T result(T::zero()); typename std::vector::const_iterator vec_it; typename std::vector::const_iterator scalar_it; for (vec_it = vec_start, scalar_it = scalar_start; vec_it != vec_end; ++vec_it, ++scalar_it) { bigint scalar_bigint = scalar_it->as_bigint(); result = result + opt_window_wnaf_exp(*vec_it, scalar_bigint, scalar_bigint.num_bits()); } assert_except(scalar_it == scalar_end); return result; } template T naive_plain_exp(typename std::vector::const_iterator vec_start, typename std::vector::const_iterator vec_end, typename std::vector::const_iterator scalar_start, typename std::vector::const_iterator scalar_end) { T result(T::zero()); typename std::vector::const_iterator vec_it; typename std::vector::const_iterator scalar_it; for (vec_it = vec_start, scalar_it = scalar_start; vec_it != vec_end; ++vec_it, ++scalar_it) { result = result + (*scalar_it) * (*vec_it); } assert_except(scalar_it == scalar_end); return result; } /* The multi-exponentiation algorithm below is a variant of the Bos-Coster algorithm [Bos and Coster, "Addition chain heuristics", CRYPTO '89]. The implementation uses suggestions from [Bernstein, Duif, Lange, Schwabe, and Yang, "High-speed high-security signatures", CHES '11]. */ template T multi_exp_inner(typename std::vector::const_iterator vec_start, typename std::vector::const_iterator vec_end, typename std::vector::const_iterator scalar_start, typename std::vector::const_iterator scalar_end) { const mp_size_t n = std::remove_reference::type::num_limbs; if (vec_start == vec_end) { return T::zero(); } if (vec_start + 1 == vec_end) { return (*scalar_start)*(*vec_start); } std::vector > opt_q; const size_t vec_len = scalar_end - scalar_start; const size_t odd_vec_len = (vec_len % 2 == 1 ? vec_len : vec_len + 1); opt_q.reserve(odd_vec_len); std::vector g; g.reserve(odd_vec_len); typename std::vector::const_iterator vec_it; typename std::vector::const_iterator scalar_it; size_t i; for (i=0, vec_it = vec_start, scalar_it = scalar_start; vec_it != vec_end; ++vec_it, ++scalar_it, ++i) { g.emplace_back(*vec_it); opt_q.emplace_back(ordered_exponent(i, scalar_it->as_bigint())); } std::make_heap(opt_q.begin(),opt_q.end()); assert_except(scalar_it == scalar_end); if (vec_len != odd_vec_len) { g.emplace_back(T::zero()); opt_q.emplace_back(ordered_exponent(odd_vec_len - 1, bigint(UINT64_C(0)))); } assert_except(g.size() % 2 == 1); assert_except(opt_q.size() == g.size()); T opt_result = T::zero(); while (true) { ordered_exponent &a = opt_q[0]; ordered_exponent &b = (opt_q[1] < opt_q[2] ? opt_q[2] : opt_q[1]); const size_t abits = a.r.num_bits(); if (b.r.is_zero()) { // opt_result = opt_result + (a.r * g[a.idx]); opt_result = opt_result + opt_window_wnaf_exp(g[a.idx], a.r, abits); break; } const size_t bbits = b.r.num_bits(); const size_t limit = (abits-bbits >= 20 ? 20 : abits-bbits); if (bbits < UINT64_C(1)< (x-y) A + y (B+A) mpn_sub_n(a.r.data, a.r.data, b.r.data, n); g[b.idx] = g[b.idx] + g[a.idx]; } // regardless of whether a was cleared or subtracted from we push it down, then take back up /* heapify A down */ size_t a_pos = 0; while (2*a_pos + 2< odd_vec_len) { // this is a max-heap so to maintain a heap property we swap with the largest of the two if (opt_q[2*a_pos+1] < opt_q[2*a_pos+2]) { std::swap(opt_q[a_pos], opt_q[2*a_pos+2]); a_pos = 2*a_pos+2; } else { std::swap(opt_q[a_pos], opt_q[2*a_pos+1]); a_pos = 2*a_pos+1; } } /* now heapify A up appropriate amount of times */ while (a_pos > 0 && opt_q[(a_pos-1)/2] < opt_q[a_pos]) { std::swap(opt_q[a_pos], opt_q[(a_pos-1)/2]); a_pos = (a_pos-1) / 2; } } return opt_result; } template T multi_exp(typename std::vector::const_iterator vec_start, typename std::vector::const_iterator vec_end, typename std::vector::const_iterator scalar_start, typename std::vector::const_iterator scalar_end, const size_t chunks, const bool use_multiexp) { const size_t total = vec_end - vec_start; if (total < chunks) { return naive_exp(vec_start, vec_end, scalar_start, scalar_end); } const size_t one = total/chunks; std::vector partial(chunks, T::zero()); if (use_multiexp) { #ifdef MULTICORE #pragma omp parallel for #endif for (size_t i = 0; i < chunks; ++i) { partial[i] = multi_exp_inner(vec_start + i*one, (i == chunks-1 ? vec_end : vec_start + (i+1)*one), scalar_start + i*one, (i == chunks-1 ? scalar_end : scalar_start + (i+1)*one)); } } else { #ifdef MULTICORE #pragma omp parallel for #endif for (size_t i = 0; i < chunks; ++i) { partial[i] = naive_exp(vec_start + i*one, (i == chunks-1 ? vec_end : vec_start + (i+1)*one), scalar_start + i*one, (i == chunks-1 ? scalar_end : scalar_start + (i+1)*one)); } } T final = T::zero(); for (size_t i = 0; i < chunks; ++i) { final = final + partial[i]; } return final; } template T multi_exp_with_mixed_addition(typename std::vector::const_iterator vec_start, typename std::vector::const_iterator vec_end, typename std::vector::const_iterator scalar_start, typename std::vector::const_iterator scalar_end, const size_t chunks, const bool use_multiexp) { assert_except(std::distance(vec_start, vec_end) == std::distance(scalar_start, scalar_end)); enter_block("Process scalar vector"); auto value_it = vec_start; auto scalar_it = scalar_start; const FieldT zero = FieldT::zero(); const FieldT one = FieldT::one(); std::vector p; std::vector g; T acc = T::zero(); size_t num_skip = 0; size_t num_add = 0; size_t num_other = 0; for (; scalar_it != scalar_end; ++scalar_it, ++value_it) { if (*scalar_it == zero) { // do nothing ++num_skip; } else if (*scalar_it == one) { #ifdef USE_MIXED_ADDITION acc = acc.mixed_add(*value_it); #else acc = acc + (*value_it); #endif ++num_add; } else { p.emplace_back(*scalar_it); g.emplace_back(*value_it); ++num_other; } } //print_indent(); printf("* Elements of w skipped: %zu (%0.2f%%)\n", num_skip, 100.*num_skip/(num_skip+num_add+num_other)); //print_indent(); printf("* Elements of w processed with special addition: %zu (%0.2f%%)\n", num_add, 100.*num_add/(num_skip+num_add+num_other)); //print_indent(); printf("* Elements of w remaining: %zu (%0.2f%%)\n", num_other, 100.*num_other/(num_skip+num_add+num_other)); leave_block("Process scalar vector"); return acc + multi_exp(g.begin(), g.end(), p.begin(), p.end(), chunks, use_multiexp); } template size_t get_exp_window_size(const size_t num_scalars) { if (T::fixed_base_exp_window_table.empty()) { #ifdef LOWMEM return 14; #else return 17; #endif } size_t window = 1; for (int64_t i = T::fixed_base_exp_window_table.size()-1; i >= 0; --i) { #ifdef DEBUG if (!inhibit_profiling_info) { printf("%ld %zu %zu\n", i, num_scalars, T::fixed_base_exp_window_table[i]); } #endif if (T::fixed_base_exp_window_table[i] != 0 && num_scalars >= T::fixed_base_exp_window_table[i]) { window = i+1; break; } } if (!inhibit_profiling_info) { print_indent(); printf("Choosing window size %zu for %zu elements\n", window, num_scalars); } #ifdef LOWMEM window = std::min((size_t)14, window); #endif return window; } template window_table get_window_table(const size_t scalar_size, const size_t window, const T &g) { const size_t in_window = UINT64_C(1)< powers_of_g(outerc, std::vector(in_window, T::zero())); T gouter = g; for (size_t outer = 0; outer < outerc; ++outer) { T ginner = T::zero(); size_t cur_in_window = outer == outerc-1 ? last_in_window : in_window; for (size_t inner = 0; inner < cur_in_window; ++inner) { powers_of_g[outer][inner] = ginner; ginner = ginner + gouter; } for (size_t i = 0; i < window; ++i) { gouter = gouter + gouter; } } return powers_of_g; } template T windowed_exp(const size_t scalar_size, const size_t window, const window_table &powers_of_g, const FieldT &pow) { const size_t outerc = (scalar_size+window-1)/window; const bigint pow_val = pow.as_bigint(); /* exp */ T res = powers_of_g[0][0]; for (size_t outer = 0; outer < outerc; ++outer) { size_t inner = 0; for (size_t i = 0; i < window; ++i) { if (pow_val.test_bit(outer*window + i)) { inner |= 1u << i; } } res = res + powers_of_g[outer][inner]; } return res; } template std::vector batch_exp(const size_t scalar_size, const size_t window, const window_table &table, const std::vector &v) { if (!inhibit_profiling_info) { print_indent(); } std::vector res(v.size(), table[0][0]); #ifdef MULTICORE #pragma omp parallel for #endif for (size_t i = 0; i < v.size(); ++i) { res[i] = windowed_exp(scalar_size, window, table, v[i]); if (!inhibit_profiling_info && (i % 10000 == 0)) { printf("."); fflush(stdout); } } if (!inhibit_profiling_info) { printf(" DONE!\n"); } return res; } template std::vector batch_exp_with_coeff(const size_t scalar_size, const size_t window, const window_table &table, const FieldT &coeff, const std::vector &v) { if (!inhibit_profiling_info) { print_indent(); } std::vector res(v.size(), table[0][0]); #ifdef MULTICORE #pragma omp parallel for #endif for (size_t i = 0; i < v.size(); ++i) { res[i] = windowed_exp(scalar_size, window, table, coeff * v[i]); if (!inhibit_profiling_info && (i % 10000 == 0)) { printf("."); fflush(stdout); } } if (!inhibit_profiling_info) { printf(" DONE!\n"); } return res; } template void batch_to_special(std::vector &vec) { enter_block("Batch-convert elements to special form"); std::vector non_zero_vec; for (size_t i = 0; i < vec.size(); ++i) { if (!vec[i].is_zero()) { non_zero_vec.emplace_back(vec[i]); } } batch_to_special_all_non_zeros(non_zero_vec); auto it = non_zero_vec.begin(); T zero_special = T::zero(); zero_special.to_special(); for (size_t i = 0; i < vec.size(); ++i) { if (!vec[i].is_zero()) { vec[i] = *it; ++it; } else { vec[i] = zero_special; } } leave_block("Batch-convert elements to special form"); } } // libsnark #endif // MULTIEXP_TCC_