From f2c103207f5137029bffd2c948d202498433eb89 Mon Sep 17 00:00:00 2001 From: Asher Dawes Date: Fri, 10 Aug 2018 17:44:52 -0700 Subject: [PATCH] Portable dev (#105) * Force portable code * Switch to old MMX instructions, avoiding SSE & SSE2 instructions. * Less agressive, leave verus code (which checks for it) using advanced instructions. * Compiling only for windows * Update .gitlab-ci.yml * Try -march=native for C++ code generation. * Tweaking machine architecture settings. * Try native alone. * Also get LIBCRYPTOCONDITIONS to -march=native * Switch other lib to native as well. * Try switching back further for CPU architecture. * Even lower end settings. * Turn on symbols. * Use sse2,3 and 4 capable x64. * Once again let verus lib use advances instructioins since it checjs via CPUID at run time. * Modify a few more makefile entries. * Switch to AMD model similar to our test system. * Get snark makefile to k8 too. * Yet another -march to modify to k8, or two of them. * Brute force k8 settings, comment non-portable code out. * Put the condition on cpuid back. * Put non-portable advenced instruction code back * Enable instructions. * Add lib for separate settings. * Update .gitlab-ci.yml * Update .gitlab-ci.yml * Update .gitlab-ci.yml * Update .gitlab-ci.yml * replacing k8-sse3 specific flags to x86-84 * updating versions --- .gitlab-ci.yml | 3 +- depends/packages/libsnark.mk | 6 ++-- libsnark.mk.patch | 4 +-- src/Makefile.am | 30 +++++++++++++------ src/cryptoconditions/Makefile.am | 2 +- .../src/include/secp256k1/Makefile.am | 2 +- src/snark/Makefile | 4 +-- 7 files changed, 32 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e0c71a676..f776d2183 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,7 +10,7 @@ variables: AGAMA_ARTIFACTS_WINDOWS: win64.zip VERUS_CLI_DEBIAN: verus-cli-v${VERSION}-beta-amd64.deb VERUS_CLI_LINUX_PORTABLE: verus-cli-linux-v$VERSION-beta.tar.gz - VERUS_CLI_WINDOWS_PORTABLE: verus-cli-windows-v$VERSION-beta.zip + VERUS_CLI_WINDOWS_PORTABLE: verus-cli-windows-v$VERSION-beta-portable.zip VERUS_CLI_MACOS_PORTABLE: verus-cli-mac-v$VERSION-beta.tar.gz @@ -154,6 +154,7 @@ license_management: artifacts: paths: [gl-license-management-report.json] + ubuntu:xenial: image: ubuntu:xenial variables: diff --git a/depends/packages/libsnark.mk b/depends/packages/libsnark.mk index cb4bc04ca..c2a620f26 100644 --- a/depends/packages/libsnark.mk +++ b/depends/packages/libsnark.mk @@ -14,15 +14,15 @@ define $(package)_set_vars $(package)_build_env+=CXXFLAGS="$($(package)_cxxflags) -DBINARY_OUTPUT -DSTATICLIB -DNO_PT_COMPRESSION=1 " endef define $(package)_build_cmds - $(MAKE) lib DEPINST=$(host_prefix) CURVE=ALT_BN128 MULTICORE=1 NO_PROCPS=1 NO_GTEST=1 NO_DOCS=1 STATIC=1 NO_SUPERCOP=1 FEATUREFLAGS=-DMONTGOMERY_OUTPUT OPTFLAGS="-O2 -march=x86-64" + $(MAKE) lib DEPINST=$(host_prefix) CURVE=ALT_BN128 MULTICORE=1 NO_PROCPS=1 NO_GTEST=1 NO_DOCS=1 STATIC=1 NO_SUPERCOP=1 FEATUREFLAGS=-DMONTGOMERY_OUTPUT OPTFLAGS="-O2 -march=x86-64 -g " endef else ifeq ($(host_os),mingw32) define $(package)_build_cmds - CXX="x86_64-w64-mingw32-g++-posix" CXXFLAGS="-DBINARY_OUTPUT -DPTW32_STATIC_LIB -DSTATICLIB -DNO_PT_COMPRESSION=1 -fopenmp" $(MAKE) lib DEPINST=$(host_prefix) CURVE=ALT_BN128 MULTICORE=1 NO_PROCPS=1 NO_GTEST=1 NO_DOCS=1 STATIC=1 NO_SUPERCOP=1 FEATUREFLAGS=-DMONTGOMERY_OUTPUT OPTFLAGS="-O2 -march=x86-64" + CXX="x86_64-w64-mingw32-g++-posix" CXXFLAGS="-DBINARY_OUTPUT -DPTW32_STATIC_LIB -DSTATICLIB -DNO_PT_COMPRESSION=1 -fopenmp" $(MAKE) lib DEPINST=$(host_prefix) CURVE=ALT_BN128 MULTICORE=1 NO_PROCPS=1 NO_GTEST=1 NO_DOCS=1 STATIC=1 NO_SUPERCOP=1 FEATUREFLAGS=-DMONTGOMERY_OUTPUT OPTFLAGS="-O2 -march=x86-64 -g " endef else define $(package)_build_cmds - CXXFLAGS="-fPIC -DBINARY_OUTPUT -DNO_PT_COMPRESSION=1" $(MAKE) lib DEPINST=$(host_prefix) CURVE=ALT_BN128 MULTICORE=1 NO_PROCPS=1 NO_GTEST=1 NO_DOCS=1 STATIC=1 NO_SUPERCOP=1 FEATUREFLAGS=-DMONTGOMERY_OUTPUT OPTFLAGS="-O2 -march=x86-64" + CXXFLAGS="-fPIC -DBINARY_OUTPUT -DNO_PT_COMPRESSION=1" $(MAKE) lib DEPINST=$(host_prefix) CURVE=ALT_BN128 MULTICORE=1 NO_PROCPS=1 NO_GTEST=1 NO_DOCS=1 STATIC=1 NO_SUPERCOP=1 FEATUREFLAGS=-DMONTGOMERY_OUTPUT OPTFLAGS="-O2 -march=x86-64 -g " endef endif diff --git a/libsnark.mk.patch b/libsnark.mk.patch index 28a60c8b6..7c555d208 100644 --- a/libsnark.mk.patch +++ b/libsnark.mk.patch @@ -18,7 +18,7 @@ - endef - else define $(package)_build_cmds - CXXFLAGS="-fPIC -DBINARY_OUTPUT -DNO_PT_COMPRESSION=1" $(MAKE) lib DEPINST=$(host_prefix) CURVE=ALT_BN128 MULTICORE=1 NO_PROCPS=1 NO_GTEST=1 NO_DOCS=1 STATIC=1 NO_SUPERCOP=1 FEATUREFLAGS=-DMONTGOMERY_OUTPUT OPTFLAGS="-O2 -march=x86-64" + CXXFLAGS="-fPIC -DBINARY_OUTPUT -DNO_PT_COMPRESSION=1" $(MAKE) lib DEPINST=$(host_prefix) CURVE=ALT_BN128 MULTICORE=1 NO_PROCPS=1 NO_GTEST=1 NO_DOCS=1 STATIC=1 NO_SUPERCOP=1 FEATUREFLAGS=-DMONTGOMERY_OUTPUT OPTFLAGS="-O2 -march=x86-64 -g " endef - @@ -44,7 +44,7 @@ $(package)_dependencies=libgmp libsodium define $(package)_build_cmds - CXXFLAGS="-fPIC -DBINARY_OUTPUT -DNO_PT_COMPRESSION=1" $(MAKE) lib DEPINST=$(host_prefix) CURVE=ALT_BN128 MULTICORE=1 NO_PROCPS=1 NO_GTEST=1 NO_DOCS=1 STATIC=1 NO_SUPERCOP=1 FEATUREFLAGS=-DMONTGOMERY_OUTPUT OPTFLAGS="-O2 -march=x86-64" + CXXFLAGS="-fPIC -DBINARY_OUTPUT -DNO_PT_COMPRESSION=1" $(MAKE) lib DEPINST=$(host_prefix) CURVE=ALT_BN128 MULTICORE=1 NO_PROCPS=1 NO_GTEST=1 NO_DOCS=1 STATIC=1 NO_SUPERCOP=1 FEATUREFLAGS=-DMONTGOMERY_OUTPUT OPTFLAGS="-O2 -march=x86-64 -g " endef define $(package)_stage_cmds diff --git a/src/Makefile.am b/src/Makefile.am index e409ce8d7..e1115d860 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -43,6 +43,7 @@ LIBBITCOIN_CLI=libbitcoin_cli.a LIBBITCOIN_UTIL=libbitcoin_util.a LIBBITCOIN_CRYPTO=crypto/libbitcoin_crypto.a LIBVERUS_CRYPTO=crypto/libverus_crypto.a +LIBVERUS_PORTABLE_CRYPTO=crypto/libverus_portable_crypto.a LIBSECP256K1=secp256k1/libsecp256k1.la LIBCRYPTOCONDITIONS=cryptoconditions/libcryptoconditions_core.la LIBSNARK=snark/libsnark.a @@ -50,7 +51,7 @@ LIBUNIVALUE=univalue/libunivalue.la LIBZCASH=libzcash.a -lcurl $(LIBSECP256K1): $(wildcard secp256k1/src/*) $(wildcard secp256k1/include/*) - $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) + $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) OPTFLAGS="-O2 -march=x86-64 -g " LIBSNARK_CXXFLAGS = -fPIC -DBINARY_OUTPUT -DNO_PT_COMPRESSION=1 -fstack-protector-all LIBSNARK_CONFIG_FLAGS = CURVE=ALT_BN128 NO_PROCPS=1 NO_DOCS=1 STATIC=1 NO_SUPERCOP=1 FEATUREFLAGS=-DMONTGOMERY_OUTPUT NO_COPY_DEPINST=1 NO_COMPILE_LIBGTEST=1 @@ -59,22 +60,23 @@ LIBSNARK_CONFIG_FLAGS += MULTICORE=1 endif $(LIBSNARK): $(wildcard snark/src/*) - $(AM_V_at) CXXFLAGS="$(LIBSNARK_CXXFLAGS)" $(MAKE) $(AM_MAKEFLAGS) -C snark/ DEPINST="$(LIBSNARK_DEPINST)" $(LIBSNARK_CONFIG_FLAGS) OPTFLAGS="-O2 -march=x86-64" + $(AM_V_at) CXXFLAGS="$(LIBSNARK_CXXFLAGS)" $(MAKE) $(AM_MAKEFLAGS) -C snark/ DEPINST="$(LIBSNARK_DEPINST)" $(LIBSNARK_CONFIG_FLAGS) OPTFLAGS="-O2 -march=x86-64 -g " libsnark-tests: $(wildcard snark/src/*) - $(AM_V_at) CXXFLAGS="$(LIBSNARK_CXXFLAGS)" $(MAKE) $(AM_MAKEFLAGS) -C snark/ check DEPINST="$(LIBSNARK_DEPINST)" $(LIBSNARK_CONFIG_FLAGS) OPTFLAGS="-O2 -march=x86-64" + $(AM_V_at) CXXFLAGS="$(LIBSNARK_CXXFLAGS)" $(MAKE) $(AM_MAKEFLAGS) -C snark/ check DEPINST="$(LIBSNARK_DEPINST)" $(LIBSNARK_CONFIG_FLAGS) OPTFLAGS="-O2 -march=x86-64 -g " $(LIBUNIVALUE): $(wildcard univalue/lib/*) - $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C univalue/ + $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C univalue/ OPTFLAGS="-O2 -march=x86-64 -g " $(LIBCRYPTOCONDITIONS): $(wildcard cryptoconditions/src/*) $(wildcard cryptoconditions/include/*) - $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) + $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) OPTFLAGS="-O2 -march=x86-64 -g " # Make is not made aware of per-object dependencies to avoid limiting building parallelization # But to build the less dependent modules first, we manually select their order here: EXTRA_LIBRARIES = \ crypto/libbitcoin_crypto.a \ crypto/libverus_crypto.a \ + crypto/libverus_portable_crypto.a \ libbitcoin_util.a \ libbitcoin_common.a \ libbitcoin_server.a \ @@ -401,12 +403,17 @@ crypto_libbitcoin_crypto_a_SOURCES += \ ${EQUIHASH_TROMP_SOURCES} endif -# Verus hash specific library -crypto_libverus_crypto_a_CPPFLAGS = -O3 -Wint-conversion -march=native -funroll-loops -fomit-frame-pointer -fPIC $(AM_CPPFLAGS) -crypto_libverus_crypto_a_CXXFLAGS = -O3 -Wint-conversion -march=native -funroll-loops -fomit-frame-pointer -fPIC $(AM_CXXFLAGS) +# Verus hash specific library - optimized +crypto_libverus_crypto_a_CPPFLAGS = -O3 -Wint-conversion -march=x86-64 -msse4 -msse4.1 -msse4.2 -mssse3 -mavx -maes -g -funroll-loops -fomit-frame-pointer -fPIC $(AM_CPPFLAGS) +crypto_libverus_crypto_a_CXXFLAGS = -O3 -Wint-conversion -march=x86-64 -msse4 -msse4.1 -msse4.2 -mssse3 -mavx -maes -g -funroll-loops -fomit-frame-pointer -fPIC $(AM_CXXFLAGS) crypto_libverus_crypto_a_SOURCES = \ crypto/haraka.h \ - crypto/haraka.c \ + crypto/haraka.c + +# Verus hash specific library - portable +crypto_libverus_portable_crypto_a_CPPFLAGS = -O3 -Wint-conversion -march=x86-64 -g -funroll-loops -fomit-frame-pointer -fPIC $(AM_CPPFLAGS) +crypto_libverus_portable_crypto_a_CXXFLAGS = -O3 -Wint-conversion -march=x86-64 -g -funroll-loops -fomit-frame-pointer -fPIC $(AM_CXXFLAGS) +crypto_libverus_portable_crypto_a_SOURCES = \ crypto/haraka_portable.h \ crypto/haraka_portable.c @@ -505,6 +512,7 @@ komodod_LDADD = \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CRYPTO) \ $(LIBVERUS_CRYPTO) \ + $(LIBVERUS_PORTABLE_CRYPTO) \ $(LIBZCASH) \ $(LIBSNARK) \ $(LIBLEVELDB) \ @@ -529,6 +537,7 @@ komodod_LDADD += \ $(EVENT_LIBS) \ $(LIBBITCOIN_CRYPTO) \ $(LIBVERUS_CRYPTO) \ + $(LIBVERUS_PORTABLE_CRYPTO) \ $(LIBZCASH_LIBS) if ENABLE_PROTON @@ -564,6 +573,7 @@ komodo_cli_LDADD = \ $(LIBZCASH) \ $(LIBBITCOIN_CRYPTO) \ $(LIBVERUS_CRYPTO) \ + $(LIBVERUS_PORTABLE_CRYPTO) \ $(LIBZCASH_LIBS) if ENABLE_WALLET @@ -572,6 +582,7 @@ wallet_utility_LDADD = \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_CRYPTO) \ $(LIBVERUS_CRYPTO) \ + $(LIBVERUS_PORTABLE_CRYPTO) \ $(LIBSECP256K1) \ $(LIBBITCOIN_UTIL) \ $(BOOST_LIBS) \ @@ -603,6 +614,7 @@ komodo_tx_LDADD = \ $(LIBSNARK) \ $(LIBBITCOIN_CRYPTO) \ $(LIBVERUS_CRYPTO) \ + $(LIBVERUS_PORTABLE_CRYPTO) \ $(LIBZCASH_LIBS) \ $(LIBCRYPTOCONDITIONS) diff --git a/src/cryptoconditions/Makefile.am b/src/cryptoconditions/Makefile.am index 52f12eee0..787b11ac6 100644 --- a/src/cryptoconditions/Makefile.am +++ b/src/cryptoconditions/Makefile.am @@ -15,7 +15,7 @@ AM_CFLAGS = -I$(top_srcdir)/src/asn -I$(top_srcdir)/include -I$(top_srcdir)/src/ LIBSECP256K1=src/include/secp256k1/libsecp256k1.la $(LIBSECP256K1): $(wildcard src/secp256k1/*) - $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) + $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) -march:x86-64 -g CRYPTOCONDITIONS_CORE=libcryptoconditions_core.la diff --git a/src/cryptoconditions/src/include/secp256k1/Makefile.am b/src/cryptoconditions/src/include/secp256k1/Makefile.am index c071fbe27..52303e4e4 100644 --- a/src/cryptoconditions/src/include/secp256k1/Makefile.am +++ b/src/cryptoconditions/src/include/secp256k1/Makefile.am @@ -71,7 +71,7 @@ endif endif libsecp256k1_la_SOURCES = src/secp256k1.c -libsecp256k1_la_CPPFLAGS = -DSECP256K1_BUILD -I$(top_srcdir)/include -I$(top_srcdir)/src $(SECP_INCLUDES) +libsecp256k1_la_CPPFLAGS = -DSECP256K1_BUILD -I$(top_srcdir)/include -I$(top_srcdir)/src $(SECP_INCLUDES) -march=x86-64 -g libsecp256k1_la_LIBADD = $(JNI_LIB) $(SECP_LIBS) $(COMMON_LIB) libsecp256k1_jni_la_SOURCES = src/java/org_bitcoin_NativeSecp256k1.c src/java/org_bitcoin_Secp256k1Context.c diff --git a/src/snark/Makefile b/src/snark/Makefile index 17dcfdc6e..fdba63aae 100644 --- a/src/snark/Makefile +++ b/src/snark/Makefile @@ -15,7 +15,7 @@ endif # To override these, use "make OPTFLAGS=..." etc. CURVE = BN128 -OPTFLAGS = -O2 -march=native -mtune=native +OPTFLAGS = -O2 -march=x86-64 -g -mtune=x86-64 FEATUREFLAGS = -DUSE_ASM -DMONTGOMERY_OUTPUT # Initialize this using "CXXFLAGS=... make". The makefile appends to that. @@ -158,7 +158,7 @@ ifeq ($(DEBUG),1) endif ifeq ($(PERFORMANCE),1) - OPTFLAGS = -O3 -march=native -mtune=native + OPTFLAGS = -O3 -march=x86-64 -g -mtune=x86-64 CXXFLAGS += -DNDEBUG # Enable link-time optimization: CXXFLAGS += -flto -fuse-linker-plugin