monero-project/monero
monero-project/monero
1
0
Fork 0
mirror of https://github.com/monero-project/monero.git synced 2024-11-17 00:07:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improved performance for epee serialization:

Browse source 
- Removed copy of field names in binary deserialization
  - Removed copy of array values in binary deserialization
  - Removed copy of string values in json deserialization
  - Removed unhelpful allocation in json string value parsing
  - Removed copy of blob data on binary and json serialization
This commit is contained in:
Lee Clagett 2019-11-04 01:06:01 +00:00
parent 411f1b0ee3
commit a9bdc6e4c4
7 changed files with 56 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
contrib/epee/include/serialization/keyvalue_serialization_overloads.h
View file

@ -46,24 +46,12 @@ namespace epee
namespace serialization
{
//-------------------------------------------------------------------------------------------------------------------
template<class t_type, class t_storage>
static bool serialize_t_val(const t_type& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
{
return stg.set_value(pname, d, hparent_section);
}
//-------------------------------------------------------------------------------------------------------------------
template<class t_type, class t_storage>
static bool unserialize_t_val(t_type& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
{
return stg.get_value(pname, d, hparent_section);
}
//-------------------------------------------------------------------------------------------------------------------
template<class t_type, class t_storage>
static bool serialize_t_val_as_blob(const t_type& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
{
std::string blob((const char *)&d, sizeof(d));
return stg.set_value(pname, blob, hparent_section);
return stg.set_value(pname, std::move(blob), hparent_section);
}
//-------------------------------------------------------------------------------------------------------------------
template<class t_type, class t_storage>
@ -114,13 +102,15 @@ namespace epee
template<class stl_container, class t_storage>
static bool serialize_stl_container_t_val (const stl_container& container, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
{
using value_type = typename stl_container::value_type;
if(!container.size()) return true;
typename stl_container::const_iterator it = container.begin();
typename t_storage::harray hval_array = stg.insert_first_value(pname, *it, hparent_section);
typename t_storage::harray hval_array = stg.insert_first_value(pname, value_type(*it), hparent_section);
CHECK_AND_ASSERT_MES(hval_array, false, "failed to insert first value to storage");
it++;
for(;it!= container.end();it++)
stg.insert_next_value(hval_array, *it);
stg.insert_next_value(hval_array, value_type(*it));
return true;
}
@ -149,7 +139,7 @@ namespace epee
*p_elem = v;
p_elem++;
}
return stg.set_value(pname, mb, hparent_section);
return stg.set_value(pname, std::move(mb), hparent_section);
}
//--------------------------------------------------------------------------------------------------------------------
template<class stl_container, class t_storage>
@ -221,7 +211,7 @@ namespace epee
template<class t_type, class t_storage>
static bool kv_serialize(const t_type& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
{
return stg.set_value(pname, d, hparent_section);
return stg.set_value(pname, t_type(d), hparent_section);
}
//-------------------------------------------------------------------------------------------------------------------
template<class t_type, class t_storage>

1
contrib/epee/include/storages/parserse_base_utils.h
View file

@ -157,7 +157,6 @@ namespace misc_utils
while (fi != buf_end && ((lut[(uint8_t)*fi] & 32)) == 0)
++fi;
val.assign(it, fi);
val.reserve(std::distance(star_end_string, buf_end));
it = fi;
for(;it != buf_end;it++)
{

52
contrib/epee/include/storages/portable_storage.h
View file

@ -28,6 +28,8 @@
#pragma once
#include <type_traits>
#include "misc_language.h"
#include "portable_storage_base.h"
#include "portable_storage_to_bin.h"
@ -59,7 +61,7 @@ namespace epee
bool get_value(const std::string& value_name, t_value& val, hsection hparent_section);
bool get_value(const std::string& value_name, storage_entry& val, hsection hparent_section);
template<class t_value>
bool set_value(const std::string& value_name, const t_value& target, hsection hparent_section);
bool set_value(const std::string& value_name, t_value&& target, hsection hparent_section);
//serial access for arrays of values --------------------------------------
//values
@ -68,9 +70,9 @@ namespace epee
template<class t_value>
bool get_next_value(harray hval_array, t_value& target);
template<class t_value>
harray insert_first_value(const std::string& value_name, const t_value& target, hsection hparent_section);
harray insert_first_value(const std::string& value_name, t_value&& target, hsection hparent_section);
template<class t_value>
bool insert_next_value(harray hval_array, const t_value& target);
bool insert_next_value(harray hval_array, t_value&& target);
//sections
harray get_first_section(const std::string& pSectionName, hsection& h_child_section, hsection hparent_section);
bool get_next_section(harray hSecArray, hsection& h_child_section);
@ -94,7 +96,7 @@ namespace epee
hsection get_root_section() {return &m_root;}
storage_entry* find_storage_entry(const std::string& pentry_name, hsection psection);
template<class entry_type>
storage_entry* insert_new_entry_get_storage_entry(const std::string& pentry_name, hsection psection, const entry_type& entry);
storage_entry* insert_new_entry_get_storage_entry(const std::string& pentry_name, hsection psection, entry_type&& entry);
hsection insert_new_section(const std::string& pentry_name, hsection psection);
@ -241,21 +243,22 @@ namespace epee
}
//---------------------------------------------------------------------------------------------------------------
template<class t_value>
bool portable_storage::set_value(const std::string& value_name, const t_value& v, hsection hparent_section)
bool portable_storage::set_value(const std::string& value_name, t_value&& v, hsection hparent_section)
{
BOOST_MPL_ASSERT(( boost::mpl::contains<boost::mpl::push_front<storage_entry::types, storage_entry>::type, t_value> ));
using t_real_value = typename std::decay<t_value>::type;
BOOST_MPL_ASSERT(( boost::mpl::contains<boost::mpl::push_front<storage_entry::types, storage_entry>::type, t_real_value> ));
TRY_ENTRY();
if(!hparent_section)
hparent_section = &m_root;
storage_entry* pentry = find_storage_entry(value_name, hparent_section);
if(!pentry)
{
pentry = insert_new_entry_get_storage_entry(value_name, hparent_section, v);
pentry = insert_new_entry_get_storage_entry(value_name, hparent_section, std::forward<t_value>(v));
if(!pentry)
return false;
return true;
}
*pentry = storage_entry(v);
*pentry = std::forward<t_value>(v);
return true;
CATCH_ENTRY("portable_storage::template<>set_value", false);
}
@ -274,11 +277,12 @@ namespace epee
}
//---------------------------------------------------------------------------------------------------------------
template<class entry_type>
storage_entry* portable_storage::insert_new_entry_get_storage_entry(const std::string& pentry_name, hsection psection, const entry_type& entry)
storage_entry* portable_storage::insert_new_entry_get_storage_entry(const std::string& pentry_name, hsection psection, entry_type&& entry)
{
static_assert(std::is_rvalue_reference<entry_type&&>(), "unexpected copy of value");
TRY_ENTRY();
CHECK_AND_ASSERT(psection, nullptr);
auto ins_res = psection->m_entries.insert(std::pair<std::string, storage_entry>(pentry_name, entry));
auto ins_res = psection->m_entries.emplace(pentry_name, std::forward<entry_type>(entry));
return &ins_res.first->second;
CATCH_ENTRY("portable_storage::insert_new_entry_get_storage_entry", nullptr);
}
@ -362,41 +366,45 @@ namespace epee
}
//---------------------------------------------------------------------------------------------------------------
template<class t_value>
harray portable_storage::insert_first_value(const std::string& value_name, const t_value& target, hsection hparent_section)
harray portable_storage::insert_first_value(const std::string& value_name, t_value&& target, hsection hparent_section)
{
using t_real_value = typename std::decay<t_value>::type;
static_assert(std::is_rvalue_reference<t_value&&>(), "unexpected copy of value");
TRY_ENTRY();
if(!hparent_section) hparent_section = &m_root;
storage_entry* pentry = find_storage_entry(value_name, hparent_section);
if(!pentry)
{
pentry = insert_new_entry_get_storage_entry(value_name, hparent_section, array_entry(array_entry_t<t_value>()));
pentry = insert_new_entry_get_storage_entry(value_name, hparent_section, array_entry(array_entry_t<t_real_value>()));
if(!pentry)
return nullptr;
}
if(pentry->type() != typeid(array_entry))
*pentry = storage_entry(array_entry(array_entry_t<t_value>()));
*pentry = storage_entry(array_entry(array_entry_t<t_real_value>()));
array_entry& arr = boost::get<array_entry>(*pentry);
if(arr.type() != typeid(array_entry_t<t_value>))
arr = array_entry(array_entry_t<t_value>());
if(arr.type() != typeid(array_entry_t<t_real_value>))
arr = array_entry(array_entry_t<t_real_value>());
array_entry_t<t_value>& arr_typed = boost::get<array_entry_t<t_value> >(arr);
arr_typed.insert_first_val(target);
array_entry_t<t_real_value>& arr_typed = boost::get<array_entry_t<t_real_value> >(arr);
arr_typed.insert_first_val(std::forward<t_value>(target));
return &arr;
CATCH_ENTRY("portable_storage::insert_first_value", nullptr);
}
//---------------------------------------------------------------------------------------------------------------
template<class t_value>
bool portable_storage::insert_next_value(harray hval_array, const t_value& target)
bool portable_storage::insert_next_value(harray hval_array, t_value&& target)
{
using t_real_value = typename std::decay<t_value>::type;
static_assert(std::is_rvalue_reference<t_value&&>(), "unexpected copy of value");
TRY_ENTRY();
CHECK_AND_ASSERT(hval_array, false);
CHECK_AND_ASSERT_MES(hval_array->type() == typeid(array_entry_t<t_value>),
false, "unexpected type in insert_next_value: " << typeid(array_entry_t<t_value>).name());
CHECK_AND_ASSERT_MES(hval_array->type() == typeid(array_entry_t<t_real_value>),
false, "unexpected type in insert_next_value: " << typeid(array_entry_t<t_real_value>).name());
array_entry_t<t_value>& arr_typed = boost::get<array_entry_t<t_value> >(*hval_array);
arr_typed.insert_next_value(target);
array_entry_t<t_real_value>& arr_typed = boost::get<array_entry_t<t_real_value> >(*hval_array);
arr_typed.insert_next_value(std::forward<t_value>(target));
return true;
CATCH_ENTRY("portable_storage::insert_next_value", false);
}

8
contrib/epee/include/storages/portable_storage_base.h
View file

@ -111,16 +111,16 @@ namespace epee
return (t_entry_type*)&(*(m_it++));//fuckoff
}
t_entry_type& insert_first_val(const t_entry_type& v)
t_entry_type& insert_first_val(t_entry_type&& v)
{
m_array.clear();
m_it = m_array.end();
return insert_next_value(v);
return insert_next_value(std::move(v));
}
t_entry_type& insert_next_value(const t_entry_type& v)
t_entry_type& insert_next_value(t_entry_type&& v)
{
m_array.push_back(v);
m_array.push_back(std::move(v));
return m_array.back();
}

6
contrib/epee/include/storages/portable_storage_from_bin.h
View file

@ -143,7 +143,7 @@ namespace epee
//TODO: add some optimization here later
while(size--)
sa.m_array.push_back(read<type_name>());
return storage_entry(array_entry(sa));
return storage_entry(array_entry(std::move(sa)));
}
inline
@ -213,7 +213,7 @@ namespace epee
{
RECURSION_LIMITATION();
section s;//use extra variable due to vs bug, line "storage_entry se(section()); " can't be compiled in visual studio
storage_entry se(s);
storage_entry se(std::move(s));
section& section_entry = boost::get<section>(se);
read(section_entry);
return se;
@ -268,7 +268,7 @@ namespace epee
//read section name string
std::string sec_name;
read_sec_name(sec_name);
sec.m_entries.insert(std::make_pair(sec_name, load_storage_entry()));
sec.m_entries.emplace(std::move(sec_name), load_storage_entry());
}
}
inline

18
contrib/epee/include/storages/portable_storage_from_json.h
View file

@ -128,20 +128,20 @@ namespace epee
errno = 0;
int64_t nval = strtoll(val.data(), NULL, 10);
if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
stg.set_value(name, nval, current_section);
stg.set_value(name, int64_t(nval), current_section);
}else
{
errno = 0;
uint64_t nval = strtoull(val.data(), NULL, 10);
if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
stg.set_value(name, nval, current_section);
stg.set_value(name, uint64_t(nval), current_section);
}
}else
{
errno = 0;
double nval = strtod(val.data(), NULL);
if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
stg.set_value(name, nval, current_section);
stg.set_value(name, double(nval), current_section);
}
state = match_state_wonder_after_value;
}else if(isalpha(*it) )
@ -219,13 +219,13 @@ namespace epee
errno = 0;
int64_t nval = strtoll(val.data(), NULL, 10);
if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
h_array = stg.insert_first_value(name, nval, current_section);
h_array = stg.insert_first_value(name, int64_t(nval), current_section);
}else
{
errno = 0;
uint64_t nval = strtoull(val.data(), NULL, 10);
if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
h_array = stg.insert_first_value(name, nval, current_section);
h_array = stg.insert_first_value(name, uint64_t(nval), current_section);
}
CHECK_AND_ASSERT_THROW_MES(h_array, " failed to insert values section entry");
}else
@ -233,7 +233,7 @@ namespace epee
errno = 0;
double nval = strtod(val.data(), NULL);
if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
h_array = stg.insert_first_value(name, nval, current_section);
h_array = stg.insert_first_value(name, double(nval), current_section);
CHECK_AND_ASSERT_THROW_MES(h_array, " failed to insert values section entry");
}
@ -310,20 +310,20 @@ namespace epee
errno = 0;
int64_t nval = strtoll(val.data(), NULL, 10);
if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
insert_res = stg.insert_next_value(h_array, nval);
insert_res = stg.insert_next_value(h_array, int64_t(nval));
}else
{
errno = 0;
uint64_t nval = strtoull(val.data(), NULL, 10);
if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
insert_res = stg.insert_next_value(h_array, nval);
insert_res = stg.insert_next_value(h_array, uint64_t(nval));
}
}else
{
errno = 0;
double nval = strtod(val.data(), NULL);
if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
insert_res = stg.insert_next_value(h_array, nval);
insert_res = stg.insert_next_value(h_array, double(nval));
}
CHECK_AND_ASSERT_THROW_MES(insert_res, "Failed to insert next value");
state = match_state_array_after_value;

6
tests/unit_tests/net.cpp
View file

@ -271,7 +271,7 @@ TEST(tor_address, epee_serializev_v2)
EXPECT_EQ(std::strlen(v2_onion), host.size());
host.push_back('k');
EXPECT_TRUE(stg.set_value("host", host, stg.open_section("tor", nullptr, false)));
EXPECT_TRUE(stg.set_value("host", std::move(host), stg.open_section("tor", nullptr, false)));
EXPECT_TRUE(command.load(stg)); // poor error reporting from `KV_SERIALIZE`
}
@ -322,7 +322,7 @@ TEST(tor_address, epee_serializev_v3)
EXPECT_EQ(std::strlen(v3_onion), host.size());
host.push_back('k');
EXPECT_TRUE(stg.set_value("host", host, stg.open_section("tor", nullptr, false)));
EXPECT_TRUE(stg.set_value("host", std::move(host), stg.open_section("tor", nullptr, false)));
EXPECT_TRUE(command.load(stg)); // poor error reporting from `KV_SERIALIZE`
}
@ -373,7 +373,7 @@ TEST(tor_address, epee_serialize_unknown)
EXPECT_EQ(std::strlen(net::tor_address::unknown_str()), host.size());
host.push_back('k');
EXPECT_TRUE(stg.set_value("host", host, stg.open_section("tor", nullptr, false)));
EXPECT_TRUE(stg.set_value("host", std::move(host), stg.open_section("tor", nullptr, false)));
EXPECT_TRUE(command.load(stg)); // poor error reporting from `KV_SERIALIZE`
}