From 3fd69311cb14fef23e3d0b1c7ee15103e590ff7c Mon Sep 17 00:00:00 2001 From: hakunamtu Date: Wed, 16 Jan 2019 22:30:25 +0300 Subject: [PATCH] Make persistent storage synchronization interval configurable --- libi2pd/Config.cpp | 1 + libi2pd/NetDb.cpp | 75 +++++++++++++++++++++++++++++++------------- libi2pd/NetDb.hpp | 4 +++ libi2pd/RouterInfo.h | 2 ++ 4 files changed, 61 insertions(+), 21 deletions(-) diff --git a/libi2pd/Config.cpp b/libi2pd/Config.cpp index 89531f5a..32bc815c 100644 --- a/libi2pd/Config.cpp +++ b/libi2pd/Config.cpp @@ -256,6 +256,7 @@ namespace config { options_description persist("Network information persisting options"); persist.add_options() ("persist.profiles", value()->default_value(true), "Persist peer profiles (default: true)") + ("persist.syncinterval", value()->default_value(60), "Peer profiles and NetDb persistent storage sync interval in seconds (0 - save only on exit)") ; m_OptionsDesc diff --git a/libi2pd/NetDb.cpp b/libi2pd/NetDb.cpp index 714c327d..7a54e606 100644 --- a/libi2pd/NetDb.cpp +++ b/libi2pd/NetDb.cpp @@ -26,7 +26,7 @@ namespace data { NetDb netdb; - NetDb::NetDb (): m_IsRunning (false), m_Thread (nullptr), m_Reseeder (nullptr), m_Storage("netDb", "r", "routerInfo-", "dat"), m_PersistProfiles (true), m_HiddenMode(false) + NetDb::NetDb (): m_IsRunning (false), m_Thread (nullptr), m_Reseeder (nullptr), m_Storage("netDb", "r", "routerInfo-", "dat"), m_PersistSyncInterval(60), m_PersistProfiles (true), m_HiddenMode(false) { } @@ -49,6 +49,7 @@ namespace data Reseed (); i2p::config::GetOption("persist.profiles", m_PersistProfiles); + i2p::config::GetOption("persist.syncinterval", m_PersistSyncInterval); m_IsRunning = true; m_Thread = new std::thread (std::bind (&NetDb::Run, this)); @@ -61,8 +62,10 @@ namespace data if (m_PersistProfiles) for (auto& it: m_RouterInfos) it.second->SaveProfile (); + SaveUpdated (); DeleteObsoleteProfiles (); m_RouterInfos.clear (); + m_UnsavedProfiles.clear (); m_Floodfills.clear (); if (m_Thread) { @@ -79,7 +82,7 @@ namespace data void NetDb::Run () { - uint32_t lastSave = 0, lastPublish = 0, lastExploratory = 0, lastManageRequest = 0, lastDestinationCleanup = 0; + uint32_t lastSave = 0, lastLeasesetsManage = 0, lastPublish = 0, lastExploratory = 0, lastManageRequest = 0, lastDestinationCleanup = 0; while (m_IsRunning) { try @@ -123,13 +126,19 @@ namespace data m_Requests.ManageRequests (); lastManageRequest = ts; } - if (ts - lastSave >= 60) // save routers, manage leasesets and validate subscriptions every minute + if (ts - lastLeasesetsManage >= 60) // manage leasesets and validate subscriptions every minute { - if (lastSave) + if (lastLeasesetsManage) { - SaveUpdated (); + RemoveExpired (); ManageLeaseSets (); } + lastLeasesetsManage = ts; + } + if (ts - lastSave >= m_PersistSyncInterval && m_PersistSyncInterval > 0) // save routers + { + if (lastSave) + SaveUpdated (); lastSave = ts; } if (ts - lastDestinationCleanup >= i2p::garlic::INCOMING_TAGS_EXPIRATION_TIMEOUT) @@ -229,6 +238,13 @@ namespace data } if (inserted) { + { + std::unique_lock l(m_UnsavedProfilesMutex); + auto it = m_UnsavedProfiles.find (ident); + if (it != m_UnsavedProfiles.end ()) + r->SetProfile(it->second); + } + LogPrint (eLogInfo, "NetDb: RouterInfo added: ", ident.ToBase64()); if (r->IsFloodfill () && r->IsReachable ()) // floodfill must be reachable { @@ -489,6 +505,7 @@ namespace data // make sure we cleanup netDb from previous attempts m_RouterInfos.clear (); m_Floodfills.clear (); + m_UnsavedProfiles.clear (); m_LastLoad = i2p::util::GetSecondsSinceEpoch(); std::vector files; @@ -501,8 +518,36 @@ namespace data void NetDb::SaveUpdated () { - int updatedCount = 0, deletedCount = 0; + int updatedCount = 0; + for (auto& it: m_RouterInfos) + { + std::string ident = it.second->GetIdentHashBase64(); + std::string path = m_Storage.Path(ident); + if (it.second->IsUpdated ()) + { + it.second->SaveToFile (path); + it.second->SetUpdated (false); + it.second->DeleteBuffer (); + updatedCount++; + } + } // m_RouterInfos iteration + + if (updatedCount > 0) + LogPrint (eLogInfo, "NetDb: saved ", updatedCount, " new/updated routers"); + + for (auto& it: m_UnsavedProfiles) + it.second->Save (it.first); + { + std::unique_lock l(m_UnsavedProfilesMutex); + m_UnsavedProfiles.clear (); + } + } + + void NetDb::RemoveExpired () + { auto total = m_RouterInfos.size (); + int deletedCount = 0; + uint64_t expirationTimeout = NETDB_MAX_EXPIRATION_TIMEOUT*1000LL; uint64_t ts = i2p::util::GetMillisecondsSinceEpoch(); // routers don't expire if less than 90 or uptime is less than 1 hour @@ -513,17 +558,6 @@ namespace data for (auto& it: m_RouterInfos) { - std::string ident = it.second->GetIdentHashBase64(); - std::string path = m_Storage.Path(ident); - if (it.second->IsUpdated ()) - { - it.second->SaveToFile (path); - it.second->SetUpdated (false); - it.second->SetUnreachable (false); - it.second->DeleteBuffer (); - updatedCount++; - continue; - } // find & mark expired routers if (it.second->UsesIntroducer ()) { @@ -537,14 +571,12 @@ namespace data if (it.second->IsUnreachable ()) { // delete RI file - m_Storage.Remove(ident); + m_Storage.Remove (it.first.ToBase64 ()); deletedCount++; if (total - deletedCount < NETDB_MIN_ROUTERS) checkForExpiration = false; } } // m_RouterInfos iteration - if (updatedCount > 0) - LogPrint (eLogInfo, "NetDb: saved ", updatedCount, " new/updated routers"); if (deletedCount > 0) { LogPrint (eLogInfo, "NetDb: deleting ", deletedCount, " unreachable routers"); @@ -555,7 +587,8 @@ namespace data { if (it->second->IsUnreachable ()) { - if (m_PersistProfiles) it->second->SaveProfile (); + if (m_PersistProfiles && it->second->HasProfile ()) + m_UnsavedProfiles[it->second->GetIdentHash ()] = it->second->GetProfile (); it = m_RouterInfos.erase (it); continue; } diff --git a/libi2pd/NetDb.hpp b/libi2pd/NetDb.hpp index 94ed2ed2..3e814746 100644 --- a/libi2pd/NetDb.hpp +++ b/libi2pd/NetDb.hpp @@ -109,6 +109,7 @@ namespace data void Load (); bool LoadRouterInfo (const std::string & path); void SaveUpdated (); + void RemoveExpired (); void Run (); // exploratory thread void Explore (int numDestinations); void Publish (); @@ -129,6 +130,8 @@ namespace data std::map > m_LeaseSets; mutable std::mutex m_RouterInfosMutex; std::map > m_RouterInfos; + mutable std::mutex m_UnsavedProfilesMutex; + std::map> m_UnsavedProfiles; mutable std::mutex m_FloodfillsMutex; std::list > m_Floodfills; @@ -145,6 +148,7 @@ namespace data friend class NetDbRequests; NetDbRequests m_Requests; + unsigned m_PersistSyncInterval; bool m_PersistProfiles; /** router info we are bootstrapping from or nullptr if we are not currently doing that*/ diff --git a/libi2pd/RouterInfo.h b/libi2pd/RouterInfo.h index f66a73fa..e90f0e2f 100644 --- a/libi2pd/RouterInfo.h +++ b/libi2pd/RouterInfo.h @@ -194,6 +194,8 @@ namespace data bool SaveToFile (const std::string& fullPath); std::shared_ptr GetProfile () const; + bool HasProfile () const { return static_cast(m_Profile); }; + void SetProfile (const std::shared_ptr& profile) { m_Profile = profile; }; void SaveProfile () { if (m_Profile) m_Profile->Save (GetIdentHash ()); }; void Update (const uint8_t * buf, int len);