diff --git a/libi2pd/Config.cpp b/libi2pd/Config.cpp
index 89531f5a..32bc815c 100644
--- a/libi2pd/Config.cpp
+++ b/libi2pd/Config.cpp
@@ -256,6 +256,7 @@ namespace config {
options_description persist("Network information persisting options");
persist.add_options()
("persist.profiles", value<bool>()->default_value(true), "Persist peer profiles (default: true)")
+ ("persist.syncinterval", value<unsigned>()->default_value(60), "Peer profiles and NetDb persistent storage sync interval in seconds (0 - save only on exit)")
;
m_OptionsDesc
diff --git a/libi2pd/NetDb.cpp b/libi2pd/NetDb.cpp
index 9d858d91..542735ec 100644
--- a/libi2pd/NetDb.cpp
+++ b/libi2pd/NetDb.cpp
@@ -26,7 +26,7 @@ namespace data
{
NetDb netdb;
- NetDb::NetDb (): m_IsRunning (false), m_Thread (nullptr), m_Reseeder (nullptr), m_Storage("netDb", "r", "routerInfo-", "dat"), m_PersistProfiles (true), m_HiddenMode(false)
+ NetDb::NetDb (): m_IsRunning (false), m_Thread (nullptr), m_Reseeder (nullptr), m_Storage("netDb", "r", "routerInfo-", "dat"), m_PersistSyncInterval(60), m_PersistProfiles (true), m_HiddenMode(false)
{
}
@@ -49,6 +49,7 @@ namespace data
Reseed ();
i2p::config::GetOption("persist.profiles", m_PersistProfiles);
+ i2p::config::GetOption("persist.syncinterval", m_PersistSyncInterval);
m_IsRunning = true;
m_Thread = new std::thread (std::bind (&NetDb::Run, this));
@@ -61,8 +62,10 @@ namespace data
if (m_PersistProfiles)
for (auto& it: m_RouterInfos)
it.second->SaveProfile ();
+ SaveUpdated ();
DeleteObsoleteProfiles ();
m_RouterInfos.clear ();
+ m_UnsavedProfiles.clear ();
m_Floodfills.clear ();
if (m_Thread)
{
@@ -79,7 +82,7 @@ namespace data
void NetDb::Run ()
{
- uint32_t lastSave = 0, lastPublish = 0, lastExploratory = 0, lastManageRequest = 0, lastDestinationCleanup = 0;
+ uint32_t lastSave = 0, lastLeasesetsManage = 0, lastPublish = 0, lastExploratory = 0, lastManageRequest = 0, lastDestinationCleanup = 0, lastProfilesCleanup = 0;
while (m_IsRunning)
{
try
@@ -123,13 +126,19 @@ namespace data
m_Requests.ManageRequests ();
lastManageRequest = ts;
}
- if (ts - lastSave >= 60) // save routers, manage leasesets and validate subscriptions every minute
+ if (ts - lastLeasesetsManage >= 60) // manage leasesets and validate subscriptions every minute
{
- if (lastSave)
+ if (lastLeasesetsManage)
{
- SaveUpdated ();
+ RemoveExpired ();
ManageLeaseSets ();
}
+ lastLeasesetsManage = ts;
+ }
+ if (ts - lastSave >= m_PersistSyncInterval && m_PersistSyncInterval > 0) // save routers
+ {
+ if (lastSave)
+ SaveUpdated ();
lastSave = ts;
}
if (ts - lastDestinationCleanup >= i2p::garlic::INCOMING_TAGS_EXPIRATION_TIMEOUT)
@@ -162,6 +171,11 @@ namespace data
lastExploratory = ts;
}
}
+ if (ts - lastProfilesCleanup >= PEER_PROFILE_EXPIRATION_TIMEOUT * 3600)
+ {
+ RemoveObsoleteProfiles ();
+ lastProfilesCleanup = ts;
+ }
}
catch (std::exception& ex)
{
@@ -238,6 +252,13 @@ namespace data
}
if (inserted)
{
+ {
+ std::unique_lock<std::mutex> l(m_UnsavedProfilesMutex);
+ auto it = m_UnsavedProfiles.find (ident);
+ if (it != m_UnsavedProfiles.end ())
+ r->SetProfile(it->second);
+ }
+
LogPrint (eLogInfo, "NetDb: RouterInfo added: ", ident.ToBase64());
if (r->IsFloodfill () && r->IsReachable ()) // floodfill must be reachable
{
@@ -495,6 +516,7 @@ namespace data
// make sure we cleanup netDb from previous attempts
m_RouterInfos.clear ();
m_Floodfills.clear ();
+ m_UnsavedProfiles.clear ();
m_LastLoad = i2p::util::GetSecondsSinceEpoch();
std::vector<std::string> files;
@@ -507,8 +529,36 @@ namespace data
void NetDb::SaveUpdated ()
{
- int updatedCount = 0, deletedCount = 0;
+ int updatedCount = 0;
+ for (auto& it: m_RouterInfos)
+ {
+ std::string ident = it.second->GetIdentHashBase64();
+ std::string path = m_Storage.Path(ident);
+ if (it.second->IsUpdated ())
+ {
+ it.second->SaveToFile (path);
+ it.second->SetUpdated (false);
+ it.second->DeleteBuffer ();
+ updatedCount++;
+ }
+ } // m_RouterInfos iteration
+
+ if (updatedCount > 0)
+ LogPrint (eLogInfo, "NetDb: saved ", updatedCount, " new/updated routers");
+
+ for (auto& it: m_UnsavedProfiles)
+ it.second->Save (it.first);
+ {
+ std::unique_lock<std::mutex> l(m_UnsavedProfilesMutex);
+ m_UnsavedProfiles.clear ();
+ }
+ }
+
+ void NetDb::RemoveExpired ()
+ {
auto total = m_RouterInfos.size ();
+ int deletedCount = 0;
+
uint64_t expirationTimeout = NETDB_MAX_EXPIRATION_TIMEOUT*1000LL;
uint64_t ts = i2p::util::GetMillisecondsSinceEpoch();
// routers don't expire if less than 90 or uptime is less than 1 hour
@@ -519,17 +569,6 @@ namespace data
for (auto& it: m_RouterInfos)
{
- std::string ident = it.second->GetIdentHashBase64();
- std::string path = m_Storage.Path(ident);
- if (it.second->IsUpdated ())
- {
- it.second->SaveToFile (path);
- it.second->SetUpdated (false);
- it.second->SetUnreachable (false);
- it.second->DeleteBuffer ();
- updatedCount++;
- continue;
- }
// find & mark expired routers
if (it.second->UsesIntroducer ())
{
@@ -543,14 +582,12 @@ namespace data
if (it.second->IsUnreachable ())
{
// delete RI file
- m_Storage.Remove(ident);
+ m_Storage.Remove (it.first.ToBase64 ());
deletedCount++;
if (total - deletedCount < NETDB_MIN_ROUTERS) checkForExpiration = false;
}
} // m_RouterInfos iteration
- if (updatedCount > 0)
- LogPrint (eLogInfo, "NetDb: saved ", updatedCount, " new/updated routers");
if (deletedCount > 0)
{
LogPrint (eLogInfo, "NetDb: deleting ", deletedCount, " unreachable routers");
@@ -561,7 +598,8 @@ namespace data
{
if (it->second->IsUnreachable ())
{
- if (m_PersistProfiles) it->second->SaveProfile ();
+ if (m_PersistProfiles && it->second->HasProfile ())
+ m_UnsavedProfiles[it->second->GetIdentHash ()] = it->second->GetProfile ();
it = m_RouterInfos.erase (it);
continue;
}
@@ -580,6 +618,21 @@ namespace data
}
}
+ void NetDb::RemoveObsoleteProfiles ()
+ {
+ DeleteObsoleteProfiles ();
+ auto now = boost::posix_time::second_clock::local_time ();
+ {
+ std::unique_lock<std::mutex> l(m_UnsavedProfilesMutex);
+ for (auto it = m_UnsavedProfiles.begin (); it != m_UnsavedProfiles.end ();)
+ if ((now - it->second->GetLastUpdateTime ()).hours () >= PEER_PROFILE_EXPIRATION_TIMEOUT)
+ it = m_UnsavedProfiles.erase (it);
+ else
+ ++it;
+ }
+ LogPrint (eLogInfo, "NetDb: obsolete profiles were removed");
+ }
+
void NetDb::RequestDestination (const IdentHash& destination, RequestedDestination::RequestComplete requestComplete)
{
auto dest = m_Requests.CreateRequest (destination, false, requestComplete); // non-exploratory
diff --git a/libi2pd/NetDb.hpp b/libi2pd/NetDb.hpp
index 93e9e48f..58999e91 100644
--- a/libi2pd/NetDb.hpp
+++ b/libi2pd/NetDb.hpp
@@ -109,6 +109,8 @@ namespace data
void Load ();
bool LoadRouterInfo (const std::string & path);
void SaveUpdated ();
+ void RemoveExpired ();
+ void RemoveObsoleteProfiles ();
void Run (); // exploratory thread
void Explore (int numDestinations);
void Publish ();
@@ -129,6 +131,8 @@ namespace data
std::map<IdentHash, std::shared_ptr<LeaseSet> > m_LeaseSets;
mutable std::mutex m_RouterInfosMutex;
std::map<IdentHash, std::shared_ptr<RouterInfo> > m_RouterInfos;
+ mutable std::mutex m_UnsavedProfilesMutex;
+ std::map<IdentHash, std::shared_ptr<RouterProfile>> m_UnsavedProfiles;
mutable std::mutex m_FloodfillsMutex;
std::list<std::shared_ptr<RouterInfo> > m_Floodfills;
@@ -145,6 +149,7 @@ namespace data
friend class NetDbRequests;
NetDbRequests m_Requests;
+ unsigned m_PersistSyncInterval;
bool m_PersistProfiles;
/** router info we are bootstrapping from or nullptr if we are not currently doing that*/
diff --git a/libi2pd/Profiling.h b/libi2pd/Profiling.h
index 4ba6702f..708cae45 100644
--- a/libi2pd/Profiling.h
+++ b/libi2pd/Profiling.h
@@ -29,6 +29,8 @@ namespace data
RouterProfile ();
RouterProfile& operator= (const RouterProfile& ) = default;
+ boost::posix_time::ptime GetLastUpdateTime () const { return m_LastUpdateTime; };
+
void Save (const IdentHash& identHash);
void Load (const IdentHash& identHash);
diff --git a/libi2pd/RouterInfo.h b/libi2pd/RouterInfo.h
index 084ad8a7..3e492a96 100644
--- a/libi2pd/RouterInfo.h
+++ b/libi2pd/RouterInfo.h
@@ -194,6 +194,8 @@ namespace data
bool SaveToFile (const std::string& fullPath);
std::shared_ptr<RouterProfile> GetProfile () const;
+ bool HasProfile () const { return static_cast<bool>(m_Profile); };
+ void SetProfile (const std::shared_ptr<RouterProfile>& profile) { m_Profile = profile; };
void SaveProfile () { if (m_Profile) m_Profile->Save (GetIdentHash ()); };
void Update (const uint8_t * buf, int len);