moved save/delete profile disk oprations to separate threads

libi2pd/NetDb.cpp

@@ -107,8 +107,8 @@ namespace data
 		i2p::util::SetThreadName("NetDB");
 
 		uint64_t lastManage = 0, lastExploratory = 0, lastManageRequest = 0;
-		uint64_t lastProfilesCleanup = i2p::util::GetMonotonicMilliseconds ();
+		uint64_t lastProfilesCleanup = i2p::util::GetMonotonicMilliseconds (), lastObsoleteProfilesCleanup = lastProfilesCleanup;
-		int16_t profilesCleanupVariance = 0;
+		int16_t profilesCleanupVariance = 0, obsoleteProfilesCleanVariance = 0;
 
 		while (m_IsRunning)
 		{
@@ -171,12 +171,39 @@ namespace data
 				if (mts >= lastProfilesCleanup + (uint64_t)(i2p::data::PEER_PROFILE_AUTOCLEAN_TIMEOUT + profilesCleanupVariance)*1000)
 				{
 					m_RouterProfilesPool.CleanUpMt ();
-					if (m_PersistProfiles) PersistProfiles ();
+					if (m_PersistProfiles)
-					DeleteObsoleteProfiles ();
+					{	
+						bool isSaving = m_SavingProfiles.valid ();
+						if (isSaving && m_SavingProfiles.wait_for(std::chrono::seconds(0)) == std::future_status::ready) // still active?
+						{
+							m_SavingProfiles.get ();
+							isSaving = false;
+						}	
+						if (!isSaving)
+							m_SavingProfiles = PersistProfiles ();
+						else
+							LogPrint (eLogWarning, "NetDb: Can't persist profiles. Profiles are being saved to disk");
+					}	
 					lastProfilesCleanup = mts;
 					profilesCleanupVariance = (rand () % (2 * i2p::data::PEER_PROFILE_AUTOCLEAN_VARIANCE) - i2p::data::PEER_PROFILE_AUTOCLEAN_VARIANCE);
 				}
 
+				if (mts >= lastObsoleteProfilesCleanup + (uint64_t)(i2p::data::PEER_PROFILE_OBSOLETE_PROFILES_CLEAN_TIMEOUT + obsoleteProfilesCleanVariance)*1000)
+				{
+					bool isDeleting = m_DeletingProfiles.valid ();
+					if (isDeleting && m_DeletingProfiles.wait_for(std::chrono::seconds(0)) == std::future_status::ready) // still active?
+					{
+						m_DeletingProfiles.get ();
+						isDeleting = false;
+					}	
+					if (!isDeleting)
+						m_DeletingProfiles = DeleteObsoleteProfiles ();	
+					else
+						LogPrint (eLogWarning, "NetDb: Can't delete profiles. Profiles are being deleted from disk");
+					lastObsoleteProfilesCleanup = mts;
+					obsoleteProfilesCleanVariance = (rand () % (2 * i2p::data::PEER_PROFILE_OBSOLETE_PROFILES_CLEAN_VARIANCE) - i2p::data::PEER_PROFILE_OBSOLETE_PROFILES_CLEAN_TIMEOUT);
+				}	
+			
 				if (mts >= lastExploratory + 30000) // exploratory every 30 seconds
 				{
 					auto numRouters = m_RouterInfos.size ();

libi2pd/NetDb.hpp

@@ -15,6 +15,7 @@
 #include <string>
 #include <thread>
 #include <mutex>
+#include <future>
 
 #include "Base.h"
 #include "Gzip.h"
@@ -177,6 +178,7 @@ namespace data
 			NetDbRequests m_Requests;
 
 			bool m_PersistProfiles;
+			std::future<void> m_SavingProfiles, m_DeletingProfiles;
 
 			/** router info we are bootstrapping from or nullptr if we are not currently doing that*/
 			std::shared_ptr<RouterInfo> m_FloodfillBootstrap;

libi2pd/Profiling.cpp

@@ -260,7 +260,13 @@ namespace data
 		g_ProfilesStorage.Init(i2p::data::GetBase64SubstitutionTable(), 64);
 	}
 
-	void PersistProfiles ()
+	static void SaveProfilesToDisk (std::list<std::pair<i2p::data::IdentHash, std::shared_ptr<RouterProfile> > >&& profiles)
+	{
+		for (auto& it: profiles)
+			if (it.second) it.second->Save (it.first);
+	}	
+		
+	std::future<void> PersistProfiles ()
 	{
 		auto ts = GetTime ();
 		std::list<std::pair<i2p::data::IdentHash, std::shared_ptr<RouterProfile> > > tmp;
@@ -278,8 +284,9 @@ namespace data
 					it++;
 			}
 		}
-		for (auto& it: tmp)
+		if (!tmp.empty ())
-			if (it.second) it.second->Save (it.first);
+			return std::async (std::launch::async, SaveProfilesToDisk, std::move (tmp));
+		return std::future<void>();
 	}
 
 	void SaveProfiles ()
@@ -287,8 +294,7 @@ namespace data
 		std::unordered_map<i2p::data::IdentHash, std::shared_ptr<RouterProfile> > tmp;
 		{
 			std::unique_lock<std::mutex> l(g_ProfilesMutex);
-			tmp = g_Profiles;
+			std::swap (tmp, g_Profiles);
-			g_Profiles.clear ();
 		}
 		auto ts = GetTime ();
 		for (auto& it: tmp)
@@ -296,7 +302,26 @@ namespace data
 				it.second->Save (it.first);
 	}
 
-	void DeleteObsoleteProfiles ()
+	static void DeleteFilesFromDisk (std::vector<std::string>&& files)
+	{
+		struct stat st;
+		std::time_t now = std::time(nullptr);
+		for (const auto& path: files) 
+		{
+			if (stat(path.c_str(), &st) != 0) 
+			{	
+				LogPrint(eLogWarning, "Profiling: Can't stat(): ", path);
+				continue;
+			}
+			if (now - st.st_mtime >= PEER_PROFILE_EXPIRATION_TIMEOUT*3600) 
+			{
+				LogPrint(eLogDebug, "Profiling: Removing expired peer profile: ", path);
+				i2p::fs::Remove(path);
+			}
+		}
+	}	
+		
+	std::future<void> DeleteObsoleteProfiles ()
 	{
 		{
 			auto ts = GetTime ();
@@ -310,21 +335,9 @@ namespace data
 			}
 		}
 
-		struct stat st;
-		std::time_t now = std::time(nullptr);
-
 		std::vector<std::string> files;
 		g_ProfilesStorage.Traverse(files);
-		for (const auto& path: files) {
+		return std::async (std::launch::async, DeleteFilesFromDisk, std::move (files));
-			if (stat(path.c_str(), &st) != 0) {
-				LogPrint(eLogWarning, "Profiling: Can't stat(): ", path);
-				continue;
-			}
-			if (now - st.st_mtime >= PEER_PROFILE_EXPIRATION_TIMEOUT*3600) {
-				LogPrint(eLogDebug, "Profiling: Removing expired peer profile: ", path);
-				i2p::fs::Remove(path);
-			}
-		}
 	}
 }
 }

libi2pd/Profiling.h

@@ -10,6 +10,7 @@
 #define PROFILING_H__
 
 #include <memory>
+#include <future>
 #include <boost/date_time/posix_time/posix_time.hpp>
 #include "Identity.h"
 
@@ -31,8 +32,10 @@ namespace data
 	const char PEER_PROFILE_USAGE_CONNECTED[] = "connected";
 	
 	const int PEER_PROFILE_EXPIRATION_TIMEOUT = 36; // in hours (1.5 days)
-	const int PEER_PROFILE_AUTOCLEAN_TIMEOUT = 3 * 3600; // in seconds (3 hours)
+	const int PEER_PROFILE_AUTOCLEAN_TIMEOUT = 1500; // in seconds (25 minutes)
-	const int PEER_PROFILE_AUTOCLEAN_VARIANCE = 3600; // in seconds (1 hour)
+	const int PEER_PROFILE_AUTOCLEAN_VARIANCE = 900; // in seconds (15 minutes)
+	const int PEER_PROFILE_OBSOLETE_PROFILES_CLEAN_TIMEOUT = 5400; // in seconds (1.5 hours)
+	const int PEER_PROFILE_OBSOLETE_PROFILES_CLEAN_VARIANCE = 2400; // in seconds (40 minutes)
 	const int PEER_PROFILE_DECLINED_RECENTLY_INTERVAL = 150; // in seconds (2.5 minutes)
 	const int PEER_PROFILE_PERSIST_INTERVAL = 3300; // in seconds (55 minutes)
 	const int PEER_PROFILE_UNREACHABLE_INTERVAL = 480; // in seconds (8 minutes)
@@ -89,9 +92,9 @@ namespace data
 	std::shared_ptr<RouterProfile> GetRouterProfile (const IdentHash& identHash);
 	bool IsRouterBanned (const IdentHash& identHash); // check only existing profiles
 	void InitProfilesStorage ();
-	void DeleteObsoleteProfiles ();
+	std::future<void> DeleteObsoleteProfiles ();
 	void SaveProfiles ();
-	void PersistProfiles ();
+	std::future<void> PersistProfiles ();
 }
 }