Merge 8a8ea9d5d4 into 59954c1d7c

Makefile

@@ -18,6 +18,7 @@ USE_AVX		:= yes
 USE_STATIC	:= no
 USE_MESHNET	:= no
 USE_UPNP	:= no
+USE_LMDB	:= yes
 
 ifeq ($(WEBSOCKETS),1)
 	NEEDED_CXXFLAGS += -DWITH_EVENTS
@@ -47,6 +48,11 @@ ifeq ($(USE_MESHNET),yes)
 	NEEDED_CXXFLAGS += -DMESHNET
 endif
 
+ifeq ($(USE_LMDB),yes)
+	NEEDED_CXXFLAGS += -DLMDB
+	LDLIBS += -llmdb
+endif
+
 NEEDED_CXXFLAGS += -I$(LIB_SRC_DIR) -I$(LIB_CLIENT_SRC_DIR)
 
 all: mk_obj_dir $(ARLIB) $(ARLIB_CLIENT) $(I2PD)

libi2pd/Config.cpp

@@ -232,6 +232,11 @@ namespace config {
 			("exploratory.outbound.quantity", value<int>()->default_value(3), "Exploratory outbound tunnels quantity")
 		;
 
+		options_description storage("Persistent storage options for NetDb, profiles, etc.");
+		storage.add_options()
+				("storage.engine", value<std::string>()->default_value("fs"), "Storage engine")
+		;
+
 		m_OptionsDesc
 			.add(general)
 			.add(limits)
@@ -249,6 +254,7 @@ namespace config {
 			.add(trust)
 			.add(websocket)
 			.add(exploratory)
+			.add(storage)
 		;
 	}
 

libi2pd/NetDb.cpp

@@ -25,7 +25,7 @@ namespace data
 {
 	NetDb netdb;
 
-	NetDb::NetDb (): m_IsRunning (false), m_Thread (nullptr), m_Reseeder (nullptr), m_Storage("netDb", "r", "routerInfo-", "dat"), m_FloodfillBootstrap(nullptr), m_HiddenMode(false)
+	NetDb::NetDb (): m_IsRunning (false), m_Thread (nullptr), m_Reseeder (nullptr), m_Storage(nullptr), m_FloodfillBootstrap(nullptr), m_HiddenMode(false)
 	{
 	}
 
@@ -37,8 +37,14 @@ namespace data
 
 	void NetDb::Start ()
 	{
-		m_Storage.SetPlace(i2p::fs::GetDataDir());
+		std::string engine;
-		m_Storage.Init(i2p::data::GetBase64SubstitutionTable(), 64);
+		i2p::config::GetOption("storage.engine", engine);
+		if (engine == "lmdb")
+			m_Storage.reset(new MdbIdentStorage("netDb.lmdb"));
+		else
+			m_Storage.reset(new FsIdentStorage("netDb", "r", "routerInfo-", "dat"));
+
+		m_Storage->Init();
 		InitProfilesStorage ();
 		m_Families.LoadCertificates ();
 		Load ();
@@ -55,9 +61,15 @@ namespace data
 	{
 		if (m_IsRunning)
 		{
-			for (auto& it: m_RouterInfos)
+			if(BeginProfilesStorageUpdate()) //TODO Error handling
-				it.second->SaveProfile ();
+			{
-			DeleteObsoleteProfiles ();
+				for (auto& it: m_RouterInfos)
+					it.second->SaveProfile ();
+				DeleteObsoleteProfiles();
+				EndProfilesStorageUpdate();
+			}
+			m_Storage->DeInit();
+			DeInitProfilesStorage();
 			m_RouterInfos.clear ();
 			m_Floodfills.clear ();
 			if (m_Thread)
@@ -360,9 +372,9 @@ namespace data
 		i2p::transport::transports.SendMessages(ih, requests);
 	}
 
-	bool NetDb::LoadRouterInfo (const std::string & path)
+	bool NetDb::LoadRouterInfo (const char *buffer, size_t len)
 	{
-		auto r = std::make_shared<RouterInfo>(path);
+		auto r = std::make_shared<RouterInfo>((uint8_t *)buffer, len, false);
 		if (r->GetRouterIdentity () && !r->IsUnreachable () &&
 				(!r->UsesIntroducer () || m_LastLoad < r->GetTimestamp () + NETDB_INTRODUCEE_EXPIRATION_TIMEOUT*1000LL)) // 1 hour
 		{
@@ -374,8 +386,7 @@ namespace data
 		}
 		else
 		{
-			LogPrint(eLogWarning, "NetDb: RI from ", path, " is invalid. Delete");
+			LogPrint(eLogWarning, "NetDb: RI from ", r->GetIdentHash(), " is invalid.");
-			i2p::fs::Remove(path);
 		}
 		return true;
 	}
@@ -387,14 +398,6 @@ namespace data
 			v(entry.first, entry.second);
 	}
 
-	void NetDb::VisitStoredRouterInfos(RouterInfoVisitor v)
-	{
-		m_Storage.Iterate([v] (const std::string & filename) {
-        auto ri = std::make_shared<i2p::data::RouterInfo>(filename);
-				v(ri);
-		});
-	}
-
 	void NetDb::VisitRouterInfos(RouterInfoVisitor v)
 	{
 		std::unique_lock<std::mutex> lock(m_RouterInfosMutex);
@@ -455,10 +458,16 @@ namespace data
 		m_Floodfills.clear ();
 
 		m_LastLoad = i2p::util::GetSecondsSinceEpoch();
-		std::vector<std::string> files;
+		m_Storage->BeginUpdate();
-		m_Storage.Traverse(files);
+		m_Storage->Iterate([&](const IdentHash &ident, const StorageRecord &record)
-		for (const auto& path : files)
+		{
-			LoadRouterInfo(path);
+			if (!LoadRouterInfo(record.data.get(), record.len))
+			{
+				if (m_Storage->Remove(ident))
+					LogPrint(eLogInfo, "Deleting invalid RI from ", ident.ToBase64());
+			}
+		});
+		m_Storage->EndUpdate();
 
 		LogPrint (eLogInfo, "NetDb: ", m_RouterInfos.size(), " routers loaded (", m_Floodfills.size (), " floodfils)");
 	}
@@ -475,13 +484,18 @@ namespace data
 			expirationTimeout = i2p::context.IsFloodfill () ? NETDB_FLOODFILL_EXPIRATION_TIMEOUT*1000LL :
 					NETDB_MIN_EXPIRATION_TIMEOUT*1000LL + (NETDB_MAX_EXPIRATION_TIMEOUT - NETDB_MIN_EXPIRATION_TIMEOUT)*1000LL*NETDB_MIN_ROUTERS/total;
 
+		m_Storage->BeginUpdate();
 		for (auto& it: m_RouterInfos)
 		{
-			std::string ident = it.second->GetIdentHashBase64();
+			IdentHash ident = it.second->GetIdentHash();
-			std::string path  = m_Storage.Path(ident);
 			if (it.second->IsUpdated ())
 			{
-				it.second->SaveToFile (path);
+				if (it.second->GetBuffer()) {
+					StorageRecord record((char *)it.second->GetBuffer(), it.second->GetBufferLen());
+					m_Storage->Store(ident, record);
+				} else {
+					LogPrint (eLogError, "RouterInfo: Can't save, m_Buffer == NULL, Bleat!", it.second->GetIdentHashBase64());
+				}
 				it.second->SetUpdated (false);
 				it.second->SetUnreachable (false);
 				it.second->DeleteBuffer ();
@@ -501,17 +515,19 @@ namespace data
 			if (it.second->IsUnreachable ())
 			{
 				// delete RI file
-				m_Storage.Remove(ident);
+				m_Storage->Remove(ident);
 				deletedCount++;
 				if (total - deletedCount < NETDB_MIN_ROUTERS) checkForExpiration = false;
 			}
 		} // m_RouterInfos iteration
+		m_Storage->EndUpdate();
 
 		if (updatedCount > 0)
 			LogPrint (eLogInfo, "NetDb: saved ", updatedCount, " new/updated routers");
 		if (deletedCount > 0)
 		{
 			LogPrint (eLogInfo, "NetDb: deleting ", deletedCount, " unreachable routers");
+			BeginProfilesStorageUpdate(); //TODO: Error handling
 			// clean up RouterInfos table
 			{
 				std::unique_lock<std::mutex> l(m_RouterInfosMutex);
@@ -526,6 +542,7 @@ namespace data
 					++it;
 				}
 			}
+			EndProfilesStorageUpdate(); //TODO: Error handling
 			// clean up expired floodfiils
 			{
 				std::unique_lock<std::mutex> l(m_FloodfillsMutex);
@@ -829,7 +846,11 @@ namespace data
 				if (router)
 				{
 					LogPrint (eLogDebug, "NetDb: requested RouterInfo ", key, " found");
-					router->LoadBuffer ();
+					if (!router->GetBuffer())
+					{
+						StorageRecord record = m_Storage->Fetch(router->GetIdentHash());
+						router->LoadBuffer((const uint8_t*)record.data.get(), record.len);
+					}
 					if (router->GetBuffer ())
 						replyMsg = CreateDatabaseStoreMsg (router);
 				}

libi2pd/NetDb.hpp

@@ -7,6 +7,7 @@
 #include <list>
 #include <string>
 #include <thread>
+#include <memory>
 #include <mutex>
 
 #include "Base.h"
@@ -19,6 +20,7 @@
 #include "Tunnel.h"
 #include "TunnelPool.h"
 #include "Reseed.h"
+#include "Storage.h"
 #include "NetDbRequests.h"
 #include "Family.h"
 
@@ -93,8 +95,6 @@ namespace data
 
 			/** visit all lease sets we currently store */
 			void VisitLeaseSets(LeaseSetVisitor v);
-			/** visit all router infos we have currently on disk, usually insanely expensive, does not access in memory RI */
-			void VisitStoredRouterInfos(RouterInfoVisitor v);
 			/** visit all router infos we have loaded in memory, cheaper than VisitLocalRouterInfos but locks access while visiting */
 			void VisitRouterInfos(RouterInfoVisitor v);
 			/** visit N random router that match using filter, then visit them with a visitor, return number of RouterInfos that were visited */
@@ -105,7 +105,7 @@ namespace data
 		private:
 
 			void Load ();
-			bool LoadRouterInfo (const std::string & path);
+			bool LoadRouterInfo (const char *buffer, size_t len);
 			void SaveUpdated ();
 			void Run (); // exploratory thread
 			void Explore (int numDestinations);
@@ -135,7 +135,7 @@ namespace data
 			GzipInflator m_Inflator;
 			Reseeder * m_Reseeder;
 			Families m_Families;
-			i2p::fs::HashedStorage m_Storage;
+			std::unique_ptr<IdentStorage> m_Storage;
 
 			friend class NetDbRequests;
 			NetDbRequests m_Requests;

libi2pd/Profiling.cpp

@@ -1,7 +1,8 @@
-#include <sys/stat.h>
+#include <memory>
 #include <boost/property_tree/ptree.hpp>
 #include <boost/property_tree/ini_parser.hpp>
 #include "Base.h"
+#include "Config.h"
 #include "FS.h"
 #include "Log.h"
 #include "Profiling.h"
@@ -10,7 +11,7 @@ namespace i2p
 {
 namespace data
 {
-	i2p::fs::HashedStorage m_ProfilesStorage("peerProfiles", "p", "profile-", "txt");
+	std::unique_ptr<IdentStorage> m_ProfilesStorage(nullptr);
 
 	RouterProfile::RouterProfile ():
 		m_LastUpdateTime (boost::posix_time::second_clock::local_time()),
@@ -45,12 +46,15 @@ namespace data
 		pt.put_child (PEER_PROFILE_SECTION_PARTICIPATION, participation);
 		pt.put_child (PEER_PROFILE_SECTION_USAGE, usage);
 
-		// save to file
+		// save to string stream
 		std::string ident = identHash.ToBase64 ();
-		std::string path = m_ProfilesStorage.Path(ident);
+		std::ostringstream ss;
 
 		try {
-			boost::property_tree::write_ini (path, pt);
+			boost::property_tree::write_ini (ss, pt);
+			std::string str = ss.str();
+			StorageRecord record(str.c_str(), str.length());
+			m_ProfilesStorage->Store(identHash, record);
 		} catch (std::exception& ex) {
 			/* boost exception verbose enough */
 			LogPrint (eLogError, "Profiling: ", ex.what ());
@@ -59,19 +63,20 @@ namespace data
 
 	void RouterProfile::Load (const IdentHash& identHash)
 	{
-		std::string ident = identHash.ToBase64 ();
+		StorageRecord record = m_ProfilesStorage->Fetch(identHash);
-		std::string path = m_ProfilesStorage.Path(ident);
-		boost::property_tree::ptree pt;
 
-		if (!i2p::fs::Exists(path))
+		if (!record.IsValid())
 		{
-			LogPrint(eLogWarning, "Profiling: no profile yet for ", ident);
+			LogPrint(eLogWarning, "Profiling: no profile yet for ", identHash.ToBase64());
 			return;
 		}
 
+		std::istringstream ss(std::string(record.data.get(), record.data.get() + record.len));
+		boost::property_tree::ptree pt;
+
 		try
 		{
-			boost::property_tree::read_ini (path, pt);
+			boost::property_tree::read_ini (ss, pt);
 		} catch (std::exception& ex)
 		{
 			/* boost exception verbose enough */
@@ -96,7 +101,7 @@ namespace data
 				}
 				catch (boost::property_tree::ptree_bad_path& ex)
 				{
-					LogPrint (eLogWarning, "Profiling: Missing section ", PEER_PROFILE_SECTION_PARTICIPATION, " in profile for ", ident);
+					LogPrint (eLogWarning, "Profiling: Missing section ", PEER_PROFILE_SECTION_PARTICIPATION, " in profile for ", identHash.ToBase64());
 				}
 				try
 				{
@@ -107,7 +112,7 @@ namespace data
 				}
 				catch (boost::property_tree::ptree_bad_path& ex)
 				{
-					LogPrint (eLogWarning, "Missing section ", PEER_PROFILE_SECTION_USAGE, " in profile for ", ident);
+					LogPrint (eLogWarning, "Missing section ", PEER_PROFILE_SECTION_USAGE, " in profile for ", identHash.ToBase64());
 				}
 			}
 			else
@@ -115,7 +120,7 @@ namespace data
 		}
 		catch (std::exception& ex)
 		{
-			LogPrint (eLogError, "Profiling: Can't read profile ", ident, " :", ex.what ());
+			LogPrint (eLogError, "Profiling: Can't read profile ", identHash.ToBase64(), " :", ex.what ());
 		}
 	}
 
@@ -169,27 +174,56 @@ namespace data
 
 	void InitProfilesStorage ()
 	{
-		m_ProfilesStorage.SetPlace(i2p::fs::GetDataDir());
+		std::string engine;
-		m_ProfilesStorage.Init(i2p::data::GetBase64SubstitutionTable(), 64);
+		i2p::config::GetOption("storage.engine", engine);
+		if (engine == "lmdb")
+			m_ProfilesStorage.reset(new MdbIdentStorage("peerProfiles.lmdb"));
+		else
+			m_ProfilesStorage.reset(new FsIdentStorage("peerProfiles", "p", "profile-", "txt"));
+		m_ProfilesStorage->Init();
+
 	}
 
 	void DeleteObsoleteProfiles ()
 	{
-		struct stat st;
+		boost::posix_time::ptime now = boost::posix_time::from_time_t(std::time(nullptr));
-		std::time_t now = std::time(nullptr);
+		m_ProfilesStorage->Iterate([now](const IdentHash &ident, const StorageRecord &record) {
+			std::istringstream ss(std::string(record.data.get(), record.data.get() + record.len));
+			boost::property_tree::ptree pt;
+			try
+			{
+				boost::property_tree::read_ini (ss, pt);
+			} catch (std::exception& ex)
+			{
+				/* boost exception verbose enough */
+				LogPrint (eLogError, "Deleting obsolete profile: ", ex.what ());
+				return;
+			}
 
-		std::vector<std::string> files;
+			auto t = pt.get (PEER_PROFILE_LAST_UPDATE_TIME, "");
-		m_ProfilesStorage.Traverse(files);
+			boost::posix_time::ptime lastUpdate = now;
-		for (const auto& path: files) {
+			if (t.length () > 0)
-			if (stat(path.c_str(), &st) != 0) {
+				lastUpdate = boost::posix_time::time_from_string (t);
-				LogPrint(eLogWarning, "Profiling: Can't stat(): ", path);
+			if((now - lastUpdate).total_seconds()/ 3600 >= PEER_PROFILE_EXPIRATION_TIMEOUT) {
-				continue;
+				LogPrint(eLogDebug, "Profiling: removing expired peer profile: ", ident.ToBase64());
+				m_ProfilesStorage->Remove(ident);
 			}
-			if (((now - st.st_mtime) / 3600) >= PEER_PROFILE_EXPIRATION_TIMEOUT) {
+		});
-				LogPrint(eLogDebug, "Profiling: removing expired peer profile: ", path);
+	}
-				i2p::fs::Remove(path);
+
-			}
+	bool BeginProfilesStorageUpdate()
-		}
+	{
+		return m_ProfilesStorage->BeginUpdate();
+	}
+
+	bool EndProfilesStorageUpdate()
+	{
+		return m_ProfilesStorage->EndUpdate();
+	}
+
+	void DeInitProfilesStorage()
+	{
+		m_ProfilesStorage->DeInit();
 	}
 }
 }

libi2pd/Profiling.h

@@ -4,6 +4,7 @@
 #include <memory>
 #include <boost/date_time/posix_time/posix_time.hpp>
 #include "Identity.h"
+#include "Storage.h"
 
 namespace i2p
 {
@@ -61,6 +62,9 @@ namespace data
 	std::shared_ptr<RouterProfile> GetRouterProfile (const IdentHash& identHash);
 	void InitProfilesStorage ();
 	void DeleteObsoleteProfiles ();
+	bool BeginProfilesStorageUpdate();
+	bool EndProfilesStorageUpdate();
+	void DeInitProfilesStorage();
 }
 }
 

libi2pd/RouterInfo.cpp

@@ -34,14 +34,19 @@ namespace data
 		ReadFromFile ();
 	}
 
-	RouterInfo::RouterInfo (const uint8_t * buf, int len):
+	RouterInfo::RouterInfo (const uint8_t * buf, int len, bool checkSig):
-		m_IsUpdated (true), m_IsUnreachable (false), m_SupportedTransports (0), m_Caps (0)
+		m_IsUpdated (false), m_IsUnreachable (false), m_SupportedTransports (0), m_Caps (0)
 	{
 		m_Addresses = boost::make_shared<Addresses>(); // create empty list
 		m_Buffer = new uint8_t[MAX_RI_BUFFER_SIZE];
 		memcpy (m_Buffer, buf, len);
 		m_BufferLen = len;
-		ReadFromBuffer (true);
+		ReadFromBuffer (checkSig);
+	}
+
+	RouterInfo::RouterInfo (const uint8_t * buf, int len): RouterInfo(buf, len, true)
+	{
+		m_IsUpdated = true;
 	}
 
 	RouterInfo::~RouterInfo ()
@@ -353,7 +358,7 @@ namespace data
 	}
 
   bool RouterInfo::IsFamily(const std::string & fam) const {
-    return m_Family == fam;
+	return m_Family == fam;
   }
 
 	void RouterInfo::ExtractCaps (const char * value)
@@ -585,6 +590,20 @@ namespace data
 		return bufbe64toh (buf + size) > m_Timestamp;
 	}
 
+	const uint8_t * RouterInfo::LoadBuffer(const uint8_t *buf, size_t len) //TODO: error handling
+	{
+		m_BufferLen = len;
+		if (m_BufferLen < 40 || m_BufferLen > MAX_RI_BUFFER_SIZE)
+		{
+			LogPrint(eLogError, "RouterInfo: File", m_FullPath, " is malformed");
+			return nullptr;
+		}
+		if (!m_Buffer)
+			m_Buffer = new uint8_t[MAX_RI_BUFFER_SIZE];
+		memcpy(m_Buffer, buf, len);
+		return m_Buffer;
+	}
+
 	const uint8_t * RouterInfo::LoadBuffer ()
 	{
 		if (!m_Buffer)

libi2pd/RouterInfo.h

@@ -131,6 +131,7 @@ namespace data
 			RouterInfo (const std::string& fullPath);
 			RouterInfo (const RouterInfo& ) = default;
 			RouterInfo& operator=(const RouterInfo& ) = default;
+			RouterInfo (const uint8_t * buf, int len, bool);
 			RouterInfo (const uint8_t * buf, int len);
 			~RouterInfo ();
 
@@ -180,6 +181,7 @@ namespace data
 
 			const uint8_t * GetBuffer () const { return m_Buffer; };
 			const uint8_t * LoadBuffer (); // load if necessary
+			const uint8_t * LoadBuffer(const uint8_t *buf, size_t len); //load from memory
 			int GetBufferLen () const { return m_BufferLen; };
 			void CreateBuffer (const PrivateKeys& privateKeys);
 

libi2pd/Storage.cpp

@@ -0,0 +1,234 @@
+#include <iostream>
+#include <ios>
+
+#include <boost/filesystem.hpp>
+
+#include "Storage.h"
+
+namespace i2p {
+namespace data {
+
+FsIdentStorage::~FsIdentStorage() {}
+IdentStorage::~IdentStorage() {}
+
+bool IdentStorage::Init() { return true; }
+bool FsIdentStorage::Init()
+{
+	m_Storage.SetPlace(i2p::fs::GetDataDir());
+	if (m_IsB32)
+		m_Storage.Init(i2p::data::GetBase32SubstitutionTable(), 32);
+	else
+		m_Storage.Init(i2p::data::GetBase64SubstitutionTable(), 64);
+	return true;
+}
+
+bool FsIdentStorage::Store(const i2p::data::IdentHash &ident, const StorageRecord &record)
+{
+	std::string strid = m_IsB32 ? ident.ToBase32() : ident.ToBase64();
+	std::string path = m_Storage.Path(strid);
+	std::ofstream ofs(path, std::ios::binary);
+	ofs.write(record.data.get(), record.len);
+	ofs.flush();
+	return true;
+}
+
+StorageRecord FsIdentStorage::Fetch(const i2p::data::IdentHash &ident)
+{
+	std::string strid = m_IsB32 ? ident.ToBase32() : ident.ToBase64();
+	std::string path = m_Storage.Path(strid);
+	if (boost::filesystem::exists(path)) {
+		std::ifstream ifs(path, std::ios::binary);
+		ifs.seekg(0, std::ios::end);
+
+		int size = ifs.tellg();
+		ifs.seekg(0, std::ios::beg);
+
+		StorageRecord result(size);
+
+		ifs.read(result.data.get(), size);
+		return result;
+	}
+
+	return StorageRecord();
+}
+
+bool FsIdentStorage::Remove(const IdentHash & ident)
+{
+	std::string strid = m_IsB32 ? ident.ToBase32() : ident.ToBase64();
+	std::string path = m_Storage.Path(strid);
+	return i2p::fs::Remove(path);
+}
+
+void FsIdentStorage::Iterate(const DVisitor &f)
+{
+	auto fv = [&f, this](const std::string &path)
+	{
+		boost::filesystem::path p(path);
+		std::string id = p.stem().string().substr(m_Fprefix.length());
+		i2p::data::IdentHash ident;
+		if (m_IsB32)
+			ident.FromBase32(id);
+		else
+			ident.FromBase64(id);
+		StorageRecord data = Fetch(ident);
+		if (data.IsValid()) f(ident,data);
+	};
+
+	m_Storage.Iterate(fv);
+}
+
+#ifdef LMDB
+//MdbIdentStorage
+MdbIdentStorage::~MdbIdentStorage()
+{
+}
+
+bool MdbIdentStorage::Init()
+{
+	m_Path = i2p::fs::GetDataDir() + i2p::fs::dirSep + m_Name;
+
+	if (!boost::filesystem::exists(m_Path))
+		boost::filesystem::create_directory(m_Path);
+
+	if (!mdb_env_create(&env))
+	{
+
+		int ret = mdb_env_open(env, m_Path.c_str(), MDB_NOTLS, 0664);
+		if (!ret)
+		{
+			return true;
+		}
+		mdb_env_close(env);
+	}
+	return false;
+}
+
+void MdbIdentStorage::DeInit()
+{
+	if (m_Initialized)
+		DeInitWrite();
+	mdb_env_close(env);
+}
+
+bool MdbIdentStorage::BeginUpdate()  //TODO: remove unneeded memory juggling
+{
+	return InitWrite();
+}
+
+bool MdbIdentStorage::EndUpdate()
+{
+	return DeInitWrite();
+}
+
+bool MdbIdentStorage::Store(const i2p::data::IdentHash &ident, const StorageRecord& record)
+{
+	if (!m_Initialized)
+	{
+		return false;
+	}
+
+	MDB_val data;
+	data.mv_data = const_cast<char*>(record.data.get());
+	data.mv_size = record.len;
+
+	MDB_val key;
+	key.mv_data = const_cast<uint8_t*>(ident.data());
+	key.mv_size = 32;
+	return !mdb_put(txn, dbi, &key, &data, 0);
+}
+
+StorageRecord MdbIdentStorage::Fetch(const i2p::data::IdentHash &ident)
+{
+	StorageRecord result;
+
+	MDB_txn *trn;
+	MDB_dbi dbh;
+	if (!mdb_txn_begin(env, NULL, MDB_RDONLY, &trn))
+	{
+		if (!mdb_open(trn, NULL, 0, &dbh))
+		{
+			MDB_val key, data;
+			key.mv_data = const_cast<uint8_t*>(ident.data());
+			key.mv_size = 32;
+			if (!mdb_get(trn, dbh, &key, &data))
+			{
+				result = StorageRecord((char*)data.mv_data, data.mv_size);
+			}
+
+			mdb_txn_abort(trn);
+			mdb_close(env, dbh);
+		}
+	}
+	return result;
+}
+
+bool MdbIdentStorage::Remove(const IdentHash &ident)
+{
+	if (!m_Initialized)
+	{
+		return false;
+	}
+
+	MDB_val key;
+	key.mv_data = const_cast<uint8_t*>(ident.data());
+	key.mv_size = 32;
+	return !mdb_del(txn, dbi, &key, NULL);
+}
+
+void MdbIdentStorage::Iterate(const DVisitor & f)
+{
+	MDB_txn *trn;
+	MDB_dbi dbh;
+	MDB_val data, key;
+	MDB_cursor *cursor;
+
+	if(!mdb_txn_begin(env, NULL, MDB_RDONLY, &trn))
+	{
+		if (!mdb_open(trn, NULL, 0, &dbh))
+		{
+			if (!mdb_cursor_open(trn, dbh, &cursor))
+			{
+				while(!mdb_cursor_get(cursor, &key, &data, MDB_NEXT))
+				{
+					StorageRecord record((char*)data.mv_data, data.mv_size);
+					IdentHash ident((uint8_t*)key.mv_data);
+					f(ident, record);
+				}
+				mdb_cursor_close(cursor);
+			}
+
+			mdb_txn_abort(trn);
+			mdb_close(env, dbh);
+		} else
+		{
+			mdb_txn_abort(trn);
+		}
+	}
+}
+
+bool MdbIdentStorage::InitWrite()
+{
+	int ret = mdb_txn_begin(env, NULL, 0, &txn);
+	if (ret)
+		return false;
+
+	ret = mdb_open(txn, NULL, 0, &dbi);
+	if (ret)
+	{
+		mdb_txn_abort(txn);
+		return false;
+	}
+	m_Initialized = true;
+	return true;
+}
+
+bool MdbIdentStorage::DeInitWrite() {
+	int ret = mdb_txn_commit(txn);
+	mdb_close(env, dbi);
+	m_Initialized = false;
+	return  !ret;
+}
+#endif
+
+} //ns data
+} //ns i2p

libi2pd/Storage.h

@@ -0,0 +1,106 @@
+#ifndef STORAGE_H
+#define STORAGE_H
+#include <memory>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <fcntl.h>
+
+#ifdef LMDB
+#include <lmdb.h>
+#endif
+
+#include "FS.h"
+#include "Identity.h"
+#include "Log.h"
+
+namespace i2p
+{
+namespace data
+{
+
+class StorageRecord
+{
+public:
+	std::shared_ptr<char[]> data;
+	size_t len;
+	StorageRecord(size_t len): data(new char[len]), len(len) {}
+	StorageRecord(const char *buf, size_t len): StorageRecord(len)
+	{
+		memcpy(data.get(), buf, len);
+	}
+
+	StorageRecord():  data(nullptr), len(0) {}
+	bool IsValid() { return len > 0; }
+};
+
+typedef std::function<void(const IdentHash&, const StorageRecord&)> DVisitor;
+
+class IdentStorage
+{
+public:
+	IdentStorage() {}
+	virtual bool Init();
+	virtual void DeInit() {}
+	virtual bool BeginUpdate() { return true; }
+	virtual bool EndUpdate() { return true; }
+	virtual bool Store(const IdentHash &, const StorageRecord&) { return true; }
+	virtual bool Remove(const IdentHash &) { return true; }
+	virtual StorageRecord Fetch(const IdentHash&) { return StorageRecord(); }
+	virtual void Iterate(const DVisitor&) {}
+	virtual ~IdentStorage();
+};
+
+class FsIdentStorage : public IdentStorage
+{
+public:
+	FsIdentStorage(const char *name, const char* dprefix, const char *fprefix, const char *suffix, bool isB32=false) :
+		m_Storage(name, dprefix, fprefix, suffix), m_Fprefix(fprefix), m_IsB32(isB32) {}
+
+	virtual bool Init();
+	virtual void DeInit() {}
+	virtual bool BeginUpdate() { return true; }
+	virtual bool EndUpdate() { return true; }
+	virtual bool Store(const IdentHash &, const StorageRecord&);
+	virtual bool Remove(const IdentHash &);
+	StorageRecord Fetch(const IdentHash&);
+	virtual void Iterate(const DVisitor&);
+	virtual ~FsIdentStorage();
+
+private:
+	i2p::fs::HashedStorage m_Storage;
+	std::string m_Fprefix;
+	bool m_IsB32;
+};
+
+#ifdef LMDB
+class MdbIdentStorage : public IdentStorage
+{
+public:
+	MdbIdentStorage(const char *name) : m_Name(name) {}
+	virtual bool Init();
+	virtual void DeInit();
+	virtual bool BeginUpdate();
+	virtual bool EndUpdate();
+	virtual bool Store(const IdentHash &, const StorageRecord&);
+	virtual bool Remove(const IdentHash &);
+	StorageRecord Fetch(const IdentHash&);
+	virtual void Iterate(const DVisitor&);
+	virtual ~MdbIdentStorage();
+private:
+	bool InitRead();
+	void DeInitRead();
+	bool InitWrite();
+
+	bool DeInitWrite();
+	bool m_Initialized = false;
+	std::string m_Path;
+	std::string m_Name;
+	MDB_env *env;
+	MDB_dbi dbi;
+	MDB_txn *txn;
+};
+#endif
+} //ns data
+} //ns i2p
+
+#endif // STORAGE_H