Mirrors/i2pd
2
0
Fork 0
mirror of https://github.com/PurpleI2P/i2pd.git synced 2025-01-22 21:37:17 +01:00
Code Issues Actions 6 Packages Projects Releases Wiki Activity

* i2p::fs migration: NetDb.*

Browse source
This commit is contained in:
hagen 2016-02-11 00:00:00 +00:00
parent 6f4271c054
commit 6d74493491
2 changed files with 74 additions and 136 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

205
NetDb.cpp
View file

@ -1,12 +1,13 @@
#include <string.h> #include <string.h>
#include "I2PEndian.h"
#include <fstream> #include <fstream>
#include <vector> #include <vector>
#include <boost/asio.hpp> #include <boost/asio.hpp>
#include <openssl/rand.h> #include <openssl/rand.h>
#include <zlib.h> #include <zlib.h>
#include "I2PEndian.h"
#include "Base.h" #include "Base.h"
#include "Log.h" #include "Log.h"
#include "FS.h"
#include "Timestamp.h" #include "Timestamp.h"
#include "I2NPProtocol.h" #include "I2NPProtocol.h"
#include "Tunnel.h" #include "Tunnel.h"
@ -22,7 +23,6 @@ namespace i2p
{ {
namespace data namespace data
{ {
const char NetDb::m_NetDbPath[] = "netDb";
NetDb netdb; NetDb netdb;
NetDb::NetDb (): m_IsRunning (false), m_Thread (nullptr), m_Reseeder (nullptr) NetDb::NetDb (): m_IsRunning (false), m_Thread (nullptr), m_Reseeder (nullptr)
@ -250,30 +250,6 @@ namespace data
return it->second->SetUnreachable (unreachable); return it->second->SetUnreachable (unreachable);
} }
// TODO: Move to reseed and/or scheduled tasks. (In java version, scheduler fix this as well as sort RIs.)
bool NetDb::CreateNetDb(boost::filesystem::path directory)
{
LogPrint (eLogInfo, "NetDb: storage directory doesn't exist, trying to create it.");
if (!boost::filesystem::create_directory (directory))
{
LogPrint (eLogError, "NetDb: failed to create directory ", directory);
return false;
}
// list of chars might appear in base64 string
const char * chars = GetBase64SubstitutionTable (); // 64 bytes
for (int i = 0; i < 64; i++)
{
auto p = directory / (std::string ("r") + chars[i]);
if (!boost::filesystem::exists (p) && !boost::filesystem::create_directory (p))
{
LogPrint (eLogError, "NetDb: failed to create directory ", p);
return false;
}
}
return true;
}
void NetDb::Reseed () void NetDb::Reseed ()
{ {
if (!m_Reseeder) if (!m_Reseeder)
@ -288,145 +264,108 @@ namespace data
LogPrint (eLogWarning, "NetDb: failed to reseed after 10 attempts"); LogPrint (eLogWarning, "NetDb: failed to reseed after 10 attempts");
} }
bool NetDb::LoadRouterInfo (const std::string & path)
{
auto r = std::make_shared<RouterInfo>(path);
if (r->GetRouterIdentity () && !r->IsUnreachable () &&
(!r->UsesIntroducer () || m_LastLoad < r->GetTimestamp () + 3600*1000LL)) // 1 hour
{
r->DeleteBuffer ();
r->ClearProperties (); // properties are not used for regular routers
m_RouterInfos[r->GetIdentHash ()] = r;
if (r->IsFloodfill ())
m_Floodfills.push_back (r);
} else {
LogPrint(eLogWarning, "NetDb: Can't load RI from ", path, ", delete");
i2p::fs::Remove(path);
}
return true;
}
void NetDb::Load () void NetDb::Load ()
{ {
boost::filesystem::path p(i2p::util::filesystem::GetDataDir() / m_NetDbPath);
if (!boost::filesystem::exists (p))
{
// seems netDb doesn't exist yet
if (!CreateNetDb(p)) return;
}
// make sure we cleanup netDb from previous attempts // make sure we cleanup netDb from previous attempts
m_RouterInfos.clear (); m_RouterInfos.clear ();
m_Floodfills.clear (); m_Floodfills.clear ();
// load routers now m_LastLoad = i2p::util::GetSecondsSinceEpoch();
uint64_t ts = i2p::util::GetMillisecondsSinceEpoch (); std::vector<std::string> files;
int numRouters = 0; i2p::fs::GetNetDB().Traverse(files);
boost::filesystem::directory_iterator end; for (auto path : files)
for (boost::filesystem::directory_iterator it (p); it != end; ++it) LoadRouterInfo(path);
{
if (boost::filesystem::is_directory (it->status())) LogPrint (eLogInfo, "NetDb: ", m_RouterInfos.size(), " routers loaded (", m_Floodfills.size (), " floodfils)");
{
for (boost::filesystem::directory_iterator it1 (it->path ()); it1 != end; ++it1)
{
#if BOOST_VERSION > 10500
const std::string& fullPath = it1->path().string();
#else
const std::string& fullPath = it1->path();
#endif
auto r = std::make_shared<RouterInfo>(fullPath);
if (r->GetRouterIdentity () && !r->IsUnreachable () &&
(!r->UsesIntroducer () || ts < r->GetTimestamp () + 3600*1000LL)) // 1 hour
{
r->DeleteBuffer ();
r->ClearProperties (); // properties are not used for regular routers
m_RouterInfos[r->GetIdentHash ()] = r;
if (r->IsFloodfill ())
m_Floodfills.push_back (r);
numRouters++;
}
else
{
if (boost::filesystem::exists (fullPath))
boost::filesystem::remove (fullPath);
}
}
}
}
LogPrint (eLogInfo, "NetDb: ", numRouters, " routers loaded (", m_Floodfills.size (), " floodfils)");
} }
void NetDb::SaveUpdated () void NetDb::SaveUpdated ()
{ {
auto GetFilePath = [](const boost::filesystem::path& directory, const RouterInfo * routerInfo) int updatedCount = 0, deletedCount = 0;
{
std::string s(routerInfo->GetIdentHashBase64());
return directory / (std::string("r") + s[0]) / ("routerInfo-" + s + ".dat");
};
boost::filesystem::path fullDirectory (i2p::util::filesystem::GetDataDir() / m_NetDbPath);
int count = 0, deletedCount = 0;
auto total = m_RouterInfos.size (); auto total = m_RouterInfos.size ();
uint64_t ts = i2p::util::GetMillisecondsSinceEpoch (); uint64_t ts = i2p::util::GetMillisecondsSinceEpoch();
for (auto it: m_RouterInfos) for (auto it: m_RouterInfos)
{ {
if (it.second->IsUpdated ()) std::string ident = it.second->GetIdentHashBase64();
{ std::string path = i2p::fs::GetNetDB().Path(ident);
std::string f = GetFilePath(fullDirectory, it.second.get()).string(); if (it.second->IsUpdated ()) {
it.second->SaveToFile (f); it.second->SaveToFile (path);
it.second->SetUpdated (false); it.second->SetUpdated (false);
it.second->SetUnreachable (false); it.second->SetUnreachable (false);
it.second->DeleteBuffer (); it.second->DeleteBuffer ();
count++; updatedCount++;
continue;
} }
else // find & mark unreachable routers
{ if (it.second->UsesIntroducer () && ts > it.second->GetTimestamp () + 3600*1000LL) {
// RouterInfo expires after 1 hour if uses introducer // RouterInfo expires after 1 hour if uses introducer
if (it.second->UsesIntroducer () && ts > it.second->GetTimestamp () + 3600*1000LL) // 1 hour it.second->SetUnreachable (true);
it.second->SetUnreachable (true); } else if (total > 75 && ts > (i2p::context.GetStartupTime () + 600)*1000LL) {
else if (total > 75 && ts > (i2p::context.GetStartupTime () + 600)*1000LL) // routers don't expire if less than 25 or uptime is less than 10 minutes // routers don't expire if less than 25 or uptime is less than 10 minutes
{ if (i2p::context.IsFloodfill ()) {
if (i2p::context.IsFloodfill ()) if (ts > it.second->GetTimestamp () + 3600*1000LL) { // 1 hour
{ it.second->SetUnreachable (true);
if (ts > it.second->GetTimestamp () + 3600*1000LL) // 1 hours total--;
{
it.second->SetUnreachable (true);
total--;
}
} }
else if (total > 300) } else if (total > 300) {
{ if (ts > it.second->GetTimestamp () + 30*3600*1000LL) { // 30 hours
if (ts > it.second->GetTimestamp () + 30*3600*1000LL) // 30 hours it.second->SetUnreachable (true);
{ total--;
it.second->SetUnreachable (true);
total--;
}
} }
else if (total > 120) } else if (total > 120) {
{ if (ts > it.second->GetTimestamp () + 72*3600*1000LL) { // 72 hours
if (ts > it.second->GetTimestamp () + 72*3600*1000LL) // 72 hours it.second->SetUnreachable (true);
{ total--;
it.second->SetUnreachable (true);
total--;
}
} }
} }
}
if (it.second->IsUnreachable ())
{ if (it.second->IsUnreachable ()) {
total--; total--;
// delete RI file // delete RI file
if (boost::filesystem::exists (GetFilePath (fullDirectory, it.second.get ()))) i2p::fs::GetNetDB().Remove(ident);
{ deletedCount++;
boost::filesystem::remove (GetFilePath (fullDirectory, it.second.get ())); // delete from floodfills list
deletedCount++; if (it.second->IsFloodfill ()) {
} std::unique_lock<std::mutex> l(m_FloodfillsMutex);
// delete from floodfills list m_Floodfills.remove (it.second);
if (it.second->IsFloodfill ())
{
std::unique_lock<std::mutex> l(m_FloodfillsMutex);
m_Floodfills.remove (it.second);
}
} }
} }
} } // m_RouterInfos iteration
if (count > 0) if (updatedCount > 0)
LogPrint (eLogInfo, "NetDb: ", count, " new/updated routers saved"); LogPrint (eLogInfo, "NetDb: saved ", updatedCount, " new/updated routers");
if (deletedCount > 0) if (deletedCount > 0)
{ {
LogPrint (eLogDebug, "NetDb: ", deletedCount, " routers deleted"); LogPrint (eLogInfo, "NetDb: deleting ", deletedCount, " unreachable routers");
// clean up RouterInfos table // clean up RouterInfos table
std::unique_lock<std::mutex> l(m_RouterInfosMutex); std::unique_lock<std::mutex> l(m_RouterInfosMutex);
for (auto it = m_RouterInfos.begin (); it != m_RouterInfos.end ();) for (auto it = m_RouterInfos.begin (); it != m_RouterInfos.end ();)
{ {
if (it->second->IsUnreachable ()) if (it->second->IsUnreachable ()) {
{
it->second->SaveProfile (); it->second->SaveProfile ();
it = m_RouterInfos.erase (it); it = m_RouterInfos.erase (it);
continue;
} }
else it++;
it++;
} }
} }
} }

5
NetDb.h
View file

@ -8,7 +8,6 @@
#include <string> #include <string>
#include <thread> #include <thread>
#include <mutex> #include <mutex>
#include <boost/filesystem.hpp>
#include "Base.h" #include "Base.h"
#include "Queue.h" #include "Queue.h"
#include "I2NPProtocol.h" #include "I2NPProtocol.h"
@ -23,7 +22,6 @@ namespace i2p
{ {
namespace data namespace data
{ {
class NetDb class NetDb
{ {
public: public:
@ -69,8 +67,8 @@ namespace data
private: private:
bool CreateNetDb(boost::filesystem::path directory);
void Load (); void Load ();
bool LoadRouterInfo (const std::string & path);
void SaveUpdated (); void SaveUpdated ();
void Run (); // exploratory thread void Run (); // exploratory thread
void Explore (int numDestinations); void Explore (int numDestinations);
@ -90,6 +88,7 @@ namespace data
std::list<std::shared_ptr<RouterInfo> > m_Floodfills; std::list<std::shared_ptr<RouterInfo> > m_Floodfills;
bool m_IsRunning; bool m_IsRunning;
uint64_t m_LastLoad;
std::thread * m_Thread; std::thread * m_Thread;
i2p::util::Queue<std::shared_ptr<const I2NPMessage> > m_Queue; // of I2NPDatabaseStoreMsg i2p::util::Queue<std::shared_ptr<const I2NPMessage> > m_Queue; // of I2NPDatabaseStoreMsg