* Added support for variable sized fd_set's, this should remove the FD_SETSIZE limit.

git-svn-id: svn://rakshasa.no/libtorrent/trunk/rtorrent@512 e378c898-3ddf-0310-93e7-cc216c733640

TODO

@@ -1,3 +1,5 @@
+See libtorrent/TODO for a list of TODO items.
+
 Polling during last phase of shutdown should be very quick, don't use
 normal timeout.
 

configure.ac

@@ -20,6 +20,8 @@ TORRENT_CHECK_EXECINFO()
 TORRENT_CHECK_CURL()
 TORRENT_OTFD()
 
+TORRENT_WITHOUT_VARIABLE_FDSET()
+
 PKG_CHECK_MODULES(STUFF, sigc++-2.0 libtorrent >= 0.7.1,
 	          CXXFLAGS="$CXXFLAGS $STUFF_CFLAGS $CURL_CFLAGS";
 		  LIBS="$LIBS $STUFF_LIBS $CURL_LIBS")

scripts/checks.m4

@@ -128,7 +128,7 @@ AC_DEFUN([TORRENT_CHECK_EPOLL], [
 
 AC_DEFUN([TORRENT_WITHOUT_EPOLL], [
   AC_ARG_WITH(epoll,
-    [  --without-epoll         Do not check for epoll support],
+    [  --without-epoll         Do not check for epoll support.],
     [
       if test "$withval" = "yes"; then
         TORRENT_CHECK_EPOLL
@@ -139,6 +139,20 @@ AC_DEFUN([TORRENT_WITHOUT_EPOLL], [
 ])
 
 
+AC_DEFUN([TORRENT_WITHOUT_VARIABLE_FDSET], [
+  AC_ARG_WITH(variable-fdset,
+
+    [  --without-variable-fdset       do not use non-portable variable sized fd_set's.],
+    [
+      if test "$withval" = "yes"; then
+        AC_DEFINE(USE_VARIABLE_FDSET, 1, defined when we allow the use of fd_set's of any size)
+      fi
+    ], [
+      AC_DEFINE(USE_VARIABLE_FDSET, 1, defined when we allow the use of fd_set's of any size)
+    ])
+])
+
+
 AC_DEFUN([TORRENT_CHECK_POSIX_FALLOCATE], [
   AC_MSG_CHECKING(for posix_fallocate)
 

scripts/common.m4

@@ -2,7 +2,7 @@ AC_DEFUN([TORRENT_CHECK_CXXFLAGS], [
 
   AC_MSG_CHECKING([for user-defined CXXFLAGS])
 
-  if test $CXXFLAGS; then
+  if test -n "$CXXFLAGS"; then
     AC_MSG_RESULT([user-defined "$CXXFLAGS"])
   else
     CXXFLAGS="-O3 -Wall"

src/core/manager.cc

@@ -67,42 +67,39 @@ connect_signal_tracker_log(Download* d, torrent::Download::SlotString s) {
 }
 
 Manager::Manager() :
+  m_pollManager(NULL),
   m_portRandom(false),
   m_portFirst(6890),
   m_portLast(6999),
   m_checkHash(true) {
+}
 
-  // Consider doing this somewhere else.
+void
+Manager::initialize_first() {
   if ((m_pollManager = PollManagerEPoll::create(sysconf(_SC_OPEN_MAX))) != NULL)
     m_logImportant.push_front("Using 'epoll' based polling");
   else if ((m_pollManager = PollManagerSelect::create(sysconf(_SC_OPEN_MAX))) != NULL)
     m_logImportant.push_front("Using 'select' based polling");
   else
     throw std::runtime_error("Could not create any PollManager");
-}
 
-Manager::~Manager() {
-  delete m_pollManager;
+  // Need to initialize this before parseing options.
+  torrent::initialize(m_pollManager->get_torrent_poll());
 }
 
 void
-Manager::initialize() {
+Manager::initialize_second() {
   torrent::Http::set_factory(m_pollManager->get_http_stack()->get_http_factory());
   m_httpQueue.slot_factory(m_pollManager->get_http_stack()->get_http_factory());
 
   CurlStack::global_init();
   listen_open();
 
-  if (torrent::get_max_open_files() + torrent::get_max_open_sockets() + 32 > FD_SETSIZE) {
-    m_logImportant.push_front("Warning: Max open sockets and files exceeds FD_SETSIZE");
-    m_logComplete.push_front("Warning: Max open sockets and files exceeds FD_SETSIZE");
+  if (torrent::get_max_open_files() + torrent::get_max_open_sockets() + 32 > m_pollManager->max_open_sockets()) {
+    m_logImportant.push_front("Warning: Max open sockets and files exceeds poll manager's max open sockets");
+    m_logComplete.push_front("Warning: Max open sockets and files exceeds poll manager's max open sockets");
   }    
 
-  if (torrent::get_max_open_files() + torrent::get_max_open_sockets() + 32 > (unsigned int)sysconf(_SC_OPEN_MAX)) {
-    m_logImportant.push_front("Warning: Max open sockets and files exceeds _SC_OPEN_MAX");
-    m_logComplete.push_front("Warning: Max open sockets and files exceeds _SC_OPEN_MAX");
-  }
-
   // Register slots to be called when a download is inserted/erased,
   // opened or closed.
   m_downloadList.slot_map_insert()["0_initialize_bencode"]  = sigc::mem_fun(*this, &Manager::initialize_bencode);
@@ -137,6 +134,8 @@ Manager::cleanup() {
 
   torrent::cleanup();
   CurlStack::global_cleanup();
+
+  delete m_pollManager;
 }
 
 void

src/core/manager.h

@@ -59,7 +59,6 @@ public:
   typedef sigc::slot0<void>                         SlotFailed;
 
   Manager();
-  ~Manager();
 
   DownloadList&       get_download_list()                 { return m_downloadList; }
   DownloadStore&      get_download_store()                { return m_downloadStore; }
@@ -75,7 +74,9 @@ public:
 
   void                set_check_hash(bool state)          { m_checkHash = state; }
 
-  void                initialize();
+  // Really should find a more descriptive name.
+  void                initialize_first();
+  void                initialize_second();
   void                cleanup();
 
   void                shutdown(bool force);

src/core/poll_manager.cc

@@ -43,17 +43,30 @@
 
 namespace core {
 
-PollManager::PollManager(int maxOpenSockets) {
-  // Add a hack here to create larger fd_set's, depending on a USE
-  // flag.
-  
-  m_maxOpenSockets = maxOpenSockets;
+PollManager::PollManager(torrent::Poll* poll) :
+  m_poll(poll) {
+
+  if (m_poll == NULL)
+    throw std::logic_error("PollManager::PollManager(...) received poll == NULL");
+
+#if defined USE_VARIABLE_FDSET
+  m_setSize = m_poll->max_open_sockets() / 8;
+  m_readSet = (fd_set*)new char[m_setSize];
+  m_writeSet = (fd_set*)new char[m_setSize];
+  m_errorSet = (fd_set*)new char[m_setSize];
+#else
+  if (m_poll->max_open_sockets() > FD_SETSIZE)
+    throw std::logic_error("PollManager::PollManager(...) received a max open sockets >= FD_SETSIZE, but USE_VARIABLE_FDSET was not defined");
+
+  m_setSize = FD_SETSIZE / 8;
   m_readSet = new fd_set;
   m_writeSet = new fd_set;
   m_errorSet = new fd_set;
+#endif
 }
 
 PollManager::~PollManager() {
+  delete m_poll;
   delete m_readSet;
   delete m_writeSet;
   delete m_errorSet;

src/core/poll_manager.h

@@ -39,14 +39,11 @@
 
 #include <sys/select.h>
 #include <sigc++/signal.h>
+#include <torrent/poll.h>
 
 #include "curl_stack.h"
 #include "utils/timer.h"
 
-namespace torrent {
-  class Poll;
-}
-
 namespace core {
 
 // CurlStack really should be somewhere else, but that won't happen
@@ -56,13 +53,13 @@ class PollManager {
 public:
   typedef sigc::signal0<void> Signal;
 
-  PollManager(int maxOpenSockets);
+  PollManager(torrent::Poll* poll);
   virtual ~PollManager();
 
-  unsigned int        get_max_open_sockets() const { return m_maxOpenSockets; }
-  CurlStack*          get_http_stack()             { return &m_httpStack; }
+  unsigned int        max_open_sockets() const     { return m_poll->max_open_sockets(); }
 
-  virtual torrent::Poll* get_torrent_poll() = 0;
+  CurlStack*          get_http_stack()             { return &m_httpStack; }
+  torrent::Poll*      get_torrent_poll()           { return m_poll; }
 
   virtual void        poll(utils::Timer timeout) = 0;
 
@@ -75,9 +72,10 @@ protected:
 
   void                check_error();
 
-  unsigned int        m_maxOpenSockets;
+  torrent::Poll*      m_poll;
   CurlStack           m_httpStack;
 
+  unsigned int        m_setSize;
   fd_set*             m_readSet;
   fd_set*             m_writeSet;
   fd_set*             m_errorSet;

src/core/poll_manager_epoll.cc

@@ -36,6 +36,7 @@
 
 #include "config.h"
 
+#include <cstring>
 #include <stdexcept>
 #include <torrent/poll_epoll.h>
 #include <torrent/torrent.h>
@@ -50,20 +51,11 @@ PollManagerEPoll::create(int maxOpenSockets) {
 
   if (p == NULL)
     return NULL;
-
-  PollManagerEPoll* manager = new PollManagerEPoll(maxOpenSockets);
-  manager->m_poll = p;
-
-  return manager;
+  else
+    return new PollManagerEPoll(p);
 }
 
 PollManagerEPoll::~PollManagerEPoll() {
-  delete m_poll;
-}
-
-torrent::Poll*
-PollManagerEPoll::get_torrent_poll() {
-  return m_poll;
 }
 
 void
@@ -73,17 +65,20 @@ PollManagerEPoll::poll(utils::Timer timeout) {
   if (m_httpStack.is_busy()) {
     // When we're using libcurl we need to use select, but as this is
     // inefficient we try avoiding it whenever possible.
+#if defined USE_VARIABLE_FDSET
+    std::memset(m_readSet, 0, m_setSize);
+    std::memset(m_writeSet, 0, m_setSize);
+    std::memset(m_errorSet, 0, m_setSize);
+#else
     FD_ZERO(m_readSet);
     FD_ZERO(m_writeSet);
     FD_ZERO(m_errorSet);
-    FD_SET(m_poll->get_fd(), m_readSet);
+#endif    
+    FD_SET(static_cast<torrent::PollEPoll*>(m_poll)->get_fd(), m_readSet);
 
-    unsigned int maxFd = std::max((unsigned int)m_poll->get_fd(),
+    unsigned int maxFd = std::max((unsigned int)static_cast<torrent::PollEPoll*>(m_poll)->get_fd(),
 				  m_httpStack.fdset(m_readSet, m_writeSet, m_errorSet));
 
-    if (maxFd >= m_maxOpenSockets)
-      throw std::runtime_error("Error polling, maxFd >= m_maxOpenSockets");
-
     timeval t = timeout.tval();
 
     if (select(maxFd + 1, m_readSet, m_writeSet, m_errorSet, &t) == -1)
@@ -91,7 +86,7 @@ PollManagerEPoll::poll(utils::Timer timeout) {
 
     m_httpStack.perform();
 
-    if (!FD_ISSET(m_poll->get_fd(), m_readSet))
+    if (!FD_ISSET(static_cast<torrent::PollEPoll*>(m_poll)->get_fd(), m_readSet))
       return;
 
     // Clear the timeout since we've already used it in the select call.
@@ -102,10 +97,10 @@ PollManagerEPoll::poll(utils::Timer timeout) {
   // function. ;)
   torrent::perform();
 
-  if (m_poll->poll(timeout.usec() / 1000) == -1)
+  if (static_cast<torrent::PollEPoll*>(m_poll)->poll(timeout.usec() / 1000) == -1)
     return check_error();
 
-  m_poll->perform();
+  static_cast<torrent::PollEPoll*>(m_poll)->perform();
   torrent::perform();
 }
 

src/core/poll_manager_epoll.h

@@ -55,9 +55,7 @@ public:
   void                poll(utils::Timer timeout);
 
 private:
-  PollManagerEPoll(int maxOpenSockets) : PollManager(maxOpenSockets) {}
-
-  torrent::PollEPoll* m_poll;
+  PollManagerEPoll(torrent::Poll* p) : PollManager(p) {}
 };
 
 }

src/core/poll_manager_select.cc

@@ -36,6 +36,7 @@
 
 #include "config.h"
 
+#include <cstring>
 #include <stdexcept>
 #include <torrent/poll_select.h>
 #include <torrent/torrent.h>
@@ -50,38 +51,32 @@ PollManagerSelect::create(int maxOpenSockets) {
 
   if (p == NULL)
     return NULL;
-
-  PollManagerSelect* manager = new PollManagerSelect(maxOpenSockets);
-  manager->m_poll = p;
-
-  return manager;
+  else
+    return new PollManagerSelect(p);
 }
 
 PollManagerSelect::~PollManagerSelect() {
-  delete m_poll;
-}
-
-torrent::Poll*
-PollManagerSelect::get_torrent_poll() {
-  return m_poll;
 }
 
 void
 PollManagerSelect::poll(utils::Timer timeout) {
   timeout = std::min(timeout, utils::Timer(torrent::get_next_timeout()));
 
+#if defined USE_VARIABLE_FDSET
+  std::memset(m_readSet, 0, m_setSize);
+  std::memset(m_writeSet, 0, m_setSize);
+  std::memset(m_errorSet, 0, m_setSize);
+#else
   FD_ZERO(m_readSet);
   FD_ZERO(m_writeSet);
   FD_ZERO(m_errorSet);
+#endif    
 
-  unsigned int maxFd = m_poll->fdset(m_readSet, m_writeSet, m_errorSet);
+  unsigned int maxFd = static_cast<torrent::PollSelect*>(m_poll)->fdset(m_readSet, m_writeSet, m_errorSet);
 
   if (m_httpStack.is_busy())
     maxFd = std::max(maxFd, m_httpStack.fdset(m_readSet, m_writeSet, m_errorSet));
 
-  if (maxFd >= m_maxOpenSockets)
-    throw std::runtime_error("Error polling, maxFd >= m_maxOpenSockets");
-
   timeval t = timeout.tval();
 
   if (select(maxFd + 1, m_readSet, m_writeSet, m_errorSet, &t) == -1)
@@ -91,7 +86,7 @@ PollManagerSelect::poll(utils::Timer timeout) {
     m_httpStack.perform();
 
   torrent::perform();
-  m_poll->perform(m_readSet, m_writeSet, m_errorSet);
+  static_cast<torrent::PollSelect*>(m_poll)->perform(m_readSet, m_writeSet, m_errorSet);
   torrent::perform();
 }
 

src/core/poll_manager_select.h

@@ -50,14 +50,10 @@ public:
   static PollManagerSelect* create(int maxOpenSockets);
   ~PollManagerSelect();
 
-  torrent::Poll*      get_torrent_poll();
-
   void                poll(utils::Timer timeout);
 
 private:
-  PollManagerSelect(int maxOpenSockets) : PollManager(maxOpenSockets) {}
-
-  torrent::PollSelect* m_poll;
+  PollManagerSelect(torrent::Poll* p) : PollManager(p) {}
 };
 
 }

src/main.cc

@@ -188,20 +188,6 @@ load_arg_torrents(ui::Control* c, char** first, char** last) {
   }
 }
 
-void
-initialize_display(ui::Control* c) {
-  display::Canvas::init();
-  display::Window::slot_adjust(sigc::mem_fun(c->get_display(), &display::Manager::adjust_layout));
-}
-
-void
-initialize_core(ui::Control* c) {
-  c->get_core().get_poll_manager()->signal_interrupted().connect(sigc::mem_fun(c->get_input_stdin(), &input::InputEvent::event_read));
-  c->get_core().get_poll_manager()->signal_interrupted().connect(sigc::ptr_fun(display::Canvas::do_update));
-
-  c->get_core().initialize();
-}
-
 int
 main(int argc, char** argv) {
   utils::Timer::update();
@@ -222,19 +208,13 @@ main(int argc, char** argv) {
     SignalHandler::set_handler(SIGBUS,  sigc::bind(sigc::ptr_fun(&do_panic), SIGBUS));
     SignalHandler::set_handler(SIGFPE,  sigc::bind(sigc::ptr_fun(&do_panic), SIGFPE));
 
-    // Need to initialize this before parseing options.
-    torrent::initialize(uiControl.get_core().get_poll_manager()->get_torrent_poll());
+    uiControl.get_core().initialize_first();
 
     if (getenv("HOME"))
       load_option_file(getenv("HOME") + std::string("/.rtorrent.rc"), &optionHandler);
 
     int firstArg = parse_options(&uiControl, &optionHandler, argc, argv);
 
-    initialize_display(&uiControl);
-    initialize_core(&uiControl);
-
-    uiControl.get_ui().init(&uiControl);
-
     uiControl.initialize();
 
     load_session_torrents(&uiControl);
@@ -260,14 +240,6 @@ main(int argc, char** argv) {
 
     uiControl.cleanup();
 
-    uiControl.get_ui().cleanup();
-    uiControl.get_core().cleanup();
-
-    display::Canvas::erase_std();
-    display::Canvas::refresh_std();
-    display::Canvas::do_update();
-    display::Canvas::cleanup();
-
   } catch (std::exception& e) {
     display::Canvas::cleanup();
 

src/ui/control.cc

@@ -38,6 +38,9 @@
 
 #include <unistd.h>
 
+#include "display/canvas.h"
+#include "display/window.h"
+
 #include "control.h"
 
 namespace ui {
@@ -55,12 +58,30 @@ Control::~Control() {
 
 void
 Control::initialize() {
+  display::Canvas::init();
+  display::Window::slot_adjust(sigc::mem_fun(m_display, &display::Manager::adjust_layout));
+
+  m_core.get_poll_manager()->signal_interrupted().connect(sigc::mem_fun(*m_inputStdin, &input::InputEvent::event_read));
+  m_core.get_poll_manager()->signal_interrupted().connect(sigc::ptr_fun(display::Canvas::do_update));
+
+  m_core.initialize_second();
+
+  m_ui.init(this);
+
   m_inputStdin->insert(m_core.get_poll_manager()->get_torrent_poll());
 }
 
 void
 Control::cleanup() {
   m_inputStdin->remove(m_core.get_poll_manager()->get_torrent_poll());
+
+  m_ui.cleanup();
+  m_core.cleanup();
+  
+  display::Canvas::erase_std();
+  display::Canvas::refresh_std();
+  display::Canvas::do_update();
+  display::Canvas::cleanup();
 }
 
 // I think it should be safe to initiate the shutdown from anywhere,