* Keep a cache listing all, including failed, torrent loads and the

mtimes of those files. This is used to ensure that e.g. the watch dir
does not try to load bad/duplicate torrents, in addition to optimizing
the pruning of those already loaded.


git-svn-id: svn://rakshasa.no/libtorrent/trunk/rtorrent@1024 e378c898-3ddf-0310-93e7-cc216c733640

src/command_download.cc

@@ -354,7 +354,7 @@ p_multicall(core::Download* download, const torrent::Object& rawArgs) {
 
 #define ADD_CD_VARIABLE_VALUE(key, firstKey, secondKey) \
   ADD_CD_SLOT_PUBLIC("d.get_" key, call_unknown, rpc::get_variable_d_fn(firstKey, secondKey), "i:", ""); \
-  ADD_CD_SLOT("d.set_" key, call_value,   rpc::set_variable_d_fn(firstKey, secondKey), "i:i", "");
+  ADD_CD_SLOT       ("d.set_" key, call_value,   rpc::set_variable_d_fn(firstKey, secondKey), "i:i", "");
 
 #define ADD_CD_VARIABLE_VALUE_PUBLIC(key, firstKey, secondKey) \
   ADD_CD_SLOT_PUBLIC("d.get_" key, call_unknown, rpc::get_variable_d_fn(firstKey, secondKey), "i:", ""); \
@@ -362,7 +362,7 @@ p_multicall(core::Download* download, const torrent::Object& rawArgs) {
 
 #define ADD_CD_VARIABLE_STRING(key, firstKey, secondKey) \
   ADD_CD_SLOT_PUBLIC("d.get_" key, call_unknown, rpc::get_variable_d_fn(firstKey, secondKey), "i:", ""); \
-  ADD_CD_SLOT("d.set_" key, call_string,  rpc::set_variable_d_fn(firstKey, secondKey), "i:s", "");
+  ADD_CD_SLOT       ("d.set_" key, call_string,  rpc::set_variable_d_fn(firstKey, secondKey), "i:s", "");
 
 #define ADD_CD_VARIABLE_STRING_PUBLIC(key, firstKey, secondKey) \
   ADD_CD_SLOT_PUBLIC("d.get_" key, call_unknown, rpc::get_variable_d_fn(firstKey, secondKey), "i:", ""); \
@@ -456,7 +456,14 @@ initialize_command_download() {
   // 2 - Download finished, hashing
   // 3 - Rehashing
   ADD_CD_VARIABLE_VALUE("hashing",       "rtorrent", "hashing");
+
+  // 'tied_to_file' is the file the download is associated with, and
+  // can be changed by the user.
+  //
+  // 'loaded_file' is the file this instance of the torrent was loaded
+  // from, and should not be changed.
   ADD_CD_VARIABLE_STRING("tied_to_file", "rtorrent", "tied_to_file");
+  ADD_CD_VARIABLE_STRING("loaded_file",  "rtorrent", "loaded_file");
 
   // The "state_changed" variable is required to be a valid unix time
   // value, it indicates the last time the torrent changed its state,

src/core/download_factory.cc

@@ -61,6 +61,14 @@
 
 namespace core {
 
+bool
+is_network_uri(const std::string& uri) {
+  return
+    std::strncmp(uri.c_str(), "http://", 7) == 0 ||
+    std::strncmp(uri.c_str(), "https://", 8) == 0 ||
+    std::strncmp(uri.c_str(), "ftp://", 6) == 0;
+}
+
 DownloadFactory::DownloadFactory(Manager* m) :
   m_manager(m),
   m_stream(NULL),
@@ -69,7 +77,8 @@ DownloadFactory::DownloadFactory(Manager* m) :
 
   m_session(false),
   m_start(false),
-  m_printLog(true) {
+  m_printLog(true),
+  m_isFile(false) {
 
   m_taskLoad.set_slot(rak::mem_fn(this, &DownloadFactory::receive_load));
   m_taskCommit.set_slot(rak::mem_fn(this, &DownloadFactory::receive_commit));
@@ -114,9 +123,7 @@ DownloadFactory::receive_load() {
   if (m_stream)
     throw torrent::internal_error("DownloadFactory::load*() called on an object with m_stream != NULL");
 
-  if (std::strncmp(m_uri.c_str(), "http://", 7) == 0 ||
-      std::strncmp(m_uri.c_str(), "https://", 8) == 0 ||
-      std::strncmp(m_uri.c_str(), "ftp://", 6) == 0) {
+  if (is_network_uri(m_uri)) {
     // Http handling here.
     m_stream = new std::stringstream;
     HttpQueue::iterator itr = m_manager->http_queue()->insert(m_uri, m_stream);
@@ -130,10 +137,14 @@ DownloadFactory::receive_load() {
     std::fstream* stream = new std::fstream(rak::path_expand(m_uri).c_str(), std::ios::in | std::ios::binary);
     m_stream = stream;
 
-    if (stream->is_open())
-      receive_loaded();
-    else
-      receive_failed("Could not open file");
+    // Since FileStatusCache checks file stats, it will automatically
+    // cull away invalid paths.
+    m_isFile = true;
+
+    if (!stream->is_open())
+      return receive_failed("Could not open file");
+
+    receive_loaded();
   }
 }
 

src/core/download_factory.h

@@ -101,6 +101,7 @@ private:
   bool                m_session;
   bool                m_start;
   bool                m_printLog;
+  bool                m_isFile;
 
   command_list_type         m_commands;
   torrent::Object::map_type m_variables;
@@ -110,6 +111,8 @@ private:
   rak::priority_item  m_taskCommit;
 };
 
+bool is_network_uri(const std::string& uri);
+
 }
 
 #endif

src/core/manager.cc

@@ -57,6 +57,7 @@
 
 #include "rpc/parse_commands.h"
 #include "utils/directory.h"
+#include "utils/file_status_cache.h"
 
 #include "globals.h"
 #include "curl_get.h"
@@ -177,10 +178,10 @@ Manager::Manager() :
 
   m_pollManager(NULL) {
 
-  m_downloadStore = new DownloadStore();
-  m_downloadList = new DownloadList();
-
-  m_httpQueue = new HttpQueue();
+  m_downloadStore   = new DownloadStore();
+  m_downloadList    = new DownloadList();
+  m_fileStatusCache = new FileStatusCache();
+  m_httpQueue       = new HttpQueue();
 }
 
 Manager::~Manager() {
@@ -188,6 +189,7 @@ Manager::~Manager() {
 
   delete m_downloadStore;
   delete m_httpQueue;
+  delete m_fileStatusCache;
 }
 
 void
@@ -400,6 +402,12 @@ Manager::receive_http_failed(std::string msg) {
 
 void
 Manager::try_create_download(const std::string& uri, int flags, const command_list_type& commands) {
+  // If the path was attempted loaded before, skip it.
+  if (!(flags & create_raw_data) &&
+      !is_network_uri(uri) &&
+      !m_fileStatusCache->insert(uri, 0))
+    return;
+
   // Adding download.
   DownloadFactory* f = new DownloadFactory(this);
 
@@ -430,7 +438,7 @@ path_expand(std::vector<std::string>* paths, const std::string& pattern) {
   std::vector<utils::Directory> nextCache;
 
   rak::split_iterator_t<std::string> first = rak::split_iterator(pattern, '/');
-  rak::split_iterator_t<std::string> last = rak::split_iterator(pattern);
+  rak::split_iterator_t<std::string> last  = rak::split_iterator(pattern);
     
   if (first == last)
     return;
@@ -489,14 +497,6 @@ Manager::try_create_download_expand(const std::string& uri, int flags, command_l
 
   path_expand(&paths, uri);
 
-  if (flags & create_tied)
-    for (std::vector<std::string>::iterator itr = paths.begin(); itr != paths.end(); )
-      if (std::find_if(m_downloadList->begin(), m_downloadList->end(), rak::bind1st(std::ptr_fun(&manager_equal_tied), *itr))
-          != m_downloadList->end())
-        itr = paths.erase(itr);
-      else
-        itr++;
-
   if (!paths.empty())
     for (std::vector<std::string>::iterator itr = paths.begin(); itr != paths.end(); ++itr)
       try_create_download(*itr, flags, commands);

src/core/manager.h

@@ -48,6 +48,10 @@ namespace torrent {
   class Bencode;
 }
 
+namespace utils {
+class FileStatusCache;
+}
+
 namespace core {
 
 class DownloadStore;
@@ -58,6 +62,8 @@ class View;
 class Manager {
 public:
   typedef DownloadList::iterator                    DListItr;
+  typedef utils::FileStatusCache                    FileStatusCache;
+
   typedef sigc::slot1<void, DownloadList::iterator> SlotReady;
   typedef sigc::slot0<void>                         SlotFailed;
 
@@ -66,6 +72,7 @@ public:
 
   DownloadList*       download_list()                     { return m_downloadList; }
   DownloadStore*      download_store()                    { return m_downloadStore; }
+  FileStatusCache*    file_status_cache()                 { return m_fileStatusCache; }
 
   HttpQueue*          http_queue()                        { return m_httpQueue; }
 
@@ -122,6 +129,7 @@ private:
 
   DownloadList*       m_downloadList;
   DownloadStore*      m_downloadStore;
+  FileStatusCache*    m_fileStatusCache;
   HttpQueue*          m_httpQueue;
 
   View*               m_hashingView;

src/utils/Makefile.am

@@ -3,6 +3,8 @@ noinst_LIBRARIES = libsub_utils.a
 libsub_utils_a_SOURCES = \
 	directory.cc \
 	directory.h \
+	file_status_cache.cc \
+	file_status_cache.h \
 	list_focus.h \
 	lockfile.cc \
 	lockfile.h \

src/utils/directory.cc

@@ -58,7 +58,6 @@ Directory::is_valid() const {
   return d;
 }
 
-// Update should take various flags and sort functors.
 bool
 Directory::update(int flags) {
   if (m_path.empty())
@@ -80,7 +79,12 @@ Directory::update(int flags) {
     itr->d_fileno = entry->d_fileno;
     itr->d_reclen = entry->d_reclen;
     itr->d_type   = entry->d_type;
+
+#ifdef DIRENT_NAMLEN_EXISTS_FOOBAR
     itr->d_name   = std::string(entry->d_name, entry->d_name + entry->d_namlen);
+#else
+    itr->d_name   = std::string(entry->d_name);
+#endif
   }
 
   closedir(d);

src/utils/directory.h

@@ -48,7 +48,7 @@ struct directory_entry {
 
   // The name and types should match POSIX.
   uint32_t            d_fileno;
-  uint16_t            d_reclen;
+  uint32_t            d_reclen;
   uint8_t             d_type;
 
   std::string         d_name;

src/utils/file_status_cache.cc

@@ -0,0 +1,71 @@
+// rTorrent - BitTorrent client
+// Copyright (C) 2005-2007, Jari Sundell
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+// 
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+// 
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+//
+// In addition, as a special exception, the copyright holders give
+// permission to link the code of portions of this program with the
+// OpenSSL library under certain conditions as described in each
+// individual source file, and distribute linked combinations
+// including the two.
+//
+// You must obey the GNU General Public License in all respects for
+// all of the code used other than OpenSSL.  If you modify file(s)
+// with this exception, you may extend this exception to your version
+// of the file(s), but you are not obligated to do so.  If you do not
+// wish to do so, delete this exception statement from your version.
+// If you delete this exception statement from all source files in the
+// program, then also delete it here.
+//
+// Contact:  Jari Sundell <jaris@ifi.uio.no>
+//
+//           Skomakerveien 33
+//           3185 Skoppum, NORWAY
+
+#include "config.h"
+
+#include <rak/file_stat.h>
+#include <rak/path.h>
+#include <torrent/exceptions.h>
+
+#include "file_status_cache.h"
+
+namespace utils {
+
+bool
+FileStatusCache::insert(const std::string& path, int flags) {
+  rak::file_stat fs;
+
+  // Should we expand somewhere else? Problem is it adds a lot of junk
+  // to the start of the paths added to the cache, causing more work
+  // during search, etc.
+  if (!fs.update(rak::path_expand(path)))
+    return false;
+
+  std::pair<iterator, bool> result = base_type::insert(value_type(path, file_status()));
+
+  // Return false if the file hasn't been modified since last time. We
+  // use 'equal to' instead of 'greater than' since the file might
+  // have been replaced by another file, and thus should be re-tried.
+  if (!result.second && result.first->second.m_mtime == (uint32_t)fs.modified_time())
+    return false;
+
+  result.first->second.m_flags = 0;
+  result.first->second.m_mtime = fs.modified_time();
+
+  return true;
+}
+
+}

src/utils/file_status_cache.h

@@ -0,0 +1,84 @@
+// rTorrent - BitTorrent client
+// Copyright (C) 2005-2007, Jari Sundell
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+// 
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+// 
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+//
+// In addition, as a special exception, the copyright holders give
+// permission to link the code of portions of this program with the
+// OpenSSL library under certain conditions as described in each
+// individual source file, and distribute linked combinations
+// including the two.
+//
+// You must obey the GNU General Public License in all respects for
+// all of the code used other than OpenSSL.  If you modify file(s)
+// with this exception, you may extend this exception to your version
+// of the file(s), but you are not obligated to do so.  If you do not
+// wish to do so, delete this exception statement from your version.
+// If you delete this exception statement from all source files in the
+// program, then also delete it here.
+//
+// Contact:  Jari Sundell <jaris@ifi.uio.no>
+//
+//           Skomakerveien 33
+//           3185 Skoppum, NORWAY
+
+#ifndef RTORRENT_UTILS_FILE_STATUS_CACHE_H
+#define RTORRENT_UTILS_FILE_STATUS_CACHE_H
+
+#include <map>
+#include <string>
+
+namespace utils {
+
+struct file_status {
+  int      m_flags;
+  uint32_t m_mtime;
+};
+
+class FileStatusCache : public std::map<std::string, file_status> {
+public:
+  typedef std::map<std::string, file_status> base_type;
+
+  using base_type::iterator;
+  using base_type::const_iterator;
+  using base_type::reverse_iterator;
+  using base_type::const_reverse_iterator;
+  using base_type::value_type;
+
+  using base_type::begin;
+  using base_type::end;
+  using base_type::rbegin;
+  using base_type::rend;
+
+  using base_type::empty;
+  using base_type::size;
+
+  using base_type::erase;
+
+  // Insert and return true if the entry does not exist or the new
+  // file's mtime is more recent.
+  bool                insert(const std::string& path, int flags);
+
+  // Add a function for pruning a sorted list of paths.
+
+  // Function for removing entries in a container that shouldn't be
+  // there based on some parameters.
+
+  // Function for pruning entries with different mtime or not existing.
+};
+
+}
+
+#endif