diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 103ab677..32516045 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -16,7 +16,8 @@ jobs: sudo apt-get install -y \ bear \ clang-tidy \ - libcurl4-openssl-dev + libcurl4-openssl-dev \ + zlib1g-dev - name: Fetch libtorrent run: | git clone https://github.com/rakshasa/libtorrent diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 293784a4..ca4c62f3 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -15,6 +15,7 @@ jobs: run: | sudo apt-get install -y \ libcppunit-dev \ + zlib1g-dev \ libcurl4-openssl-dev - name: Fetch libtorrent run: | diff --git a/configure.ac b/configure.ac index 965dc71e..2fb288aa 100644 --- a/configure.ac +++ b/configure.ac @@ -48,6 +48,7 @@ if test "x$ax_cv_ncursesw" != xyes && test "x$ax_cv_ncurses" != xyes; then fi PKG_CHECK_MODULES([CPPUNIT], [cppunit],, [no_cppunit="yes"]) +PKG_CHECK_MODULES([ZLIB], [zlib]) PKG_CHECK_MODULES([DEPENDENCIES], [libtorrent >= 0.16.9]) AC_LANG_PUSH(C++) @@ -66,9 +67,9 @@ AC_DEFINE(USER_AGENT, [std::string(PACKAGE "/" VERSION "/") + torrent::version() dnl Only update global build variables immediately before generating the output, dnl to avoid affecting the global build environment for other autoconf checks. -LIBS="$PTHREAD_LIBS $CURSES_LIB $CURSES_LIBS $DEPENDENCIES_LIBS $LIBS" -CFLAGS="$CFLAGS $PTHREAD_CFLAGS $DEPENDENCIES_CFLAGS $CURSES_CFLAGS" -CXXFLAGS="$CXXFLAGS $PTHREAD_CFLAGS $DEPENDENCIES_CFLAGS $CURSES_CFLAGS" +LIBS="$PTHREAD_LIBS $CURSES_LIB $CURSES_LIBS $ZLIB_LIBS $DEPENDENCIES_LIBS $LIBS" +CFLAGS="$CFLAGS $PTHREAD_CFLAGS $CURSES_CFLAGS $ZLIB_CFLAGS $DEPENDENCIES_CFLAGS" +CXXFLAGS="$CXXFLAGS $PTHREAD_CFLAGS $CURSES_CFLAGS $ZLIB_CFLAGS $DEPENDENCIES_CFLAGS" TORRENT_CHECK_POPCOUNT() diff --git a/src/Makefile.am b/src/Makefile.am index 097872a6..8b1d103d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -169,6 +169,8 @@ libsub_root_a_SOURCES = \ utils/file_status_cache.cc \ utils/file_status_cache.h \ utils/functional.h \ + utils/gzip.cc \ + utils/gzip.h \ utils/list_focus.h \ utils/lockfile.cc \ utils/lockfile.h \ diff --git a/src/command_network.cc b/src/command_network.cc index 46283da7..e333b7e4 100644 --- a/src/command_network.cc +++ b/src/command_network.cc @@ -298,9 +298,14 @@ initialize_command_network() { CMD2_VAR_BOOL ("network.scgi.dont_route", false); CMD2_ANY ("network.scgi.open_systemd", [](auto, auto) { return apply_scgi_systemd(); }); - CMD2_ANY_STRING ("network.xmlrpc.dialect.set", [](const auto&, const auto& arg) { return apply_xmlrpc_dialect(arg); }) - CMD2_ANY ("network.xmlrpc.size_limit", [](const auto&, const auto&) { return rpc::rpc.size_limit(); }); - CMD2_ANY_VALUE_V ("network.xmlrpc.size_limit.set", [](const auto&, const auto& arg) { return rpc::rpc.set_size_limit(arg); }); + CMD2_ANY ("network.scgi.use_gzip", [](const auto&, const auto&) { return rpc::rpc.scgi_allow_compression(); }); + CMD2_ANY_VALUE_V ("network.scgi.use_gzip.set", [](const auto&, const auto& arg) { return rpc::rpc.set_scgi_allow_compression(arg); }); + CMD2_ANY ("network.scgi.gzip.min_size", [](const auto&, const auto&) { return rpc::rpc.scgi_min_compress_size(); }); + CMD2_ANY_VALUE_V ("network.scgi.gzip.min_size.set", [](const auto&, const auto& arg) { return rpc::rpc.set_scgi_min_compress_size(arg); }); + + CMD2_ANY_STRING ("network.xmlrpc.dialect.set", [](const auto&, const auto& arg) { return apply_xmlrpc_dialect(arg); }) + CMD2_ANY ("network.xmlrpc.size_limit", [](const auto&, const auto&) { return rpc::rpc.size_limit(); }); + CMD2_ANY_VALUE_V ("network.xmlrpc.size_limit.set", [](const auto&, const auto& arg) { return rpc::rpc.set_size_limit(arg); }); CMD2_VAR_BOOL ("network.rpc.use_xmlrpc", true); CMD2_VAR_BOOL ("network.rpc.use_jsonrpc", true); diff --git a/src/rpc/rpc_manager.h b/src/rpc/rpc_manager.h index b81d5cb2..3f32e2a7 100644 --- a/src/rpc/rpc_manager.h +++ b/src/rpc/rpc_manager.h @@ -1,6 +1,7 @@ #ifndef RTORRENT_RPC_MANAGER_H #define RTORRENT_RPC_MANAGER_H +#include #include #include #include @@ -55,37 +56,44 @@ public: RpcManager() = default; ~RpcManager() = default; - bool is_handlers_initialized() const { return m_handlers_initialized; } + bool is_handlers_initialized() const { return m_handlers_initialized; } - void initialize_handlers(); - void cleanup(); + void initialize_handlers(); + void cleanup(); - int64_t size_limit() { return m_xmlrpc.size_limit(); }; - void set_size_limit(uint64_t size) { m_xmlrpc.set_size_limit(size); }; + int64_t size_limit() { return m_xmlrpc.size_limit(); }; + void set_size_limit(uint64_t size) { m_xmlrpc.set_size_limit(size); }; - int dialect() { return m_xmlrpc.dialect(); } - void set_dialect(int dialect) { m_xmlrpc.set_dialect(dialect); } + int dialect() { return m_xmlrpc.dialect(); } + void set_dialect(int dialect) { m_xmlrpc.set_dialect(dialect); } - bool is_type_enabled(RPCType type) const; - void set_type_enabled(RPCType type, bool enabled); + bool is_type_enabled(RPCType type) const; + void set_type_enabled(RPCType type, bool enabled); - bool process(RPCType type, const char* in_buffer, uint32_t length, slot_response_callback callback); - bool process_untrusted(RPCType type, const char* in_buffer, uint32_t length, slot_response_callback callback); + bool process(RPCType type, const char* in_buffer, uint32_t length, slot_response_callback callback); + bool process_untrusted(RPCType type, const char* in_buffer, uint32_t length, slot_response_callback callback); - void insert_command(const char* name, const char* parm, const char* doc); - void mark_safe(const std::string& key); + void insert_command(const char* name, const char* parm, const char* doc); + void mark_safe(const std::string& key); - slot_download& slot_find_download() { return m_slot_find_download; } - slot_file& slot_find_file() { return m_slot_find_file; } - slot_tracker& slot_find_tracker() { return m_slot_find_tracker; } - slot_peer& slot_find_peer() { return m_slot_find_peer; } + bool scgi_allow_compression() const { return m_scgi_allow_compression; } + void set_scgi_allow_compression(bool allow) { m_scgi_allow_compression = allow; } + + unsigned int scgi_min_compress_size() const { return m_scgi_min_compress_size; } + void set_scgi_min_compress_size(unsigned int size) { m_scgi_min_compress_size = size; } + + slot_download& slot_find_download() { return m_slot_find_download; } + slot_file& slot_find_file() { return m_slot_find_file; } + slot_tracker& slot_find_tracker() { return m_slot_find_tracker; } + slot_peer& slot_find_peer() { return m_slot_find_peer; } // Trusted/untrusted XMLRPC connection model. + // // When an SCGI request includes the UNTRUSTED_CONNECTION header, // commands without flag_untrusted_safe are blocked. - bool is_trusted() const; + bool is_trusted() const; - static void object_to_target(const torrent::Object& obj, int callFlags, rpc::target_type* target, std::function* deleter); + static void object_to_target(const torrent::Object& obj, int callFlags, rpc::target_type* target, std::function* deleter); private: bool m_trusted{true}; @@ -97,6 +105,9 @@ private: bool m_is_jsonrpc_enabled{true}; bool m_is_xmlrpc_enabled{true}; + std::atomic m_scgi_allow_compression{false}; + std::atomic m_scgi_min_compress_size{0}; + slot_download m_slot_find_download; slot_file m_slot_find_file; slot_tracker m_slot_find_tracker; diff --git a/src/rpc/scgi.h b/src/rpc/scgi.h index d6e8cfd0..b475cf20 100644 --- a/src/rpc/scgi.h +++ b/src/rpc/scgi.h @@ -2,7 +2,6 @@ #define RTORRENT_RPC_SCGI_H #include -#include #include #include @@ -27,10 +26,10 @@ public: void stop(); - const std::string& path() const { return m_path; } + const std::string& path() const { return m_path; } - int log_fd() const { return m_logFd; } - void set_log_fd(int fd) { m_logFd = fd; } + int log_fd() const { return m_logFd; } + void set_log_fd(int fd) { m_logFd = fd; } void event_read() override; void event_write() override; diff --git a/src/rpc/scgi_task.cc b/src/rpc/scgi_task.cc index 868f5350..75603cc6 100644 --- a/src/rpc/scgi_task.cc +++ b/src/rpc/scgi_task.cc @@ -2,13 +2,14 @@ #include "rpc/scgi_task.h" +#include +#include +#include #include #include -#include #include #include #include -#include #include #include #include @@ -17,8 +18,9 @@ #include "control.h" #include "globals.h" -#include "scgi.h" #include "rpc/parse_commands.h" +#include "rpc/scgi.h" +#include "utils/gzip.h" namespace rpc { @@ -26,16 +28,22 @@ void SCgiTask::open(SCgi* parent, int fd) { set_file_descriptor(fd); - m_buffer.reset(new char[default_buffer_size + 1]); - m_parent = parent; - m_buffer_size = default_buffer_size; - m_position = m_buffer.get(); - m_body = nullptr; + m_position = 0; + m_body = 0; + + m_content_length = 0; + m_content_type = XML; + m_accepts_compression = false; torrent::this_thread::poll()->open(this); torrent::this_thread::poll()->insert_read(this); torrent::this_thread::poll()->insert_error(this); + + auto lock = std::lock_guard(m_result_mutex); + + // Leave room for terminating nul byte for parsing the header. + m_buffer.resize(default_buffer_size + 1); } void @@ -64,14 +72,15 @@ SCgiTask::close() { set_file_descriptor(-1); }); + // The callbacks are guaranteed to be finished/canceled at this point. auto lock = std::lock_guard(m_result_mutex); - m_buffer = nullptr; + m_buffer.clear(); } void SCgiTask::event_read() { - int bytes = ::recv(m_fileDesc, m_position, m_buffer_size - (m_position - m_buffer.get()), 0); + int bytes = ::recv(m_fileDesc, m_buffer.data() + m_position, m_buffer.size() - m_position, 0); if (bytes <= 0) { if (bytes == 0 || !(errno == EAGAIN || errno == EINTR)) @@ -82,104 +91,45 @@ SCgiTask::event_read() { // The buffer has space to nul-terminate to ease the parsing below. m_position += bytes; - *m_position = '\0'; - if (m_body == NULL) { - // Don't bother caching the parsed values, as we're likely to - // receive all the data we need the first time. - char* current; + if (m_content_length == 0) { + // While reading the header, we leave room in the buffer for a nul byte. + m_buffer[m_position] = '\0'; - int header_size = strtol(m_buffer.get(), ¤t, 0); + // Don't bother caching the parsed values, as we're likely to receive all the data we need the + // first time. + unsigned int header_size{}; - if (current == m_position) + auto [current, ec] = std::from_chars(m_buffer.data(), m_buffer.data() + m_position, header_size); + + // If the request doesn't start with an integer or if it didn't end in ':', then close the + // connection. + + if (current == m_buffer.data()) return; - // If the request doesn't start with an integer or if it didn't - // end in ':', then close the connection. - if (current == m_buffer.get() || *current != ':' || header_size < 17 || header_size > max_header_size) + if (*current != ':' || header_size < 17 || header_size > max_header_size) goto event_read_failed; - if (std::distance(++current, m_position) < header_size + 1) + current++; + + // The header size starts after the ':' and ends at the first ','. + if ((m_buffer.data() + m_position) - current < static_cast(header_size) + 1) return; - // We'll parse this fully below, but the SCGI spec requires it to - // be the first header. - if (std::memcmp(current, "CONTENT_LENGTH", 15) != 0) + // Check for ',' after the header, if it's not there, then close the connection. + if (*(current + header_size) != ',') goto event_read_failed; - std::string content_type = ""; - size_t content_length = 0; - const char* header_end = current + header_size; - - // Assume trusted until we find the UNTRUSTED_CONNECTION header. - m_trusted = true; - - // Parse out the null-terminated header keys and values, with - // checks to ensure it doesn't scan beyond the limits of the - // header - while (current < header_end) { - char* key = current; - char* key_end = static_cast(std::memchr(current, '\0', header_end - current)); - - if (!key_end) - goto event_read_failed; - - current = key_end + 1; - - if (current >= header_end) - goto event_read_failed; - - char* value = current; - char* value_end = static_cast(std::memchr(current, '\0', header_end - current)); - - if (!value_end) - goto event_read_failed; - - current = value_end + 1; - - if (strcmp(key, "CONTENT_LENGTH") == 0) { - char* content_pos; - content_length = strtol(value, &content_pos, 10); - if (*content_pos != '\0' || content_length <= 0 || content_length > max_content_size) - goto event_read_failed; - } else if (strcmp(key, "CONTENT_TYPE") == 0) { - content_type = value; - } else if (strcmp(key, "UNTRUSTED_CONNECTION") == 0 && strcmp(value, "1") == 0) { - m_trusted = false; - } - } - - if (current != header_end) + // The header must start with "CONTENT_LENGTH" followed by a null byte. + if (std::memcmp(current, "CONTENT_LENGTH", 14+1) != 0) goto event_read_failed; - if (content_length <= 0) + if (!parse_headers(current, header_size)) goto event_read_failed; - - m_body = current + 1; - header_size = std::distance(m_buffer.get(), m_body); - - if (!detect_content_type(content_type)) - goto event_read_failed; - - if ((unsigned int)(content_length + header_size) < m_buffer_size) { - m_buffer_size = content_length + header_size; - - } else if ((unsigned int)content_length <= default_buffer_size) { - m_buffer_size = content_length; - - std::memmove(m_buffer.get(), m_body, std::distance(m_body, m_position)); - m_position = m_buffer.get() + std::distance(m_body, m_position); - m_body = m_buffer.get(); - - } else { - realloc_buffer((m_buffer_size = content_length) + 1, m_body, std::distance(m_body, m_position)); - - m_position = m_buffer.get() + std::distance(m_body, m_position); - m_body = m_buffer.get(); - } } - if ((unsigned int)std::distance(m_buffer.get(), m_position) != m_buffer_size) + if (m_position < m_body + m_content_length) return; torrent::this_thread::poll()->remove_read(this); @@ -189,13 +139,13 @@ SCgiTask::event_read() { // Clean up logging, this is just plain ugly... // write(m_logFd, "\n---\n", sizeof("\n---\n")); - result = ::write(m_parent->log_fd(), m_buffer.get(), m_buffer_size); + result = ::write(m_parent->log_fd(), m_buffer.data() + m_body, m_position - m_body); result = ::write(m_parent->log_fd(), "\n---\n", sizeof("\n---\n")); } - lt_log_print_dump(torrent::LOG_RPC_DUMP, m_body, m_buffer_size - std::distance(m_buffer.get(), m_body), "scgi", "RPC read.", 0); + lt_log_print_dump(torrent::LOG_RPC_DUMP, m_buffer.data() + m_body, m_content_length, "scgi", "RPC read.", 0); - receive_call(m_body, m_buffer_size - std::distance(m_buffer.get(), m_body)); + receive_call(m_buffer.data() + m_body, m_content_length); return; event_read_failed: @@ -205,19 +155,18 @@ event_read_failed: void SCgiTask::event_write() { - int bytes = ::send(m_fileDesc, m_position, m_buffer_size, 0); + int bytes = ::send(m_fileDesc, m_buffer.data() + m_position, m_buffer.size() - m_position, 0); if (bytes == -1) { - if (!(errno == EAGAIN || errno == EINTR)) + if (!(errno == EAGAIN || errno == EINTR || errno == EPIPE)) close(); return; } m_position += bytes; - m_buffer_size -= bytes; - if (bytes == 0 || m_buffer_size == 0) + if (bytes == 0 || m_position == m_buffer.size()) return close(); } @@ -226,6 +175,95 @@ SCgiTask::event_error() { close(); } +bool +SCgiTask::parse_headers(const char* current, unsigned int header_length) { + std::string content_type; + + const char* header_end = current + header_length; + + // Parse out the null-terminated header keys and values, with + // checks to ensure it doesn't scan beyond the limits of the + // header + while (current < header_end) { + auto* key = current; + auto* key_end = static_cast(std::memchr(current, '\0', header_end - current)); + + if (key_end == nullptr) + return false; + + current = key_end + 1; + + if (current >= header_end) + return false; + + auto* value = current; + auto* value_end = static_cast(std::memchr(current, '\0', header_end - current)); + + if (value_end == nullptr) + return false; + + current = value_end + 1; + + if (std::strncmp(key, "CONTENT_LENGTH", 14+1) == 0) { + auto [content_pos, ec] = std::from_chars(value, value_end, m_content_length); + + if (*content_pos != '\0' || m_content_length <= 0 || m_content_length > max_content_size) + return false; + + } else if (std::strncmp(key, "CONTENT_TYPE", 12+1) == 0) { + content_type = value; + + } else if (std::strncmp(key, "ACCEPT_ENCODING", 15+1) == 0) { + std::string accept_encoding(value, value_end - value); + + if (accept_encoding.find("gzip") != std::string::npos) + m_accepts_compression = true; + + } else if (std::strncmp(key, "UNTRUSTED_CONNECTION", 20+1) == 0) { + if (std::strncmp(value, "1", 1+1) == 0) + m_trusted = false; + else if (std::strncmp(value, "0", 1+1) == 0) + ; // Default is trusted, so do nothing. + else + return false; + } + } + + if (current != header_end) + return false; + + if (m_content_length == 0) + return false; + + // Move past the ',' that ends the header. + current++; + + if (current > m_buffer.data() + m_buffer.size()) + throw torrent::internal_error("SCgiTask::event_read() header parsing overflow : body start is beyond buffer end"); + + m_body = current - m_buffer.data(); + + if (!detect_content_type(content_type)) + return false; + + if (m_body + m_content_length > m_buffer.size()) { + if (m_content_length > default_buffer_size) { + std::vector tmp(m_content_length); + + std::memcpy(tmp.data(), m_buffer.data() + m_body, m_position - m_body); + m_buffer.swap(tmp); + + } else { + std::memmove(m_buffer.data(), m_buffer.data() + m_body, m_position - m_body); + } + + m_position = m_position - m_body; + m_body = 0; + } + + return true; +} + static inline bool scgi_match_content_type(const std::string& content_type, const char* type) { std::string::size_type pos = content_type.find_first_of(" ;"); @@ -241,7 +279,8 @@ SCgiTask::detect_content_type(const std::string& content_type) { if (content_type.empty()) { // If no CONTENT_TYPE was supplied, peek at the body to check if it's JSON // { is a single request object, while [ is a batch array - if (*m_body == '{' || *m_body == '[') + + if (m_buffer[m_body] == '{' || m_buffer[m_body] == '[') m_content_type = ContentType::JSON; else m_content_type = ContentType::XML; @@ -260,36 +299,9 @@ SCgiTask::detect_content_type(const std::string& content_type) { return true; } -// If bufferSize is zero then memcpy won't do anything. -void -SCgiTask::realloc_buffer(uint32_t size, const char* buffer, uint32_t bufferSize) { - auto tmp = new char[size]; - - std::memcpy(tmp, buffer, bufferSize); - - m_buffer.reset(tmp); -} - void SCgiTask::receive_call(const char* buffer, uint32_t length) { - // TODO: Rewrite RpcManager.process to pass the result buffer instead of having to copy it. - - auto scgi_thread = torrent::utils::Thread::self(); - bool trusted = m_trusted; - - auto result_callback = [this, scgi_thread](const char* b, uint32_t l) { - receive_write(b, l); - - scgi_thread->callback_interrupt_polling(this, [this]() { - // Only need to lock once here as a memory barrier. - m_result_mutex.lock(); - m_result_mutex.unlock(); - - torrent::this_thread::poll()->insert_write(this); - }); - }; - - auto lock = std::lock_guard(m_result_mutex); + assert(torrent::utils::Thread::self() == scgi_thread::thread()); RpcManager::RPCType rpc_type; @@ -304,49 +316,121 @@ SCgiTask::receive_call(const char* buffer, uint32_t length) { throw torrent::internal_error("SCgiTask::receive_call(...) received bad input."); } - torrent::main_thread::thread()->callback_interrupt_polling(this, [buffer, length, result_callback, trusted, rpc_type]() { - auto callback = [result_callback](const char* b, uint32_t l) { - result_callback(b, l); - return true; - }; + // TODO: Rewrite RpcManager.process to pass the result buffer instead of having to copy it. + // TODO: Also allow us to request RpcManager.process to reserve space for a header. + // TODO: Completely remove the mutex, and align m_buffer? - if (trusted) - rpc.process(rpc_type, buffer, length, callback); + auto result_callback = [this](const char* b, uint32_t l) { + receive_write(b, l); + + // Memory barrier for the result data. + // std::atomic_thread_fence(std::memory_order_release); + m_result_mutex.lock(); + m_result_mutex.unlock(); + + scgi_thread::thread()->callback_interrupt_polling(this, [this]() { + if (!is_open()) + return; + + torrent::this_thread::poll()->insert_write(this); + + // Memory barrier for the result data. + // std::atomic_thread_fence(std::memory_order_acquire); + m_result_mutex.lock(); + m_result_mutex.unlock(); + }); + + return true; + }; + + // Memory barrier for the input data. + // std::atomic_thread_fence(std::memory_order_release); + m_result_mutex.lock(); + m_result_mutex.unlock(); + + torrent::main_thread::thread()->callback_interrupt_polling(this, [this, rpc_type, buffer, length, result_callback]() { + // Memory barrier for the input data. + // std::atomic_thread_fence(std::memory_order_acquire); + m_result_mutex.lock(); + m_result_mutex.unlock(); + + if (m_trusted) + rpc.process(rpc_type, buffer, length, result_callback); else - rpc.process_untrusted(rpc_type, buffer, length, callback); + rpc.process_untrusted(rpc_type, buffer, length, result_callback); }); } void SCgiTask::receive_write(const char* buffer, uint32_t length) { - if (buffer == NULL || length > (100 << 20)) + assert(torrent::utils::Thread::self() == torrent::main_thread::thread()); + + if (buffer == nullptr || length > (100 << 20)) throw torrent::internal_error("SCgiTask::receive_write(...) received bad input."); - auto lock = std::lock_guard(m_result_mutex); - - // Need to cast due to a bug in MacOSX gcc-4.0.1. - if (length + 256 > std::max(m_buffer_size, (unsigned int)default_buffer_size)) - realloc_buffer(length + 256, NULL, 0); - - const auto header = m_content_type == ContentType::JSON - ? "Status: 200 OK\r\nContent-Type: application/json\r\nContent-Length: %i\r\n\r\n" - : "Status: 200 OK\r\nContent-Type: text/xml\r\nContent-Length: %i\r\n\r\n"; - - // Who ever bothers to check the return value? - int headerSize = snprintf(m_buffer.get(), m_buffer_size, header, length); - - m_position = m_buffer.get(); - m_buffer_size = length + headerSize; - - std::memcpy(m_buffer.get() + headerSize, buffer, length); + // Main thread callback already locked this mutex. + // auto lock = std::lock_guard(m_result_mutex); + // Write to log prior to possible compression if (m_parent->log_fd() >= 0) { - [[maybe_unused]] int result; - result = write(m_parent->log_fd(), m_buffer.get(), m_buffer_size); + int result [[maybe_unused]]; + // Clean up logging, this is just plain ugly... + // write(m_logFd, "\n---\n", sizeof("\n---\n")); + result = write(m_parent->log_fd(), buffer, length); result = write(m_parent->log_fd(), "\n---\n", sizeof("\n---\n")); } - lt_log_print_dump(torrent::LOG_RPC_DUMP, m_buffer.get(), m_buffer_size, "scgi", "RPC write.", 0); + lt_log_print_dump(torrent::LOG_RPC_DUMP, buffer, length, "scgi", "RPC write.", 0); + + if (m_accepts_compression && rpc.scgi_allow_compression() && length > rpc.scgi_min_compress_size()) + gzip_response(buffer, length); + else + plaintext_response(buffer, length); +} + +// We don't know the size of the content-length string, so leave sufficient space for the header +// strings plus max length of content-length. + +void +SCgiTask::plaintext_response(const char* buffer, uint32_t content_length) { + auto header_first = content_type() == ContentType::XML ? header_xml : header_json; + auto header_first_size = content_type() == ContentType::XML ? header_xml_size : header_json_size; + auto length_str = std::to_string(content_length); + + m_buffer.resize(header_first_size + length_str.size() + header_last_size + content_length); + + std::memcpy(m_buffer.data(), header_first, header_first_size); + std::memcpy(m_buffer.data() + header_first_size, length_str.data(), length_str.size()); + std::memcpy(m_buffer.data() + header_first_size + length_str.size(), header_last, header_last_size); + std::memcpy(m_buffer.data() + header_first_size + length_str.size() + header_last_size, buffer, content_length); + + m_position = 0; +} + +void +SCgiTask::gzip_response(const char* buffer, uint32_t content_length) { + auto header_first = content_type() == ContentType::XML ? header_xml : header_json; + auto header_first_size = content_type() == ContentType::XML ? header_xml_size : header_json_size; + + unsigned int body_offset = header_first_size + 20 + header_last_size; + + utils::gzip_compress_to_vector(buffer, content_length, m_buffer, body_offset); + + auto length_str = std::to_string(m_buffer.size() - body_offset); + auto header_size = header_first_size + length_str.size() + header_last_size; + auto header_start = body_offset - header_size; + + if (header_size > body_offset) + throw torrent::internal_error("SCgiTask::gzip_response(...) header overflow : start position is negative"); + + if (header_start > body_offset) + throw torrent::internal_error("SCgiTask::gzip_response(...) header overflow : start position is beyond buffer start"); + + std::memcpy(m_buffer.data() + header_start, header_first, header_first_size); + std::memcpy(m_buffer.data() + header_start + header_first_size, length_str.data(), length_str.size()); + std::memcpy(m_buffer.data() + header_start + header_first_size + length_str.size(), header_last, header_last_size); + + m_position = header_start; } } // namespace rpc diff --git a/src/rpc/scgi_task.h b/src/rpc/scgi_task.h index 717752a3..fea62b6c 100644 --- a/src/rpc/scgi_task.h +++ b/src/rpc/scgi_task.h @@ -3,6 +3,7 @@ #include #include +#include #include namespace rpc { @@ -11,9 +12,9 @@ class SCgi; class SCgiTask : public torrent::Event { public: - static const unsigned int default_buffer_size = 2047; - static const int max_header_size = 2000; - static const int max_content_size = (2 << 23); + static constexpr int default_buffer_size = 8191; + static constexpr int max_header_size = 2000; + static constexpr int max_content_size = (2 << 23); enum ContentType { XML, JSON }; @@ -36,23 +37,35 @@ public: void event_error() override; private: + static constexpr char header_xml[] = "Status: 200 OK\r\nContent-Type: text/xml\r\nContent-Length: "; + static constexpr char header_json[] = "Status: 200 OK\r\nContent-Type: application/json\r\nContent-Length: "; + static constexpr char header_last[] = "\r\n\r\n"; + + static constexpr size_t header_xml_size = sizeof(header_xml) - 1; + static constexpr size_t header_json_size = sizeof(header_json) - 1; + static constexpr size_t header_last_size = sizeof(header_last) - 1; + + bool parse_headers(const char* current, unsigned int header_length); bool detect_content_type(const std::string& content_type); - void realloc_buffer(uint32_t size, const char* buffer, uint32_t bufferSize); void receive_call(const char* buffer, uint32_t length); void receive_write(const char* buffer, uint32_t length); + void plaintext_response(const char* buffer, uint32_t content_length); + void gzip_response(const char* buffer, uint32_t content_length); + SCgi* m_parent{}; std::mutex m_result_mutex; - std::unique_ptr m_buffer; - char* m_position{}; - char* m_body{}; + std::vector m_buffer; + unsigned int m_position{}; + unsigned int m_body{}; - unsigned int m_buffer_size{0}; + unsigned int m_content_length{}; + ContentType m_content_type{XML}; - ContentType m_content_type{ XML }; + bool m_accepts_compression{}; bool m_trusted{true}; }; diff --git a/src/utils/gzip.cc b/src/utils/gzip.cc new file mode 100644 index 00000000..b0a416f0 --- /dev/null +++ b/src/utils/gzip.cc @@ -0,0 +1,41 @@ +#include "config.h" + +#include "utils/gzip.h" + +#include +#include + +namespace utils { + +void +gzip_compress_to_vector(const char* buffer, unsigned int length, std::vector& output, unsigned int offset) { + z_stream zs{}; + zs.zalloc = Z_NULL; + zs.zfree = Z_NULL; + zs.opaque = Z_NULL; + + constexpr int window_bits = 15; + constexpr int gzip_encoding = 16; + constexpr int gzip_level = 6; + + if (deflateInit2(&zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED, window_bits | gzip_encoding, gzip_level, Z_DEFAULT_STRATEGY) != Z_OK) + throw torrent::internal_error("gzip_compress_to_vector(...) could not initialize gzip deflate."); + + auto max_response_size = deflateBound(&zs, length); + + output.resize(offset + max_response_size); + + zs.next_in = (Bytef*)buffer; + zs.avail_in = length; + zs.next_out = (Bytef*)(output.data() + offset); + zs.avail_out = max_response_size; + + int ret = deflate(&zs, Z_FINISH); + + if (ret != Z_STREAM_END) + throw torrent::internal_error("gzip_compress_to_vector(...) deflate did not return Z_STREAM_END: " + std::to_string(ret)); + + output.resize(offset + max_response_size - zs.avail_out); +} + +} // namespace utils diff --git a/src/utils/gzip.h b/src/utils/gzip.h new file mode 100644 index 00000000..03c0891e --- /dev/null +++ b/src/utils/gzip.h @@ -0,0 +1,12 @@ +#ifndef RTORRENT_UTILS_GZIP_H +#define RTORRENT_UTILS_GZIP_H + +#include + +namespace utils { + +void gzip_compress_to_vector(const char* buffer, unsigned int length, std::vector& output, unsigned int offset = 0); + +} // namespace utils + +#endif