Hanlde jacket torrent url

This commit is contained in:
Alexander
2026-07-19 17:39:19 +02:00
parent 6ef2b76977
commit 6309b87323
12 changed files with 624 additions and 23 deletions
Generated
+15 -1
View File
@@ -2593,6 +2593,17 @@ dependencies = [
"syn",
]
[[package]]
name = "sha1"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8"
dependencies = [
"cfg-if",
"cpufeatures 0.2.17",
"digest 0.10.7",
]
[[package]]
name = "sha1"
version = "0.11.0"
@@ -2818,7 +2829,7 @@ dependencies = [
"log",
"percent-encoding",
"serde",
"sha1",
"sha1 0.11.0",
"sha2 0.11.0",
"sqlx-core",
"thiserror 2.0.18",
@@ -3217,9 +3228,12 @@ name = "torad"
version = "0.1.0"
dependencies = [
"anyhow",
"async-trait",
"clap",
"librqbit",
"librqbit-core",
"reqwest",
"sha1 0.10.7",
"sqlx",
"tokio",
"tokio-stream",
+3
View File
@@ -5,9 +5,12 @@ edition = "2024"
[dependencies]
anyhow = "1.0.103"
async-trait = "0.1"
clap = { version = "4.6.1", features = ["derive", "env"] }
librqbit = { version = "8.1.1", default-features = false, features = ["rust-tls", "http-api-client"] }
librqbit-core = { version = "5.0.0", default-features = false }
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
sha1 = "0.10"
sqlx = { version = "0.9.0", features = ["postgres", "runtime-tokio", "tls-rustls", "uuid"] }
tokio = { version = "1.52.3", features = ["rt-multi-thread", "macros", "net", "signal"] }
tokio-stream = { version = "0.1.18", features = ["net"] }
-17
View File
@@ -1,17 +0,0 @@
use anyhow::{Context, Result};
use librqbit::Magnet;
pub fn info_hash(source: &str) -> Result<String> {
if source.starts_with("magnet:") {
let parsed = Magnet::parse(source).context("failed to parse magnet link")?;
let id20 = parsed
.as_id20()
.context("magnet link has no v1 (BTIH) info hash")?;
return Ok(id20.as_string());
}
let bytes =
std::fs::read(source).with_context(|| format!("failed to read torrent file: {source}"))?;
let parsed = librqbit::torrent_from_bytes_ext::<Vec<u8>>(&bytes)
.context("failed to parse .torrent file")?;
Ok(parsed.meta.info_hash.as_string())
}
+1 -1
View File
@@ -1,5 +1,5 @@
mod db;
mod magnet;
mod source;
mod torrents;
use std::path::PathBuf;
+32
View File
@@ -0,0 +1,32 @@
//! Handler for local filesystem paths. Reads the file and delegates to the
//! shared tolerant [`super::info_hash`] extractor. Used when clients pass a
//! path to a `.torrent` file on disk rather than a URL.
use anyhow::{Context, Result};
use async_trait::async_trait;
use super::info_hash;
use super::{Resolved, SourceHandler};
pub(crate) struct FileHandler;
#[async_trait]
impl SourceHandler for FileHandler {
fn matches(&self, source: &str) -> bool {
!source.starts_with("magnet:")
&& !source.starts_with("http://")
&& !source.starts_with("https://")
}
async fn resolve(&self, source: &str) -> Result<Resolved> {
let bytes = tokio::fs::read(source)
.await
.with_context(|| format!("failed to read torrent file: {source}"))?;
let info_hash =
info_hash::from_bytes(&bytes).context("failed to extract info_hash from file")?;
Ok(Resolved {
info_hash,
rewritten_source: None,
})
}
}
+120
View File
@@ -0,0 +1,120 @@
//! Shared HTTP-fetch helper used by handlers that need to download `.torrent`
//! bytes or follow `Location` redirects manually.
//!
//! reqwest's default policy follows 3xx only for `http`/`https`. Jackett (and
//! some indexers) 302-redirect to `magnet:` URIs — those would error out under
//! reqwest's default policy. We disable auto-following entirely and inspect
//! `Location` ourselves so the caller can route to the magnet handler.
use anyhow::{Context, Result, bail};
use reqwest::StatusCode;
use reqwest::header::LOCATION;
use tracing::{debug, warn};
/// Maximum number of HTTP redirect hops before giving up.
const MAX_REDIRECTS: u8 = 5;
/// Truncation limit for error-response bodies logged for diagnostics.
const ERROR_BODY_LOG_LIMIT: usize = 500;
pub(super) enum HttpFetchOutcome {
/// Server returned 2xx with a body — assumed to be bencoded `.torrent` bytes.
Bytes(Vec<u8>),
/// Server returned a 3xx whose `Location` is a `magnet:` URI.
Magnet(String),
}
pub(super) async fn fetch(client: &reqwest::Client, source: &str) -> Result<HttpFetchOutcome> {
let mut current = source.to_string();
for hop in 0..MAX_REDIRECTS {
let resp = client
.get(&current)
.send()
.await
.with_context(|| format!("HTTP GET failed for {current}"))?;
let status = resp.status();
debug!(url = %current, status = %status, hop, "HTTP response");
if status.is_redirection() {
let loc = resp
.headers()
.get(LOCATION)
.and_then(|v| v.to_str().ok())
.unwrap_or("");
if loc.is_empty() {
bail!("redirect with empty Location header from {current}");
}
if loc.starts_with("magnet:") {
debug!(magnet = %loc, "redirected to magnet");
return Ok(HttpFetchOutcome::Magnet(loc.to_string()));
}
if loc.starts_with("http://") || loc.starts_with("https://") {
current = loc.to_string();
continue;
}
bail!("unsupported redirect target from {current}: {loc}");
}
if !status.is_success() {
// Read the body so error responses are useful in logs.
let body = resp.text().await.unwrap_or_default();
let preview: String = body.chars().take(ERROR_BODY_LOG_LIMIT).collect();
warn!(url = %current, status = %status, body = %preview, "non-success HTTP status");
bail!(
"HTTP {status} from {current}: {preview}",
status = status,
current = current,
preview = preview,
);
}
let content_type = resp
.headers()
.get(reqwest::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.unwrap_or("")
.to_string();
let bytes = resp
.bytes()
.await
.with_context(|| format!("failed reading response body from {current}"))?
.to_vec();
debug!(url = %current, content_type = %content_type, bytes = bytes.len(), "received body");
// Detect obvious non-torrent responses so callers get a clear error
// instead of an opaque bencode-parse failure downstream.
if looks_like_html(&bytes) {
bail!(
"received HTML from {current} (likely a Jackett error page); first 200 bytes: {}",
String::from_utf8_lossy(&bytes[..bytes.len().min(200)])
);
}
if status == StatusCode::OK && bytes.is_empty() {
bail!("empty response body from {current}");
}
return Ok(HttpFetchOutcome::Bytes(bytes));
}
bail!("exceeded {MAX_REDIRECTS} redirects while resolving {source}");
}
/// Cheap heuristic: true if the first non-whitespace bytes look like an HTML
/// document (`<!doctype`, `<html`, or any `<` tag opener). Jackett returns
/// HTML error pages with HTTP 200 on auth/path failures, so catching this
/// here yields a clearer error than letting bencode parsing fail.
fn looks_like_html(bytes: &[u8]) -> bool {
let prefix: &[u8] = &bytes[..bytes.len().min(200)];
let trimmed = prefix
.iter()
.copied()
.skip_while(|b| b.is_ascii_whitespace())
.collect::<Vec<_>>();
let lower = trimmed
.iter()
.take_while(|b| b.is_ascii())
.map(|b| b.to_ascii_lowercase())
.collect::<Vec<_>>();
lower.starts_with(b"<!doctype")
|| lower.starts_with(b"<html")
|| lower.starts_with(b"<head")
|| lower.starts_with(b"<body")
|| lower.starts_with(b"<!--")
}
+53
View File
@@ -0,0 +1,53 @@
//! Handler for generic HTTP(S) `.torrent` URLs that are not Jackett proxy
//! links (covered by [`super::jackett::JackettHandler`]). Same redirect +
//! tolerant-parse logic, just a broader match.
use anyhow::{Context, Result};
use async_trait::async_trait;
use librqbit::Magnet;
use super::http_fetch::{self, HttpFetchOutcome};
use super::info_hash;
use super::{Resolved, SourceHandler};
pub(crate) struct HttpTorrentHandler {
client: reqwest::Client,
}
impl HttpTorrentHandler {
pub(crate) fn new(client: reqwest::Client) -> Self {
Self { client }
}
}
#[async_trait]
impl SourceHandler for HttpTorrentHandler {
fn matches(&self, source: &str) -> bool {
source.starts_with("http://") || source.starts_with("https://")
}
async fn resolve(&self, source: &str) -> Result<Resolved> {
let outcome = http_fetch::fetch(&self.client, source).await?;
match outcome {
HttpFetchOutcome::Magnet(magnet) => {
let parsed = Magnet::parse(&magnet)
.with_context(|| format!("failed to parse redirected magnet: {magnet}"))?;
let id20 = parsed
.as_id20()
.context("redirected magnet has no v1 (BTIH) info hash")?;
Ok(Resolved {
info_hash: id20.as_string(),
rewritten_source: Some(magnet),
})
}
HttpFetchOutcome::Bytes(bytes) => {
let info_hash = info_hash::from_bytes(&bytes)
.context("failed to extract info_hash from .torrent body")?;
Ok(Resolved {
info_hash,
rewritten_source: None,
})
}
}
}
}
+214
View File
@@ -0,0 +1,214 @@
//! Tolerant extraction of the BTIH v1 info-hash from bencoded `.torrent`
//! bytes.
//!
//! Strategy: try librqbit's strict parser first (correct, well-tested). If it
//! rejects the bytes — which happens in practice because some indexers
//! (rutracker, filelist) emit non-spec bencode with duplicate dict keys (see
//! <https://github.com/Jackett/Jackett/issues/16492>) — fall back to a
//! byte-span SHA1 of the `info` value, which doesn't care about dict-key
//! validity at all.
//!
//! The info-hash *is* by definition the SHA1 of the bencoded `info` value's
//! byte range, so walking the value's byte boundaries (rather than fully
//! parsing it) yields the correct hash even for spec-violating files.
use anyhow::{Context, Result};
use librqbit::torrent_from_bytes_ext;
use sha1::{Digest, Sha1};
/// Extracts the v1 info-hash from `.torrent` bytes.
pub(super) fn from_bytes(bytes: &[u8]) -> Result<String> {
// Fast path: spec-compliant torrent, librqbit's parser handles it.
if let Ok(parsed) = torrent_from_bytes_ext::<Vec<u8>>(bytes) {
return Ok(parsed.meta.info_hash.as_string());
}
// Fallback: locate the `info` value's byte span and SHA1 it directly.
let span = locate_info_value(bytes)
.context("could not locate `info` value in .torrent bytes; not a valid torrent file")?;
let mut hasher = Sha1::new();
hasher.update(&bytes[span]);
let digest = hasher.finalize();
Ok(hex_encode(&digest))
}
/// Locates the byte range `[start, end)` of the bencoded value associated
/// with the top-level dict key `info`. Returns `None` if the bytes aren't a
/// `.torrent`-shaped dict containing that key.
///
/// Walks the top-level dict properly (key → value → key → value…) instead of
/// substring-scanning for `4:info`, so an `info` substring appearing inside
/// some other value's payload (e.g. a filename) can't trigger a false match.
fn locate_info_value(bytes: &[u8]) -> Option<std::ops::Range<usize>> {
let mut i = 0;
// Top-level value must be a dict.
if *bytes.get(i)? != b'd' {
return None;
}
i += 1;
while i < bytes.len() {
let b = bytes[i];
if b == b'e' {
// Dict ended without finding `info`.
return None;
}
// Dict keys are always byte strings.
if !b.is_ascii_digit() {
return None;
}
let (key_len, colon_at) = read_string_header(bytes, i)?;
let key_start = colon_at + 1;
let key_end = key_start.checked_add(key_len)?;
if key_end > bytes.len() {
return None;
}
let key = &bytes[key_start..key_end];
i = key_end;
if key == b"info" {
let value_end = walk_value(bytes, i)?;
return Some(i..value_end);
}
// Skip past this value to reach the next key.
i = walk_value(bytes, i)?;
}
None
}
/// Walks one bencoded value starting at `start` and returns the index just
/// past its last byte. Handles dicts (`d...e`), lists (`l...e`), integers
/// (`i...e`), and byte strings (`<len>:<bytes>`). Returns `None` on
/// malformed input or unexpected end-of-buffer.
fn walk_value(bytes: &[u8], start: usize) -> Option<usize> {
let mut i = start;
let first = *bytes.get(i)?;
match first {
b'd' | b'l' => {
// Container: track depth, walk until matching `e`.
i += 1;
let mut depth: u32 = 1;
while depth > 0 {
let b = *bytes.get(i)?;
match b {
b'd' | b'l' => {
depth += 1;
i += 1;
}
b'e' => {
depth -= 1;
i += 1;
}
b'0'..=b'9' => {
// Byte string: read length prefix up to `:`, then skip
// the length-prefixed payload.
let (end, colon_at) = read_string_header(bytes, i)?;
i = colon_at + 1;
let payload_end = i.checked_add(end)?;
if payload_end > bytes.len() {
return None;
}
i = payload_end;
}
b'i' => {
// Integer: skip to next `e`.
i += 1;
while i < bytes.len() && bytes[i] != b'e' {
i += 1;
}
if i >= bytes.len() {
return None;
}
i += 1; // consume `e`
}
_ => return None,
}
if i > bytes.len() + 1 {
return None;
}
}
Some(i)
}
b'i' => {
i += 1;
while i < bytes.len() && bytes[i] != b'e' {
i += 1;
}
if i >= bytes.len() {
return None;
}
Some(i + 1)
}
b'0'..=b'9' => {
let (len, colon_at) = read_string_header(bytes, i)?;
let payload_end = colon_at + 1 + len;
if payload_end > bytes.len() {
return None;
}
Some(payload_end)
}
_ => None,
}
}
/// Parses a bencode string-length prefix `<digits>:` at position `i`.
/// Returns `(length, index_of_colon)`.
fn read_string_header(bytes: &[u8], i: usize) -> Option<(usize, usize)> {
let mut j = i;
while j < bytes.len() && bytes[j].is_ascii_digit() {
j += 1;
}
if j >= bytes.len() || bytes[j] != b':' {
return None;
}
let len: usize = std::str::from_utf8(&bytes[i..j]).ok()?.parse().ok()?;
// Sanity-cap to avoid overflow / huge allocations on garbage input.
if len > bytes.len() {
return None;
}
Some((len, j))
}
fn hex_encode(bytes: &[u8]) -> String {
const HEX: &[u8; 16] = b"0123456789abcdef";
let mut out = String::with_capacity(bytes.len() * 2);
for b in bytes {
out.push(HEX[(b >> 4) as usize] as char);
out.push(HEX[(b & 0x0f) as usize] as char);
}
out
}
#[cfg(test)]
mod tests {
use super::*;
/// Synthetic minimal torrent: `d4:infod6:lengthi1e4:name1:aee`.
/// info_hash is SHA1 of the bytes between `4:info` and the matching outer
/// `e`, i.e. SHA1("d6:lengthi1e4:name1:ae").
#[test]
fn extracts_hash_from_minimal_torrent() {
let torrent = b"d4:infod6:lengthi1e4:name1:aee";
let resolved = from_bytes(torrent).expect("must extract info_hash");
// Manually compute expected hash for the same info span.
let info_span = locate_info_value(torrent).unwrap();
let mut h = Sha1::new();
h.update(&torrent[info_span.clone()]);
let expected = hex_encode(&h.finalize());
assert_eq!(resolved, expected);
assert_eq!(resolved.len(), 40);
}
#[test]
fn locate_finds_info_value() {
// `d8:announce9:example.x4:infod6:lengthi1e4:name1:aee` — a torrent
// with a non-info key before `info`, to verify the walker skips the
// preceding value correctly.
let torrent = b"d8:announce9:example.x4:infod6:lengthi1e4:name1:aee";
let span = locate_info_value(torrent).expect("info value must be found");
assert_eq!(&torrent[span.clone()], b"d6:lengthi1e4:name1:ae");
}
#[test]
fn rejects_garbage() {
let garbage = b"this is not a torrent";
assert!(locate_info_value(garbage).is_none());
}
}
+72
View File
@@ -0,0 +1,72 @@
//! Handler for Jackett's `/dl/{indexer}/?jackett_apikey=...&path=...` proxy URLs.
//!
//! Jackett's download endpoint is not a static file: it re-authenticates to
//! the upstream indexer, fetches the details page, and either returns the
//! `.torrent` bytes (HTTP 200) or 302-redirects to a `magnet:` URI when the
//! indexer has no file download. Some indexers (notably rutracker and
//! filelist) also produce non-spec bencode with duplicate dict keys, which
//! librqbit's strict parser rejects — so we tolerate that case via a
//! byte-span SHA1 fallback (see [`super::info_hash`]).
//!
//! See:
//! - <https://github.com/Jackett/Jackett/issues/8252> (proxy-link semantics)
//! - <https://github.com/Jackett/Jackett/issues/16492> (non-spec .torrent files)
use anyhow::{Context, Result};
use async_trait::async_trait;
use librqbit::Magnet;
use super::http_fetch::{self, HttpFetchOutcome};
use super::info_hash;
use super::{Resolved, SourceHandler};
/// Matches any HTTP(S) URL whose query string carries `jackett_apikey=`,
/// which uniquely identifies a Jackett proxy download link regardless of the
/// host/port Jackett is deployed on.
const JACKETT_MARKER: &str = "jackett_apikey=";
pub(crate) struct JackettHandler {
client: reqwest::Client,
}
impl JackettHandler {
pub(crate) fn new(client: reqwest::Client) -> Self {
Self { client }
}
}
#[async_trait]
impl SourceHandler for JackettHandler {
fn matches(&self, source: &str) -> bool {
(source.starts_with("http://") || source.starts_with("https://"))
&& source.contains(JACKETT_MARKER)
}
async fn resolve(&self, source: &str) -> Result<Resolved> {
let outcome = http_fetch::fetch(&self.client, source).await?;
match outcome {
HttpFetchOutcome::Magnet(magnet) => {
let parsed = Magnet::parse(&magnet)
.with_context(|| format!("failed to parse redirected magnet: {magnet}"))?;
let id20 = parsed
.as_id20()
.context("redirected magnet has no v1 (BTIH) info hash")?;
// Store the magnet instead of the original Jackett URL: the
// magnet is stable, doesn't depend on Jackett being reachable
// at poller time, and is what librqbit will actually use.
Ok(Resolved {
info_hash: id20.as_string(),
rewritten_source: Some(magnet),
})
}
HttpFetchOutcome::Bytes(bytes) => {
let info_hash = info_hash::from_bytes(&bytes)
.context("failed to extract info_hash from Jackett .torrent body")?;
Ok(Resolved {
info_hash,
rewritten_source: None,
})
}
}
}
}
+28
View File
@@ -0,0 +1,28 @@
//! Handler for `magnet:` URIs. Extracts the BTIH v1 info-hash from the
//! `xt=urn:btih:` parameter; no network IO.
use anyhow::{Context, Result};
use async_trait::async_trait;
use librqbit::Magnet;
use super::{Resolved, SourceHandler};
pub(super) struct MagnetHandler;
#[async_trait]
impl SourceHandler for MagnetHandler {
fn matches(&self, source: &str) -> bool {
source.starts_with("magnet:")
}
async fn resolve(&self, source: &str) -> Result<Resolved> {
let parsed = Magnet::parse(source).context("failed to parse magnet link")?;
let id20 = parsed
.as_id20()
.context("magnet link has no v1 (BTIH) info hash")?;
Ok(Resolved {
info_hash: id20.as_string(),
rewritten_source: None,
})
}
}
+78
View File
@@ -0,0 +1,78 @@
//! Torrent-source resolution.
//!
//! A "source" is the opaque string a client hands to `add` — it may be a
//! magnet URI, an HTTP(S) URL pointing at a `.torrent` file (Jackett's `/dl/`
//! proxy, a direct indexer download, etc.), or a local file path.
//!
//! Each kind of source is handled by a [`SourceHandler`] implementation. The
//! [`SourceResolver`] holds an ordered list of handlers and dispatches to the
//! first one that claims the source via [`SourceHandler::matches`].
//!
//! Order matters: more-specific handlers (magnet, Jackett) must precede more
//! general ones (HTTP, file).
use anyhow::{Context, Result};
use async_trait::async_trait;
pub(crate) mod file;
mod http_fetch;
pub(crate) mod http_torrent;
pub(crate) mod info_hash;
pub(crate) mod jackett;
pub(crate) mod magnet;
/// Result of resolving a source string into an identifying info-hash.
///
/// `rewritten_source`, when set, replaces the original source for downstream
/// storage. This lets a Jackett URL that 302-redirects to a magnet be stored
/// as the magnet — cheaper and more reliable for the poller to re-fetch later.
pub(crate) struct Resolved {
pub info_hash: String,
pub rewritten_source: Option<String>,
}
/// A handler for one kind of torrent source.
#[async_trait]
pub(crate) trait SourceHandler: Send + Sync {
/// Return `true` if this handler recognises the source string.
fn matches(&self, source: &str) -> bool;
/// Extract the info-hash (and optionally rewrite the source) from the
/// given source string. Called only when [`matches`](Self::matches)
/// returned `true`.
async fn resolve(&self, source: &str) -> Result<Resolved>;
}
/// Dispatches a source string to the first matching [`SourceHandler`].
pub(crate) struct SourceResolver {
handlers: Vec<Box<dyn SourceHandler>>,
}
impl SourceResolver {
pub(crate) fn new() -> Result<Self> {
// Manual redirect policy: Jackett 302-redirects to `magnet:` URIs that
// reqwest would refuse to follow; we want to inspect `Location`
// ourselves so the magnet handler can take over.
let http_client = reqwest::Client::builder()
.redirect(reqwest::redirect::Policy::none())
.build()
.context("failed to build HTTP client")?;
Ok(Self {
handlers: vec![
Box::new(magnet::MagnetHandler),
Box::new(jackett::JackettHandler::new(http_client.clone())),
Box::new(http_torrent::HttpTorrentHandler::new(http_client)),
Box::new(file::FileHandler),
],
})
}
pub(crate) async fn resolve(&self, source: &str) -> Result<Resolved> {
for handler in &self.handlers {
if handler.matches(source) {
return handler.resolve(source).await;
}
}
anyhow::bail!("no handler matched source: {source}")
}
}
+8 -4
View File
@@ -19,7 +19,7 @@ use tracing::{error, info, warn};
use uuid::Uuid;
use crate::db::{self, TorrentRow, TorrentState};
use crate::magnet;
use crate::source::SourceResolver;
const POLL_INTERVAL: Duration = Duration::from_secs(2);
const BYTES_PER_MIB: f64 = 1_048_576.0;
@@ -34,6 +34,7 @@ pub struct TorrentManager {
pool: PgPool,
session: Arc<Session>,
default_output_dir: PathBuf,
source: SourceResolver,
tracked: Mutex<HashMap<Uuid, Arc<ManagedTorrent>>>,
}
@@ -48,21 +49,24 @@ impl TorrentManager {
let session = Session::new(download_dir.clone())
.await
.context("failed to create librqbit session")?;
let source = SourceResolver::new().context("failed to build source resolver")?;
Ok(Arc::new(Self {
pool,
session,
default_output_dir: download_dir,
source,
tracked: Mutex::new(HashMap::new()),
}))
}
pub async fn add(&self, magnet: &str, output_dir: Option<&str>) -> Result<TorrentRow> {
let info_hash = magnet::info_hash(magnet)?;
pub async fn add(&self, source: &str, output_dir: Option<&str>) -> Result<TorrentRow> {
let resolved = self.source.resolve(source).await?;
let stored_source = resolved.rewritten_source.as_deref().unwrap_or(source);
let output_path = output_dir
.filter(|s| !s.is_empty())
.map(str::to_string)
.unwrap_or_else(|| self.default_output_dir.display().to_string());
db::insert_pending(&self.pool, &info_hash, magnet, &output_path).await
db::insert_pending(&self.pool, &resolved.info_hash, stored_source, &output_path).await
}
pub async fn get_status(&self, id: Uuid) -> Result<Option<TorrentStatus>> {