diff --git a/melib/src/email/parser.rs b/melib/src/email/parser.rs index 8771161f..8d7adddb 100644 --- a/melib/src/email/parser.rs +++ b/melib/src/email/parser.rs @@ -781,6 +781,11 @@ pub mod generic { ))(input) } + /// Invalid version of [`ctext`] that accepts non-ascii characters. + fn ctext_invalid(input: &[u8]) -> IResult<&[u8], ()> { + map(is_not("()\\"), |_| ())(input) + } + ///```text /// ctext = %d33-39 / ; Printable US-ASCII /// %d42-91 / ; characters not including @@ -804,8 +809,10 @@ pub mod generic { )); } input = context("comment()", opt(fws))(input)?.0; - while let Ok((_input, _)) = - context("comment()", alt((ctext, map(quoted_pair, |_| ()))))(input) + while let Ok((_input, _)) = context( + "comment()", + alt((ctext, ctext_invalid, map(quoted_pair, |_| ()))), + )(input) { input = _input; } diff --git a/melib/src/nntp/connection.rs b/melib/src/nntp/connection.rs index ca10c40e..69da06dd 100644 --- a/melib/src/nntp/connection.rs +++ b/melib/src/nntp/connection.rs @@ -296,10 +296,9 @@ impl NntpStream { ret: &mut String, is_multiline: bool, expected_reply_code: &[&str], - ) -> Result<()> { + ) -> Result { self.read_lines(ret, is_multiline, expected_reply_code) - .await?; - Ok(()) + .await } pub async fn read_lines( @@ -307,7 +306,7 @@ impl NntpStream { ret: &mut String, is_multiline: bool, expected_reply_code: &[&str], - ) -> Result<()> { + ) -> Result { let mut buf: Vec = vec![0; Connection::IO_BUF_SIZE]; ret.clear(); let mut last_line_idx: usize = 0; @@ -337,7 +336,14 @@ impl NntpStream { if let Some(mut pos) = ret[last_line_idx..].rfind("\r\n") { if !is_multiline { break; - } else if let Some(pos) = ret.find("\r\n.\r\n") { + } + if !matches!( + expected_reply_code.iter().position(|r| ret.starts_with(r)), + Some(0) | None + ) { + break; + } + if let Some(pos) = ret.find("\r\n.\r\n") { ret.replace_range(pos + "\r\n".len()..pos + "\r\n.\r\n".len(), ""); break; } @@ -355,27 +361,24 @@ impl NntpStream { } } } - //debug!("returning nntp response:\n{:?}", &ret); - Ok(()) + ret.split_whitespace() + .next() + .map(str::parse) + .and_then(std::result::Result::ok) + .ok_or_else(|| Error::new(format!("Internal error: {}", ret))) } pub async fn send_command(&mut self, command: &[u8]) -> Result<()> { - debug!("sending: {}", unsafe { - std::str::from_utf8_unchecked(command) - }); if let Err(err) = try_await(async move { let command = command.trim(); self.stream.write_all(command).await?; self.stream.write_all(b"\r\n").await?; self.stream.flush().await?; - debug!("sent: {}", unsafe { - std::str::from_utf8_unchecked(command) - }); Ok(()) }) .await { - debug!("stream send_command err {:?}", err); + log::debug!("stream send_command err {:?}", err); Err(err) } else { Ok(()) @@ -404,12 +407,11 @@ impl NntpStream { } self.stream.write_all(b".\r\n").await?; self.stream.flush().await?; - debug!("sent data block {} bytes", data.len()); Ok(()) }) .await { - debug!("stream send_multiline_data_block err {:?}", err); + log::debug!("stream send_multiline_data_block err {:?}", err); Err(err) } else { Ok(()) @@ -456,7 +458,7 @@ impl NntpConnection { ret: &'a mut String, is_multiline: bool, expected_reply_code: &'static [&str], - ) -> Pin> + Send + 'a>> { + ) -> Pin> + Send + 'a>> { Box::pin(async move { ret.clear(); self.stream @@ -484,9 +486,9 @@ impl NntpConnection { try_await(async { self.stream.as_mut()?.send_command(command).await }).await { self.stream = Err(err.clone()); - debug!(err.kind); + log::debug!("{:?}", err.kind); if err.kind.is_network() { - debug!(self.connect().await)?; + self.connect().await?; } Err(err) } else { @@ -540,7 +542,7 @@ impl NntpConnection { pub fn command_to_replycodes(c: &str) -> &'static [&'static str] { if c.starts_with("OVER") { - &["224 "] + &["224 ", "423 "] } else if c.starts_with("LIST") { &["215 "] } else if c.starts_with("POST") { diff --git a/melib/src/nntp/mod.rs b/melib/src/nntp/mod.rs index b336d790..389cb63f 100644 --- a/melib/src/nntp/mod.rs +++ b/melib/src/nntp/mod.rs @@ -241,7 +241,7 @@ impl MailBackend for NntpType { mailbox_hash, uid_store: self.uid_store.clone(), connection: self.connection.clone(), - high_low_total: None, + total_low_high: None, }; Ok(Box::pin(async_stream::try_stream! { { @@ -876,7 +876,7 @@ struct FetchState { mailbox_hash: MailboxHash, connection: Arc>, uid_store: Arc, - high_low_total: Option<(usize, usize, usize)>, + total_low_high: Option<(usize, usize, usize)>, } impl FetchState { @@ -885,13 +885,14 @@ impl FetchState { mailbox_hash, ref connection, ref uid_store, - ref mut high_low_total, + ref mut total_low_high, } = self; let mailbox_hash = *mailbox_hash; let mut res = String::with_capacity(8 * 1024); let mut conn = connection.lock().await; let mut unseen = LazyCountSet::new(); - if high_low_total.is_none() { + + if total_low_high.is_none() { conn.select_group(mailbox_hash, true, &mut res).await?; /* * Parameters @@ -911,37 +912,46 @@ impl FetchState { ))); } let total = usize::from_str(s[1]).unwrap_or(0); - let _low = usize::from_str(s[2]).unwrap_or(0); + let low = usize::from_str(s[2]).unwrap_or(0); let high = usize::from_str(s[3]).unwrap_or(0); - *high_low_total = Some((high, _low, total)); + *total_low_high = Some((total, low, high)); { let f = &uid_store.mailboxes.lock().await[&mailbox_hash]; f.exists.lock().unwrap().set_not_yet_seen(total); }; } - let (high, low, _) = high_low_total.unwrap(); - if high <= low { - return Ok(None); - } - const CHUNK_SIZE: usize = 50000; - let new_low = std::cmp::max(low, high.saturating_sub(CHUNK_SIZE)); - high_low_total.as_mut().unwrap().0 = new_low; - // [ref:FIXME]: server might not implement OVER capability - conn.send_command(format!("OVER {}-{}", new_low, high).as_bytes()) - .await?; - conn.read_response(&mut res, true, command_to_replycodes("OVER")) - .await - .chain_err_summary(|| { - format!( - "{} Could not select newsgroup: expected OVER response but got: {}", - &uid_store.account_name, res - ) - })?; - let mut ret = Vec::with_capacity(high - new_low); - //hash_index: Arc>>, - //uid_index: Arc>>, + let (low, new_low) = loop { + let (_, low, high) = total_low_high.unwrap(); + if high <= low { + return Ok(None); + } + const CHUNK_SIZE: usize = 50000; + let new_low = std::cmp::max(low, std::cmp::min(high, low.saturating_add(CHUNK_SIZE))); + total_low_high.as_mut().unwrap().1 = new_low; + + // [ref:FIXME]: server might not implement OVER capability + conn.send_command(format!("OVER {}-{}", low, new_low).as_bytes()) + .await?; + let reply_code = conn + .read_response(&mut res, true, command_to_replycodes("OVER")) + .await + .chain_err_summary(|| { + format!( + "{} Could not select newsgroup: expected OVER response but got: {}", + &uid_store.account_name, res + ) + })?; + if reply_code == 423 { + // No articles in this range, so move on to next chunk. + continue; + } + break (low, new_low); + }; + + let mut ret = Vec::with_capacity(new_low - low); let mut latest_article: Option = None; + { let mut message_id_lck = uid_store.message_id_index.lock().await; let mut uid_index_lck = uid_store.uid_index.lock().await;