browse/sync: trashing / deleting of mails

src/bin/browse.rs

@@ -11,7 +11,6 @@ use cursive::views::{Checkbox, LinearLayout, NamedView, OnEventView, Panel, Resi
 use cursive_tree_view::{Placement, TreeEntry, TreeView};
 use inboxid::*;
 use io::Write;
-use imap::types::Flag;
 use itertools::Itertools;
 use log::error;
 use mailparse::{MailHeaderMap, ParsedMail};
@@ -227,13 +226,14 @@ fn show_listing(mailbox: &str) -> Result<()> {
 	let tree = tree.on_select(tree_on_select).with_name("tree").scrollable().with_name("tree_scroller");
 	let update_flags2 = Arc::clone(&update_flags);
 	let update_flags3 = Arc::clone(&update_flags);
+	let update_flags4 = Arc::clone(&update_flags);
+	let update_flags5 = Arc::clone(&update_flags);
 	let tree = OnEventView::new(tree)
 		.on_event('r', move |siv| {
 			siv.call_on_name("tree", |tree: &mut MailTreeView| {
 				if let Some(r) = tree.row() {
 					let mail = tree.borrow_item_mut(r).unwrap();
-					mail.add_flag(Flag::Seen);
+					mail.mark_as_read(true);
-					mail.remove_flag2('U');
 					// TODO error handling
 					let _ = mail.save_flags(&maildir);
 					let _ = update_flags2.lock().execute(params![mail.get_flags(), mail.id.to_i64()]);
@@ -244,13 +244,35 @@ fn show_listing(mailbox: &str) -> Result<()> {
 			siv.call_on_name("tree", |tree: &mut MailTreeView| {
 				if let Some(r) = tree.row() {
 					let mail = tree.borrow_item_mut(r).unwrap();
-					mail.remove_flag(Flag::Seen);
+					mail.mark_as_read(false);
-					mail.add_flag2('U');
 					// TODO error handling
 					let _ = mail.save_flags(&maildir);
 					let _ = update_flags3.lock().execute(params![mail.get_flags(), mail.id.to_i64()]);
 				}
 			});
+		})
+		.on_event('t', move |siv| {
+			siv.call_on_name("tree", |tree: &mut MailTreeView| {
+				if let Some(r) = tree.row() {
+					let mail = tree.borrow_item_mut(r).unwrap();
+					mail.mark_as_read(true);
+					mail.add_flag2(TRASHED);
+					// TODO error handling
+					let _ = mail.save_flags(&maildir);
+					let _ = update_flags4.lock().execute(params![mail.get_flags(), mail.id.to_i64()]);
+				}
+			});
+		})
+		.on_event('d', move |siv| {
+			siv.call_on_name("tree", |tree: &mut MailTreeView| {
+				if let Some(r) = tree.row() {
+					let mail = tree.borrow_item_mut(r).unwrap();
+					mail.add_flag2(DELETE);
+					// TODO error handling
+					let _ = mail.save_flags(&maildir);
+					let _ = update_flags5.lock().execute(params![mail.get_flags(), mail.id.to_i64()]);
+				}
+			});
 		});
 	let tree_resized = ResizedView::new(SizeConstraint::AtMost(120), SizeConstraint::Free, tree);
 	let mail_info = MailInfoView::new().with_name("mail_info");

src/bin/rebuild-db.rs

@@ -2,7 +2,6 @@ use std::env;
 
 use inboxid::*;
 use itertools::Itertools;
-use mailparse::MailHeaderMap;
 use rusqlite::params;
 
 fn main() -> Result<()> {
@@ -20,15 +19,15 @@ fn main() -> Result<()> {
 		for x in maildir.list_cur() {
 			mails.push(x?);
 		}
+		for x in maildir.list_new() {
+			mails.push(x?);
+		}
 		println!("acquired {} mails", mails.len());
 		let mut mails = maildir.get_mails(&mut mails)?;
 		mails.sort_by_key(|x| x.date);
 		for mail in mails {
 			let headers = mail.get_headers();
-			let mut message_id = headers.get_all_values("Message-ID").join(" ");
+			let message_id = headers.message_id(&mailbox, mail.id);
-			if message_id.is_empty() {
-				message_id = format!("<{}_{}_{}@no-message-id>", mailbox, mail.id.uid_validity, mail.id.uid);
-			}
 			save_mail.execute(params![&mailbox, mail.id.to_i64(), message_id, mail.get_flags()])?;
 		}
 	}

src/bin/sync.rs

@@ -1,12 +1,13 @@
 use std::{borrow::Cow, collections::HashMap, env};
 
+use anyhow::Context;
 use imap::types::{Flag, NameAttribute};
 use itertools::Itertools;
 use maildir::Maildir;
 
 use inboxid::*;
 use mailparse::{MailHeaderMap, parse_header, parse_headers};
-use rusqlite::params;
+use rusqlite::{Row, params, types::FromSql};
 
 const TRASH: NameAttribute = NameAttribute::Custom(Cow::Borrowed("\\Trash"));
 
@@ -49,32 +50,80 @@ fn sync(
 		let mut mails = HashMap::new();
 		let messages = imap_session.uid_fetch("1:*", "(FLAGS BODY[HEADER.FIELDS (MESSAGE-ID)])")?;
 		for m in messages.iter() {
+			let id = MaildirID::new(uid_validity, m.uid.unwrap());
 			let flags = m.flags();
 			if flags.contains(&Flag::Deleted) {
 				continue;
 			}
 			let header = m.header().unwrap();
 			let message_id = parse_header(header).map(|x| x.0.get_value())
-				.unwrap_or_else(|_| format!("<{}_{}_{}@no-message-id>", mailbox, uid_validity, m.uid.unwrap()));
+				.unwrap_or_else(|_| fallback_mid(mailbox, id));
-			let uid = m.uid.unwrap();
-			let full_uid = ((uid_validity as u64) << 32) | uid as u64;
 			let flags = flags.iter().map(|x| remove_cow(x)).collect_vec();
-			mails.insert(message_id, (uid_validity, uid, full_uid, flags));
+			mails.insert(message_id, (id.uid_validity, id.uid, id.to_u64(), flags));
 		}
 		remote.insert(mailbox, mails);
 	}
 
 	let mut have_mail = db.prepare("SELECT mailbox, uid, flags FROM mail WHERE message_id = ?")?;
 	let mut delete_mail = db.prepare("DELETE FROM mail WHERE mailbox = ? AND uid = ?")?;
-	let mut all_mail = db.prepare("SELECT uid, message_id FROM mail WHERE mailbox = ?")?;
+	let mut all_mail = db.prepare("SELECT uid, message_id, flags FROM mail WHERE mailbox = ?")?;
 	let mut save_mail = db.prepare("INSERT INTO mail VALUES (?,?,?,?)")?;
 	let mut maildirs: HashMap<&str, Maildir> = names.iter().map(|&x| (x.name(), get_maildir(x.name()).unwrap())).collect();
+	macro_rules! ensure_mailbox {
+		($name:expr) => {{
+			if !maildirs.contains_key($name) {
+				maildirs.insert($name, get_maildir($name)?);
+			}
+			&maildirs[$name]
+		}}
+	}
+	let mut printed_trash_warning = false;
+	let trash_dir = names.iter().filter(|x| x.attributes().iter().any(|x| *x == TRASH)).map(|x| x.name()).next();
 	let mut to_remove: HashMap<&str, _> = HashMap::new();
 	for &name in &names {
 		let mailbox = name.name();
+		let is_trash = name.attributes().iter().any(|x| *x == TRASH);
 		let remote_mails = remote.get_mut(mailbox).unwrap();
-		let resp = imap_session.select(mailbox)?;
+		println!("selecting {}", mailbox);
-		let uid_validity = resp.uid_validity.unwrap();
+		imap_session.select(mailbox).context("select failed")?;
+		let all_mails = all_mail.query_map(params![mailbox], map3rows::<i64, String, String>)?;
+		let mut deleted_some = false;
+		for x in all_mails {
+			let (uid, mid, flags) = x?;
+			let uid: MaildirID = uid.into();
+			if flags.contains(TRASHED) && !is_trash {
+				if let Some(trash_dir) = trash_dir {
+					println!("trashing: {}/{}", mailbox, uid);
+					if remote_mails.contains_key(&mid) {
+						imap_session.uid_mv(uid.to_imap(), trash_dir)?;
+					} else {
+						println!("Warning: only trashing locally!");
+					}
+					let gone = ensure_mailbox!(".gone");
+					let uid_name = uid.to_string();
+					let _ = maildir_cp(&maildirs[mailbox], gone, &uid_name, &uid_name);
+					maildirs[mailbox].delete(&uid_name)?;
+					delete_mail.execute(params![mailbox, uid.to_i64()])?;
+				} else if !printed_trash_warning {
+					println!("Warning: unable to trash mail, no trash folder found!");
+					printed_trash_warning = true;
+				}
+			} else if flags.contains(DELETE) {
+				println!("deleting: {}/{}", mailbox, uid);
+				if remote_mails.contains_key(&mid) {
+					imap_session.uid_store(uid.to_imap(), "+FLAGS.SILENT (\\Deleted)")?;
+				} else {
+					println!("Warning: only deleting locally!");
+				}
+				remote_mails.remove(&mid);
+				delete_mail.execute(params![mailbox, uid.to_i64()])?;
+				maildirs[mailbox].delete(&uid.to_string())?;
+				deleted_some = true;
+			}
+		}
+		if deleted_some {
+			imap_session.expunge().context("expunge failed")?;
+		}
 
 		let mut to_fetch = Vec::new();
 		for (message_id, entry) in remote_mails.iter_mut() {
@@ -84,20 +133,25 @@ fn sync(
 				load_i64(row.get::<_, i64>(1)?),
 				row.get::<_, String>(2)?
 			)))?.map(|x| x.unwrap()).collect_vec();
-			if let Some((_, full_uid, flags)) = local.iter().filter(|x| x.0 == mailbox && x.1 == *full_uid).next() {
+			macro_rules! update_flags {
-				let uid = (full_uid << 32) >> 32;
+				($full_uid:expr, $flags:expr) => {
-				let local_s = flags.contains('S');
+					let uid = ($full_uid << 32) >> 32;
-				let local_u = flags.contains('U');
+					let local_s = $flags.contains('S');
-				let remote_s = remote_flags.contains(&Flag::Seen);
+					let local_u = $flags.contains(UNREAD);
-				if local_s && !remote_s {
+					let remote_s = remote_flags.contains(&Flag::Seen);
-					println!("setting Seen flag on {}/{}", mailbox, uid);
+					if local_s && !remote_s {
-					imap_session.uid_store(uid.to_string(), "+FLAGS.SILENT (\\Seen)")?;
+						println!("setting Seen flag on {}/{}", mailbox, uid);
-					remote_flags.push(Flag::Seen);
+						imap_session.uid_store(uid.to_string(), "+FLAGS.SILENT (\\Seen)")?;
-				} else if local_u && remote_s {
+						remote_flags.push(Flag::Seen);
-					println!("removing Seen flag on {}/{}", mailbox, uid);
+					} else if local_u && remote_s {
-					imap_session.uid_store(uid.to_string(), "-FLAGS.SILENT (\\Seen)")?;
+						println!("removing Seen flag on {}/{}", mailbox, uid);
-					remote_flags.remove(remote_flags.iter().position(|x| x == &Flag::Seen).unwrap());
+						imap_session.uid_store(uid.to_string(), "-FLAGS.SILENT (\\Seen)")?;
+						remote_flags.remove(remote_flags.iter().position(|x| x == &Flag::Seen).unwrap());
+					}
 				}
+			}
+			if let Some((_, full_uid, flags)) = local.iter().filter(|x| x.0 == mailbox && x.1 == *full_uid).next() {
+				update_flags!(full_uid, flags);
 				continue;
 			}
 			if !local.is_empty() {
@@ -114,7 +168,8 @@ fn sync(
 				let maildir2 = &maildirs[mailbox];
 				maildir2.store_cur_from_path(&new_id, flags, name)?;
 				save_mail.execute(params![mailbox, new_uid.to_i64(), message_id, flags])?;
-			} else if !name.attributes().iter().any(|x| *x == TRASH) { // do not fetch trashed mail
+				update_flags!(new_uid.to_u64(), flags);
+			} else if !is_trash { // do not fetch trashed mail
 				to_fetch.push(uid2);
 			}
 		}
@@ -129,16 +184,17 @@ fn sync(
 			for mail in fetch.iter() {
 				let uid = mail.uid.unwrap();
 				println!("fetching: {}/{}", mailbox, uid);
-				let id = gen_id(uid_validity, uid);
+				let id = MaildirID::new(uid_validity, uid);
-				if !maildir.exists(&id) {
+				let id_name = id.to_string();
+				if !maildir.exists(&id_name) {
 					let mail_data = mail.body().unwrap_or_default();
 					let flags = imap_flags_to_maildir("".into(), mail.flags());
-					maildir.store_cur_with_id_flags(&id, &flags, mail_data)?;
+					maildir.store_cur_with_id_flags(&id_name, &flags, mail_data)?;
 
 					let headers = parse_headers(&mail_data)?.0;
 					let mut message_id = headers.get_all_values("Message-ID").join(" ");
 					if message_id.is_empty() {
-						message_id = format!("<{}_{}_{}@no-message-id>", mailbox, uid_validity, uid);
+						message_id = headers.message_id(mailbox, id);
 					}
 					let full_uid = ((uid_validity as u64) << 32) | uid as u64;
 					save_mail.execute(params![mailbox, store_i64(full_uid), message_id, flags])?;
@@ -152,7 +208,7 @@ fn sync(
 			let (uid1, uid2, _, ref flags) = remote_mails[message_id];
 			let id = gen_id(uid1, uid2);
 			let _ = maildir.update_flags(&id, |f| {
-				let f = f.replace('U', "");
+				let f = f.replace(UNREAD, "");
 				let f = imap_flags_to_maildir(f, flags);
 				Maildir::normalize_flags(&f)
 			});
@@ -176,13 +232,10 @@ fn sync(
 		for &(uid1, uid2, uid) in &to_remove[mailbox] {
 			let uid_name = gen_id(uid1, uid2);
 			println!("removing: {}/{}", mailbox, uid_name);
-			if !maildirs.contains_key(".gone") {
+			let gone = ensure_mailbox!(".gone");
-				maildirs.insert(".gone", get_maildir(".gone")?);
-			}
 			let maildir = &maildirs[mailbox];
-			let name = maildir.find_filename(&uid_name).unwrap();
-			let _ = maildirs[".gone"].store_new_from_path(&format!("{}_{}", mailbox, uid_name), name);
 			// hardlink should only fail if the mail was already deleted
+			let _ = maildir_cp(maildir, gone, &uid_name, &uid_name);
 			maildir.delete(&uid_name)?;
 			delete_mail.execute(params![mailbox, store_i64(uid)])?;
 		}
@@ -193,3 +246,10 @@ fn sync(
 
 	Ok(())
 }
+
+pub fn map3rows<A: FromSql, B: FromSql, C: FromSql>(row: &Row) -> rusqlite::Result<(A, B, C)> {
+	let a = row.get::<_, A>(0)?;
+	let b = row.get::<_, B>(1)?;
+	let c = row.get::<_, C>(2)?;
+	Ok((a, b, c))
+}

src/lib.rs

@@ -21,12 +21,16 @@ use serde_derive::{Deserialize, Serialize};
 pub type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
 pub type ImapSession = Session<StreamOwned<ClientSession, TcpStream>>;
 
+pub const UNREAD: char = 'U';
+pub const TRASHED: char = 'T';
+pub const DELETE: char = 'E'; // Exterminate
+
 pub fn connect(host: &str, port: u16, user: &str, password: &str) -> Result<ImapSession> {
 	println!("connecting..");
-	let stream = TcpStream::connect((host, port))?;
+	let stream = TcpStream::connect((host, port)).context("TCP connect failed")?;
-	let tls = RustlsConnector::new_with_native_certs()?;
+	let tls = RustlsConnector::new_with_native_certs().context("TLS configuration failed")?;
 	println!("initializing TLS..");
-	let tlsstream = tls.connect(host, stream)?;
+	let tlsstream = tls.connect(host, stream).context("TLS connection failed")?;
 	println!("initializing client..");
 	let client = imap::Client::new(tlsstream);
 
@@ -83,23 +87,44 @@ impl TryFrom<&str> for MaildirID {
 	}
 }
 
-impl ToString for MaildirID {
+impl From<i64> for MaildirID {
-	fn to_string(&self) -> String {
+	fn from(x: i64) -> Self {
-		format!("{}_{}", self.uid_validity, self.uid)
+		let x = load_i64(x);
+		Self::new((x >> 32) as u32, ((x << 32) >> 32) as u32)
 	}
 }
 
+impl Display for MaildirID {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		write!(f, "{}_{}", self.uid_validity, self.uid)
+    }
+}
+
 impl MaildirID {
-	pub fn new(x: u32, y: u32) -> Self {
+	pub fn new(uid_validity: u32, uid: u32) -> Self {
 		Self {
-			uid_validity: x,
+			uid_validity,
-			uid: y
+			uid
 		}
 	}
 
-	pub fn to_i64(&self) -> i64 {
+	pub fn to_u64(&self) -> u64 {
-		store_i64(((self.uid_validity as u64) << 32) | self.uid as u64)
+		((self.uid_validity as u64) << 32) | self.uid as u64
 	}
+
+	pub fn to_i64(&self) -> i64 {
+		store_i64(self.to_u64())
+	}
+
+	pub fn to_imap(&self) -> String {
+		self.uid.to_string()
+	}
+}
+
+pub fn maildir_cp(maildir1: &Maildir, maildir2: &Maildir, id1: &str, id2: &str) -> Result<()> {
+	let name = maildir1.find_filename(id1).context("mail not found")?;
+	maildir2.store_new_from_path(id2, name)?;
+	Ok(())
 }
 
 pub struct EasyMail<'a> {
@@ -166,6 +191,16 @@ impl EasyMail<'_> {
 		*f = f.replace(flag, "");
 	}
 
+	pub fn mark_as_read(&self, read: bool) {
+		if read {
+			self.add_flag(Flag::Seen);
+			self.remove_flag2(UNREAD);
+		} else {
+			self.remove_flag(Flag::Seen);
+			self.add_flag2(UNREAD);
+		}
+	}
+
 	pub fn save_flags(&self, maildir: &Maildir) -> Result<()> {
 		maildir.set_flags(&self.id.to_string(), &self.flags.read())?;
 		Ok(())
@@ -175,10 +210,6 @@ impl EasyMail<'_> {
 		self.flags.read().clone()
 	}
 
-	pub fn get_header(&self, header: &str) -> String {
-		self.get_headers().get_all_values(header).join(" ")
-	}
-
 	pub fn get_header_values(&self, header: &str) -> Vec<String> {
 		self.get_headers().get_all_values(header)
 	}
@@ -298,6 +329,7 @@ pub trait MailExtension {
 	fn get_tree_structure<'a>(&'a self, graph: &mut Graph<&'a ParsedMail<'a>, ()>, parent: Option<NodeIndex>);
 	fn print_tree_structure(&self, depth: usize, counter: &mut usize);
 	fn get_tree_part(&self, counter: &mut usize, target: usize) -> Option<&ParsedMail>;
+	fn get_header(&self, header: &str) -> String;
 }
 
 impl MailExtension for ParsedMail<'_> {
@@ -337,6 +369,34 @@ impl MailExtension for ParsedMail<'_> {
 		}
 		None
 	}
+
+	fn get_header(&self, header: &str) -> String {
+		self.get_headers().get_header(header)
+	}
+}
+
+pub trait HeadersExtension {
+	fn message_id(&self, mailbox: &str, id: MaildirID) -> String;
+	fn get_header(&self, header: &str) -> String;
+}
+
+impl<T: MailHeaderMap + ?Sized> HeadersExtension for T {
+	fn message_id(&self, mailbox: &str, id: MaildirID) -> String {
+		let mid = self.get_header("Message-ID");
+		if mid.is_empty() {
+			fallback_mid(mailbox, id)
+		} else {
+			mid
+		}
+	}
+
+	fn get_header(&self, header: &str) -> String {
+		self.get_all_values(header).join(" ")
+	}
+}
+
+pub fn fallback_mid(mailbox: &str, id: MaildirID) -> String {
+	format!("<{}_{}_{}@no-message-id>", mailbox, id.uid_validity, id.uid)
 }
 
 pub trait MaildirExtension {