Store flags in the index DB + 'U' unread flag

This commit is contained in:
FliegendeWurst 2021-04-08 16:02:04 +02:00 committed by Arne Keller
parent efd5575346
commit 95b6abc572
3 changed files with 95 additions and 53 deletions

View File

@ -6,24 +6,34 @@ use mailparse::MailHeaderMap;
use rusqlite::params; use rusqlite::params;
fn main() -> Result<()> { fn main() -> Result<()> {
let db = get_db()?; let mut db = get_db()?;
let mut delete_mail = db.prepare("DELETE FROM mail WHERE mailbox = ?")?; let tx = db.transaction()?;
let mut save_mail = db.prepare("INSERT INTO mail VALUES (?,?,?)")?; {
let mut delete_mail = tx.prepare("DELETE FROM mail WHERE mailbox = ?")?;
let mut save_mail = tx.prepare("INSERT INTO mail VALUES (?,?,?,?)")?;
let mailboxes = env::args().skip(1).collect_vec(); let mailboxes = env::args().skip(1).collect_vec();
for mailbox in mailboxes { for mailbox in mailboxes {
println!("reading {}..", mailbox);
let maildir = get_maildir(&mailbox)?; let maildir = get_maildir(&mailbox)?;
delete_mail.execute(params![&mailbox])?; delete_mail.execute(params![&mailbox])?;
let mut mails = Vec::new(); let mut mails = Vec::new();
for x in maildir.list_cur() { for x in maildir.list_cur() {
mails.push(x?); mails.push(x?);
} }
println!("acquired {} mails", mails.len());
let mut mails = maildir.get_mails(&mut mails)?; let mut mails = maildir.get_mails(&mut mails)?;
mails.sort_by_key(|x| x.date); mails.sort_by_key(|x| x.date);
for mail in mails { for mail in mails {
let headers = mail.get_headers(); let headers = mail.get_headers();
let message_id = headers.get_all_values("Message-ID").join(" "); let mut message_id = headers.get_all_values("Message-ID").join(" ");
save_mail.execute(params![&mailbox, mail.id.to_i64(), message_id])?; if message_id.is_empty() {
message_id = format!("<{}_{}_{}@no-message-id>", mailbox, mail.id.uid_validity, mail.id.uid);
}
save_mail.execute(params![&mailbox, mail.id.to_i64(), message_id, mail.flags])?;
} }
} }
}
tx.commit()?;
db.execute("VACUUM", params![])?;
Ok(()) Ok(())
} }

View File

@ -64,45 +64,60 @@ fn sync(
remote.insert(mailbox, mails); remote.insert(mailbox, mails);
} }
let mut have_mail = db.prepare("SELECT mailbox, uid FROM mail WHERE message_id = ?")?; let mut have_mail = db.prepare("SELECT mailbox, uid, flags FROM mail WHERE message_id = ?")?;
let mut delete_mail = db.prepare("DELETE FROM mail WHERE mailbox = ? AND uid = ?")?; let mut delete_mail = db.prepare("DELETE FROM mail WHERE mailbox = ? AND uid = ?")?;
let mut all_mail = db.prepare("SELECT uid, message_id FROM mail WHERE mailbox = ?")?; let mut all_mail = db.prepare("SELECT uid, message_id FROM mail WHERE mailbox = ?")?;
let mut save_mail = db.prepare("INSERT INTO mail VALUES (?,?,?)")?; let mut save_mail = db.prepare("INSERT INTO mail VALUES (?,?,?,?)")?;
let mut maildirs: HashMap<&str, Maildir> = names.iter().map(|&x| (x.name(), get_maildir(x.name()).unwrap())).collect(); let mut maildirs: HashMap<&str, Maildir> = names.iter().map(|&x| (x.name(), get_maildir(x.name()).unwrap())).collect();
let mut to_remove: HashMap<&str, _> = HashMap::new(); let mut to_remove: HashMap<&str, _> = HashMap::new();
for &name in &names { for &name in &names {
let mailbox = name.name(); let mailbox = name.name();
let remote_mails = &remote[mailbox]; let remote_mails = remote.get_mut(mailbox).unwrap();
let resp = imap_session.examine(mailbox)?;
let uid_validity = resp.uid_validity.unwrap();
let mut to_fetch = Vec::new(); let mut to_fetch = Vec::new();
for message_id in remote_mails.keys() { for (message_id, entry) in remote_mails.iter_mut() {
let (uid1, uid2, full_uid, ref _flags) = remote_mails[message_id]; let (uid1, uid2, full_uid, remote_flags) = entry;
let local = have_mail.query_map(params![message_id], |row| Ok((row.get::<_, String>(0)?, load_i64(row.get::<_, i64>(1)?))))?.map(|x| x.unwrap()).collect_vec(); let local = have_mail.query_map(params![message_id], |row| Ok((
if local.iter().any(|x| x.0 == mailbox && x.1 == full_uid) { row.get::<_, String>(0)?,
load_i64(row.get::<_, i64>(1)?),
row.get::<_, String>(2)?
)))?.map(|x| x.unwrap()).collect_vec();
if let Some((_, full_uid, flags)) = local.iter().filter(|x| x.0 == mailbox && x.1 == *full_uid).next() {
let uid = (full_uid << 32) >> 32;
let local_s = flags.contains('S');
let local_u = flags.contains('U');
let remote_s = remote_flags.contains(&Flag::Seen);
if local_s && !remote_s {
imap_session.uid_store(uid.to_string(), "+FLAGS.SILENT (\\Seen)")?;
remote_flags.push(Flag::Seen);
} else if local_u && remote_s {
imap_session.uid_store(uid.to_string(), "-FLAGS.SILENT (\\Seen)")?;
remote_flags.remove(remote_flags.iter().position(|x| x == &Flag::Seen).unwrap());
}
continue; continue;
} }
if !local.is_empty() { if !local.is_empty() {
let (inbox, full_uid) = &local[0]; let (inbox, full_uid, flags) = &local[0];
let local_uid1 = (full_uid >> 32) as u32; let local_uid1 = (full_uid >> 32) as u32;
let local_uid2 = ((full_uid << 32) >> 32) as u32; let local_uid2 = ((full_uid << 32) >> 32) as u32;
let local_id = gen_id(local_uid1, local_uid2); let local_id = gen_id(local_uid1, local_uid2);
let new_uid = MaildirID::new(uid1, uid2); let new_uid = MaildirID::new(*uid1, *uid2);
let new_id = new_uid.to_string(); let new_id = new_uid.to_string();
// hardlink mail // hardlink mail
let maildir1 = &maildirs[&**inbox]; let maildir1 = &maildirs[&**inbox];
println!("hardlinking: {}/{} -> {}/{}", inbox, local_id, mailbox, new_id); println!("hardlinking: {}/{} -> {}/{}", inbox, local_id, mailbox, new_id);
let name = maildir1.find_filename(&local_id).unwrap(); let name = maildir1.find_filename(&local_id).unwrap();
let maildir2 = &maildirs[mailbox]; let maildir2 = &maildirs[mailbox];
maildir2.store_cur_from_path(&new_id, name)?; maildir2.store_cur_from_path(&new_id, flags, name)?;
save_mail.execute(params![mailbox, new_uid.to_i64(), message_id])?; save_mail.execute(params![mailbox, new_uid.to_i64(), message_id, flags])?;
} else if !name.attributes().iter().any(|x| *x == TRASH) { // do not fetch trashed mail } else if !name.attributes().iter().any(|x| *x == TRASH) { // do not fetch trashed mail
to_fetch.push(uid2); to_fetch.push(uid2);
} }
} }
if !to_fetch.is_empty() { if !to_fetch.is_empty() {
let maildir = &maildirs[mailbox]; let maildir = &maildirs[mailbox];
let resp = imap_session.examine(mailbox)?;
let uid_validity = resp.uid_validity.unwrap();
let fetch_range = to_fetch.into_iter().map(|x| x.to_string()).join(","); let fetch_range = to_fetch.into_iter().map(|x| x.to_string()).join(",");
let fetch = imap_session.uid_fetch(fetch_range, "RFC822")?; let fetch = imap_session.uid_fetch(fetch_range, "RFC822")?;
@ -113,7 +128,8 @@ fn sync(
let id = gen_id(uid_validity, uid); let id = gen_id(uid_validity, uid);
if !maildir.exists(&id) { if !maildir.exists(&id) {
let mail_data = mail.body().unwrap_or_default(); let mail_data = mail.body().unwrap_or_default();
maildir.store_cur_with_id(&id, mail_data)?; let flags = imap_flags_to_maildir("".into(), mail.flags());
maildir.store_cur_with_id_flags(&id, &flags, mail_data)?;
let headers = parse_headers(&mail_data)?.0; let headers = parse_headers(&mail_data)?.0;
let mut message_id = headers.get_all_values("Message-ID").join(" "); let mut message_id = headers.get_all_values("Message-ID").join(" ");
@ -121,7 +137,7 @@ fn sync(
message_id = format!("<{}_{}_{}@no-message-id>", mailbox, uid_validity, uid); message_id = format!("<{}_{}_{}@no-message-id>", mailbox, uid_validity, uid);
} }
let full_uid = ((uid_validity as u64) << 32) | uid as u64; let full_uid = ((uid_validity as u64) << 32) | uid as u64;
save_mail.execute(params![mailbox, store_i64(full_uid), message_id])?; save_mail.execute(params![mailbox, store_i64(full_uid), message_id, flags])?;
} else { } else {
println!("warning: DB outdated, downloaded mail again"); println!("warning: DB outdated, downloaded mail again");
} }
@ -132,22 +148,8 @@ fn sync(
let (uid1, uid2, _, ref flags) = remote_mails[message_id]; let (uid1, uid2, _, ref flags) = remote_mails[message_id];
let id = gen_id(uid1, uid2); let id = gen_id(uid1, uid2);
let _ = maildir.update_flags(&id, |f| { let _ = maildir.update_flags(&id, |f| {
let mut f = f.to_owned(); let f = f.replace('U', "");
if flags.contains(&Flag::Seen) { let f = imap_flags_to_maildir(f, flags);
f.push('S');
} else {
f = f.replace('S', "");
}
if flags.contains(&Flag::Answered) {
f.push('R');
} else {
f = f.replace('R', "");
}
if flags.contains(&Flag::Flagged) {
f.push('F');
} else {
f = f.replace('F', "");
}
Maildir::normalize_flags(&f) Maildir::normalize_flags(&f)
}); });
} }
@ -175,7 +177,8 @@ fn sync(
} }
let maildir = &maildirs[mailbox]; let maildir = &maildirs[mailbox];
let name = maildir.find_filename(&uid_name).unwrap(); let name = maildir.find_filename(&uid_name).unwrap();
maildirs[".gone"].store_new_from_path(&format!("{}_{}", mailbox, uid_name), name)?; let _ = maildirs[".gone"].store_new_from_path(&format!("{}_{}", mailbox, uid_name), name);
// hardlink should only fail if the mail was already deleted
maildir.delete(&uid_name)?; maildir.delete(&uid_name)?;
delete_mail.execute(params![mailbox, store_i64(uid)])?; delete_mail.execute(params![mailbox, store_i64(uid)])?;
} }

View File

@ -52,7 +52,8 @@ pub fn get_db() -> Result<Connection> {
CREATE TABLE IF NOT EXISTS mail( CREATE TABLE IF NOT EXISTS mail(
mailbox STRING NOT NULL, mailbox STRING NOT NULL,
uid INTEGER NOT NULL, uid INTEGER NOT NULL,
message_id STRING NOT NULL message_id STRING NOT NULL,
flags STRING NOT NULL
)", params![])?; )", params![])?;
Ok(conn) Ok(conn)
@ -64,7 +65,7 @@ pub fn gen_id(uid_validity: u32, uid: u32) -> String {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct MaildirID { pub struct MaildirID {
uid_validity: u32, pub uid_validity: u32,
pub uid: u32, pub uid: u32,
} }
@ -105,7 +106,8 @@ pub struct EasyMail<'a> {
mail: Option<ParsedMail<'a>>, mail: Option<ParsedMail<'a>>,
pub id: MaildirID, pub id: MaildirID,
pub flags: String, pub flags: String,
from: SingleInfo, from: Option<SingleInfo>,
from_raw: String,
pub subject: String, pub subject: String,
pub date: DateTime<Local>, pub date: DateTime<Local>,
pub date_iso: String, pub date_iso: String,
@ -117,10 +119,8 @@ impl EasyMail<'_> {
mail: None, mail: None,
id: MaildirID::new(0, 0), id: MaildirID::new(0, 0),
flags: "S".to_owned(), flags: "S".to_owned(),
from: SingleInfo { from: None,
display_name: None, from_raw: String::new(),
addr: String::new()
},
subject, subject,
date: Local.from_utc_datetime(&NaiveDateTime::from_timestamp(0, 0)), date: Local.from_utc_datetime(&NaiveDateTime::from_timestamp(0, 0)),
date_iso: "????-??-??".to_owned() date_iso: "????-??-??".to_owned()
@ -132,13 +132,17 @@ impl EasyMail<'_> {
} }
pub fn from(&self) -> String { pub fn from(&self) -> String {
let name = self.from.display_name.as_deref().unwrap_or_default(); if let Some(from) = self.from.as_ref() {
let name = from.display_name.as_deref().unwrap_or_default();
if let Some(config) = CONFIG.get() { if let Some(config) = CONFIG.get() {
if config.read().browse.show_email_addresses { if config.read().browse.show_email_addresses {
return format!("{} <{}>", name, self.from.addr); return format!("{} <{}>", name, from.addr);
} }
} }
name.to_owned() name.to_owned()
} else {
self.from_raw.clone()
}
} }
pub fn get_header(&self, header: &str) -> String { pub fn get_header(&self, header: &str) -> String {
@ -173,8 +177,10 @@ impl Eq for EasyMail<'_> {}
impl Hash for EasyMail<'_> { impl Hash for EasyMail<'_> {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) { fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
self.id.hash(state); self.id.hash(state);
self.from.display_name.hash(state); if let Some(from) = self.from.as_ref() {
self.from.addr.hash(state); from.display_name.hash(state);
from.addr.hash(state);
}
self.subject.hash(state); self.subject.hash(state);
} }
} }
@ -326,7 +332,8 @@ impl MaildirExtension for Maildir {
let flags = maile.flags().to_owned(); let flags = maile.flags().to_owned();
let mail = maile.parsed()?; let mail = maile.parsed()?;
let headers = mail.get_headers(); let headers = mail.get_headers();
let from = addrparse(&headers.get_all_values("From").join(" "))?.extract_single_info().context("failed to extract from")?; let from_raw = headers.get_all_values("From").join(" ");
let from = addrparse(&from_raw).map(|x| x.extract_single_info()).ok().flatten();
let subject = headers.get_all_values("Subject").join(" "); let subject = headers.get_all_values("Subject").join(" ");
let date = headers.get_all_values("Date").join(" "); let date = headers.get_all_values("Date").join(" ");
let date = dateparse(&date).map(|x| let date = dateparse(&date).map(|x|
@ -337,6 +344,7 @@ impl MaildirExtension for Maildir {
flags, flags,
id, id,
from, from,
from_raw,
subject, subject,
date_iso: date.format("%Y-%m-%d %H:%M").to_string(), date_iso: date.format("%Y-%m-%d %H:%M").to_string(),
date, date,
@ -353,7 +361,8 @@ impl MaildirExtension for Maildir {
let flags = maile.flags().to_owned(); let flags = maile.flags().to_owned();
let mail = maile.parsed()?; let mail = maile.parsed()?;
let headers = mail.get_headers(); let headers = mail.get_headers();
let from = addrparse(&headers.get_all_values("From").join(" "))?.extract_single_info().context("failed to extract from")?; let from_raw = headers.get_all_values("From").join(" ");
let from = addrparse(&from_raw).map(|x| x.extract_single_info()).ok().flatten();
let subject = headers.get_all_values("Subject").join(" "); let subject = headers.get_all_values("Subject").join(" ");
let date = headers.get_all_values("Date").join(" "); let date = headers.get_all_values("Date").join(" ");
let date = dateparse(&date).map(|x| let date = dateparse(&date).map(|x|
@ -364,6 +373,7 @@ impl MaildirExtension for Maildir {
flags, flags,
id, id,
from, from,
from_raw,
subject, subject,
date_iso: date.format("%Y-%m-%d %H:%M").to_string(), date_iso: date.format("%Y-%m-%d %H:%M").to_string(),
date, date,
@ -534,3 +544,22 @@ pub fn parse_effect(effect: &str) -> Option<Effect> {
fn default_unread_style() -> Style { fn default_unread_style() -> Style {
Effect::Reverse.into() Effect::Reverse.into()
} }
pub fn imap_flags_to_maildir(mut f: String, flags: &[Flag]) -> String {
if flags.contains(&Flag::Seen) {
f.push('S');
} else {
f = f.replace('S', "");
}
if flags.contains(&Flag::Answered) {
f.push('R');
} else {
f = f.replace('R', "");
}
if flags.contains(&Flag::Flagged) {
f.push('F');
} else {
f = f.replace('F', "");
}
f
}