From 9e746fb7d7b9fbc8de37f5620da2cdf0dc77b516 Mon Sep 17 00:00:00 2001 From: FliegendeWurst <2012gdwu+github@posteo.de> Date: Mon, 29 Mar 2021 10:52:14 +0200 Subject: [PATCH] sync: synchronize remote mailboxes to local --- src/bin/fetch.rs | 21 +------ src/bin/sync.rs | 146 +++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 44 +++++++++++++- 3 files changed, 191 insertions(+), 20 deletions(-) create mode 100644 src/bin/sync.rs diff --git a/src/bin/fetch.rs b/src/bin/fetch.rs index 6b56b67..1d3623a 100644 --- a/src/bin/fetch.rs +++ b/src/bin/fetch.rs @@ -1,4 +1,4 @@ -use std::{cmp, env, fs, io, time::Duration}; +use std::{cmp, env, time::Duration}; use itertools::Itertools; use maildir::Maildir; @@ -73,7 +73,7 @@ fn fetch_inbox_top( let uid = mail.uid.unwrap(); largest_uid = cmp::max(largest_uid, uid); println!("mail {:?}", uid); - let id = format!("{}_{}", uid_validity, uid); + let id = gen_id(uid_validity, uid); let uid = ((uid_validity as u64) << 32) | uid as u64; if !maildir.exists(&id).unwrap_or(false) { let mail_data = mail.body().unwrap_or_default(); @@ -81,7 +81,7 @@ fn fetch_inbox_top( let headers = parse_headers(&mail_data)?.0; let message_id = headers.get_all_values("Message-ID").join(" "); - save_mail.execute(params![mailbox, uid, message_id])?; + save_mail.execute(params![mailbox, store_i64(uid), message_id])?; } } let uid = cmp::max(uid_next - 1, largest_uid); @@ -92,18 +92,3 @@ fn fetch_inbox_top( Ok(()) } - -trait MaildirExtension { - fn get_file(&self, name: &str) -> std::result::Result; - fn save_file(&self, name: &str, content: &str) -> std::result::Result<(), io::Error>; -} - -impl MaildirExtension for Maildir { - fn get_file(&self, name: &str) -> std::result::Result { - fs::read_to_string(self.path().join(name)) - } - - fn save_file(&self, name: &str, content: &str) -> std::result::Result<(), io::Error> { - fs::write(self.path().join(name), content) - } -} diff --git a/src/bin/sync.rs b/src/bin/sync.rs new file mode 100644 index 0000000..261515b --- /dev/null +++ b/src/bin/sync.rs @@ -0,0 +1,146 @@ +use std::{collections::HashMap, env}; + +use imap::types::Flag; +use itertools::Itertools; +use maildir::Maildir; + +use inboxid::*; +use mailparse::{MailHeaderMap, parse_header, parse_headers}; +use rusqlite::params; + +fn main() -> Result<()> { + let host = env::var("MAILHOST").expect("missing envvar MAILHOST"); + let user = env::var("MAILUSER").expect("missing envvar MAILUSER"); + let password = env::var("MAILPASSWORD").expect("missing envvar MAILPASSWORD"); + let port = 993; + + sync(&host, &user, &password, port) +} + +fn sync( + host: &str, + user: &str, + password: &str, + port: u16, +) -> Result<()> { + let db = get_db()?; + let mut imap_session = connect(host, port, user, password)?; + println!("getting capabilities.."); + let caps = imap_session.capabilities()?; + println!("capabilities: {}", caps.iter().map(|x| format!("{:?}", x)).join(" ")); + + let mut names = Vec::new(); + let list = imap_session.list(None, Some("*"))?; + for x in list.iter() { + println!("{:?}", x); + names.push(x.name()); + } + names = vec!["INBOX", "nebenan"]; + + let mut remote = HashMap::new(); + + for &mailbox in &names { + println!("indexing {}", mailbox); + let resp = imap_session.examine(mailbox)?; + let uid_validity = resp.uid_validity.unwrap(); + + let mut mails = HashMap::new(); + let messages = imap_session.uid_fetch("1:*", "(FLAGS BODY[HEADER.FIELDS (MESSAGE-ID)])")?; + for m in messages.iter() { + let flags = m.flags(); + if flags.contains(&Flag::Deleted) { + continue; + } + let header = m.header().unwrap(); + let header = parse_header(header)?.0; + let uid = m.uid.unwrap(); + let full_uid = ((uid_validity as u64) << 32) | uid as u64; + let flags = flags.iter().map(|x| remove_cow(x)).collect_vec(); + mails.insert(header.get_value(), (uid_validity, uid, full_uid, flags)); + } + remote.insert(mailbox, mails); + } + + let mut have_mail = db.prepare("SELECT mailbox, uid FROM mail WHERE message_id = ?")?; + let mut delete_mail = db.prepare("DELETE FROM mail WHERE mailbox = ? AND uid = ?")?; + let mut all_mail = db.prepare("SELECT uid, message_id FROM mail WHERE mailbox = ?")?; + let mut save_mail = db.prepare("INSERT INTO mail VALUES (?,?,?)")?; + let mut maildirs: HashMap<&str, Maildir> = names.iter().map(|&x| (x, get_maildir(x).unwrap())).collect(); + for &mailbox in &names { + let remote_mails = &remote[mailbox]; + + let mut to_fetch = Vec::new(); + for message_id in remote_mails.keys() { + let (uid1, uid2, full_uid, ref _flags) = remote_mails[message_id]; + let local = have_mail.query_map(params![message_id], |row| Ok((row.get::<_, String>(0)?, load_i64(row.get::<_, i64>(1)?))))?.map(|x| x.unwrap()).collect_vec(); + if local.iter().any(|x| x.0 == mailbox && x.1 == full_uid) { + continue; + } + if !local.is_empty() { + let (inbox, full_uid) = &local[0]; + let local_uid1 = (full_uid >> 32) as u32; + let local_uid2 = ((full_uid << 32) >> 32) as u32; + let local_id = gen_id(local_uid1, local_uid2); + // hardlink mail + let maildir1 = &maildirs[&**inbox]; + let name = maildir1.find_filename(&local_id).unwrap(); + let maildir2 = &maildirs[mailbox]; + let new_id = gen_id(uid1, uid2); + println!("hardlinking: {}/{} -> {}/{}", inbox, local_id, mailbox, new_id); + maildir2.store_new_from_path(&new_id, name)?; + save_mail.execute(params![mailbox, store_i64(*full_uid), message_id])?; + } else { + to_fetch.push(uid2); + } + } + if !to_fetch.is_empty() { + let maildir = &maildirs[mailbox]; + let resp = imap_session.examine(mailbox)?; + let uid_validity = resp.uid_validity.unwrap(); + + let fetch_range = to_fetch.into_iter().map(|x| x.to_string()).join(","); + let fetch = imap_session.uid_fetch(fetch_range, "RFC822")?; + + for mail in fetch.iter() { + let uid = mail.uid.unwrap(); + println!("fetching: {}/{}", mailbox, uid); + let id = gen_id(uid_validity, uid); + if !maildir.exists(&id).unwrap_or(false) { + let mail_data = mail.body().unwrap_or_default(); + maildir.store_new_with_id(&id, mail_data)?; + + let headers = parse_headers(&mail_data)?.0; + let message_id = headers.get_all_values("Message-ID").join(" "); + let full_uid = ((uid_validity as u64) << 32) | uid as u64; + save_mail.execute(params![mailbox, store_i64(full_uid), message_id])?; + } else { + println!("warning: DB outdated, downloaded mail again"); + } + } + } + let mails = all_mail.query_map(params![mailbox], |row| + Ok((load_i64(row.get::<_, i64>(0)?), row.get::<_, String>(1)?)))? + .map(|x| x.unwrap()).collect_vec(); + for (uid, message_id) in mails { + let uid1 = (uid >> 32) as u32; + let uid2 = ((uid << 32) >> 32) as u32; + if !remote_mails.contains_key(&message_id) { + let uid_name = gen_id(uid1, uid2); + println!("removing: {}/{}", mailbox, uid_name); + if !maildirs.contains_key(".gone") { + maildirs.insert(".gone", get_maildir(".gone")?); + } + let maildir = &maildirs[mailbox]; + let name = maildir.find_filename(&uid_name).unwrap(); + maildirs[".gone"].store_new_from_path(&format!("{}_{}", mailbox, uid_name), name)?; + maildir.delete(&uid_name)?; + delete_mail.execute(params![mailbox, uid])?; + } + } + } + + // be nice to the server and log out + imap_session.logout()?; + + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index fd47afe..8b13984 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,6 @@ -use std::{env, net::TcpStream}; +use std::{borrow::Cow, env, fs, io, net::TcpStream}; -use imap::Session; +use imap::{Session, types::Flag}; use maildir::Maildir; use rusqlite::{Connection, params}; use rustls_connector::{RustlsConnector, rustls::{ClientSession, StreamOwned}}; @@ -43,3 +43,43 @@ pub fn get_db() -> Result { Ok(conn) } + +pub fn gen_id(uid_validity: u32, uid: u32) -> String { + format!("{}_{}", uid_validity, uid) +} + +pub trait MaildirExtension { + fn get_file(&self, name: &str) -> std::result::Result; + fn save_file(&self, name: &str, content: &str) -> std::result::Result<(), io::Error>; +} + +impl MaildirExtension for Maildir { + fn get_file(&self, name: &str) -> std::result::Result { + fs::read_to_string(self.path().join(name)) + } + + fn save_file(&self, name: &str, content: &str) -> std::result::Result<(), io::Error> { + fs::write(self.path().join(name), content) + } +} + +pub fn store_i64(x: u64) -> i64 { + unsafe { std::mem::transmute(x) } +} + +pub fn load_i64(x: i64) -> u64 { + unsafe { std::mem::transmute(x) } +} + +pub fn remove_cow<'a>(x: &Flag<'a>) -> Flag<'static> { + match x { + Flag::Custom(x) => Flag::Custom(Cow::Owned(x.to_string())), + Flag::Seen => Flag::Seen, + Flag::Answered => Flag::Answered, + Flag::Flagged => Flag::Flagged, + Flag::Deleted => Flag::Deleted, + Flag::Draft => Flag::Draft, + Flag::Recent => Flag::Recent, + Flag::MayCreate => Flag::MayCreate, + } +}