sync: synchronize remote mailboxes to local

This commit is contained in:
FliegendeWurst 2021-03-29 10:52:14 +02:00 committed by Arne Keller
parent 094c42d696
commit 9e746fb7d7
3 changed files with 191 additions and 20 deletions

View File

@ -1,4 +1,4 @@
use std::{cmp, env, fs, io, time::Duration}; use std::{cmp, env, time::Duration};
use itertools::Itertools; use itertools::Itertools;
use maildir::Maildir; use maildir::Maildir;
@ -73,7 +73,7 @@ fn fetch_inbox_top(
let uid = mail.uid.unwrap(); let uid = mail.uid.unwrap();
largest_uid = cmp::max(largest_uid, uid); largest_uid = cmp::max(largest_uid, uid);
println!("mail {:?}", uid); println!("mail {:?}", uid);
let id = format!("{}_{}", uid_validity, uid); let id = gen_id(uid_validity, uid);
let uid = ((uid_validity as u64) << 32) | uid as u64; let uid = ((uid_validity as u64) << 32) | uid as u64;
if !maildir.exists(&id).unwrap_or(false) { if !maildir.exists(&id).unwrap_or(false) {
let mail_data = mail.body().unwrap_or_default(); let mail_data = mail.body().unwrap_or_default();
@ -81,7 +81,7 @@ fn fetch_inbox_top(
let headers = parse_headers(&mail_data)?.0; let headers = parse_headers(&mail_data)?.0;
let message_id = headers.get_all_values("Message-ID").join(" "); let message_id = headers.get_all_values("Message-ID").join(" ");
save_mail.execute(params![mailbox, uid, message_id])?; save_mail.execute(params![mailbox, store_i64(uid), message_id])?;
} }
} }
let uid = cmp::max(uid_next - 1, largest_uid); let uid = cmp::max(uid_next - 1, largest_uid);
@ -92,18 +92,3 @@ fn fetch_inbox_top(
Ok(()) Ok(())
} }
trait MaildirExtension {
fn get_file(&self, name: &str) -> std::result::Result<String, io::Error>;
fn save_file(&self, name: &str, content: &str) -> std::result::Result<(), io::Error>;
}
impl MaildirExtension for Maildir {
fn get_file(&self, name: &str) -> std::result::Result<String, io::Error> {
fs::read_to_string(self.path().join(name))
}
fn save_file(&self, name: &str, content: &str) -> std::result::Result<(), io::Error> {
fs::write(self.path().join(name), content)
}
}

146
src/bin/sync.rs Normal file
View File

@ -0,0 +1,146 @@
use std::{collections::HashMap, env};
use imap::types::Flag;
use itertools::Itertools;
use maildir::Maildir;
use inboxid::*;
use mailparse::{MailHeaderMap, parse_header, parse_headers};
use rusqlite::params;
fn main() -> Result<()> {
let host = env::var("MAILHOST").expect("missing envvar MAILHOST");
let user = env::var("MAILUSER").expect("missing envvar MAILUSER");
let password = env::var("MAILPASSWORD").expect("missing envvar MAILPASSWORD");
let port = 993;
sync(&host, &user, &password, port)
}
fn sync(
host: &str,
user: &str,
password: &str,
port: u16,
) -> Result<()> {
let db = get_db()?;
let mut imap_session = connect(host, port, user, password)?;
println!("getting capabilities..");
let caps = imap_session.capabilities()?;
println!("capabilities: {}", caps.iter().map(|x| format!("{:?}", x)).join(" "));
let mut names = Vec::new();
let list = imap_session.list(None, Some("*"))?;
for x in list.iter() {
println!("{:?}", x);
names.push(x.name());
}
names = vec!["INBOX", "nebenan"];
let mut remote = HashMap::new();
for &mailbox in &names {
println!("indexing {}", mailbox);
let resp = imap_session.examine(mailbox)?;
let uid_validity = resp.uid_validity.unwrap();
let mut mails = HashMap::new();
let messages = imap_session.uid_fetch("1:*", "(FLAGS BODY[HEADER.FIELDS (MESSAGE-ID)])")?;
for m in messages.iter() {
let flags = m.flags();
if flags.contains(&Flag::Deleted) {
continue;
}
let header = m.header().unwrap();
let header = parse_header(header)?.0;
let uid = m.uid.unwrap();
let full_uid = ((uid_validity as u64) << 32) | uid as u64;
let flags = flags.iter().map(|x| remove_cow(x)).collect_vec();
mails.insert(header.get_value(), (uid_validity, uid, full_uid, flags));
}
remote.insert(mailbox, mails);
}
let mut have_mail = db.prepare("SELECT mailbox, uid FROM mail WHERE message_id = ?")?;
let mut delete_mail = db.prepare("DELETE FROM mail WHERE mailbox = ? AND uid = ?")?;
let mut all_mail = db.prepare("SELECT uid, message_id FROM mail WHERE mailbox = ?")?;
let mut save_mail = db.prepare("INSERT INTO mail VALUES (?,?,?)")?;
let mut maildirs: HashMap<&str, Maildir> = names.iter().map(|&x| (x, get_maildir(x).unwrap())).collect();
for &mailbox in &names {
let remote_mails = &remote[mailbox];
let mut to_fetch = Vec::new();
for message_id in remote_mails.keys() {
let (uid1, uid2, full_uid, ref _flags) = remote_mails[message_id];
let local = have_mail.query_map(params![message_id], |row| Ok((row.get::<_, String>(0)?, load_i64(row.get::<_, i64>(1)?))))?.map(|x| x.unwrap()).collect_vec();
if local.iter().any(|x| x.0 == mailbox && x.1 == full_uid) {
continue;
}
if !local.is_empty() {
let (inbox, full_uid) = &local[0];
let local_uid1 = (full_uid >> 32) as u32;
let local_uid2 = ((full_uid << 32) >> 32) as u32;
let local_id = gen_id(local_uid1, local_uid2);
// hardlink mail
let maildir1 = &maildirs[&**inbox];
let name = maildir1.find_filename(&local_id).unwrap();
let maildir2 = &maildirs[mailbox];
let new_id = gen_id(uid1, uid2);
println!("hardlinking: {}/{} -> {}/{}", inbox, local_id, mailbox, new_id);
maildir2.store_new_from_path(&new_id, name)?;
save_mail.execute(params![mailbox, store_i64(*full_uid), message_id])?;
} else {
to_fetch.push(uid2);
}
}
if !to_fetch.is_empty() {
let maildir = &maildirs[mailbox];
let resp = imap_session.examine(mailbox)?;
let uid_validity = resp.uid_validity.unwrap();
let fetch_range = to_fetch.into_iter().map(|x| x.to_string()).join(",");
let fetch = imap_session.uid_fetch(fetch_range, "RFC822")?;
for mail in fetch.iter() {
let uid = mail.uid.unwrap();
println!("fetching: {}/{}", mailbox, uid);
let id = gen_id(uid_validity, uid);
if !maildir.exists(&id).unwrap_or(false) {
let mail_data = mail.body().unwrap_or_default();
maildir.store_new_with_id(&id, mail_data)?;
let headers = parse_headers(&mail_data)?.0;
let message_id = headers.get_all_values("Message-ID").join(" ");
let full_uid = ((uid_validity as u64) << 32) | uid as u64;
save_mail.execute(params![mailbox, store_i64(full_uid), message_id])?;
} else {
println!("warning: DB outdated, downloaded mail again");
}
}
}
let mails = all_mail.query_map(params![mailbox], |row|
Ok((load_i64(row.get::<_, i64>(0)?), row.get::<_, String>(1)?)))?
.map(|x| x.unwrap()).collect_vec();
for (uid, message_id) in mails {
let uid1 = (uid >> 32) as u32;
let uid2 = ((uid << 32) >> 32) as u32;
if !remote_mails.contains_key(&message_id) {
let uid_name = gen_id(uid1, uid2);
println!("removing: {}/{}", mailbox, uid_name);
if !maildirs.contains_key(".gone") {
maildirs.insert(".gone", get_maildir(".gone")?);
}
let maildir = &maildirs[mailbox];
let name = maildir.find_filename(&uid_name).unwrap();
maildirs[".gone"].store_new_from_path(&format!("{}_{}", mailbox, uid_name), name)?;
maildir.delete(&uid_name)?;
delete_mail.execute(params![mailbox, uid])?;
}
}
}
// be nice to the server and log out
imap_session.logout()?;
Ok(())
}

View File

@ -1,6 +1,6 @@
use std::{env, net::TcpStream}; use std::{borrow::Cow, env, fs, io, net::TcpStream};
use imap::Session; use imap::{Session, types::Flag};
use maildir::Maildir; use maildir::Maildir;
use rusqlite::{Connection, params}; use rusqlite::{Connection, params};
use rustls_connector::{RustlsConnector, rustls::{ClientSession, StreamOwned}}; use rustls_connector::{RustlsConnector, rustls::{ClientSession, StreamOwned}};
@ -43,3 +43,43 @@ pub fn get_db() -> Result<Connection> {
Ok(conn) Ok(conn)
} }
pub fn gen_id(uid_validity: u32, uid: u32) -> String {
format!("{}_{}", uid_validity, uid)
}
pub trait MaildirExtension {
fn get_file(&self, name: &str) -> std::result::Result<String, io::Error>;
fn save_file(&self, name: &str, content: &str) -> std::result::Result<(), io::Error>;
}
impl MaildirExtension for Maildir {
fn get_file(&self, name: &str) -> std::result::Result<String, io::Error> {
fs::read_to_string(self.path().join(name))
}
fn save_file(&self, name: &str, content: &str) -> std::result::Result<(), io::Error> {
fs::write(self.path().join(name), content)
}
}
pub fn store_i64(x: u64) -> i64 {
unsafe { std::mem::transmute(x) }
}
pub fn load_i64(x: i64) -> u64 {
unsafe { std::mem::transmute(x) }
}
pub fn remove_cow<'a>(x: &Flag<'a>) -> Flag<'static> {
match x {
Flag::Custom(x) => Flag::Custom(Cow::Owned(x.to_string())),
Flag::Seen => Flag::Seen,
Flag::Answered => Flag::Answered,
Flag::Flagged => Flag::Flagged,
Flag::Deleted => Flag::Deleted,
Flag::Draft => Flag::Draft,
Flag::Recent => Flag::Recent,
Flag::MayCreate => Flag::MayCreate,
}
}