gian/imap-migration-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix duplicate detection failing with iCloud IMAP

Browse Source 
Match on Message-ID only instead of requiring both Message-ID
and exact RFC822.SIZE, which fails when servers modify messages
on storage.
This commit is contained in:
Nathan Moinvaziri 2026-01-28 20:04:58 -08:00
parent 927fff17c7
commit 1fc5e13e9d
5 changed files with 24 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/imap_common.py
View File

@ -128,9 +128,9 @@ def get_msg_details(imap_conn, uid):
return msg_id, size, subject
def message_exists_in_folder(dest_conn, msg_id, src_size):
def message_exists_in_folder(dest_conn, msg_id):
"""
Checks if a message with the given Message-ID and RFC822.SIZE exists in the CURRENTLY SELECTED folder of dest_conn.
Checks if a message with the given Message-ID exists in the CURRENTLY SELECTED folder of dest_conn.
Returns True if found, False otherwise.
"""
if not msg_id:
@ -143,23 +143,9 @@ def message_exists_in_folder(dest_conn, msg_id, src_size):
return False
dest_ids = data[0].split()
if not dest_ids:
return False
for did in dest_ids:
resp, items = dest_conn.fetch(did, "(RFC822.SIZE)")
if resp == "OK":
for item in items:
if isinstance(item, bytes):
content = item.decode("utf-8", errors="ignore")
else:
content = item[0].decode("utf-8", errors="ignore")
size_match = re.search(r"RFC822\.SIZE\s+(\d+)", content)
if size_match and int(size_match.group(1)) == src_size:
return True
return len(dest_ids) > 0
except Exception:
return False
return False
def sanitize_filename(filename):

4
src/migrate_imap_emails.py
View File

@ -257,8 +257,8 @@ def process_batch(uids, folder_name, src_conf, dest_conf, delete_from_source, tr
size_str = f"{size / 1024:.1f}KB" if size else "0KB"
is_duplicate = False
if msg_id and size:
is_duplicate = imap_common.message_exists_in_folder(dest, msg_id, size)
if msg_id:
is_duplicate = imap_common.message_exists_in_folder(dest, msg_id)
if is_duplicate:
safe_print(f"[{folder_name}] {'SKIP (Dup)':<18} | {size_str:<8} | {subject[:40]}")

10
src/restore_imap_emails.py
View File

@ -238,15 +238,15 @@ def get_eml_files(folder_path):
return eml_files
def email_exists_in_folder(imap_conn, message_id, size):
def email_exists_in_folder(imap_conn, message_id):
"""
Check if an email with the given Message-ID and size exists in the currently selected folder.
Check if an email with the given Message-ID exists in the currently selected folder.
"""
if not message_id:
return False
try:
return imap_common.message_exists_in_folder(imap_conn, message_id, size)
return imap_common.message_exists_in_folder(imap_conn, message_id)
except Exception:
return False
@ -272,7 +272,7 @@ def upload_email(dest, folder_name, raw_content, date_str, message_id, subject,
# Check for duplicates
size = len(raw_content)
if message_id and email_exists_in_folder(dest, message_id, size):
if message_id and email_exists_in_folder(dest, message_id):
return False # Already exists
# Upload with original date and flags
@ -414,7 +414,7 @@ def process_restore_batch(eml_files, folder_name, dest_conf, manifest, apply_lab
# Select and check for duplicate
dest.select(f'"{label_folder}"')
if not email_exists_in_folder(dest, message_id, size):
if not email_exists_in_folder(dest, message_id):
dest.append(f'"{label_folder}"', flags, date_str, raw_content)
safe_print(f" -> Applied label: {label}")
# If email exists in this label folder, sync flags

22
test/test_imap_common.py
View File

@ -274,7 +274,7 @@ class TestMessageExistsInFolder:
def test_no_message_id(self):
"""Test returns False when message_id is None."""
mock_conn = Mock()
result = imap_common.message_exists_in_folder(mock_conn, None, 100)
result = imap_common.message_exists_in_folder(mock_conn, None)
assert result is False
def test_search_fails(self):
@ -282,7 +282,7 @@ class TestMessageExistsInFolder:
mock_conn = Mock()
mock_conn.search.return_value = ("NO", [])
result = imap_common.message_exists_in_folder(mock_conn, "<msg-id>", 100)
result = imap_common.message_exists_in_folder(mock_conn, "<msg-id>")
assert result is False
def test_no_matches(self):
@ -290,25 +290,23 @@ class TestMessageExistsInFolder:
mock_conn = Mock()
mock_conn.search.return_value = ("OK", [b""])
result = imap_common.message_exists_in_folder(mock_conn, "<msg-id>", 100)
result = imap_common.message_exists_in_folder(mock_conn, "<msg-id>")
assert result is False
def test_match_found_same_size(self):
"""Test returns True when message with same ID and size found."""
def test_match_found(self):
"""Test returns True when message with same ID found."""
mock_conn = Mock()
mock_conn.search.return_value = ("OK", [b"1"])
mock_conn.fetch.return_value = ("OK", [b"1 (RFC822.SIZE 100)"])
result = imap_common.message_exists_in_folder(mock_conn, "<msg-id>", 100)
result = imap_common.message_exists_in_folder(mock_conn, "<msg-id>")
assert result is True
def test_match_found_different_size(self):
"""Test returns False when message with same ID but different size found."""
def test_search_exception(self):
"""Test returns False when search raises an exception."""
mock_conn = Mock()
mock_conn.search.return_value = ("OK", [b"1"])
mock_conn.fetch.return_value = ("OK", [b"1 (RFC822.SIZE 200)"])
mock_conn.search.side_effect = Exception("Connection error")
result = imap_common.message_exists_in_folder(mock_conn, "<msg-id>", 100)
result = imap_common.message_exists_in_folder(mock_conn, "<msg-id>")
assert result is False

8
test/test_restore_imap_emails.py
View File

@ -396,7 +396,7 @@ class TestEmailExistsInFolder:
mock_conn = MagicMock()
monkeypatch.setattr("imap_common.message_exists_in_folder", lambda *args: True)
result = restore_imap_emails.email_exists_in_folder(mock_conn, "<test@test.com>", 1000)
result = restore_imap_emails.email_exists_in_folder(mock_conn, "<test@test.com>")
assert result is True
def test_email_exists_false(self, monkeypatch):
@ -404,14 +404,14 @@ class TestEmailExistsInFolder:
mock_conn = MagicMock()
monkeypatch.setattr("imap_common.message_exists_in_folder", lambda *args: False)
result = restore_imap_emails.email_exists_in_folder(mock_conn, "<test@test.com>", 1000)
result = restore_imap_emails.email_exists_in_folder(mock_conn, "<test@test.com>")
assert result is False
def test_email_exists_no_message_id(self, monkeypatch):
"""Test with no message ID."""
mock_conn = MagicMock()
result = restore_imap_emails.email_exists_in_folder(mock_conn, None, 1000)
result = restore_imap_emails.email_exists_in_folder(mock_conn, None)
assert result is False
def test_email_exists_exception(self, monkeypatch):
@ -422,7 +422,7 @@ class TestEmailExistsInFolder:
lambda *args: (_ for _ in ()).throw(Exception("Error")),
)
result = restore_imap_emails.email_exists_in_folder(mock_conn, "<test@test.com>", 1000)
result = restore_imap_emails.email_exists_in_folder(mock_conn, "<test@test.com>")
assert result is False