diff --git a/app/.dockerignore b/app/.dockerignore
new file mode 100644
index 0000000..f749263
--- /dev/null
+++ b/app/.dockerignore
@@ -0,0 +1,17 @@
+.idea/
+*.pyc
+db.sqlite
+.env
+.pytest_cache
+.vscode
+.DS_Store
+config
+adhoc
+static/node_modules
+db.sqlite-journal
+static/upload
+venv/
+.venv
+.coverage
+htmlcov
+.git/
\ No newline at end of file
diff --git a/app/.flake8 b/app/.flake8
new file mode 100644
index 0000000..c57a1d7
--- /dev/null
+++ b/app/.flake8
@@ -0,0 +1,26 @@
+[flake8]
+max-line-length = 88
+select = C,E,F,W,B,B902,B903,B904,B950
+extend-ignore =
+ # For black compatibility
+ E203,
+ E501,
+ # Ignore "f-string is missing placeholders"
+ F541,
+ # allow bare except
+ E722, B001
+exclude =
+ .git,
+ __pycache__,
+ .pytest_cache,
+ .venv,
+ static,
+ templates,
+ # migrations are generated by alembic
+ migrations,
+ docs,
+ shell.py
+
+per-file-ignores =
+ # ignore unused imports in __init__
+ __init__.py:F401
diff --git a/app/.gitattributes b/app/.gitattributes
new file mode 100644
index 0000000..2fcb5b2
--- /dev/null
+++ b/app/.gitattributes
@@ -0,0 +1,3 @@
+# https://github.com/github/linguist#overrides
+static/* linguist-vendored
+docs/* linguist-documentation
diff --git a/app/.github/CODEOWNERS b/app/.github/CODEOWNERS
new file mode 100644
index 0000000..a0ee70e
--- /dev/null
+++ b/app/.github/CODEOWNERS
@@ -0,0 +1,2 @@
+## code changes will send PR to following users
+* @acasajus @cquintana92 @nguyenkims
\ No newline at end of file
diff --git a/app/.github/FUNDING.yml b/app/.github/FUNDING.yml
new file mode 100644
index 0000000..52283b1
--- /dev/null
+++ b/app/.github/FUNDING.yml
@@ -0,0 +1 @@
+open_collective: simplelogin
diff --git a/app/.github/ISSUE_TEMPLATE/bug_report.md b/app/.github/ISSUE_TEMPLATE/bug_report.md
new file mode 100644
index 0000000..2fcf331
--- /dev/null
+++ b/app/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,39 @@
+---
+name: Bug report
+about: Create a report to help us improve SimpleLogin.
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+Please note that this is only for bug report.
+
+For help on your account, please reach out to us at hi[at]simplelogin.io. Please make sure to check out [our FAQ](https://simplelogin.io/faq/) that contains frequently asked questions.
+
+
+For feature request, you can use our [forum](https://github.com/simple-login/app/discussions/categories/feature-request).
+
+For self-hosted question/issue, please ask in [self-hosted forum](https://github.com/simple-login/app/discussions/categories/self-hosting-question)
+
+## Prerequisites
+- [ ] I have searched open and closed issues to make sure that the bug has not yet been reported.
+
+## Bug report
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Environment (If applicable):**
+ - OS: Linux, Mac, Windows
+ - Browser: Firefox, Chrome, Brave, Safari
+ - Version [e.g. 78]
+
+**Additional context**
+Add any other context about the problem here.
diff --git a/app/.github/changelog_configuration.json b/app/.github/changelog_configuration.json
new file mode 100644
index 0000000..7519812
--- /dev/null
+++ b/app/.github/changelog_configuration.json
@@ -0,0 +1,23 @@
+{
+ "template": "${{CHANGELOG}}\n\nUncategorized
\n\n${{UNCATEGORIZED}}\n
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
![]() Dung Nguyen Van |
+ ![]() Giuseppe Federico |
+ Ninh Dinh |
+ Tung Nguyen V. N. |
+ ![]() Son Nguyen Kim |
+ Raymond Nook |
+ Sibren Vasse |
+ Sylvia van Os |
+
{html_header} | +
+ {decode_text(payload, encoding)} + | +
+ This email failed anti-phishing checks when it was received by SimpleLogin, be careful with its content. + More info on anti-phishing measure +
+ """ + + # do not quarantine an email if fails DMARC but has a small rspamd score + if ( + config.MIN_RSPAMD_SCORE_FOR_FAILED_DMARC is not None + and spam_result.rspamd_score < config.MIN_RSPAMD_SCORE_FOR_FAILED_DMARC + and spam_result.dmarc + in ( + DmarcCheckResult.quarantine, + DmarcCheckResult.reject, + ) + ): + LOG.w( + f"email fails DMARC but has a small rspamd score, from contact {contact.email} to alias {alias.email}." + f"mail_from:{envelope.mail_from}, from_header: {from_header}" + ) + changed_msg = add_header( + msg, + warning_plain_text, + warning_html, + ) + return changed_msg, None + + if spam_result.dmarc == DmarcCheckResult.soft_fail: + LOG.w( + f"dmarc forward: soft_fail from contact {contact.email} to alias {alias.email}." + f"mail_from:{envelope.mail_from}, from_header: {from_header}" + ) + changed_msg = add_header( + msg, + warning_plain_text, + warning_html, + ) + return changed_msg, None + + if spam_result.dmarc in ( + DmarcCheckResult.quarantine, + DmarcCheckResult.reject, + ): + LOG.w( + f"dmarc forward: put email from {contact} to {alias} to quarantine. {spam_result.event_data()}, " + f"mail_from:{envelope.mail_from}, from_header: {msg[headers.FROM]}" + ) + email_log = quarantine_dmarc_failed_forward_email(alias, contact, envelope, msg) + Notification.create( + user_id=alias.user_id, + title=f"{alias.email} has a new mail in quarantine", + message=Notification.render( + "notification/message-quarantine.html", alias=alias + ), + commit=True, + ) + user = alias.user + send_email_with_rate_control( + user, + ALERT_QUARANTINE_DMARC, + user.email, + f"An email sent to {alias.email} has been quarantined", + render( + "transactional/message-quarantine-dmarc.txt.jinja2", + from_header=from_header, + alias=alias, + refused_email_url=email_log.get_dashboard_url(), + ), + render( + "transactional/message-quarantine-dmarc.html", + from_header=from_header, + alias=alias, + refused_email_url=email_log.get_dashboard_url(), + ), + max_nb_alert=10, + ignore_smtp_error=True, + ) + return msg, status.E215 + + return msg, None + + +def quarantine_dmarc_failed_forward_email(alias, contact, envelope, msg) -> EmailLog: + add_or_replace_header(msg, headers.SL_DIRECTION, "Forward") + msg[headers.SL_ENVELOPE_FROM] = envelope.mail_from + random_name = str(uuid.uuid4()) + s3_report_path = f"refused-emails/full-{random_name}.eml" + s3.upload_email_from_bytesio( + s3_report_path, BytesIO(message_to_bytes(msg)), f"full-{random_name}" + ) + refused_email = RefusedEmail.create( + full_report_path=s3_report_path, user_id=alias.user_id, flush=True + ) + return EmailLog.create( + user_id=alias.user_id, + mailbox_id=alias.mailbox_id, + contact_id=contact.id, + alias_id=alias.id, + message_id=str(msg[headers.MESSAGE_ID]), + refused_email_id=refused_email.id, + is_spam=True, + blocked=True, + commit=True, + ) + + +def apply_dmarc_policy_for_reply_phase( + alias_from: Alias, contact_recipient: Contact, envelope: Envelope, msg: Message +) -> Optional[str]: + spam_result = SpamdResult.extract_from_headers(msg, Phase.reply) + if not DMARC_CHECK_ENABLED or not spam_result: + return None + + if spam_result.dmarc not in ( + DmarcCheckResult.quarantine, + DmarcCheckResult.reject, + DmarcCheckResult.soft_fail, + ): + return None + + LOG.w( + f"dmarc reply: Put email from {alias_from.email} to {contact_recipient} into quarantine. {spam_result.event_data()}, " + f"mail_from:{envelope.mail_from}, from_header: {msg[headers.FROM]}" + ) + send_email_with_rate_control( + alias_from.user, + ALERT_DMARC_FAILED_REPLY_PHASE, + alias_from.user.email, + f"Attempt to send an email to your contact {contact_recipient.email} from {envelope.mail_from}", + render( + "transactional/spoof-reply.txt.jinja2", + contact=contact_recipient, + alias=alias_from, + sender=envelope.mail_from, + ), + render( + "transactional/spoof-reply.html", + contact=contact_recipient, + alias=alias_from, + sender=envelope.mail_from, + ), + ) + return status.E215 diff --git a/app/app/handler/provider_complaint.py b/app/app/handler/provider_complaint.py new file mode 100644 index 0000000..64af81c --- /dev/null +++ b/app/app/handler/provider_complaint.py @@ -0,0 +1,353 @@ +import uuid +from abc import ABC, abstractmethod +from dataclasses import dataclass +from io import BytesIO +from mailbox import Message +from typing import Optional, Union + +from app import s3 +from app.config import ( + ALERT_COMPLAINT_REPLY_PHASE, + ALERT_COMPLAINT_TRANSACTIONAL_PHASE, + ALERT_COMPLAINT_FORWARD_PHASE, +) +from app.email import headers +from app.email_utils import ( + parse_full_address, + save_email_for_debugging, + to_bytes, + render, + send_email_with_rate_control, + parse_address_list, + get_header_unicode, + get_verp_info_from_email, +) +from app.log import LOG +from app.models import ( + User, + Alias, + DeletedAlias, + DomainDeletedAlias, + Contact, + ProviderComplaint, + Phase, + ProviderComplaintState, + RefusedEmail, + VerpType, + EmailLog, + Mailbox, +) + + +@dataclass +class OriginalMessageInformation: + sender_address: str + rcpt_address: str + mailbox_address: Optional[str] + + +class ProviderComplaintOrigin(ABC): + @classmethod + @abstractmethod + def get_original_addresses( + cls, message: Message + ) -> Optional[OriginalMessageInformation]: + pass + + @classmethod + def _get_mailbox_id(cls, return_path: Optional[str]) -> Optional[Mailbox]: + if not return_path: + return None + _, return_path = parse_full_address(get_header_unicode(return_path)) + verp_data = get_verp_info_from_email(return_path) + if not verp_data: + return None + verp_type, email_log_id = verp_data + if verp_type == VerpType.transactional: + return None + email_log = EmailLog.get_by(id=email_log_id) + if email_log: + return email_log.mailbox.email + return None + + @classmethod + def sanitize_addresses_and_extract_mailbox_id( + cls, rcpt_header: Optional[str], message: Message + ) -> Optional[OriginalMessageInformation]: + """ + If the rcpt_header is not None, use it as the valid rcpt address, otherwise try to extract it from the To header + of the original message, since in the original message there can be more than one recipients. + There can only be one sender so that one can safely be extracted from the message headers. + """ + try: + if not rcpt_header: + rcpt_header = message[headers.TO] + rcpt_list = parse_address_list(get_header_unicode(rcpt_header)) + if not rcpt_list: + saved_file = save_email_for_debugging(message, "NoRecipientComplaint") + LOG.w(f"Cannot find rcpt. Saved to {saved_file or 'nowhere'}") + return None + rcpt_address = rcpt_list[0][1] + _, sender_address = parse_full_address( + get_header_unicode(message[headers.FROM]) + ) + + return OriginalMessageInformation( + sender_address, + rcpt_address, + cls._get_mailbox_id(message[headers.RETURN_PATH]), + ) + except ValueError: + saved_file = save_email_for_debugging(message, "ComplaintOriginalAddress") + LOG.w(f"Cannot parse from header. Saved to {saved_file or 'nowhere'}") + return None + + @classmethod + @abstractmethod + def name(cls): + pass + + +class ProviderComplaintYahoo(ProviderComplaintOrigin): + @classmethod + def get_original_message(cls, message: Message) -> Optional[Message]: + # 1st part is the container + # 2nd has empty body + # 6th is the original message + current_part = 0 + for part in message.walk(): + current_part += 1 + if current_part == 6: + return part + return None + + @classmethod + def get_feedback_report(cls, message: Message) -> Optional[Message]: + """ + Find a report that yahoo embeds in the complaint. It has content type 'message/feedback-report' + """ + for part in message.walk(): + if part["content-type"] == "message/feedback-report": + content = part.get_payload() + if not content: + continue + return content[0] + return None + + @classmethod + def get_original_addresses( + cls, message: Message + ) -> Optional[OriginalMessageInformation]: + """ + Try to get the proper recipient from the report that yahoo adds as a port of the complaint. If we cannot find + the rcpt in the report or we can't find the report, use the first address in the original message from + """ + report = cls.get_feedback_report(message) + original = cls.get_original_message(message) + rcpt_header = report[headers.YAHOO_ORIGINAL_RECIPIENT] + return cls.sanitize_addresses_and_extract_mailbox_id(rcpt_header, original) + + @classmethod + def name(cls): + return "yahoo" + + +class ProviderComplaintHotmail(ProviderComplaintOrigin): + @classmethod + def get_original_message(cls, message: Message) -> Optional[Message]: + # 1st part is the container + # 2nd has empty body + # 3rd is the original message + current_part = 0 + for part in message.walk(): + current_part += 1 + if current_part == 3: + return part + return None + + @classmethod + def get_original_addresses( + cls, message: Message + ) -> Optional[OriginalMessageInformation]: + """ + Try to get the proper recipient from original x-simplelogin-envelope-to header we add on delivery. + If we can't find the header, use the first address in the original message from""" + original = cls.get_original_message(message) + rcpt_header = original[headers.SL_ENVELOPE_TO] + return cls.sanitize_addresses_and_extract_mailbox_id(rcpt_header, original) + + @classmethod + def name(cls): + return "hotmail" + + +def handle_hotmail_complaint(message: Message) -> bool: + return handle_complaint(message, ProviderComplaintHotmail()) + + +def handle_yahoo_complaint(message: Message) -> bool: + return handle_complaint(message, ProviderComplaintYahoo()) + + +def find_alias_with_address(address: str) -> Optional[Union[Alias, DomainDeletedAlias]]: + return Alias.get_by(email=address) or DomainDeletedAlias.get_by(email=address) + + +def is_deleted_alias(address: str) -> bool: + return DeletedAlias.get_by(email=address) is not None + + +def handle_complaint(message: Message, origin: ProviderComplaintOrigin) -> bool: + msg_info = origin.get_original_addresses(message) + if not msg_info: + return False + + user = User.get_by(email=msg_info.rcpt_address) + if user: + LOG.d(f"Handle provider {origin.name()} complaint for {user}") + report_complaint_to_user_in_transactional_phase(user, origin, msg_info) + return True + + alias = find_alias_with_address(msg_info.sender_address) + # the email is during a reply phase, from=alias and to=destination + if alias: + LOG.i( + f"Complaint from {origin.name} during reply phase {alias} -> {msg_info.rcpt_address}, {user}" + ) + report_complaint_to_user_in_reply_phase( + alias, msg_info.rcpt_address, origin, msg_info + ) + store_provider_complaint(alias, message) + return True + + if is_deleted_alias(msg_info.sender_address): + LOG.i(f"Complaint is for deleted alias. Do nothing") + return True + + contact = Contact.get_by(reply_email=msg_info.sender_address) + if contact: + alias = contact.alias + else: + alias = find_alias_with_address(msg_info.rcpt_address) + + if is_deleted_alias(msg_info.rcpt_address): + LOG.i(f"Complaint is for deleted alias. Do nothing") + return True + + if not alias: + LOG.e( + f"Cannot find alias for address {msg_info.rcpt_address} or contact with reply {msg_info.sender_address}" + ) + return False + + report_complaint_to_user_in_forward_phase(alias, origin, msg_info) + return True + + +def report_complaint_to_user_in_reply_phase( + alias: Union[Alias, DomainDeletedAlias], + to_address: str, + origin: ProviderComplaintOrigin, + msg_info: OriginalMessageInformation, +): + capitalized_name = origin.name().capitalize() + mailbox_email = msg_info.mailbox_address + if not mailbox_email: + if type(alias) is Alias: + mailbox_email = alias.mailbox.email + else: + mailbox_email = alias.domain.mailboxes[0].email + send_email_with_rate_control( + alias.user, + f"{ALERT_COMPLAINT_REPLY_PHASE}_{origin.name()}", + mailbox_email, + f"Abuse report from {capitalized_name}", + render( + "transactional/provider-complaint-reply-phase.txt.jinja2", + user=alias.user, + alias=alias, + destination=to_address, + provider=capitalized_name, + ), + max_nb_alert=1, + nb_day=7, + ) + + +def report_complaint_to_user_in_transactional_phase( + user: User, origin: ProviderComplaintOrigin, msg_info: OriginalMessageInformation +): + capitalized_name = origin.name().capitalize() + send_email_with_rate_control( + user, + f"{ALERT_COMPLAINT_TRANSACTIONAL_PHASE}_{origin.name()}", + msg_info.mailbox_address or user.email, + f"Abuse report from {capitalized_name}", + render( + "transactional/provider-complaint-to-user.txt.jinja2", + user=user, + provider=capitalized_name, + ), + render( + "transactional/provider-complaint-to-user.html", + user=user, + provider=capitalized_name, + ), + max_nb_alert=1, + nb_day=7, + ) + + +def report_complaint_to_user_in_forward_phase( + alias: Union[Alias, DomainDeletedAlias], + origin: ProviderComplaintOrigin, + msg_info: OriginalMessageInformation, +): + capitalized_name = origin.name().capitalize() + user = alias.user + + mailbox_email = msg_info.mailbox_address + if not mailbox_email: + if type(alias) is Alias: + mailbox_email = alias.mailbox.email + else: + mailbox_email = alias.domain.mailboxes[0].email + send_email_with_rate_control( + user, + f"{ALERT_COMPLAINT_FORWARD_PHASE}_{origin.name()}", + mailbox_email, + f"Abuse report from {capitalized_name}", + render( + "transactional/provider-complaint-forward-phase.txt.jinja2", + email=mailbox_email, + provider=capitalized_name, + ), + render( + "transactional/provider-complaint-forward-phase.html", + email=mailbox_email, + provider=capitalized_name, + ), + max_nb_alert=1, + nb_day=7, + ) + + +def store_provider_complaint(alias, message): + email_name = f"reply-{uuid.uuid4().hex}.eml" + full_report_path = f"provider_complaint/{email_name}" + s3.upload_email_from_bytesio( + full_report_path, BytesIO(to_bytes(message)), email_name + ) + refused_email = RefusedEmail.create( + full_report_path=full_report_path, + user_id=alias.user_id, + path=email_name, + commit=True, + ) + ProviderComplaint.create( + user_id=alias.user_id, + state=ProviderComplaintState.new.value, + phase=Phase.reply.value, + refused_email_id=refused_email.id, + commit=True, + ) diff --git a/app/app/handler/spamd_result.py b/app/app/handler/spamd_result.py new file mode 100644 index 0000000..f2ff9a6 --- /dev/null +++ b/app/app/handler/spamd_result.py @@ -0,0 +1,135 @@ +from __future__ import annotations +from typing import Dict, Optional + +import newrelic.agent + +from app.email import headers +from app.log import LOG +from app.models import EnumE, Phase +from email.message import Message + + +class DmarcCheckResult(EnumE): + allow = 0 + soft_fail = 1 + quarantine = 2 + reject = 3 + not_available = 4 + bad_policy = 5 + + @staticmethod + def get_string_dict(): + return { + "DMARC_POLICY_ALLOW": DmarcCheckResult.allow, + "DMARC_POLICY_SOFTFAIL": DmarcCheckResult.soft_fail, + "DMARC_POLICY_QUARANTINE": DmarcCheckResult.quarantine, + "DMARC_POLICY_REJECT": DmarcCheckResult.reject, + "DMARC_NA": DmarcCheckResult.not_available, + "DMARC_BAD_POLICY": DmarcCheckResult.bad_policy, + } + + +class SPFCheckResult(EnumE): + allow = 0 + fail = 1 + soft_fail = 1 + neutral = 2 + temp_error = 3 + not_available = 4 + perm_error = 5 + + @staticmethod + def get_string_dict(): + return { + "R_SPF_ALLOW": SPFCheckResult.allow, + "R_SPF_FAIL": SPFCheckResult.fail, + "R_SPF_SOFTFAIL": SPFCheckResult.soft_fail, + "R_SPF_NEUTRAL": SPFCheckResult.neutral, + "R_SPF_DNSFAIL": SPFCheckResult.temp_error, + "R_SPF_NA": SPFCheckResult.not_available, + "R_SPF_PERMFAIL": SPFCheckResult.perm_error, + } + + +class SpamdResult: + def __init__(self, phase: Phase = Phase.unknown): + self.phase: Phase = phase + self.dmarc: DmarcCheckResult = DmarcCheckResult.not_available + self.spf: SPFCheckResult = SPFCheckResult.not_available + self.rspamd_score = -1 + + def set_dmarc_result(self, dmarc_result: DmarcCheckResult): + self.dmarc = dmarc_result + + def set_spf_result(self, spf_result: SPFCheckResult): + self.spf = spf_result + + def event_data(self) -> Dict: + return { + "header": "present", + "dmarc": self.dmarc.name, + "spf": self.spf.name, + "phase": self.phase.name, + } + + @classmethod + def extract_from_headers( + cls, msg: Message, phase: Phase = Phase.unknown + ) -> Optional[SpamdResult]: + cached = cls._get_from_message(msg) + if cached: + return cached + + spam_result_header = msg.get_all(headers.SPAMD_RESULT) + if not spam_result_header: + return None + + spam_entries = [ + entry.strip() for entry in str(spam_result_header[-1]).split("\n") + ] + + for entry_pos in range(len(spam_entries)): + sep = spam_entries[entry_pos].find("(") + if sep > -1: + spam_entries[entry_pos] = spam_entries[entry_pos][:sep] + + spamd_result = SpamdResult(phase) + + for header_value, dmarc_result in DmarcCheckResult.get_string_dict().items(): + if header_value in spam_entries: + spamd_result.set_dmarc_result(dmarc_result) + break + for header_value, spf_result in SPFCheckResult.get_string_dict().items(): + if header_value in spam_entries: + spamd_result.set_spf_result(spf_result) + break + + # parse the rspamd score + try: + score_line = spam_entries[0] # e.g. "default: False [2.30 / 13.00];" + spamd_result.rspamd_score = float( + score_line[(score_line.find("[") + 1) : score_line.find("]")] + .split("/")[0] + .strip() + ) + except (IndexError, ValueError): + LOG.e("cannot parse rspamd score") + + cls._store_in_message(spamd_result, msg) + return spamd_result + + @classmethod + def _store_in_message(cls, check: SpamdResult, msg: Message): + msg.spamd_check = check + + @classmethod + def _get_from_message(cls, msg: Message) -> Optional[SpamdResult]: + return getattr(msg, "spamd_check", None) + + @classmethod + def send_to_new_relic(cls, msg: Message): + check = cls._get_from_message(msg) + if check: + newrelic.agent.record_custom_event("SpamdCheck", check.event_data()) + else: + newrelic.agent.record_custom_event("SpamdCheck", {"header": "missing"}) diff --git a/app/app/handler/unsubscribe_encoder.py b/app/app/handler/unsubscribe_encoder.py new file mode 100644 index 0000000..8d74829 --- /dev/null +++ b/app/app/handler/unsubscribe_encoder.py @@ -0,0 +1,149 @@ +import base64 +import enum +import hashlib +import json +from dataclasses import dataclass +from typing import Optional, Union + +import itsdangerous + +from app import config +from app.log import LOG + +UNSUB_PREFIX = "un" + + +class UnsubscribeAction(enum.Enum): + UnsubscribeNewsletter = 1 + DisableAlias = 2 + DisableContact = 3 + OriginalUnsubscribeMailto = 4 + + +@dataclass +class UnsubscribeOriginalData: + alias_id: int + recipient: str + subject: str + + +@dataclass +class UnsubscribeData: + action: UnsubscribeAction + data: Union[UnsubscribeOriginalData, int] + + +@dataclass +class UnsubscribeLink: + link: str + via_email: bool + + +class UnsubscribeEncoder: + @staticmethod + def encode( + action: UnsubscribeAction, data: Union[int, UnsubscribeOriginalData] + ) -> UnsubscribeLink: + if config.UNSUBSCRIBER: + return UnsubscribeLink(UnsubscribeEncoder.encode_mailto(action, data), True) + return UnsubscribeLink(UnsubscribeEncoder.encode_url(action, data), False) + + @classmethod + def encode_subject( + cls, action: UnsubscribeAction, data: Union[int, UnsubscribeOriginalData] + ) -> str: + if ( + action != UnsubscribeAction.OriginalUnsubscribeMailto + and type(data) is not int + ): + raise ValueError(f"Data has to be an int for an action of type {action}") + if action == UnsubscribeAction.OriginalUnsubscribeMailto: + if type(data) is not UnsubscribeOriginalData: + raise ValueError( + f"Data has to be an UnsubscribeOriginalData for an action of type {action}" + ) + # Initial 0 is the version number. If we need to add support for extra use-cases we can bump up this number + data = (0, data.alias_id, data.recipient, data.subject) + payload = (action.value, data) + serialized_data = ( + base64.urlsafe_b64encode(json.dumps(payload).encode("utf-8")) + .rstrip(b"=") + .decode("utf-8") + ) + signed_data = cls._get_signer().sign(serialized_data).decode("utf-8") + encoded_request = f"{UNSUB_PREFIX}.{signed_data}" + if len(encoded_request) > 256: + LOG.e("Encoded request is longer than 256 chars") + return encoded_request + + @staticmethod + def encode_mailto( + action: UnsubscribeAction, data: Union[int, UnsubscribeOriginalData] + ) -> str: + subject = UnsubscribeEncoder.encode_subject(action, data) + return f"mailto:{config.UNSUBSCRIBER}?subject={subject}" + + @staticmethod + def encode_url( + action: UnsubscribeAction, data: Union[int, UnsubscribeOriginalData] + ) -> str: + if action == UnsubscribeAction.DisableAlias: + return f"{config.URL}/dashboard/unsubscribe/{data}" + if action == UnsubscribeAction.DisableContact: + return f"{config.URL}/dashboard/block_contact/{data}" + if action in ( + UnsubscribeAction.UnsubscribeNewsletter, + UnsubscribeAction.OriginalUnsubscribeMailto, + ): + encoded = UnsubscribeEncoder.encode_subject(action, data) + return f"{config.URL}/dashboard/unsubscribe/encoded?data={encoded}" + + @staticmethod + def _get_signer() -> itsdangerous.Signer: + return itsdangerous.Signer( + config.UNSUBSCRIBE_SECRET, digest_method=hashlib.sha3_224 + ) + + @classmethod + def decode_subject(cls, data: str) -> Optional[UnsubscribeData]: + if data.find(UNSUB_PREFIX) == -1: + try: + # subject has the format {alias.id}= + if data.endswith("="): + alias_id = int(data[:-1]) + return UnsubscribeData(UnsubscribeAction.DisableAlias, alias_id) + # {contact.id}_ + elif data.endswith("_"): + contact_id = int(data[:-1]) + return UnsubscribeData(UnsubscribeAction.DisableContact, contact_id) + # {user.id}* + elif data.endswith("*"): + user_id = int(data[:-1]) + return UnsubscribeData( + UnsubscribeAction.UnsubscribeNewsletter, user_id + ) + else: + # some email providers might strip off the = suffix + alias_id = int(data) + return UnsubscribeData(UnsubscribeAction.DisableAlias, alias_id) + except ValueError: + return None + + signer = cls._get_signer() + try: + verified_data = signer.unsign(data[len(UNSUB_PREFIX) + 1 :]) + except itsdangerous.BadSignature: + return None + try: + padded_data = verified_data + (b"=" * (-len(verified_data) % 4)) + payload = json.loads(base64.urlsafe_b64decode(padded_data)) + except ValueError: + return None + action = UnsubscribeAction(payload[0]) + action_data = payload[1] + if action == UnsubscribeAction.OriginalUnsubscribeMailto: + # Skip version number in action_data[0] for now it's always 0 + action_data = UnsubscribeOriginalData( + action_data[1], action_data[2], action_data[3] + ) + return UnsubscribeData(action, action_data) diff --git a/app/app/handler/unsubscribe_generator.py b/app/app/handler/unsubscribe_generator.py new file mode 100644 index 0000000..09e82e7 --- /dev/null +++ b/app/app/handler/unsubscribe_generator.py @@ -0,0 +1,98 @@ +import urllib +from email.message import Message + +from app.email import headers +from app.email_utils import add_or_replace_header, delete_header +from app.handler.unsubscribe_encoder import ( + UnsubscribeEncoder, + UnsubscribeAction, + UnsubscribeData, + UnsubscribeOriginalData, +) +from app.models import Alias, Contact, UnsubscribeBehaviourEnum + + +class UnsubscribeGenerator: + def _generate_header_with_original_behaviour( + self, alias: Alias, message: Message + ) -> Message: + """ + Generate a header that will encode the original unsub request. To do so + 1. Look if there's an original List_Unsubscribe headers, otherwise do nothing + 2. Header has the form+ +3x has some data structure changes that cannot be automatically upgraded from 2x. +Once you have upgraded your installation to 3x, please run the following scripts to make your data fully compatible with 3x + +First connect to your SimpleLogin container shell: + +```bash +docker exec -it sl-app python shell.py +``` + +Then copy and run this below script: + +```python +from app.extensions import db +from app.models import AliasUsedOn, Contact, EmailLog + +for auo in AliasUsedOn.query.all(): + auo.user_id = auo.alias.user_id +db.session.commit() + +for contact in Contact.query.all(): + contact.user_id = contact.alias.user_id +db.session.commit() + +for email_log in EmailLog.query.all(): + email_log.user_id = email_log.contact.user_id + +db.session.commit() +``` + +
++ +2.1.0 comes with PGP support. If you use PGP, please follow these steps to enable this feature: + +1) In your home directory (where `dkim.key` is located), create directory to store SimpleLogin data + +```bash +mkdir sl +mkdir sl/pgp # to store PGP key +mkdir sl/db # to store database +``` + +2) Then add this line to your config simplelogin.env file + +``` +GNUPGHOME=/sl/pgp # where to store PGP keys +``` + +Now you can follow the usual steps to upgrade SimpleLogin. + +
++ +2.0.0 comes with mailbox feature that requires running a script that puts all existing users to "full-mailbox" mode. + +1) First please make sure to upgrade to 1.0.5 which is the latest version before 2.0.0. + +2) Then connect to your SimpleLogin container shell: + +```bash +docker exec -it sl-app python shell.py +``` + +3) Finally copy and run this below script: + +```python +"""This ad-hoc script is to be run when upgrading from 1.0.5 to 2.0.0 +""" +from app.extensions import db +from app.log import LOG +from app.models import Mailbox, Alias, User + +for user in User.query.all(): + if user.default_mailbox_id: + # already run the migration on this user + continue + + # create a default mailbox + default_mb = Mailbox.get_by(user_id=user.id, email=user.email) + if not default_mb: + LOG.d("create default mailbox for user %s", user) + default_mb = Mailbox.create(user_id=user.id, email=user.email, verified=True) + db.session.commit() + + # assign existing alias to this mailbox + for gen_email in Alias.query.filter_by(user_id=user.id): + if not gen_email.mailbox_id: + LOG.d("Set alias %s mailbox to default mailbox", gen_email) + gen_email.mailbox_id = default_mb.id + + # finally set user to full_mailbox + user.full_mailbox = True + user.default_mailbox_id = default_mb.id + db.session.commit() +``` +
+