diff --git a/app/Dockerfile b/app/Dockerfile index 61638f7..7c89939 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -23,10 +23,10 @@ COPY poetry.lock pyproject.toml ./ # Install and setup poetry RUN pip install -U pip \ && apt-get update \ - && apt install -y curl netcat gcc python3-dev gnupg git libre2-dev \ + && apt install -y curl netcat-traditional gcc python3-dev gnupg git libre2-dev \ && curl -sSL https://install.python-poetry.org | python3 - \ # Remove curl and netcat from the image - && apt-get purge -y curl netcat \ + && apt-get purge -y curl netcat-traditional \ # Run poetry && poetry config virtualenvs.create false \ && poetry install --no-interaction --no-ansi --no-root \ diff --git a/app/app/alias_suffix.py b/app/app/alias_suffix.py index 40c5462..6a2c69e 100644 --- a/app/app/alias_suffix.py +++ b/app/app/alias_suffix.py @@ -162,8 +162,6 @@ def get_alias_suffixes( or user.default_alias_public_domain_id != sl_domain.id ): alias_suffixes.append(alias_suffix) - # If no default domain mark it as found - default_domain_found = user.default_alias_public_domain_id is None else: default_domain_found = True alias_suffixes.insert(0, alias_suffix) diff --git a/app/app/dashboard/views/mailbox.py b/app/app/dashboard/views/mailbox.py index 0be92f4..6e7c203 100644 --- a/app/app/dashboard/views/mailbox.py +++ b/app/app/dashboard/views/mailbox.py @@ -1,3 +1,7 @@ +import base64 +import binascii +import json + import arrow from flask import render_template, request, redirect, url_for, flash from flask_login import login_required, current_user @@ -180,7 +184,9 @@ def mailbox_route(): def send_verification_email(user, mailbox): s = TimestampSigner(MAILBOX_SECRET) - mailbox_id_signed = s.sign(str(mailbox.id)).decode() + encoded_data = json.dumps([mailbox.id, mailbox.email]).encode("utf-8") + b64_data = base64.urlsafe_b64encode(encoded_data) + mailbox_id_signed = s.sign(b64_data).decode() verification_url = ( URL + "/dashboard/mailbox_verify" + f"?mailbox_id={mailbox_id_signed}" ) @@ -205,22 +211,34 @@ def send_verification_email(user, mailbox): @dashboard_bp.route("/mailbox_verify") def mailbox_verify(): s = TimestampSigner(MAILBOX_SECRET) - mailbox_id = request.args.get("mailbox_id") - + mailbox_verify_request = request.args.get("mailbox_id") try: - r_id = int(s.unsign(mailbox_id, max_age=900)) + mailbox_raw_data = s.unsign(mailbox_verify_request, max_age=900) except Exception: flash("Invalid link. Please delete and re-add your mailbox", "error") return redirect(url_for("dashboard.mailbox_route")) - else: - mailbox = Mailbox.get(r_id) - if not mailbox: - flash("Invalid link", "error") - return redirect(url_for("dashboard.mailbox_route")) + try: + decoded_data = base64.urlsafe_b64decode(mailbox_raw_data) + except binascii.Error: + flash("Invalid link. Please delete and re-add your mailbox", "error") + return redirect(url_for("dashboard.mailbox_route")) + mailbox_data = json.loads(decoded_data) + if not isinstance(mailbox_data, list) or len(mailbox_data) != 2: + flash("Invalid link. Please delete and re-add your mailbox", "error") + return redirect(url_for("dashboard.mailbox_route")) + mailbox_id = mailbox_data[0] + mailbox = Mailbox.get(mailbox_id) + if not mailbox: + flash("Invalid link", "error") + return redirect(url_for("dashboard.mailbox_route")) + mailbox_email = mailbox_data[1] + if mailbox_email != mailbox.email: + flash("Invalid link", "error") + return redirect(url_for("dashboard.mailbox_route")) - mailbox.verified = True - Session.commit() + mailbox.verified = True + Session.commit() - LOG.d("Mailbox %s is verified", mailbox) + LOG.d("Mailbox %s is verified", mailbox) - return render_template("dashboard/mailbox_validation.html", mailbox=mailbox) + return render_template("dashboard/mailbox_validation.html", mailbox=mailbox) diff --git a/app/app/email/headers.py b/app/app/email/headers.py index 800a5a2..788693b 100644 --- a/app/app/email/headers.py +++ b/app/app/email/headers.py @@ -20,6 +20,7 @@ X_SPAM_STATUS = "X-Spam-Status" LIST_UNSUBSCRIBE = "List-Unsubscribe" LIST_UNSUBSCRIBE_POST = "List-Unsubscribe-Post" RETURN_PATH = "Return-Path" +AUTHENTICATION_RESULTS = "Authentication-Results" # headers used to DKIM sign in order of preference DKIM_HEADERS = [ @@ -32,6 +33,7 @@ DKIM_HEADERS = [ SL_DIRECTION = "X-SimpleLogin-Type" SL_EMAIL_LOG_ID = "X-SimpleLogin-EmailLog-ID" SL_ENVELOPE_FROM = "X-SimpleLogin-Envelope-From" +SL_ORIGINAL_FROM = "X-SimpleLogin-Original-From" SL_ENVELOPE_TO = "X-SimpleLogin-Envelope-To" SL_CLIENT_IP = "X-SimpleLogin-Client-IP" diff --git a/app/app/handler/unsubscribe_generator.py b/app/app/handler/unsubscribe_generator.py index 09e82e7..288002e 100644 --- a/app/app/handler/unsubscribe_generator.py +++ b/app/app/handler/unsubscribe_generator.py @@ -9,6 +9,7 @@ from app.handler.unsubscribe_encoder import ( UnsubscribeData, UnsubscribeOriginalData, ) +from app.log import LOG from app.models import Alias, Contact, UnsubscribeBehaviourEnum @@ -30,6 +31,7 @@ class UnsubscribeGenerator: """ unsubscribe_data = message[headers.LIST_UNSUBSCRIBE] if not unsubscribe_data: + LOG.info("Email has no unsubscribe header") return message raw_methods = [method.strip() for method in unsubscribe_data.split(",")] mailto_unsubs = None @@ -44,7 +46,9 @@ class UnsubscribeGenerator: if url_data.scheme == "mailto": query_data = urllib.parse.parse_qs(url_data.query) mailto_unsubs = (url_data.path, query_data.get("subject", [""])[0]) + LOG.debug(f"Unsub is mailto to {mailto_unsubs}") else: + LOG.debug(f"Unsub has {url_data.scheme} scheme") other_unsubs.append(method) # If there are non mailto unsubscribe methods, use those in the header if other_unsubs: @@ -56,18 +60,19 @@ class UnsubscribeGenerator: add_or_replace_header( message, headers.LIST_UNSUBSCRIBE_POST, "List-Unsubscribe=One-Click" ) + LOG.debug(f"Adding click unsub methods to header {other_unsubs}") return message - if not mailto_unsubs: + elif not mailto_unsubs: + LOG.debug("No unsubs. Deleting all unsub headers") message = delete_header(message, headers.LIST_UNSUBSCRIBE) message = delete_header(message, headers.LIST_UNSUBSCRIBE_POST) return message - return self._add_unsubscribe_header( - message, - UnsubscribeData( - UnsubscribeAction.OriginalUnsubscribeMailto, - UnsubscribeOriginalData(alias.id, mailto_unsubs[0], mailto_unsubs[1]), - ), + unsub_data = UnsubscribeData( + UnsubscribeAction.OriginalUnsubscribeMailto, + UnsubscribeOriginalData(alias.id, mailto_unsubs[0], mailto_unsubs[1]), ) + LOG.debug(f"Adding unsub data {unsub_data}") + return self._add_unsubscribe_header(message, unsub_data) def _add_unsubscribe_header( self, message: Message, unsub: UnsubscribeData diff --git a/app/app/models.py b/app/app/models.py index 0c252f7..3b4df67 100644 --- a/app/app/models.py +++ b/app/app/models.py @@ -445,7 +445,7 @@ class User(Base, ModelMixin, UserMixin, PasswordOracle): random_alias_suffix = sa.Column( sa.Integer, nullable=False, - default=AliasSuffixEnum.random_string.value, + default=AliasSuffixEnum.word.value, server_default=str(AliasSuffixEnum.random_string.value), ) @@ -514,9 +514,8 @@ class User(Base, ModelMixin, UserMixin, PasswordOracle): server_default=BlockBehaviourEnum.return_2xx.name, ) - # to keep existing behavior, the server default is TRUE whereas for new user, the default value is FALSE include_header_email_header = sa.Column( - sa.Boolean, default=False, nullable=False, server_default="1" + sa.Boolean, default=True, nullable=False, server_default="1" ) # bitwise flags. Allow for future expansion diff --git a/app/email_handler.py b/app/email_handler.py index 470d4d2..f28c320 100644 --- a/app/email_handler.py +++ b/app/email_handler.py @@ -846,22 +846,23 @@ def forward_email_to_mailbox( f"""Email sent to {alias.email} from an invalid address and cannot be replied""", ) - delete_all_headers_except( - msg, - [ - headers.FROM, - headers.TO, - headers.CC, - headers.SUBJECT, - headers.DATE, - # do not delete original message id - headers.MESSAGE_ID, - # References and In-Reply-To are used for keeping the email thread - headers.REFERENCES, - headers.IN_REPLY_TO, - ] - + headers.MIME_HEADERS, - ) + headers_to_keep = [ + headers.FROM, + headers.TO, + headers.CC, + headers.SUBJECT, + headers.DATE, + # do not delete original message id + headers.MESSAGE_ID, + # References and In-Reply-To are used for keeping the email thread + headers.REFERENCES, + headers.IN_REPLY_TO, + headers.LIST_UNSUBSCRIBE, + headers.LIST_UNSUBSCRIBE_POST, + ] + headers.MIME_HEADERS + if user.include_header_email_header: + headers_to_keep.append(headers.AUTHENTICATION_RESULTS) + delete_all_headers_except(msg, headers_to_keep) # create PGP email if needed if mailbox.pgp_enabled() and user.is_premium() and not alias.disable_pgp: @@ -898,6 +899,7 @@ def forward_email_to_mailbox( msg[headers.SL_EMAIL_LOG_ID] = str(email_log.id) if user.include_header_email_header: msg[headers.SL_ENVELOPE_FROM] = envelope.mail_from + msg[headers.SL_ORIGINAL_FROM] = contact.website_email # when an alias isn't in the To: header, there's no way for users to know what alias has received the email msg[headers.SL_ENVELOPE_TO] = alias.email diff --git a/app/templates/dashboard/setting.html b/app/templates/dashboard/setting.html index ad4b7ee..550995a 100644 --- a/app/templates/dashboard/setting.html +++ b/app/templates/dashboard/setting.html @@ -684,7 +684,8 @@ SimpleLogin forwards emails to your mailbox from the reverse-alias and not from the original sender address.
- If this option is enabled, the original sender addresses is stored in the email header X-SimpleLogin-Envelope-From. + If this option is enabled, the original sender addresses is stored in the email header X-SimpleLogin-Envelope-From + and the original From header is stored in X-SimpleLogin-Original-From. You can choose to display this header in your email client.
As email headers aren't encrypted, your mailbox service can know the sender address via this header. diff --git a/app/templates/footer.html b/app/templates/footer.html index 70fe1be..903e04e 100644 --- a/app/templates/footer.html +++ b/app/templates/footer.html @@ -286,6 +286,7 @@ }, async mounted() { + Object.freeze(Object.prototype); let that = this; let res = await fetch(`/api/notifications?page=${that.page}`, { method: "GET", diff --git a/app/tests/api/test_setting.py b/app/tests/api/test_setting.py index 3f545b3..199a840 100644 --- a/app/tests/api/test_setting.py +++ b/app/tests/api/test_setting.py @@ -17,7 +17,7 @@ def test_get_setting(flask_client): "notification": True, "random_alias_default_domain": "sl.local", "sender_format": "AT", - "random_alias_suffix": "random_string", + "random_alias_suffix": "word", } @@ -95,11 +95,13 @@ def test_get_setting_domains_v2(flask_client): def test_update_settings_random_alias_suffix(flask_client): user = login(flask_client) # default random_alias_suffix is random_string - assert user.random_alias_suffix == AliasSuffixEnum.random_string.value + assert user.random_alias_suffix == AliasSuffixEnum.word.value r = flask_client.patch("/api/setting", json={"random_alias_suffix": "invalid"}) assert r.status_code == 400 - r = flask_client.patch("/api/setting", json={"random_alias_suffix": "word"}) + r = flask_client.patch( + "/api/setting", json={"random_alias_suffix": "random_string"} + ) assert r.status_code == 200 - assert user.random_alias_suffix == AliasSuffixEnum.word.value + assert user.random_alias_suffix == AliasSuffixEnum.random_string.value diff --git a/app/tests/example_emls/replacement_on_forward_phase.eml b/app/tests/example_emls/replacement_on_forward_phase.eml new file mode 100644 index 0000000..9e9f626 --- /dev/null +++ b/app/tests/example_emls/replacement_on_forward_phase.eml @@ -0,0 +1,65 @@ +Received: by mail-ed1-f49.google.com with SMTP id ej4so13657316edb.7 + for ; Mon, 27 Jun 2022 08:48:15 -0700 (PDT) +X-Gm-Message-State: AJIora8exR9DGeRFoKAtjzwLtUpH5hqx6Zt3tm8n4gUQQivGQ3fELjUV + yT7RQIfeW9Kv2atuOcgtmGYVU4iQ8VBeLmK1xvOYL4XpXfrT7ZrJNQ== +Authentication-Results: mx.google.com; + dkim=pass header.i=@matera.eu header.s=fnt header.b=XahYMey7; + dkim=pass header.i=@sendgrid.info header.s=smtpapi header.b="QOCS/yjt"; + spf=pass (google.com: domain of bounces+14445963-ab4e-csyndic.quartz=gmail.com@front-mail.matera.eu designates 168.245.4.42 as permitted sender) smtp.mailfrom="bounces+14445963-ab4e-csyndic.quartz=gmail.com@front-mail.matera.eu"; + dmarc=pass (p=NONE sp=NONE dis=NONE) header.from=matera.eu +Received: from out.frontapp.com (unknown) + by geopod-ismtpd-3-0 (SG) + with ESMTP id d2gM2N7PT7W8d2-UEC4ESA + for ; + Mon, 27 Jun 2022 15:48:11.014 +0000 (UTC) +Content-Type: multipart/alternative; + boundary="----sinikael-?=_1-16563448907660.10629093370416887" +In-Reply-To: + +References: + + + + + +From: {{ sender_address }} +To: {{ recipient_address }} +CC: {{ cc_address }} +Subject: Something +Message-ID: +X-Mailer: Front (1.0; +https://frontapp.com; + +msgid=af07e94a66ece6564ae30a2aaac7a34c@frontapp.com) +X-Feedback-ID: 14445963:SG +X-SG-EID: + =?us-ascii?Q?XtlxQDg5i3HqMzQY2Upg19JPZBVl1RybInUUL2yta9uBoIU4KU1FMJ5DjWrz6g?= + =?us-ascii?Q?fJUK5Qmneg2uc46gwp5BdHdp6Foaq5gg3xJriv3?= + =?us-ascii?Q?9OA=2FWRifeylU9O+ngdNbOKXoeJAkROmp2mCgw9x?= + =?us-ascii?Q?uud+EclOT9mYVtbZsydOLLm6Y2PPswQl8lnmiku?= + =?us-ascii?Q?DAhkG15HTz2FbWGWNDFb7VrSsN5ddjAscr6sIHw?= + =?us-ascii?Q?S48R5fnXmfhPbmlCgqFjr0FGphfuBdNAt6z6w8a?= + =?us-ascii?Q?o9u1EYDIX7zWHZ+Tr3eyw=3D=3D?= +X-SG-ID: + =?us-ascii?Q?N2C25iY2uzGMFz6rgvQsb8raWjw0ZPf1VmjsCkspi=2FI9PhcvqXQTpKqqyZkvBe?= + =?us-ascii?Q?+2RscnQ4WPkA+BN1vYgz1rezTVIqgp+rlWrKk8o?= + =?us-ascii?Q?HoB5dzpX6HKWtWCVRi10zwlDN1+pJnySoIUrlaT?= + =?us-ascii?Q?PA2aqQKmMQbjTl0CUAFryR8hhHcxdS0cQowZSd7?= + =?us-ascii?Q?XNjJWLvCGF7ODwg=2FKr+4yRE8UvULS2nrdO2wWyQ?= + =?us-ascii?Q?AiFHdPdZsRlgNomEo=3D?= +X-Spamd-Result: default: False [-2.00 / 13.00]; + ARC_ALLOW(-1.00)[google.com:s=arc-20160816:i=1]; + MIME_GOOD(-0.10)[multipart/alternative,text/plain]; + REPLYTO_ADDR_EQ_FROM(0.00)[]; + FORGED_RECIPIENTS_FORWARDING(0.00)[]; + NEURAL_HAM(-0.00)[-0.981]; + FREEMAIL_TO(0.00)[gmail.com]; + RCVD_TLS_LAST(0.00)[]; + FREEMAIL_ENVFROM(0.00)[gmail.com]; + MIME_TRACE(0.00)[0:+,1:+,2:~]; + RWL_MAILSPIKE_POSSIBLE(0.00)[209.85.208.49:from] + +------sinikael-?=_1-16563448907660.10629093370416887 +Content-Type: text/plain; charset=utf-8 +Content-Transfer-Encoding: quoted-printable + +From {{ sender_address }} To {{ recipient_address }} +------sinikael-?=_1-16563448907660.10629093370416887-- diff --git a/app/tests/handler/test_preserved_headers.py b/app/tests/handler/test_preserved_headers.py new file mode 100644 index 0000000..f34af50 --- /dev/null +++ b/app/tests/handler/test_preserved_headers.py @@ -0,0 +1,39 @@ +from aiosmtpd.smtp import Envelope + +import email_handler +from app.db import Session +from app.email import headers, status +from app.mail_sender import mail_sender +from app.models import Alias +from tests.utils import create_new_user, load_eml_file, random_email + + +@mail_sender.store_emails_test_decorator +def test_original_headers_from_preserved(): + user = create_new_user() + alias = Alias.create_new_random(user) + Session.flush() + assert user.include_header_email_header + original_sender_address = random_email() + msg = load_eml_file( + "replacement_on_forward_phase.eml", + { + "sender_address": original_sender_address, + "recipient_address": alias.email, + "cc_address": random_email(), + }, + ) + envelope = Envelope() + envelope.mail_from = f"env.{original_sender_address}" + envelope.rcpt_tos = [alias.email] + result = email_handler.MailHandler()._handle(envelope, msg) + assert result == status.E200 + send_requests = mail_sender.get_stored_emails() + assert len(send_requests) == 1 + request = send_requests[0] + assert request.msg[headers.SL_ENVELOPE_FROM] == envelope.mail_from + assert request.msg[headers.SL_ORIGINAL_FROM] == original_sender_address + assert ( + request.msg[headers.AUTHENTICATION_RESULTS] + == msg[headers.AUTHENTICATION_RESULTS] + ) diff --git a/app/tests/test_alias_suffixes.py b/app/tests/test_alias_suffixes.py index 9706a13..870baa1 100644 --- a/app/tests/test_alias_suffixes.py +++ b/app/tests/test_alias_suffixes.py @@ -131,3 +131,22 @@ def test_suffixes_are_valid(): if len(match.groups()) >= 1: has_prefix += 1 assert has_prefix > 0 + + +def test_get_default_domain_is_only_shown_once(): + user = create_new_user() + default_domain = SLDomain.filter_by(hidden=False).order_by(SLDomain.order).first() + user.default_alias_public_domain_id = default_domain.id + Session.flush() + options = AliasOptions( + show_sl_domains=True, show_partner_domains=get_proton_partner() + ) + suffixes = get_alias_suffixes(user, alias_options=options) + found_default = False + found_domains = set() + for suffix in suffixes: + assert suffix.domain not in found_domains + found_domains.add(suffix.domain) + if default_domain.domain == suffix.domain: + found_default = True + assert found_default