diff --git a/app/.github/workflows/main.yml b/app/.github/workflows/main.yml index 8465553..d8e144d 100644 --- a/app/.github/workflows/main.yml +++ b/app/.github/workflows/main.yml @@ -107,7 +107,7 @@ jobs: - name: Prepare version file run: | - scripts/generate-build-info.sh ${{ github.sha }} + scripts/generate-build-info.sh ${{ github.sha }} ${{ github.ref_name }} cat app/build_info.py - name: Test with pytest @@ -164,7 +164,7 @@ jobs: - name: Prepare version file run: | - scripts/generate-build-info.sh ${{ github.sha }} + scripts/generate-build-info.sh ${{ github.sha }} ${{ github.ref_name }} cat app/build_info.py - name: Build image and publish to Docker Registry diff --git a/app/app/api/views/setting.py b/app/app/api/views/setting.py index f58c7fb..bf8c1b6 100644 --- a/app/app/api/views/setting.py +++ b/app/app/api/views/setting.py @@ -12,7 +12,7 @@ from app.models import ( SenderFormatEnum, AliasSuffixEnum, ) -from app.proton.utils import perform_proton_account_unlink +from app.proton.proton_unlink import perform_proton_account_unlink def setting_to_dict(user: User): diff --git a/app/app/api/views/user_info.py b/app/app/api/views/user_info.py index e52e05a..97371e1 100644 --- a/app/app/api/views/user_info.py +++ b/app/app/api/views/user_info.py @@ -12,7 +12,7 @@ from app.dashboard.views.index import get_stats from app.db import Session from app.image_validation import detect_image_format, ImageFormat from app.models import ApiKey, File, PartnerUser, User -from app.proton.utils import get_proton_partner +from app.proton.proton_partner import get_proton_partner from app.session import logout_session from app.utils import random_string diff --git a/app/app/auth/views/proton.py b/app/app/auth/views/proton.py index 8de7776..52e2fb3 100644 --- a/app/app/auth/views/proton.py +++ b/app/app/auth/views/proton.py @@ -23,7 +23,7 @@ from app.proton.proton_callback_handler import ( ProtonCallbackHandler, Action, ) -from app.proton.utils import get_proton_partner +from app.proton.proton_partner import get_proton_partner from app.utils import sanitize_next_url, sanitize_scheme _authorization_base_url = PROTON_BASE_URL + "/oauth/authorize" diff --git a/app/app/build_info.py b/app/app/build_info.py index c73f88e..94d612e 100644 --- a/app/app/build_info.py +++ b/app/app/build_info.py @@ -1,2 +1,3 @@ SHA1 = "dev" BUILD_TIME = "1652365083" +VERSION = SHA1 diff --git a/app/app/config.py b/app/app/config.py index 9ec2cb6..7d6ff97 100644 --- a/app/app/config.py +++ b/app/app/config.py @@ -62,6 +62,17 @@ def get_env_dict(env_var: str) -> dict[str, str]: return result +def get_env_csv(env_var: str, default: Optional[str]) -> list[str]: + """ + Get an env variable and convert it into a list of strings separated by, + Syntax is: val1,val2 + """ + value = os.getenv(env_var, default) + if not value: + return [] + return [field.strip() for field in value.split(",") if field.strip()] + + config_file = os.environ.get("CONFIG") if config_file: config_file = get_abs_path(config_file) @@ -171,6 +182,14 @@ FIRST_ALIAS_DOMAIN = os.environ.get("FIRST_ALIAS_DOMAIN") or EMAIL_DOMAIN # e.g. [(10, "mx1.hostname."), (10, "mx2.hostname.")] EMAIL_SERVERS_WITH_PRIORITY = sl_getenv("EMAIL_SERVERS_WITH_PRIORITY") +PROTON_MX_SERVERS = get_env_csv( + "PROTON_MX_SERVERS", "mail.protonmail.ch., mailsec.protonmail.ch." +) + +PROTON_EMAIL_DOMAINS = get_env_csv( + "PROTON_EMAIL_DOMAINS", "proton.me, protonmail.com, protonmail.ch, proton.ch, pm.me" +) + # disable the alias suffix, i.e. the ".random_word" part DISABLE_ALIAS_SUFFIX = "DISABLE_ALIAS_SUFFIX" in os.environ diff --git a/app/app/dashboard/views/account_setting.py b/app/app/dashboard/views/account_setting.py index efd1649..ee46103 100644 --- a/app/app/dashboard/views/account_setting.py +++ b/app/app/dashboard/views/account_setting.py @@ -39,7 +39,7 @@ from app.models import ( SenderFormatEnum, UnsubscribeBehaviourEnum, ) -from app.proton.utils import perform_proton_account_unlink +from app.proton.proton_unlink import perform_proton_account_unlink from app.utils import ( random_string, CSRFValidationForm, diff --git a/app/app/dashboard/views/enter_sudo.py b/app/app/dashboard/views/enter_sudo.py index 8f6b80a..18115d6 100644 --- a/app/app/dashboard/views/enter_sudo.py +++ b/app/app/dashboard/views/enter_sudo.py @@ -11,7 +11,7 @@ from app.dashboard.base import dashboard_bp from app.extensions import limiter from app.log import LOG from app.models import PartnerUser, SocialAuth -from app.proton.utils import get_proton_partner +from app.proton.proton_partner import get_proton_partner from app.utils import sanitize_next_url _SUDO_GAP = 120 diff --git a/app/app/dashboard/views/pricing.py b/app/app/dashboard/views/pricing.py index f9cf4a0..6e36056 100644 --- a/app/app/dashboard/views/pricing.py +++ b/app/app/dashboard/views/pricing.py @@ -22,7 +22,7 @@ from app.models import ( PartnerUser, PartnerSubscription, ) -from app.proton.utils import get_proton_partner +from app.proton.proton_partner import get_proton_partner @dashboard_bp.route("/pricing", methods=["GET", "POST"]) diff --git a/app/app/dashboard/views/setting.py b/app/app/dashboard/views/setting.py index c912dc4..6fcec6f 100644 --- a/app/app/dashboard/views/setting.py +++ b/app/app/dashboard/views/setting.py @@ -41,7 +41,8 @@ from app.models import ( PartnerSubscription, UnsubscribeBehaviourEnum, ) -from app.proton.utils import get_proton_partner, can_unlink_proton_account +from app.proton.proton_partner import get_proton_partner +from app.proton.proton_unlink import can_unlink_proton_account from app.utils import ( random_string, CSRFValidationForm, diff --git a/app/app/dns_utils.py b/app/app/dns_utils.py index 02ca784..d2fbd85 100644 --- a/app/app/dns_utils.py +++ b/app/app/dns_utils.py @@ -115,9 +115,20 @@ class InMemoryDNSClient(DNSClient): return self.txt_records.get(hostname, []) -def get_network_dns_client() -> NetworkDNSClient: +global_dns_client: Optional[DNSClient] = None + + +def get_network_dns_client() -> DNSClient: + global global_dns_client + if global_dns_client is not None: + return global_dns_client return NetworkDNSClient(NAMESERVERS) +def set_global_dns_client(dns_client: Optional[DNSClient]): + global global_dns_client + global_dns_client = dns_client + + def get_mx_domains(hostname: str) -> dict[int, list[str]]: return get_network_dns_client().get_mx_domains(hostname) diff --git a/app/app/events/event_dispatcher.py b/app/app/events/event_dispatcher.py index 2cf29b7..586e6f7 100644 --- a/app/app/events/event_dispatcher.py +++ b/app/app/events/event_dispatcher.py @@ -8,7 +8,7 @@ from app.errors import ProtonPartnerNotSetUp from app.events.generated import event_pb2 from app.log import LOG from app.models import User, PartnerUser, SyncEvent -from app.proton.utils import get_proton_partner +from app.proton.proton_partner import get_proton_partner from typing import Optional NOTIFICATION_CHANNEL = "simplelogin_sync_events" diff --git a/app/app/events/generated/event_pb2.py b/app/app/events/generated/event_pb2.py index f56beb6..babfc8d 100644 --- a/app/app/events/generated/event_pb2.py +++ b/app/app/events/generated/event_pb2.py @@ -24,7 +24,7 @@ _sym_db = _symbol_database.Default() -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0b\x65vent.proto\x12\x12simplelogin_events\":\n\x0fUserPlanChanged\x12\x15\n\rplan_end_time\x18\x01 \x01(\r\x12\x10\n\x08lifetime\x18\x02 \x01(\x08\"\r\n\x0bUserDeleted\"\\\n\x0c\x41liasCreated\x12\n\n\x02id\x18\x01 \x01(\r\x12\r\n\x05\x65mail\x18\x02 \x01(\t\x12\x0c\n\x04note\x18\x03 \x01(\t\x12\x0f\n\x07\x65nabled\x18\x04 \x01(\x08\x12\x12\n\ncreated_at\x18\x05 \x01(\r\"T\n\x12\x41liasStatusChanged\x12\n\n\x02id\x18\x01 \x01(\r\x12\r\n\x05\x65mail\x18\x02 \x01(\t\x12\x0f\n\x07\x65nabled\x18\x03 \x01(\x08\x12\x12\n\ncreated_at\x18\x04 \x01(\r\")\n\x0c\x41liasDeleted\x12\n\n\x02id\x18\x01 \x01(\r\x12\r\n\x05\x65mail\x18\x02 \x01(\t\"D\n\x10\x41liasCreatedList\x12\x30\n\x06\x65vents\x18\x01 \x03(\x0b\x32 .simplelogin_events.AliasCreated\"\x93\x03\n\x0c\x45ventContent\x12?\n\x10user_plan_change\x18\x01 \x01(\x0b\x32#.simplelogin_events.UserPlanChangedH\x00\x12\x37\n\x0cuser_deleted\x18\x02 \x01(\x0b\x32\x1f.simplelogin_events.UserDeletedH\x00\x12\x39\n\ralias_created\x18\x03 \x01(\x0b\x32 .simplelogin_events.AliasCreatedH\x00\x12\x45\n\x13\x61lias_status_change\x18\x04 \x01(\x0b\x32&.simplelogin_events.AliasStatusChangedH\x00\x12\x39\n\ralias_deleted\x18\x05 \x01(\x0b\x32 .simplelogin_events.AliasDeletedH\x00\x12\x41\n\x11\x61lias_create_list\x18\x06 \x01(\x0b\x32$.simplelogin_events.AliasCreatedListH\x00\x42\t\n\x07\x63ontent\"y\n\x05\x45vent\x12\x0f\n\x07user_id\x18\x01 \x01(\r\x12\x18\n\x10\x65xternal_user_id\x18\x02 \x01(\t\x12\x12\n\npartner_id\x18\x03 \x01(\r\x12\x31\n\x07\x63ontent\x18\x04 \x01(\x0b\x32 .simplelogin_events.EventContentb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0b\x65vent.proto\x12\x12simplelogin_events\":\n\x0fUserPlanChanged\x12\x15\n\rplan_end_time\x18\x01 \x01(\r\x12\x10\n\x08lifetime\x18\x02 \x01(\x08\"\r\n\x0bUserDeleted\"\\\n\x0c\x41liasCreated\x12\n\n\x02id\x18\x01 \x01(\r\x12\r\n\x05\x65mail\x18\x02 \x01(\t\x12\x0c\n\x04note\x18\x03 \x01(\t\x12\x0f\n\x07\x65nabled\x18\x04 \x01(\x08\x12\x12\n\ncreated_at\x18\x05 \x01(\r\"T\n\x12\x41liasStatusChanged\x12\n\n\x02id\x18\x01 \x01(\r\x12\r\n\x05\x65mail\x18\x02 \x01(\t\x12\x0f\n\x07\x65nabled\x18\x03 \x01(\x08\x12\x12\n\ncreated_at\x18\x04 \x01(\r\")\n\x0c\x41liasDeleted\x12\n\n\x02id\x18\x01 \x01(\r\x12\r\n\x05\x65mail\x18\x02 \x01(\t\"D\n\x10\x41liasCreatedList\x12\x30\n\x06\x65vents\x18\x01 \x03(\x0b\x32 .simplelogin_events.AliasCreated\"\x0e\n\x0cUserUnlinked\"\xce\x03\n\x0c\x45ventContent\x12?\n\x10user_plan_change\x18\x01 \x01(\x0b\x32#.simplelogin_events.UserPlanChangedH\x00\x12\x37\n\x0cuser_deleted\x18\x02 \x01(\x0b\x32\x1f.simplelogin_events.UserDeletedH\x00\x12\x39\n\ralias_created\x18\x03 \x01(\x0b\x32 .simplelogin_events.AliasCreatedH\x00\x12\x45\n\x13\x61lias_status_change\x18\x04 \x01(\x0b\x32&.simplelogin_events.AliasStatusChangedH\x00\x12\x39\n\ralias_deleted\x18\x05 \x01(\x0b\x32 .simplelogin_events.AliasDeletedH\x00\x12\x41\n\x11\x61lias_create_list\x18\x06 \x01(\x0b\x32$.simplelogin_events.AliasCreatedListH\x00\x12\x39\n\ruser_unlinked\x18\x07 \x01(\x0b\x32 .simplelogin_events.UserUnlinkedH\x00\x42\t\n\x07\x63ontent\"y\n\x05\x45vent\x12\x0f\n\x07user_id\x18\x01 \x01(\r\x12\x18\n\x10\x65xternal_user_id\x18\x02 \x01(\t\x12\x12\n\npartner_id\x18\x03 \x01(\r\x12\x31\n\x07\x63ontent\x18\x04 \x01(\x0b\x32 .simplelogin_events.EventContentb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -43,8 +43,10 @@ if not _descriptor._USE_C_DESCRIPTORS: _globals['_ALIASDELETED']._serialized_end=331 _globals['_ALIASCREATEDLIST']._serialized_start=333 _globals['_ALIASCREATEDLIST']._serialized_end=401 - _globals['_EVENTCONTENT']._serialized_start=404 - _globals['_EVENTCONTENT']._serialized_end=807 - _globals['_EVENT']._serialized_start=809 - _globals['_EVENT']._serialized_end=930 + _globals['_USERUNLINKED']._serialized_start=403 + _globals['_USERUNLINKED']._serialized_end=417 + _globals['_EVENTCONTENT']._serialized_start=420 + _globals['_EVENTCONTENT']._serialized_end=882 + _globals['_EVENT']._serialized_start=884 + _globals['_EVENT']._serialized_end=1005 # @@protoc_insertion_point(module_scope) diff --git a/app/app/events/generated/event_pb2.pyi b/app/app/events/generated/event_pb2.pyi index 58914b1..83369fc 100644 --- a/app/app/events/generated/event_pb2.pyi +++ b/app/app/events/generated/event_pb2.pyi @@ -57,21 +57,27 @@ class AliasCreatedList(_message.Message): events: _containers.RepeatedCompositeFieldContainer[AliasCreated] def __init__(self, events: _Optional[_Iterable[_Union[AliasCreated, _Mapping]]] = ...) -> None: ... +class UserUnlinked(_message.Message): + __slots__ = () + def __init__(self) -> None: ... + class EventContent(_message.Message): - __slots__ = ("user_plan_change", "user_deleted", "alias_created", "alias_status_change", "alias_deleted", "alias_create_list") + __slots__ = ("user_plan_change", "user_deleted", "alias_created", "alias_status_change", "alias_deleted", "alias_create_list", "user_unlinked") USER_PLAN_CHANGE_FIELD_NUMBER: _ClassVar[int] USER_DELETED_FIELD_NUMBER: _ClassVar[int] ALIAS_CREATED_FIELD_NUMBER: _ClassVar[int] ALIAS_STATUS_CHANGE_FIELD_NUMBER: _ClassVar[int] ALIAS_DELETED_FIELD_NUMBER: _ClassVar[int] ALIAS_CREATE_LIST_FIELD_NUMBER: _ClassVar[int] + USER_UNLINKED_FIELD_NUMBER: _ClassVar[int] user_plan_change: UserPlanChanged user_deleted: UserDeleted alias_created: AliasCreated alias_status_change: AliasStatusChanged alias_deleted: AliasDeleted alias_create_list: AliasCreatedList - def __init__(self, user_plan_change: _Optional[_Union[UserPlanChanged, _Mapping]] = ..., user_deleted: _Optional[_Union[UserDeleted, _Mapping]] = ..., alias_created: _Optional[_Union[AliasCreated, _Mapping]] = ..., alias_status_change: _Optional[_Union[AliasStatusChanged, _Mapping]] = ..., alias_deleted: _Optional[_Union[AliasDeleted, _Mapping]] = ..., alias_create_list: _Optional[_Union[AliasCreatedList, _Mapping]] = ...) -> None: ... + user_unlinked: UserUnlinked + def __init__(self, user_plan_change: _Optional[_Union[UserPlanChanged, _Mapping]] = ..., user_deleted: _Optional[_Union[UserDeleted, _Mapping]] = ..., alias_created: _Optional[_Union[AliasCreated, _Mapping]] = ..., alias_status_change: _Optional[_Union[AliasStatusChanged, _Mapping]] = ..., alias_deleted: _Optional[_Union[AliasDeleted, _Mapping]] = ..., alias_create_list: _Optional[_Union[AliasCreatedList, _Mapping]] = ..., user_unlinked: _Optional[_Union[UserUnlinked, _Mapping]] = ...) -> None: ... class Event(_message.Message): __slots__ = ("user_id", "external_user_id", "partner_id", "content") diff --git a/app/app/fake_data.py b/app/app/fake_data.py index 78c5050..6f43db7 100644 --- a/app/app/fake_data.py +++ b/app/app/fake_data.py @@ -37,7 +37,7 @@ from app.models import ( PartnerSubscription, ) from app.pgp_utils import load_public_key -from app.proton.utils import get_proton_partner +from app.proton.proton_partner import get_proton_partner def fake_data(): diff --git a/app/app/jobs/send_event_job.py b/app/app/jobs/send_event_job.py index 3f5ca61..32dd7d3 100644 --- a/app/app/jobs/send_event_job.py +++ b/app/app/jobs/send_event_job.py @@ -14,7 +14,7 @@ from app.models import ( Job, PartnerUser, ) -from app.proton.utils import get_proton_partner +from app.proton.proton_partner import get_proton_partner from events.event_sink import EventSink diff --git a/app/app/mailbox_utils.py b/app/app/mailbox_utils.py index 69599c0..9ed2f02 100644 --- a/app/app/mailbox_utils.py +++ b/app/app/mailbox_utils.py @@ -2,7 +2,9 @@ import dataclasses import secrets from enum import Enum from typing import Optional + import arrow +from sqlalchemy.exc import IntegrityError from app import config from app.config import JOB_DELETE_MAILBOX @@ -351,6 +353,7 @@ def request_mailbox_email_change( check_email_for_mailbox(new_email, user) if email_ownership_verified: mailbox.email = new_email + mailbox.new_email = None else: mailbox.new_email = new_email emit_user_audit_log( @@ -358,7 +361,12 @@ def request_mailbox_email_change( action=UserAuditLogAction.UpdateMailbox, message=f"Updated mailbox {mailbox.id} email ({new_email}) pre-verified({email_ownership_verified}", ) - Session.commit() + try: + Session.commit() + except IntegrityError: + LOG.i(f"This email {new_email} is already pending for some mailbox") + Session.rollback() + raise MailboxError("Email already in use") if email_ownership_verified: LOG.i(f"User {user} as created a pre-verified mailbox with {new_email}") diff --git a/app/app/models.py b/app/app/models.py index 2f57293..3032ba5 100644 --- a/app/app/models.py +++ b/app/app/models.py @@ -2838,24 +2838,20 @@ class Mailbox(Base, ModelMixin): return len(alias_ids) def is_proton(self) -> bool: - if ( - self.email.endswith("@proton.me") - or self.email.endswith("@protonmail.com") - or self.email.endswith("@protonmail.ch") - or self.email.endswith("@proton.ch") - or self.email.endswith("@pm.me") - ): - return True + for proton_email_domain in config.PROTON_EMAIL_DOMAINS: + if self.email.endswith(f"@{proton_email_domain}"): + return True from app.email_utils import get_email_local_part mx_domains = get_mx_domains(get_email_local_part(self.email)) + + proton_mx_domains = config.PROTON_MX_SERVERS # Proton is the first domain - if mx_domains and mx_domains[0].domain in ( - "mail.protonmail.ch.", - "mailsec.protonmail.ch.", - ): - return True + for prio in mx_domains: + for mx_domain in mx_domains[prio]: + if mx_domain in proton_mx_domains: + return True return False diff --git a/app/app/monitor/views.py b/app/app/monitor/views.py index a248261..6642846 100644 --- a/app/app/monitor/views.py +++ b/app/app/monitor/views.py @@ -1,4 +1,4 @@ -from app.build_info import SHA1 +from app.build_info import SHA1, VERSION from app.monitor.base import monitor_bp @@ -7,6 +7,11 @@ def git_sha1(): return SHA1 +@monitor_bp.route("/version") +def version(): + return VERSION + + @monitor_bp.route("/live") def live(): return "live" diff --git a/app/app/monitor_utils.py b/app/app/monitor_utils.py new file mode 100644 index 0000000..4780ee5 --- /dev/null +++ b/app/app/monitor_utils.py @@ -0,0 +1,8 @@ +from app.build_info import VERSION +import newrelic.agent + + +def send_version_event(service: str): + newrelic.agent.record_custom_event( + "ServiceVersion", {"service": service, "version": VERSION} + ) diff --git a/app/app/proton/proton_partner.py b/app/app/proton/proton_partner.py new file mode 100644 index 0000000..7b02e83 --- /dev/null +++ b/app/app/proton/proton_partner.py @@ -0,0 +1,23 @@ +from typing import Optional + +from app.db import Session +from app.errors import ProtonPartnerNotSetUp +from app.models import Partner + +PROTON_PARTNER_NAME = "Proton" +_PROTON_PARTNER: Optional[Partner] = None + + +def get_proton_partner() -> Partner: + global _PROTON_PARTNER + if _PROTON_PARTNER is None: + partner = Partner.get_by(name=PROTON_PARTNER_NAME) + if partner is None: + raise ProtonPartnerNotSetUp + Session.expunge(partner) + _PROTON_PARTNER = partner + return _PROTON_PARTNER + + +def is_proton_partner(partner: Partner) -> bool: + return partner.name == PROTON_PARTNER_NAME diff --git a/app/app/proton/utils.py b/app/app/proton/proton_unlink.py similarity index 64% rename from app/app/proton/utils.py rename to app/app/proton/proton_unlink.py index 0bd4ff3..69f5cb9 100644 --- a/app/app/proton/utils.py +++ b/app/app/proton/proton_unlink.py @@ -1,31 +1,13 @@ -from typing import Optional - from newrelic import agent from app.db import Session -from app.errors import ProtonPartnerNotSetUp +from app.events.event_dispatcher import EventDispatcher +from app.events.generated.event_pb2 import EventContent, UserUnlinked from app.log import LOG -from app.models import Partner, PartnerUser, User +from app.models import User, PartnerUser +from app.proton.proton_partner import get_proton_partner from app.user_audit_log_utils import emit_user_audit_log, UserAuditLogAction -PROTON_PARTNER_NAME = "Proton" -_PROTON_PARTNER: Optional[Partner] = None - - -def get_proton_partner() -> Partner: - global _PROTON_PARTNER - if _PROTON_PARTNER is None: - partner = Partner.get_by(name=PROTON_PARTNER_NAME) - if partner is None: - raise ProtonPartnerNotSetUp - Session.expunge(partner) - _PROTON_PARTNER = partner - return _PROTON_PARTNER - - -def is_proton_partner(partner: Partner) -> bool: - return partner.name == PROTON_PARTNER_NAME - def can_unlink_proton_account(user: User) -> bool: return (user.flags & User.FLAG_CREATED_FROM_PARTNER) == 0 @@ -45,6 +27,9 @@ def perform_proton_account_unlink(current_user: User) -> bool: action=UserAuditLogAction.UnlinkAccount, message=f"User has unlinked the account (email={partner_user.partner_email} | external_user_id={partner_user.external_user_id})", ) + EventDispatcher.send_event( + partner_user.user, EventContent(user_unlinked=UserUnlinked()) + ) PartnerUser.delete(partner_user.id) Session.commit() agent.record_custom_event("AccountUnlinked", {"partner": proton_partner.name}) diff --git a/app/cron.py b/app/cron.py index 76513f9..e22d46e 100644 --- a/app/cron.py +++ b/app/cron.py @@ -59,7 +59,7 @@ from app.models import ( ApiToCookieToken, ) from app.pgp_utils import load_public_key_and_check, PGPException -from app.proton.utils import get_proton_partner +from app.proton.proton_partner import get_proton_partner from app.user_audit_log_utils import emit_user_audit_log, UserAuditLogAction from app.utils import sanitize_email from server import create_light_app diff --git a/app/email_handler.py b/app/email_handler.py index 114ab31..f732fb7 100644 --- a/app/email_handler.py +++ b/app/email_handler.py @@ -167,6 +167,7 @@ from app.models import ( VerpType, SLDomain, ) +from app.monitor_utils import send_version_event from app.pgp_utils import ( PGPException, sign_data_with_pgpy, @@ -2360,6 +2361,7 @@ class MailHandler: "Custom/nb_rcpt_tos", len(envelope.rcpt_tos) ) + send_version_event("email_handler") with create_light_app().app_context(): return_status = handle(envelope, msg) elapsed = time.time() - start @@ -2395,6 +2397,7 @@ def main(port: int): controller.start() LOG.d("Start mail controller %s %s", controller.hostname, controller.port) + send_version_event("email_handler") if LOAD_PGP_EMAIL_HANDLER: LOG.w("LOAD PGP keys") diff --git a/app/event_listener.py b/app/event_listener.py index 5c87460..038ef8d 100644 --- a/app/event_listener.py +++ b/app/event_listener.py @@ -4,6 +4,7 @@ from sys import argv, exit from app.config import EVENT_LISTENER_DB_URI from app.log import LOG +from app.monitor_utils import send_version_event from events import event_debugger from events.runner import Runner from events.event_source import DeadLetterEventSource, PostgresEventSource @@ -30,9 +31,11 @@ def main(mode: Mode, dry_run: bool, max_retries: int): if mode == Mode.DEAD_LETTER: LOG.i("Using DeadLetterEventSource") source = DeadLetterEventSource(max_retries) + service_name = "event_listener_dead_letter" elif mode == Mode.LISTENER: LOG.i("Using PostgresEventSource") source = PostgresEventSource(EVENT_LISTENER_DB_URI) + service_name = "event_listener" else: raise ValueError(f"Invalid mode: {mode}") @@ -43,7 +46,8 @@ def main(mode: Mode, dry_run: bool, max_retries: int): LOG.i("Starting with HttpEventSink") sink = HttpEventSink() - runner = Runner(source=source, sink=sink) + send_version_event(service_name) + runner = Runner(source=source, sink=sink, service_name=service_name) runner.run() diff --git a/app/events/runner.py b/app/events/runner.py index 0fe7bff..2fa4730 100644 --- a/app/events/runner.py +++ b/app/events/runner.py @@ -4,20 +4,24 @@ import newrelic.agent from app.log import LOG from app.db import Session from app.models import SyncEvent +from app.monitor_utils import send_version_event from events.event_sink import EventSink from events.event_source import EventSource class Runner: - def __init__(self, source: EventSource, sink: EventSink): + def __init__(self, source: EventSource, sink: EventSink, service_name: str = ""): self.__source = source self.__sink = sink + self.__service_name = service_name def run(self): self.__source.run(self.__on_event) @newrelic.agent.background_task() def __on_event(self, event: SyncEvent): + if self.__service_name: + send_version_event(self.__service_name) try: event_created_at = event.created_at start_time = arrow.now() diff --git a/app/init_app.py b/app/init_app.py index ced9087..3e3333c 100644 --- a/app/init_app.py +++ b/app/init_app.py @@ -6,7 +6,7 @@ from app.db import Session from app.log import LOG from app.models import Mailbox, Contact, SLDomain, Partner from app.pgp_utils import load_public_key -from app.proton.utils import PROTON_PARTNER_NAME +from app.proton.proton_partner import PROTON_PARTNER_NAME from server import create_light_app diff --git a/app/job_runner.py b/app/job_runner.py index af99fe8..85b32ad 100644 --- a/app/job_runner.py +++ b/app/job_runner.py @@ -21,6 +21,7 @@ from app.jobs.export_user_data_job import ExportUserDataJob from app.jobs.send_event_job import SendEventToWebhookJob from app.log import LOG from app.models import User, Job, BatchImport, Mailbox, CustomDomain, JobState +from app.monitor_utils import send_version_event from app.user_audit_log_utils import emit_user_audit_log, UserAuditLogAction from server import create_light_app @@ -189,6 +190,7 @@ SimpleLogin team. def process_job(job: Job): + send_version_event("job_runner") if job.name == config.JOB_ONBOARDING_1: user_id = job.payload.get("user_id") user = User.get(user_id) @@ -334,6 +336,7 @@ def get_jobs_to_run() -> List[Job]: if __name__ == "__main__": + send_version_event("job_runner") while True: # wrap in an app context to benefit from app setup like database cleanup, sentry integration, etc with create_light_app().app_context(): diff --git a/app/proto/event.proto b/app/proto/event.proto index 89621f8..1d29ab4 100644 --- a/app/proto/event.proto +++ b/app/proto/event.proto @@ -34,6 +34,9 @@ message AliasCreatedList { repeated AliasCreated events = 1; } +message UserUnlinked { +} + message EventContent { oneof content { UserPlanChanged user_plan_change = 1; @@ -42,6 +45,7 @@ message EventContent { AliasStatusChanged alias_status_change = 4; AliasDeleted alias_deleted = 5; AliasCreatedList alias_create_list = 6; + UserUnlinked user_unlinked = 7; } } diff --git a/app/scripts/generate-build-info.sh b/app/scripts/generate-build-info.sh index 000b4f4..1149d36 100755 --- a/app/scripts/generate-build-info.sh +++ b/app/scripts/generate-build-info.sh @@ -4,12 +4,14 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" || exit 1; pwd -P)" REPO_ROOT=$(echo "${SCRIPT_DIR}" | sed 's:scripts::g') BUILD_INFO_FILE="${REPO_ROOT}/app/build_info.py" -if [[ -z "$1" ]]; then - echo "This script needs to be invoked with the version as an argument" +if [[ -z "$2" ]]; then + echo "Invalid usage. Usage: $0 SHA VERSION" exit 1 fi -VERSION="$1" -echo "SHA1 = \"${VERSION}\"" > $BUILD_INFO_FILE +SHA="$1" +echo "SHA1 = \"${SHA}\"" > $BUILD_INFO_FILE BUILD_TIME=$(date +%s) echo "BUILD_TIME = \"${BUILD_TIME}\"" >> $BUILD_INFO_FILE +VERSION="$2" +echo "VERSION = \"${VERSION}\"" >> $BUILD_INFO_FILE diff --git a/app/server.py b/app/server.py index ece49e4..bc54cf7 100644 --- a/app/server.py +++ b/app/server.py @@ -99,6 +99,7 @@ from app.models import ( InvalidMailboxDomain, ) from app.monitor.base import monitor_bp +from app.monitor_utils import send_version_event from app.newsletter_utils import send_newsletter_to_user from app.oauth.base import oauth_bp from app.onboarding.base import onboarding_bp @@ -295,6 +296,7 @@ def set_index_page(app): newrelic.agent.record_custom_event( "HttpResponseStatus", {"code": res.status_code} ) + send_version_event("app") return res diff --git a/app/tests/api/test_user_info.py b/app/tests/api/test_user_info.py index dd539d8..b260df8 100644 --- a/app/tests/api/test_user_info.py +++ b/app/tests/api/test_user_info.py @@ -3,7 +3,7 @@ from flask import url_for from app import config from app.db import Session from app.models import User, PartnerUser -from app.proton.utils import get_proton_partner +from app.proton.proton_partner import get_proton_partner from tests.api.utils import get_new_user_and_api_key from tests.utils import login, random_token, random_email diff --git a/app/tests/cron/test_get_alias_for_hibp.py b/app/tests/cron/test_get_alias_for_hibp.py index 370c6ff..9f645f3 100644 --- a/app/tests/cron/test_get_alias_for_hibp.py +++ b/app/tests/cron/test_get_alias_for_hibp.py @@ -14,7 +14,7 @@ from app.models import ( PartnerSubscription, User, ) -from app.proton.utils import get_proton_partner +from app.proton.proton_partner import get_proton_partner from tests.utils import create_new_user, random_token diff --git a/app/tests/events/event_test_utils.py b/app/tests/events/event_test_utils.py index d7b6e96..1979ff0 100644 --- a/app/tests/events/event_test_utils.py +++ b/app/tests/events/event_test_utils.py @@ -1,7 +1,7 @@ from app.events.event_dispatcher import Dispatcher from app.events.generated import event_pb2 from app.models import PartnerUser, User -from app.proton.utils import get_proton_partner +from app.proton.proton_partner import get_proton_partner from tests.utils import create_new_user, random_token from typing import Tuple diff --git a/app/tests/jobs/test_send_event_to_webhook.py b/app/tests/jobs/test_send_event_to_webhook.py index dacc480..245240a 100644 --- a/app/tests/jobs/test_send_event_to_webhook.py +++ b/app/tests/jobs/test_send_event_to_webhook.py @@ -4,7 +4,7 @@ from app import config from app.events.generated.event_pb2 import EventContent, AliasDeleted from app.jobs.send_event_job import SendEventToWebhookJob from app.models import PartnerUser -from app.proton.utils import get_proton_partner +from app.proton.proton_partner import get_proton_partner from events.event_sink import ConsoleEventSink from tests.utils import create_new_user, random_token diff --git a/app/tests/models/test_mailbox.py b/app/tests/models/test_mailbox.py new file mode 100644 index 0000000..68f5d1f --- /dev/null +++ b/app/tests/models/test_mailbox.py @@ -0,0 +1,37 @@ +from app import config +from app.dns_utils import set_global_dns_client, InMemoryDNSClient +from app.email_utils import get_email_local_part +from app.models import Mailbox +from tests.utils import create_new_user, random_email + +dns_client = InMemoryDNSClient() + + +def setup_module(): + set_global_dns_client(dns_client) + + +def teardown_module(): + set_global_dns_client(None) + + +def test_is_proton_with_email_domain(): + user = create_new_user() + mailbox = Mailbox.create( + user_id=user.id, email=f"test@{config.PROTON_EMAIL_DOMAINS[0]}" + ) + assert mailbox.is_proton() + mailbox = Mailbox.create(user_id=user.id, email="a@b.c") + assert not mailbox.is_proton() + + +def test_is_proton_with_mx_domain(): + email = random_email() + dns_client.set_mx_records( + get_email_local_part(email), {10: config.PROTON_MX_SERVERS} + ) + user = create_new_user() + mailbox = Mailbox.create(user_id=user.id, email=email) + assert mailbox.is_proton() + dns_client.set_mx_records(get_email_local_part(email), {10: ["nowhere.net"]}) + assert not mailbox.is_proton() diff --git a/app/tests/models/test_user.py b/app/tests/models/test_user.py index 211cb3f..11c39eb 100644 --- a/app/tests/models/test_user.py +++ b/app/tests/models/test_user.py @@ -2,7 +2,7 @@ import arrow from app import config from app.db import Session from app.models import User, Job, PartnerSubscription, PartnerUser, ManualSubscription -from app.proton.utils import get_proton_partner +from app.proton.proton_partner import get_proton_partner from tests.utils import random_email, random_token diff --git a/app/tests/proton/test_account_linking.py b/app/tests/proton/test_account_linking.py index d2511c0..8aa9c73 100644 --- a/app/tests/proton/test_account_linking.py +++ b/app/tests/proton/test_account_linking.py @@ -7,7 +7,7 @@ from app.account_linking import ( ) from app.db import Session from app.models import User, PartnerUser, PartnerSubscription -from app.proton.utils import get_proton_partner +from app.proton.proton_partner import get_proton_partner from app.utils import random_string from tests.utils import random_email diff --git a/app/tests/proton/test_proton_callback_handler.py b/app/tests/proton/test_proton_callback_handler.py index adf8711..00edd4c 100644 --- a/app/tests/proton/test_proton_callback_handler.py +++ b/app/tests/proton/test_proton_callback_handler.py @@ -11,7 +11,7 @@ from app.proton.proton_callback_handler import ( generate_account_not_allowed_to_log_in, ) from app.models import User, PartnerUser, Job, JobState -from app.proton.utils import get_proton_partner +from app.proton.proton_partner import get_proton_partner from app.utils import random_string from typing import Optional from tests.utils import random_email diff --git a/app/tests/test_account_linking.py b/app/tests/test_account_linking.py index a452423..b38448f 100644 --- a/app/tests/test_account_linking.py +++ b/app/tests/test_account_linking.py @@ -19,7 +19,7 @@ from app.account_linking import ( from app.db import Session from app.errors import AccountAlreadyLinkedToAnotherPartnerException from app.models import Partner, PartnerUser, User, UserAuditLog -from app.proton.utils import get_proton_partner +from app.proton.proton_partner import get_proton_partner from app.user_audit_log_utils import UserAuditLogAction from app.utils import random_string, canonicalize_email from tests.utils import random_email diff --git a/app/tests/test_alias_suffixes.py b/app/tests/test_alias_suffixes.py index 870baa1..b1be294 100644 --- a/app/tests/test_alias_suffixes.py +++ b/app/tests/test_alias_suffixes.py @@ -3,7 +3,7 @@ import re from app.alias_suffix import get_alias_suffixes from app.db import Session from app.models import SLDomain, PartnerUser, AliasOptions, CustomDomain -from app.proton.utils import get_proton_partner +from app.proton.proton_partner import get_proton_partner from init_app import add_sl_domains from tests.utils import create_new_user, random_token diff --git a/app/tests/test_coupon_utils.py b/app/tests/test_coupon_utils.py index e947591..743f4ef 100644 --- a/app/tests/test_coupon_utils.py +++ b/app/tests/test_coupon_utils.py @@ -18,7 +18,7 @@ from app.models import ( PartnerSubscription, PartnerUser, ) -from app.proton.utils import get_proton_partner +from app.proton.proton_partner import get_proton_partner from tests.utils import create_new_user, random_string, random_email diff --git a/app/tests/test_custom_domain_validation.py b/app/tests/test_custom_domain_validation.py index c417fe2..fbeb54d 100644 --- a/app/tests/test_custom_domain_validation.py +++ b/app/tests/test_custom_domain_validation.py @@ -6,7 +6,7 @@ from app.custom_domain_validation import CustomDomainValidation from app.db import Session from app.dns_utils import InMemoryDNSClient from app.models import CustomDomain, User -from app.proton.utils import get_proton_partner +from app.proton.proton_partner import get_proton_partner from app.utils import random_string from tests.utils import create_new_user, random_domain diff --git a/app/tests/test_domains.py b/app/tests/test_domains.py index 5783aa0..58c28e3 100644 --- a/app/tests/test_domains.py +++ b/app/tests/test_domains.py @@ -1,6 +1,6 @@ from app.db import Session from app.models import SLDomain, PartnerUser, AliasOptions -from app.proton.utils import get_proton_partner +from app.proton.proton_partner import get_proton_partner from init_app import add_sl_domains from tests.utils import create_new_user, random_token diff --git a/app/tests/test_mailbox_utils.py b/app/tests/test_mailbox_utils.py index 86d928a..ed33c00 100644 --- a/app/tests/test_mailbox_utils.py +++ b/app/tests/test_mailbox_utils.py @@ -25,7 +25,6 @@ from app.user_audit_log_utils import UserAuditLogAction from app.utils import random_string, canonicalize_email from tests.utils import create_new_user, random_email - user: Optional[User] = None @@ -598,3 +597,47 @@ def test_change_mailbox_verified_address(flask_client): assert changed_mailbox.email == mail2 assert out.activation is None assert 0 == len(mail_sender.get_stored_emails()) + + +def test_change_mailbox_email_duplicate(flask_client): + user = create_new_user() + domain = f"{random_string(10)}.com" + mail1 = f"mail_1@{domain}" + mbox = Mailbox.create(email=mail1, user_id=user.id, verified=True, flush=True) + mail2 = f"mail_2@{domain}" + request_mailbox_email_change(user, mbox, mail2, email_ownership_verified=True) + with pytest.raises(mailbox_utils.MailboxError): + request_mailbox_email_change(user, mbox, mail2, email_ownership_verified=True) + + +def test_change_mailbox_email_duplicate_in_another_mailbox(flask_client): + user = create_new_user() + domain = f"{random_string(10)}.com" + mail1 = f"mail_1@{domain}" + mbox1 = Mailbox.create(email=mail1, user_id=user.id, verified=True, flush=True) + mail2 = f"mail_2@{domain}" + mbox2 = Mailbox.create(email=mail2, user_id=user.id, verified=True, flush=True) + mail3 = f"mail_3@{domain}" + request_mailbox_email_change(user, mbox1, mail3) + with pytest.raises(mailbox_utils.MailboxError): + request_mailbox_email_change(user, mbox2, mail3) + + +def test_change_mailbox_verified_email_clears_pending_email(flask_client): + user = create_new_user() + domain = f"{random_string(10)}.com" + mail = f"mail_1@{domain}" + mbox1 = Mailbox.create( + email=mail, + new_email=f"oldpending_{mail}", + user_id=user.id, + verified=True, + flush=True, + ) + new_email = f"new_{mail}" + out = request_mailbox_email_change( + user, mbox1, new_email, email_ownership_verified=True + ) + assert out.activation is None + assert out.mailbox.email == new_email + assert out.mailbox.new_email is None diff --git a/app/tests/utils.py b/app/tests/utils.py index 179cbb2..b7798d0 100644 --- a/app/tests/utils.py +++ b/app/tests/utils.py @@ -10,7 +10,7 @@ import jinja2 from flask import url_for from app.models import User, PartnerUser -from app.proton.utils import get_proton_partner +from app.proton.proton_partner import get_proton_partner from app.utils import random_string