From a5801551d0ce90a55504d870e2cd32aa44f41b68 Mon Sep 17 00:00:00 2001 From: MrMeeb Date: Tue, 4 Feb 2025 12:00:06 +0000 Subject: [PATCH] 4.65.1 --- app/app/admin_model.py | 28 +- app/app/alias_mailbox_utils.py | 1 + app/app/api/serializer.py | 9 +- app/app/coupon_utils.py | 19 +- app/app/custom_domain_validation.py | 141 +++++++--- app/app/dashboard/views/domain_detail.py | 4 +- app/app/dashboard/views/mailbox_detail.py | 4 +- app/app/dns_utils.py | 56 +--- app/app/email_utils.py | 6 +- app/app/log.py | 18 +- app/app/models.py | 57 ++-- app/app/request_utils.py | 6 + app/cron.py | 4 +- .../2025_013015_d3ff8848c930_index_cleanup.py | 91 ++++++ .../2025_013114_97edba8794f8_index_cleanup.py | 23 ++ .../2025_020316_20e7d3ca289a_contact_index.py | 27 ++ app/server.py | 2 + app/templates/admin/custom_domain_search.html | 261 +++++++++++------- .../dashboard/domain_detail/dns.html | 12 +- app/tests/test_alias_mailbox_utils.py | 10 +- app/tests/test_coupon_utils.py | 58 +++- app/tests/test_custom_domain_utils.py | 14 +- app/tests/test_custom_domain_validation.py | 98 ++++--- app/tests/test_dns_utils.py | 63 +++-- 24 files changed, 714 insertions(+), 298 deletions(-) create mode 100644 app/app/request_utils.py create mode 100644 app/migrations/versions/2025_013015_d3ff8848c930_index_cleanup.py create mode 100644 app/migrations/versions/2025_013114_97edba8794f8_index_cleanup.py create mode 100644 app/migrations/versions/2025_020316_20e7d3ca289a_contact_index.py diff --git a/app/app/admin_model.py b/app/app/admin_model.py index f49654b..6338641 100644 --- a/app/app/admin_model.py +++ b/app/app/admin_model.py @@ -1,21 +1,25 @@ from __future__ import annotations + from typing import Optional, List import arrow import sqlalchemy -from flask_admin import BaseView -from flask_admin.form import SecureForm -from flask_admin.model.template import EndpointLinkRowAction -from markupsafe import Markup - -from app import models, s3, config from flask import redirect, url_for, request, flash, Response +from flask_admin import BaseView from flask_admin import expose, AdminIndexView from flask_admin.actions import action from flask_admin.contrib import sqla +from flask_admin.form import SecureForm +from flask_admin.model.template import EndpointLinkRowAction from flask_login import current_user +from markupsafe import Markup -from app.custom_domain_validation import CustomDomainValidation, DomainValidationResult +from app import models, s3, config +from app.custom_domain_validation import ( + CustomDomainValidation, + DomainValidationResult, + ExpectedValidationRecords, +) from app.db import Session from app.dns_utils import get_network_dns_client from app.events.event_dispatcher import EventDispatcher @@ -929,13 +933,13 @@ class EmailSearchAdmin(BaseView): class CustomDomainWithValidationData: def __init__(self, domain: CustomDomain): self.domain: CustomDomain = domain - self.ownership_expected: Optional[str] = None + self.ownership_expected: Optional[ExpectedValidationRecords] = None self.ownership_validation: Optional[DomainValidationResult] = None - self.mx_expected: Optional[str] = None + self.mx_expected: Optional[dict[int, ExpectedValidationRecords]] = None self.mx_validation: Optional[DomainValidationResult] = None - self.spf_expected: Optional[str] = None + self.spf_expected: Optional[ExpectedValidationRecords] = None self.spf_validation: Optional[DomainValidationResult] = None - self.dkim_expected: {str: str} = {} + self.dkim_expected: {str: ExpectedValidationRecords} = {} self.dkim_validation: {str: str} = {} @@ -990,7 +994,6 @@ class CustomDomainSearchResult: custom_domain ) out.domains.append(validation_data) - print(validation_data.dkim_expected, validation_data.dkim_validation) return out @@ -1020,7 +1023,6 @@ class CustomDomainSearchAdmin(BaseView): if cd is not None: user = cd.user search = CustomDomainSearchResult.from_user(user) - print("NEW", search.domains) return self.render( "admin/custom_domain_search.html", diff --git a/app/app/alias_mailbox_utils.py b/app/app/alias_mailbox_utils.py index 1388460..c7f25ff 100644 --- a/app/app/alias_mailbox_utils.py +++ b/app/app/alias_mailbox_utils.py @@ -36,6 +36,7 @@ def set_mailboxes_for_alias( Mailbox.user_id == user_id, Mailbox.verified == True, # noqa: E712 ) + .order_by(Mailbox.id.asc()) .all() ) if len(mailboxes) != len(mailbox_ids): diff --git a/app/app/api/serializer.py b/app/app/api/serializer.py index 1c89330..e233ff9 100644 --- a/app/app/api/serializer.py +++ b/app/app/api/serializer.py @@ -191,15 +191,8 @@ def get_alias_infos_with_pagination_v3( q = q.order_by(Alias.email.desc()) else: # default sorting - latest_activity = case( - [ - (Alias.created_at > EmailLog.created_at, Alias.created_at), - (Alias.created_at < EmailLog.created_at, EmailLog.created_at), - ], - else_=Alias.created_at, - ) q = q.order_by(Alias.pinned.desc()) - q = q.order_by(latest_activity.desc()) + q = q.order_by(func.greatest(Alias.created_at, EmailLog.created_at).desc()) q = q.limit(page_limit).offset(page_id * page_size) diff --git a/app/app/coupon_utils.py b/app/app/coupon_utils.py index bc8266e..3aded1f 100644 --- a/app/app/coupon_utils.py +++ b/app/app/coupon_utils.py @@ -9,7 +9,14 @@ from app.email_utils import send_email from app.events.event_dispatcher import EventDispatcher from app.events.generated.event_pb2 import EventContent, UserPlanChanged from app.log import LOG -from app.models import User, ManualSubscription, Coupon, LifetimeCoupon +from app.models import ( + User, + ManualSubscription, + Coupon, + LifetimeCoupon, + PartnerSubscription, + PartnerUser, +) from app.user_audit_log_utils import emit_user_audit_log, UserAuditLogAction @@ -87,6 +94,16 @@ def redeem_coupon(coupon_code: str, user: User) -> Optional[Coupon]: def redeem_lifetime_coupon(coupon_code: str, user: User) -> Optional[Coupon]: + if user.lifetime: + return None + partner_sub = ( + Session.query(PartnerSubscription) + .join(PartnerUser, PartnerUser.id == PartnerSubscription.partner_user_id) + .filter(PartnerUser.user_id == user.id, PartnerSubscription.lifetime == True) # noqa: E712 + .first() + ) + if partner_sub is not None: + return None coupon: LifetimeCoupon = LifetimeCoupon.get_by(code=coupon_code) if not coupon: return None diff --git a/app/app/custom_domain_validation.py b/app/app/custom_domain_validation.py index 90bed3b..fc3ec15 100644 --- a/app/app/custom_domain_validation.py +++ b/app/app/custom_domain_validation.py @@ -5,9 +5,7 @@ from app import config from app.constants import DMARC_RECORD from app.db import Session from app.dns_utils import ( - MxRecord, DNSClient, - is_mx_equivalent, get_network_dns_client, ) from app.models import CustomDomain @@ -21,6 +19,39 @@ class DomainValidationResult: errors: [str] +@dataclass +class ExpectedValidationRecords: + recommended: str + allowed: list[str] + + +def is_mx_equivalent( + mx_domains: dict[int, list[str]], + expected_mx_domains: dict[int, ExpectedValidationRecords], +) -> bool: + """ + Compare mx_domains with ref_mx_domains to see if they are equivalent. + mx_domains and ref_mx_domains are list of (priority, domain) + + The priority order is taken into account but not the priority number. + For example, [(1, domain1), (2, domain2)] is equivalent to [(10, domain1), (20, domain2)] + """ + + expected_prios = [] + for prio in expected_mx_domains: + expected_prios.append(prio) + + if len(expected_prios) != len(mx_domains): + return False + + for prio_position, prio_value in enumerate(sorted(mx_domains.keys())): + for domain in mx_domains[prio_value]: + if domain not in expected_mx_domains[expected_prios[prio_position]].allowed: + return False + + return True + + class CustomDomainValidation: def __init__( self, @@ -37,59 +68,88 @@ class CustomDomainValidation: or config.PARTNER_CUSTOM_DOMAIN_VALIDATION_PREFIXES ) - def get_ownership_verification_record(self, domain: CustomDomain) -> str: - prefix = "sl" + def get_ownership_verification_record( + self, domain: CustomDomain + ) -> ExpectedValidationRecords: + prefixes = ["sl"] if ( domain.partner_id is not None and domain.partner_id in self._partner_domain_validation_prefixes ): - prefix = self._partner_domain_validation_prefixes[domain.partner_id] + prefixes.insert( + 0, self._partner_domain_validation_prefixes[domain.partner_id] + ) if not domain.ownership_txt_token: domain.ownership_txt_token = random_string(30) Session.commit() - return f"{prefix}-verification={domain.ownership_txt_token}" + valid = [ + f"{prefix}-verification={domain.ownership_txt_token}" for prefix in prefixes + ] + return ExpectedValidationRecords(recommended=valid[0], allowed=valid) - def get_expected_mx_records(self, domain: CustomDomain) -> list[MxRecord]: - records = [] + def get_expected_mx_records( + self, domain: CustomDomain + ) -> dict[int, ExpectedValidationRecords]: + records = {} if domain.partner_id is not None and domain.partner_id in self._partner_domains: domain = self._partner_domains[domain.partner_id] - records.append(MxRecord(10, f"mx1.{domain}.")) - records.append(MxRecord(20, f"mx2.{domain}.")) - else: - # Default ones - for priority, domain in config.EMAIL_SERVERS_WITH_PRIORITY: - records.append(MxRecord(priority, domain)) + records[10] = [f"mx1.{domain}."] + records[20] = [f"mx2.{domain}."] + # Default ones + for priority, domain in config.EMAIL_SERVERS_WITH_PRIORITY: + if priority not in records: + records[priority] = [] + records[priority].append(domain) - return records + return { + priority: ExpectedValidationRecords( + recommended=records[priority][0], allowed=records[priority] + ) + for priority in records + } - def get_expected_spf_domain(self, domain: CustomDomain) -> str: + def get_expected_spf_domain( + self, domain: CustomDomain + ) -> ExpectedValidationRecords: + records = [] if domain.partner_id is not None and domain.partner_id in self._partner_domains: - return self._partner_domains[domain.partner_id] + records.append(self._partner_domains[domain.partner_id]) else: - return config.EMAIL_DOMAIN + records.append(config.EMAIL_DOMAIN) + return ExpectedValidationRecords(recommended=records[0], allowed=records) def get_expected_spf_record(self, domain: CustomDomain) -> str: spf_domain = self.get_expected_spf_domain(domain) - return f"v=spf1 include:{spf_domain} ~all" + return f"v=spf1 include:{spf_domain.recommended} ~all" - def get_dkim_records(self, domain: CustomDomain) -> {str: str}: + def get_dkim_records( + self, domain: CustomDomain + ) -> {str: ExpectedValidationRecords}: """ Get a list of dkim records to set up. Depending on the custom_domain, whether if it's from a partner or not, it will return the default ones or the partner ones. """ # By default use the default domain - dkim_domain = self.dkim_domain + dkim_domains = [self.dkim_domain] if domain.partner_id is not None: - # Domain is from a partner. Retrieve the partner config and use that domain if exists - dkim_domain = self._partner_domains.get(domain.partner_id, dkim_domain) + # Domain is from a partner. Retrieve the partner config and use that domain as preferred if it exists + partner_domain = self._partner_domains.get(domain.partner_id, None) + if partner_domain is not None: + dkim_domains.insert(0, partner_domain) - return { - f"{key}._domainkey": f"{key}._domainkey.{dkim_domain}" - for key in ("dkim", "dkim02", "dkim03") - } + output = {} + for key in ("dkim", "dkim02", "dkim03"): + records = [ + f"{key}._domainkey.{dkim_domain}" for dkim_domain in dkim_domains + ] + output[f"{key}._domainkey"] = ExpectedValidationRecords( + recommended=records[0], allowed=records + ) + + return output def validate_dkim_records(self, custom_domain: CustomDomain) -> dict[str, str]: """ @@ -102,7 +162,7 @@ class CustomDomainValidation: for prefix, expected_record in expected_records.items(): custom_record = f"{prefix}.{custom_domain.domain}" dkim_record = self._dns_client.get_cname_record(custom_record) - if dkim_record == expected_record: + if dkim_record in expected_record.allowed: correct_records[prefix] = custom_record else: invalid_records[custom_record] = dkim_record or "empty" @@ -138,11 +198,15 @@ class CustomDomainValidation: Check if the custom_domain has added the ownership verification records """ txt_records = self._dns_client.get_txt_record(custom_domain.domain) - expected_verification_record = self.get_ownership_verification_record( + expected_verification_records = self.get_ownership_verification_record( custom_domain ) - - if expected_verification_record in txt_records: + found = False + for verification_record in expected_verification_records.allowed: + if verification_record in txt_records: + found = True + break + if found: custom_domain.ownership_verified = True emit_user_audit_log( user=custom_domain.user, @@ -161,10 +225,11 @@ class CustomDomainValidation: expected_mx_records = self.get_expected_mx_records(custom_domain) if not is_mx_equivalent(mx_domains, expected_mx_records): - return DomainValidationResult( - success=False, - errors=[f"{record.priority} {record.domain}" for record in mx_domains], - ) + errors = [] + for prio in mx_domains: + for mx_domain in mx_domains[prio]: + errors.append(f"{prio} {mx_domain}") + return DomainValidationResult(success=False, errors=errors) else: custom_domain.verified = True emit_user_audit_log( @@ -180,7 +245,7 @@ class CustomDomainValidation: ) -> DomainValidationResult: spf_domains = self._dns_client.get_spf_domain(custom_domain.domain) expected_spf_domain = self.get_expected_spf_domain(custom_domain) - if expected_spf_domain in spf_domains: + if len(set(expected_spf_domain.allowed).intersection(set(spf_domains))) > 0: custom_domain.spf_verified = True emit_user_audit_log( user=custom_domain.user, @@ -221,8 +286,8 @@ class CustomDomainValidation: self, txt_records: List[str], custom_domain: CustomDomain ) -> List[str]: final_records = [] - verification_record = self.get_ownership_verification_record(custom_domain) + verification_records = self.get_ownership_verification_record(custom_domain) for record in txt_records: - if record != verification_record: + if record not in verification_records.allowed: final_records.append(record) return final_records diff --git a/app/app/dashboard/views/domain_detail.py b/app/app/dashboard/views/domain_detail.py index 2606f6d..d239afb 100644 --- a/app/app/dashboard/views/domain_detail.py +++ b/app/app/dashboard/views/domain_detail.py @@ -5,8 +5,8 @@ from flask_login import login_required, current_user from flask_wtf import FlaskForm from wtforms import StringField, validators, IntegerField -from app.constants import DMARC_RECORD from app.config import EMAIL_SERVERS_WITH_PRIORITY, EMAIL_DOMAIN +from app.constants import DMARC_RECORD from app.custom_domain_utils import delete_custom_domain, set_custom_domain_mailboxes from app.custom_domain_validation import CustomDomainValidation from app.dashboard.base import dashboard_bp @@ -137,7 +137,7 @@ def domain_detail_dns(custom_domain_id): return render_template( "dashboard/domain_detail/dns.html", EMAIL_SERVERS_WITH_PRIORITY=EMAIL_SERVERS_WITH_PRIORITY, - ownership_record=domain_validator.get_ownership_verification_record( + ownership_records=domain_validator.get_ownership_verification_record( custom_domain ), expected_mx_records=domain_validator.get_expected_mx_records(custom_domain), diff --git a/app/app/dashboard/views/mailbox_detail.py b/app/app/dashboard/views/mailbox_detail.py index ebc642a..070ad4b 100644 --- a/app/app/dashboard/views/mailbox_detail.py +++ b/app/app/dashboard/views/mailbox_detail.py @@ -267,12 +267,13 @@ def cancel_mailbox_change_route(mailbox_id): @dashboard_bp.route("/mailbox/confirm_change") +@login_required +@limiter.limit("3/minute") def mailbox_confirm_email_change_route(): mailbox_id = request.args.get("mailbox_id") code = request.args.get("code") if code: - print("HAS OCO", code) try: mailbox = mailbox_utils.verify_mailbox_code(current_user, mailbox_id, code) flash("Successfully changed mailbox email", "success") @@ -280,7 +281,6 @@ def mailbox_confirm_email_change_route(): url_for("dashboard.mailbox_detail_route", mailbox_id=mailbox.id) ) except mailbox_utils.MailboxError as e: - print(e) flash(f"Cannot verify mailbox: {e.msg}", "error") return redirect(url_for("dashboard.mailbox_route")) else: diff --git a/app/app/dns_utils.py b/app/app/dns_utils.py index 202f109..02ca784 100644 --- a/app/app/dns_utils.py +++ b/app/app/dns_utils.py @@ -1,5 +1,4 @@ from abc import ABC, abstractmethod -from dataclasses import dataclass from typing import List, Optional import dns.resolver @@ -9,42 +8,13 @@ from app.config import NAMESERVERS _include_spf = "include:" -@dataclass -class MxRecord: - priority: int - domain: str - - -def is_mx_equivalent( - mx_domains: List[MxRecord], ref_mx_domains: List[MxRecord] -) -> bool: - """ - Compare mx_domains with ref_mx_domains to see if they are equivalent. - mx_domains and ref_mx_domains are list of (priority, domain) - - The priority order is taken into account but not the priority number. - For example, [(1, domain1), (2, domain2)] is equivalent to [(10, domain1), (20, domain2)] - """ - mx_domains = sorted(mx_domains, key=lambda x: x.priority) - ref_mx_domains = sorted(ref_mx_domains, key=lambda x: x.priority) - - if len(mx_domains) < len(ref_mx_domains): - return False - - for actual, expected in zip(mx_domains, ref_mx_domains): - if actual.domain != expected.domain: - return False - - return True - - class DNSClient(ABC): @abstractmethod def get_cname_record(self, hostname: str) -> Optional[str]: pass @abstractmethod - def get_mx_domains(self, hostname: str) -> List[MxRecord]: + def get_mx_domains(self, hostname: str) -> dict[int, list[str]]: pass def get_spf_domain(self, hostname: str) -> List[str]: @@ -88,21 +58,24 @@ class NetworkDNSClient(DNSClient): except Exception: return None - def get_mx_domains(self, hostname: str) -> List[MxRecord]: + def get_mx_domains(self, hostname: str) -> dict[int, list[str]]: """ return list of (priority, domain name) sorted by priority (lowest priority first) domain name ends with a "." at the end. """ + ret = {} try: answers = self._resolver.resolve(hostname, "MX", search=True) - ret = [] for a in answers: record = a.to_text() # for ex '20 alt2.aspmx.l.google.com.' parts = record.split(" ") - ret.append(MxRecord(priority=int(parts[0]), domain=parts[1])) - return sorted(ret, key=lambda x: x.priority) + prio = int(parts[0]) + if prio not in ret: + ret[prio] = [] + ret[prio].append(parts[1]) except Exception: - return [] + pass + return ret def get_txt_record(self, hostname: str) -> List[str]: try: @@ -119,14 +92,14 @@ class NetworkDNSClient(DNSClient): class InMemoryDNSClient(DNSClient): def __init__(self): self.cname_records: dict[str, Optional[str]] = {} - self.mx_records: dict[str, List[MxRecord]] = {} + self.mx_records: dict[int, dict[int, list[str]]] = {} self.spf_records: dict[str, List[str]] = {} self.txt_records: dict[str, List[str]] = {} def set_cname_record(self, hostname: str, cname: str): self.cname_records[hostname] = cname - def set_mx_records(self, hostname: str, mx_list: List[MxRecord]): + def set_mx_records(self, hostname: str, mx_list: dict[int, list[str]]): self.mx_records[hostname] = mx_list def set_txt_record(self, hostname: str, txt_list: List[str]): @@ -135,9 +108,8 @@ class InMemoryDNSClient(DNSClient): def get_cname_record(self, hostname: str) -> Optional[str]: return self.cname_records.get(hostname) - def get_mx_domains(self, hostname: str) -> List[MxRecord]: - mx_list = self.mx_records.get(hostname, []) - return sorted(mx_list, key=lambda x: x.priority) + def get_mx_domains(self, hostname: str) -> dict[int, list[str]]: + return self.mx_records.get(hostname, {}) def get_txt_record(self, hostname: str) -> List[str]: return self.txt_records.get(hostname, []) @@ -147,5 +119,5 @@ def get_network_dns_client() -> NetworkDNSClient: return NetworkDNSClient(NAMESERVERS) -def get_mx_domains(hostname: str) -> List[MxRecord]: +def get_mx_domains(hostname: str) -> dict[int, list[str]]: return get_network_dns_client().get_mx_domains(hostname) diff --git a/app/app/email_utils.py b/app/app/email_utils.py index 221f7d6..731b098 100644 --- a/app/app/email_utils.py +++ b/app/app/email_utils.py @@ -657,7 +657,11 @@ def get_mx_domain_list(domain) -> [str]: """ priority_domains = get_mx_domains(domain) - return [d.domain[:-1] for d in priority_domains] + mx_domains = [] + for prio in priority_domains: + for domain in priority_domains[prio]: + mx_domains.append(domain[:-1]) + return mx_domains def personal_email_already_used(email_address: str) -> bool: diff --git a/app/app/log.py b/app/app/log.py index a674698..2d22081 100644 --- a/app/app/log.py +++ b/app/app/log.py @@ -10,7 +10,7 @@ from app.config import ( # this format allows clickable link to code source in PyCharm _log_format = ( - "%(asctime)s - %(name)s - %(levelname)s - %(process)d - " + "%(asctime)s - %(name)s - %(levelname)s - %(process)d - %(request_id)s" '"%(pathname)s:%(lineno)d" - %(funcName)s() - %(message_id)s - %(message)s' ) _log_formatter = logging.Formatter(_log_format) @@ -37,6 +37,21 @@ class EmailHandlerFilter(logging.Filter): return _MESSAGE_ID +class RequestIdFilter(logging.Filter): + """automatically add request-id to keep track of a request""" + + def filter(self, record): + from flask import g, has_request_context + + request_id = "" + if has_request_context(): + ctx_request_id = getattr(g, "request_id") + if ctx_request_id: + request_id = f"{ctx_request_id} - " + record.request_id = request_id + return True + + def _get_console_handler(): console_handler = logging.StreamHandler(sys.stdout) console_handler.setFormatter(_log_formatter) @@ -54,6 +69,7 @@ def _get_logger(name) -> logging.Logger: logger.addHandler(_get_console_handler()) logger.addFilter(EmailHandlerFilter()) + logger.addFilter(RequestIdFilter()) # no propagation to avoid propagating to root logger logger.propagate = False diff --git a/app/app/models.py b/app/app/models.py index 32d017e..2f57293 100644 --- a/app/app/models.py +++ b/app/app/models.py @@ -32,7 +32,6 @@ from app import config, rate_limiter from app import s3 from app.db import Session from app.dns_utils import get_mx_domains - from app.errors import ( AliasInTrashError, DirectoryInTrashError, @@ -362,7 +361,7 @@ class User(Base, ModelMixin, UserMixin, PasswordOracle): sa.Boolean, default=True, nullable=False, server_default="1" ) - activated = sa.Column(sa.Boolean, default=False, nullable=False, index=True) + activated = sa.Column(sa.Boolean, default=False, nullable=False) # an account can be disabled if having harmful behavior disabled = sa.Column(sa.Boolean, default=False, nullable=False, server_default="0") @@ -576,6 +575,12 @@ class User(Base, ModelMixin, UserMixin, PasswordOracle): "ix_users_default_alias_custom_domain_id", default_alias_custom_domain_id ), sa.Index("ix_users_profile_picture_id", profile_picture_id), + sa.Index( + "idx_users_email_trgm", + "email", + postgresql_ops={"email": "gin_trgm_ops"}, + postgresql_using="gin", + ), ) @property @@ -1924,13 +1929,16 @@ class Contact(Base, ModelMixin): __table_args__ = ( sa.UniqueConstraint("alias_id", "website_email", name="uq_contact"), + sa.Index("ix_contact_user_id_id", "user_id", "id"), ) user_id = sa.Column( - sa.ForeignKey(User.id, ondelete="cascade"), nullable=False, index=True + sa.ForeignKey(User.id, ondelete="cascade"), + nullable=False, ) alias_id = sa.Column( - sa.ForeignKey(Alias.id, ondelete="cascade"), nullable=False, index=True + sa.ForeignKey(Alias.id, ondelete="cascade"), + nullable=False, ) name = sa.Column( @@ -2115,11 +2123,10 @@ class EmailLog(Base, ModelMixin): Index("ix_email_log_mailbox_id", "mailbox_id"), Index("ix_email_log_bounced_mailbox_id", "bounced_mailbox_id"), Index("ix_email_log_refused_email_id", "refused_email_id"), + Index("ix_email_log_user_id_email_log_id", "user_id", "id"), ) - user_id = sa.Column( - sa.ForeignKey(User.id, ondelete="cascade"), nullable=False, index=True - ) + user_id = sa.Column(sa.ForeignKey(User.id, ondelete="cascade"), nullable=False) contact_id = sa.Column( sa.ForeignKey(Contact.id, ondelete="cascade"), nullable=False, index=True ) @@ -2395,7 +2402,8 @@ class AliasUsedOn(Base, ModelMixin): ) alias_id = sa.Column( - sa.ForeignKey(Alias.id, ondelete="cascade"), nullable=False, index=True + sa.ForeignKey(Alias.id, ondelete="cascade"), + nullable=False, ) user_id = sa.Column(sa.ForeignKey(User.id, ondelete="cascade"), nullable=False) @@ -2418,10 +2426,7 @@ class ApiKey(Base, ModelMixin): user = orm.relationship(User) - __table_args__ = ( - sa.Index("ix_api_key_code", "code"), - sa.Index("ix_api_key_user_id", "user_id"), - ) + __table_args__ = (sa.Index("ix_api_key_user_id", "user_id"),) @classmethod def create(cls, user_id, name=None, **kwargs): @@ -2581,7 +2586,6 @@ class AutoCreateRule(Base, ModelMixin): sa.UniqueConstraint( "custom_domain_id", "order", name="uq_auto_create_rule_order" ), - sa.Index("ix_auto_create_rule_custom_domain_id", "custom_domain_id"), ) custom_domain_id = sa.Column( @@ -2764,7 +2768,6 @@ class Job(Base, ModelMixin): nullable=False, server_default=str(JobState.ready.value), default=JobState.ready.value, - index=True, ) attempts = sa.Column(sa.Integer, nullable=False, server_default="0", default=0) taken_at = sa.Column(ArrowType, nullable=True) @@ -2777,9 +2780,7 @@ class Job(Base, ModelMixin): class Mailbox(Base, ModelMixin): __tablename__ = "mailbox" - user_id = sa.Column( - sa.ForeignKey(User.id, ondelete="cascade"), nullable=False, index=True - ) + user_id = sa.Column(sa.ForeignKey(User.id, ondelete="cascade"), nullable=False) email = sa.Column(sa.String(256), nullable=False, index=True) verified = sa.Column(sa.Boolean, default=False, nullable=False) force_spf = sa.Column(sa.Boolean, default=True, server_default="1", nullable=False) @@ -2808,6 +2809,13 @@ class Mailbox(Base, ModelMixin): __table_args__ = ( sa.UniqueConstraint("user_id", "email", name="uq_mailbox_user"), sa.Index("ix_mailbox_pgp_finger_print", "pgp_finger_print"), + # index on email column using pg_trgm + Index( + "ix_mailbox_email_trgm_idx", + "email", + postgresql_ops={"email": "gin_trgm_ops"}, + postgresql_using="gin", + ), ) user = orm.relationship(User, foreign_keys=[user_id]) @@ -3010,7 +3018,11 @@ class SentAlert(Base, ModelMixin): to_email = sa.Column(sa.String(256), nullable=False) alert_type = sa.Column(sa.String(256), nullable=False) - __table_args__ = (sa.Index("ix_sent_alert_user_id", "user_id"),) + __table_args__ = ( + sa.Index("ix_sent_alert_user_id", "user_id"), + sa.Index("ix_sent_alert_to_email", "to_email"), + sa.Index("ix_sent_alert_alert_type", "alert_type"), + ) class AliasMailbox(Base, ModelMixin): @@ -3020,7 +3032,8 @@ class AliasMailbox(Base, ModelMixin): ) alias_id = sa.Column( - sa.ForeignKey(Alias.id, ondelete="cascade"), nullable=False, index=True + sa.ForeignKey(Alias.id, ondelete="cascade"), + nullable=False, ) mailbox_id = sa.Column( sa.ForeignKey(Mailbox.id, ondelete="cascade"), nullable=False, index=True @@ -3035,7 +3048,8 @@ class AliasHibp(Base, ModelMixin): __table_args__ = (sa.UniqueConstraint("alias_id", "hibp_id", name="uq_alias_hibp"),) alias_id = sa.Column( - sa.Integer(), sa.ForeignKey("alias.id", ondelete="cascade"), index=True + sa.Integer(), + sa.ForeignKey("alias.id", ondelete="cascade"), ) hibp_id = sa.Column( sa.Integer(), sa.ForeignKey("hibp.id", ondelete="cascade"), index=True @@ -3751,7 +3765,8 @@ class PartnerUser(Base, ModelMixin): index=True, ) partner_id = sa.Column( - sa.ForeignKey("partner.id", ondelete="cascade"), nullable=False, index=True + sa.ForeignKey("partner.id", ondelete="cascade"), + nullable=False, ) external_user_id = sa.Column(sa.String(128), unique=False, nullable=False) partner_email = sa.Column(sa.String(255), unique=False, nullable=True) diff --git a/app/app/request_utils.py b/app/app/request_utils.py new file mode 100644 index 0000000..b1fb66a --- /dev/null +++ b/app/app/request_utils.py @@ -0,0 +1,6 @@ +from random import randbytes +from base64 import b64encode + + +def generate_request_id() -> str: + return b64encode(randbytes(6)).decode() diff --git a/app/cron.py b/app/cron.py index af86bb6..76513f9 100644 --- a/app/cron.py +++ b/app/cron.py @@ -14,9 +14,9 @@ from sqlalchemy.sql import Insert, text from app import s3, config from app.alias_utils import nb_email_log_for_mailbox from app.api.views.apple import verify_receipt -from app.custom_domain_validation import CustomDomainValidation +from app.custom_domain_validation import CustomDomainValidation, is_mx_equivalent from app.db import Session -from app.dns_utils import get_mx_domains, is_mx_equivalent +from app.dns_utils import get_mx_domains from app.email_utils import ( send_email, send_trial_end_soon_email, diff --git a/app/migrations/versions/2025_013015_d3ff8848c930_index_cleanup.py b/app/migrations/versions/2025_013015_d3ff8848c930_index_cleanup.py new file mode 100644 index 0000000..59aa29f --- /dev/null +++ b/app/migrations/versions/2025_013015_d3ff8848c930_index_cleanup.py @@ -0,0 +1,91 @@ +"""index cleanup + +Revision ID: d3ff8848c930 +Revises: 085f77996ce3 +Create Date: 2025-01-30 15:00:02.995813 + +""" +from alembic import op + + +# revision identifiers, used by Alembic. +revision = "d3ff8848c930" +down_revision = "085f77996ce3" +branch_labels = None +depends_on = None + + +def upgrade(): + with op.get_context().autocommit_block(): + op.drop_index("ix_alias_hibp_alias_id", table_name="alias_hibp") + op.drop_index("ix_alias_mailbox_alias_id", table_name="alias_mailbox") + op.drop_index("ix_alias_used_on_alias_id", table_name="alias_used_on") + op.drop_index("ix_api_key_code", table_name="api_key") + op.drop_index( + "ix_auto_create_rule_custom_domain_id", table_name="auto_create_rule" + ) + op.drop_index("ix_contact_alias_id", table_name="contact") + op.create_index( + "ix_email_log_user_id_email_log_id", + "email_log", + ["user_id", "id"], + unique=False, + ) + op.drop_index("ix_job_state", table_name="job") + op.create_index( + "ix_mailbox_email_trgm_idx", + "mailbox", + ["email"], + unique=False, + postgresql_ops={"email": "gin_trgm_ops"}, + postgresql_using="gin", + ) + op.drop_index("ix_partner_user_partner_id", table_name="partner_user") + op.create_index( + "ix_sent_alert_alert_type", "sent_alert", ["alert_type"], unique=False + ) + op.create_index( + "ix_sent_alert_to_email", "sent_alert", ["to_email"], unique=False + ) + op.create_index( + "idx_users_email_trgm", + "users", + ["email"], + unique=False, + postgresql_ops={"email": "gin_trgm_ops"}, + postgresql_using="gin", + ) + op.drop_index("ix_users_activated", table_name="users") + op.drop_index("ix_mailbox_user_id", table_name="users") + + +def downgrade(): + with op.get_context().autocommit_block(): + op.create_index("ix_users_activated", "users", ["activated"], unique=False) + op.drop_index("idx_users_email_trgm", table_name="users") + op.drop_index("ix_sent_alert_to_email", table_name="sent_alert") + op.drop_index("ix_sent_alert_alert_type", table_name="sent_alert") + op.create_index( + "ix_partner_user_partner_id", "partner_user", ["partner_id"], unique=False + ) + op.drop_index("ix_mailbox_email_trgm_idx", table_name="mailbox") + op.create_index("ix_job_state", "job", ["state"], unique=False) + op.drop_index("ix_email_log_user_id_email_log_id", table_name="email_log") + op.create_index("ix_contact_alias_id", "contact", ["alias_id"], unique=False) + op.create_index( + "ix_auto_create_rule_custom_domain_id", + "auto_create_rule", + ["custom_domain_id"], + unique=False, + ) + op.create_index("ix_api_key_code", "api_key", ["code"], unique=False) + op.create_index( + "ix_alias_used_on_alias_id", "alias_used_on", ["alias_id"], unique=False + ) + op.create_index( + "ix_alias_mailbox_alias_id", "alias_mailbox", ["alias_id"], unique=False + ) + op.create_index( + "ix_alias_hibp_alias_id", "alias_hibp", ["alias_id"], unique=False + ) + op.create_index("ix_mailbox_user_id", "users", ["user_id"], unique=False) diff --git a/app/migrations/versions/2025_013114_97edba8794f8_index_cleanup.py b/app/migrations/versions/2025_013114_97edba8794f8_index_cleanup.py new file mode 100644 index 0000000..7152f4b --- /dev/null +++ b/app/migrations/versions/2025_013114_97edba8794f8_index_cleanup.py @@ -0,0 +1,23 @@ +"""index cleanup + +Revision ID: 97edba8794f8 +Revises: d3ff8848c930 +Create Date: 2025-01-31 14:42:22.590597 + +""" +from alembic import op + + +# revision identifiers, used by Alembic. +revision = '97edba8794f8' +down_revision = 'd3ff8848c930' +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_index('ix_email_log_user_id', table_name='email_log') + + +def downgrade(): + op.create_index('ix_email_log_user_id', 'email_log', ['user_id'], unique=False) diff --git a/app/migrations/versions/2025_020316_20e7d3ca289a_contact_index.py b/app/migrations/versions/2025_020316_20e7d3ca289a_contact_index.py new file mode 100644 index 0000000..c28edf9 --- /dev/null +++ b/app/migrations/versions/2025_020316_20e7d3ca289a_contact_index.py @@ -0,0 +1,27 @@ +"""contact index + +Revision ID: 20e7d3ca289a +Revises: 97edba8794f8 +Create Date: 2025-02-03 16:52:06.775032 + +""" +from alembic import op + + +# revision identifiers, used by Alembic. +revision = '20e7d3ca289a' +down_revision = '97edba8794f8' +branch_labels = None +depends_on = None + + +def upgrade(): + with op.get_context().autocommit_block(): + op.create_index('ix_contact_user_id_id', 'contact', ['user_id', 'id'], unique=False) + op.drop_index('ix_contact_user_id', table_name='contact') + + +def downgrade(): + with op.get_context().autocommit_block(): + op.create_index('ix_contact_user_id', 'contact', ['user_id'], unique=False) + op.drop_index('ix_contact_user_id_id', table_name='contact') diff --git a/app/server.py b/app/server.py index 507b885..ece49e4 100644 --- a/app/server.py +++ b/app/server.py @@ -106,6 +106,7 @@ from app.payments.coinbase import setup_coinbase_commerce from app.payments.paddle import setup_paddle_callback from app.phone.base import phone_bp from app.redis_services import initialize_redis_services +from app.request_utils import generate_request_id from app.sentry_utils import sentry_before_send if SENTRY_DSN: @@ -263,6 +264,7 @@ def set_index_page(app): and not request.path.startswith("/_debug_toolbar") ): g.start_time = time.time() + g.request_id = generate_request_id() # to handle the referral url that has ?slref=code part ref_code = request.args.get("slref") diff --git a/app/templates/admin/custom_domain_search.html b/app/templates/admin/custom_domain_search.html index 0ab0665..4de9bbd 100644 --- a/app/templates/admin/custom_domain_search.html +++ b/app/templates/admin/custom_domain_search.html @@ -1,118 +1,181 @@ {% extends 'admin/master.html' %} +{% block head_css %} + + {{ super() }} + +{% endblock %} {% macro show_user(user) -%} -

User {{ user.email }} with ID {{ user.id }}.

- - - - - - - - - - - - - - - - {% if user.activated %} +

+ User {{ user.email }} with ID {{ user.id }}. +

+
User IDEmailVerifiedStatusPaidPremium
{{ user.id }} - {{ user.email }} -
+ + + + + + + + + + + + + + + {% if user.activated %} - - {% else %} - - {% endif %} - {% if user.disabled %} + + {% else %} + + {% endif %} + {% if user.disabled %} - - {% else %} - - {% endif %} - - - - -
User IDEmailVerifiedStatusPaidPremium
{{ user.id }} + {{ user.email }} + ActivatedPendingActivatedPendingDisabledEnabled{{ "yes" if user.is_paid() else "No" }}{{ "yes" if user.is_premium() else "No" }}
+ Disabled + {% else %} + Enabled + {% endif %} + {{ "yes" if user.is_paid() else "No" }} + {{ "yes" if user.is_premium() else "No" }} + + + {%- endmacro %} - - {% macro show_verification(title, expected, errors) -%} - {% if not expected %} -

{{ title }} Verified

- {% else %} -

{{ title }}

-

Expected

-

{{expected}}

-

Current response

-