4.51.0

app/app/contact_utils.py

@@ -0,0 +1,89 @@
+from dataclasses import dataclass
+from enum import Enum
+from typing import Optional
+
+from sqlalchemy.exc import IntegrityError
+
+from app.db import Session
+from app.email_utils import generate_reply_email
+from app.email_validation import is_valid_email
+from app.log import LOG
+from app.models import Contact, Alias
+from app.utils import sanitize_email
+
+
+class ContactCreateError(Enum):
+    InvalidEmail = "Invalid email"
+
+
+@dataclass
+class ContactCreateResult:
+    contact: Optional[Contact]
+    error: Optional[ContactCreateError]
+
+
+def __update_contact_if_needed(
+    contact: Contact, name: Optional[str], mail_from: Optional[str]
+) -> ContactCreateResult:
+    if name and contact.name != name:
+        LOG.d(f"Setting {contact} name to {name}")
+        contact.name = name
+        Session.commit()
+    if mail_from and contact.mail_from is None:
+        LOG.d(f"Setting {contact} mail_from to {mail_from}")
+        contact.mail_from = mail_from
+        Session.commit()
+    return ContactCreateResult(contact, None)
+
+
+def create_contact(
+    email: str,
+    name: Optional[str],
+    alias: Alias,
+    mail_from: Optional[str] = None,
+    allow_empty_email: bool = False,
+    automatic_created: bool = False,
+    from_partner: bool = False,
+) -> ContactCreateResult:
+    if name is not None:
+        name = name[: Contact.MAX_NAME_LENGTH]
+    if name is not None and "\x00" in name:
+        LOG.w("Cannot use contact name because has \\x00")
+        name = ""
+    if not is_valid_email(email):
+        LOG.w(f"invalid contact email {email}")
+        if not allow_empty_email:
+            return ContactCreateResult(None, ContactCreateError.InvalidEmail)
+        LOG.d("Create a contact with invalid email for %s", alias)
+        # either reuse a contact with empty email or create a new contact with empty email
+        email = ""
+    email = sanitize_email(email, not_lower=True)
+    contact = Contact.get_by(alias_id=alias.id, website_email=email)
+    if contact is not None:
+        return __update_contact_if_needed(contact, name, mail_from)
+    reply_email = generate_reply_email(email, alias)
+    try:
+        flags = Contact.FLAG_PARTNER_CREATED if from_partner else 0
+        contact = Contact.create(
+            user_id=alias.user_id,
+            alias_id=alias.id,
+            website_email=email,
+            name=name,
+            reply_email=reply_email,
+            mail_from=mail_from,
+            automatic_created=automatic_created,
+            flags=flags,
+            invalid_email=email == "",
+            commit=True,
+        )
+        LOG.d(
+            f"Created contact {contact} for alias {alias} with email {email} invalid_email={contact.invalid_email}"
+        )
+    except IntegrityError:
+        Session.rollback()
+        LOG.info(
+            f"Contact with email {email} for alias_id {alias.id} already existed, fetching from DB"
+        )
+        contact = Contact.get_by(alias_id=alias.id, website_email=email)
+        return __update_contact_if_needed(contact, name, mail_from)
+    return ContactCreateResult(contact, None)

app/app/custom_domain_utils.py

@@ -1,13 +1,15 @@
+import arrow
 import re
 
 from dataclasses import dataclass
 from enum import Enum
 from typing import Optional
 
+from app.config import JOB_DELETE_DOMAIN
 from app.db import Session
 from app.email_utils import get_email_domain_part
 from app.log import LOG
-from app.models import User, CustomDomain, SLDomain, Mailbox
+from app.models import User, CustomDomain, SLDomain, Mailbox, Job
 
 _ALLOWED_DOMAIN_REGEX = re.compile(r"^(?!-)[A-Za-z0-9-]{1,63}(?<!-)$")
 
@@ -126,3 +128,15 @@ def create_custom_domain(
         success=True,
         instance=new_custom_domain,
     )
+
+
+def delete_custom_domain(domain: CustomDomain):
+    # Schedule delete domain job
+    LOG.w("schedule delete domain job for %s", domain)
+    domain.pending_deletion = True
+    Job.create(
+        name=JOB_DELETE_DOMAIN,
+        payload={"custom_domain_id": domain.id},
+        run_at=arrow.now(),
+        commit=True,
+    )

app/app/dashboard/views/domain_detail.py

@@ -1,17 +1,16 @@
 import re
 
-import arrow
 from flask import render_template, request, redirect, url_for, flash
 from flask_login import login_required, current_user
 from flask_wtf import FlaskForm
 from wtforms import StringField, validators, IntegerField
 
 from app.constants import DMARC_RECORD
-from app.config import EMAIL_SERVERS_WITH_PRIORITY, EMAIL_DOMAIN, JOB_DELETE_DOMAIN
+from app.config import EMAIL_SERVERS_WITH_PRIORITY, EMAIL_DOMAIN
+from app.custom_domain_utils import delete_custom_domain
 from app.custom_domain_validation import CustomDomainValidation
 from app.dashboard.base import dashboard_bp
 from app.db import Session
-from app.log import LOG
 from app.models import (
     CustomDomain,
     Alias,
@@ -20,7 +19,6 @@ from app.models import (
     DomainMailbox,
     AutoCreateRule,
     AutoCreateRuleMailbox,
-    Job,
 )
 from app.regex_utils import regex_match
 from app.utils import random_string, CSRFValidationForm
@@ -263,16 +261,8 @@ def domain_detail(custom_domain_id):
 
         elif request.form.get("form-name") == "delete":
             name = custom_domain.domain
-            LOG.d("Schedule deleting %s", custom_domain)
 
-            # Schedule delete domain job
-            LOG.w("schedule delete domain job for %s", custom_domain)
-            Job.create(
-                name=JOB_DELETE_DOMAIN,
-                payload={"custom_domain_id": custom_domain.id},
-                run_at=arrow.now(),
-                commit=True,
-            )
+            delete_custom_domain(custom_domain)
 
             flash(
                 f"{name} scheduled for deletion."

app/app/events/event_dispatcher.py

@@ -70,9 +70,7 @@ class EventDispatcher:
 
         partner_user = EventDispatcher.__partner_user(user.id)
         if not partner_user:
-            LOG.i(
-                f"Not sending events because there's no partner user for  user {user}"
-            )
+            LOG.i(f"Not sending events because there's no partner user for user {user}")
             return
 
         event = event_pb2.Event(
@@ -84,7 +82,9 @@ class EventDispatcher:
 
         serialized = event.SerializeToString()
         dispatcher.send(serialized)
-        newrelic.agent.record_custom_metric("Custom/events_stored", 1)
+
+        event_type = content.WhichOneof("content")
+        newrelic.agent.record_custom_event("EventStoredToDb", {"type": event_type})
         LOG.i("Sent event to the dispatcher")
 
     @staticmethod

app/app/models.py

@@ -1863,6 +1863,8 @@ class Contact(Base, ModelMixin):
 
     MAX_NAME_LENGTH = 512
 
+    FLAG_PARTNER_CREATED = 1 << 0
+
     __tablename__ = "contact"
 
     __table_args__ = (
@@ -1921,6 +1923,9 @@ class Contact(Base, ModelMixin):
     # whether contact is created automatically during the forward phase
     automatic_created = sa.Column(sa.Boolean, nullable=True, default=False)
 
+    # contact flags
+    flags = sa.Column(sa.Integer, nullable=False, default=0, server_default="0")
+
     @property
     def email(self):
         return self.website_email
@@ -2427,6 +2432,10 @@ class CustomDomain(Base, ModelMixin):
         server_default=None,
     )
 
+    pending_deletion = sa.Column(
+        sa.Boolean, nullable=False, default=False, server_default="0"
+    )
+
     __table_args__ = (
         Index(
             "ix_unique_domain",  # Index name

app/email_handler.py

@@ -52,7 +52,7 @@ from flanker.addresslib import address
 from flanker.addresslib.address import EmailAddress
 from sqlalchemy.exc import IntegrityError
 
-from app import pgp_utils, s3, config
+from app import pgp_utils, s3, config, contact_utils
 from app.alias_utils import try_auto_create, change_alias_status
 from app.config import (
     EMAIL_DOMAIN,
@@ -195,79 +195,16 @@ def get_or_create_contact(from_header: str, mail_from: str, alias: Alias) -> Con
                 mail_from,
             )
             contact_email = mail_from
-
-    if not is_valid_email(contact_email):
-        LOG.w(
-            "invalid contact email %s. Parse from %s %s",
-            contact_email,
-            from_header,
-            mail_from,
-        )
-        # either reuse a contact with empty email or create a new contact with empty email
-        contact_email = ""
-
-    contact_email = sanitize_email(contact_email, not_lower=True)
-
-    if contact_name and "\x00" in contact_name:
-        LOG.w("issue with contact name %s", contact_name)
-        contact_name = ""
-
-    contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
-    if contact:
-        if contact.name != contact_name:
-            LOG.d(
-                "Update contact %s name %s to %s",
-                contact,
-                contact.name,
-                contact_name,
-            )
-            contact.name = contact_name
-            Session.commit()
-
-        # contact created in the past does not have mail_from and from_header field
-        if not contact.mail_from and mail_from:
-            LOG.d(
-                "Set contact mail_from %s: %s to %s",
-                contact,
-                contact.mail_from,
-                mail_from,
-            )
-            contact.mail_from = mail_from
-            Session.commit()
-    else:
-        alias_id = alias.id
-        try:
-            contact_email_for_reply = (
-                contact_email if is_valid_email(contact_email) else ""
-            )
-            contact = Contact.create(
-                user_id=alias.user_id,
-                alias_id=alias_id,
-                website_email=contact_email,
-                name=contact_name,
-                mail_from=mail_from,
-                reply_email=generate_reply_email(contact_email_for_reply, alias),
-                automatic_created=True,
-            )
-            if not contact_email:
-                LOG.d("Create a contact with invalid email for %s", alias)
-                contact.invalid_email = True
-
-            LOG.d(
-                "create contact %s for %s, reverse alias:%s",
-                contact_email,
-                alias,
-                contact.reply_email,
-            )
-
-            Session.commit()
-        except IntegrityError:
-            LOG.info(
-                f"Contact with email {contact_email} for alias_id {alias_id} already existed, fetching from DB"
-            )
-            contact = Contact.get_by(alias_id=alias_id, website_email=contact_email)
-
-    return contact
+    contact_result = contact_utils.create_contact(
+        email=contact_email,
+        name=contact_name,
+        alias=alias,
+        mail_from=mail_from,
+        allow_empty_email=True,
+        automatic_created=True,
+        from_partner=False,
+    )
+    return contact_result.contact
 
 
 def get_or_create_reply_to_contact(
@@ -292,33 +229,7 @@ def get_or_create_reply_to_contact(
         )
         return None
 
-    contact = Contact.get_by(alias_id=alias.id, website_email=contact_address)
-    if contact:
-        return contact
-    else:
-        LOG.d(
-            "create contact %s for alias %s via reply-to header %s",
-            contact_address,
-            alias,
-            reply_to_header,
-        )
-
-        try:
-            contact = Contact.create(
-                user_id=alias.user_id,
-                alias_id=alias.id,
-                website_email=contact_address,
-                name=contact_name,
-                reply_email=generate_reply_email(contact_address, alias),
-                automatic_created=True,
-            )
-            Session.commit()
-        except IntegrityError:
-            LOG.w("Contact %s %s already exist", alias, contact_address)
-            Session.rollback()
-            contact = Contact.get_by(alias_id=alias.id, website_email=contact_address)
-
-    return contact
+    return contact_utils.create_contact(contact_address, contact_name, alias).contact
 
 
 def replace_header_when_forward(msg: Message, alias: Alias, header: str):

app/event_listener.py

@@ -9,7 +9,7 @@ from events.runner import Runner
 from events.event_source import DeadLetterEventSource, PostgresEventSource
 from events.event_sink import ConsoleEventSink, HttpEventSink
 
-_DEFAULT_MAX_RETRIES = 100
+_DEFAULT_MAX_RETRIES = 10
 
 
 class Mode(Enum):

app/events/event_sink.py

@@ -27,7 +27,9 @@ class HttpEventSink(EventSink):
             headers={"Content-Type": "application/x-protobuf"},
             verify=not EVENT_WEBHOOK_SKIP_VERIFY_SSL,
         )
-        newrelic.agent.record_custom_event("event_sent", {"http_code": res.status_code})
+        newrelic.agent.record_custom_event(
+            "EventSentToPartner", {"http_code": res.status_code}
+        )
         if res.status_code != 200:
             LOG.warning(
                 f"Failed to send event to webhook: {res.status_code} {res.text}"

app/job_runner.py

@@ -3,7 +3,7 @@ Run scheduled jobs.
 Not meant for running job at precise time (+- 1h)
 """
 import time
-from typing import List
+from typing import List, Optional
 
 import arrow
 from sqlalchemy.sql.expression import or_, and_
@@ -240,7 +240,7 @@ def process_job(job: Job):
 
     elif job.name == config.JOB_DELETE_DOMAIN:
         custom_domain_id = job.payload.get("custom_domain_id")
-        custom_domain = CustomDomain.get(custom_domain_id)
+        custom_domain: Optional[CustomDomain] = CustomDomain.get(custom_domain_id)
         if not custom_domain:
             return
 
@@ -252,16 +252,17 @@ def process_job(job: Job):
 
         LOG.d("Domain %s deleted", domain_name)
 
-        send_email(
-            user.email,
-            f"Your domain {domain_name} has been deleted",
-            f"""Domain {domain_name} along with its aliases are deleted successfully.
+        if custom_domain.partner_id is None:
+            send_email(
+                user.email,
+                f"Your domain {domain_name} has been deleted",
+                f"""Domain {domain_name} along with its aliases are deleted successfully.
 
-Regards,
-SimpleLogin team.
-""",
-            retries=3,
-        )
+    Regards,
+    SimpleLogin team.
+    """,
+                retries=3,
+            )
     elif job.name == config.JOB_SEND_USER_REPORT:
         export_job = ExportUserDataJob.create_from_job(job)
         if export_job:

app/migrations/versions/2024_091915_88dd7a0abf54_contact_flags_custom_domain_pending_deletion.py

@@ -0,0 +1,31 @@
+"""contact.flags and custom_domain.pending_deletion
+
+Revision ID: 88dd7a0abf54
+Revises: 2441b7ff5da9
+Create Date: 2024-09-19 15:41:20.910374
+
+"""
+import sqlalchemy_utils
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '88dd7a0abf54'
+down_revision = '2441b7ff5da9'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('contact', sa.Column('flags', sa.Integer(), server_default='0', nullable=False))
+    op.add_column('custom_domain', sa.Column('pending_deletion', sa.Boolean(), server_default='0', nullable=False))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('custom_domain', 'pending_deletion')
+    op.drop_column('contact', 'flags')
+    # ### end Alembic commands ###

app/monitoring.py

@@ -125,6 +125,21 @@ def log_events_pending_dead_letter():
     )
 
 
+@newrelic.agent.background_task()
+def log_failed_events():
+    r = Session.execute(
+        """
+        SELECT COUNT(*)
+        FROM sync_event
+        WHERE retries >= 10;
+        """,
+    )
+    failed_events = list(r)[0][0]
+
+    LOG.d("number of failed events %s", failed_events)
+    newrelic.agent.record_custom_metric("Custom/sync_events_failed", failed_events)
+
+
 if __name__ == "__main__":
     exporter = MetricExporter(get_newrelic_license())
     while True:
@@ -132,6 +147,7 @@ if __name__ == "__main__":
         log_nb_db_connection()
         log_pending_to_process_events()
         log_events_pending_dead_letter()
+        log_failed_events()
         Session.close()
 
         exporter.run()

app/tests/test_contact_utils.py

@@ -0,0 +1,117 @@
+from typing import Optional
+
+import pytest
+from app.contact_utils import create_contact, ContactCreateError
+from app.db import Session
+from app.models import (
+    Alias,
+    Contact,
+)
+from tests.utils import create_new_user, random_email, random_token
+
+
+def create_provider():
+    # name auto_created from_partner
+    yield ["name", "a@b.c", True, True]
+    yield [None, None, True, True]
+    yield [None, None, False, True]
+    yield [None, None, True, False]
+    yield [None, None, False, False]
+
+
+@pytest.mark.parametrize(
+    "name, mail_from, automatic_created, from_partner", create_provider()
+)
+def test_create_contact(
+    name: Optional[str],
+    mail_from: Optional[str],
+    automatic_created: bool,
+    from_partner: bool,
+):
+    user = create_new_user()
+    alias = Alias.create_new_random(user)
+    Session.commit()
+    email = random_email()
+    contact_result = create_contact(
+        email,
+        name,
+        alias,
+        mail_from=mail_from,
+        automatic_created=automatic_created,
+        from_partner=from_partner,
+    )
+    assert contact_result.error is None
+    contact = contact_result.contact
+    assert contact.user_id == user.id
+    assert contact.alias_id == alias.id
+    assert contact.website_email == email
+    assert contact.name == name
+    assert contact.mail_from == mail_from
+    assert contact.automatic_created == automatic_created
+    assert not contact.invalid_email
+    expected_flags = Contact.FLAG_PARTNER_CREATED if from_partner else 0
+    assert contact.flags == expected_flags
+
+
+def test_create_contact_email_email_not_allowed():
+    user = create_new_user()
+    alias = Alias.create_new_random(user)
+    Session.commit()
+    contact_result = create_contact("", "", alias)
+    assert contact_result.contact is None
+    assert contact_result.error == ContactCreateError.InvalidEmail
+
+
+def test_create_contact_email_email_allowed():
+    user = create_new_user()
+    alias = Alias.create_new_random(user)
+    Session.commit()
+    contact_result = create_contact("", "", alias, allow_empty_email=True)
+    assert contact_result.error is None
+    assert contact_result.contact is not None
+    assert contact_result.contact.website_email == ""
+    assert contact_result.contact.invalid_email
+
+
+def test_do_not_allow_invalid_email():
+    user = create_new_user()
+    alias = Alias.create_new_random(user)
+    Session.commit()
+    contact_result = create_contact("potato", "", alias)
+    assert contact_result.contact is None
+    assert contact_result.error == ContactCreateError.InvalidEmail
+    contact_result = create_contact("asdf\x00@gmail.com", "", alias)
+    assert contact_result.contact is None
+    assert contact_result.error == ContactCreateError.InvalidEmail
+
+
+def test_update_name_for_existing():
+    user = create_new_user()
+    alias = Alias.create_new_random(user)
+    Session.commit()
+    email = random_email()
+    contact_result = create_contact(email, "", alias)
+    assert contact_result.error is None
+    assert contact_result.contact is not None
+    assert contact_result.contact.name == ""
+    name = random_token()
+    contact_result = create_contact(email, name, alias)
+    assert contact_result.error is None
+    assert contact_result.contact is not None
+    assert contact_result.contact.name == name
+
+
+def test_update_mail_from_for_existing():
+    user = create_new_user()
+    alias = Alias.create_new_random(user)
+    Session.commit()
+    email = random_email()
+    contact_result = create_contact(email, "", alias)
+    assert contact_result.error is None
+    assert contact_result.contact is not None
+    assert contact_result.contact.mail_from is None
+    mail_from = random_email()
+    contact_result = create_contact(email, "", alias, mail_from=mail_from)
+    assert contact_result.error is None
+    assert contact_result.contact is not None
+    assert contact_result.contact.mail_from == mail_from