Compare commits

...

4 Commits

Author SHA1 Message Date
fae9d7bc17 4.52.0
All checks were successful
Build-Release-Image / Build-Image (linux/arm64) (push) Successful in 4m44s
Build-Release-Image / Build-Image (linux/amd64) (push) Successful in 4m31s
Build-Release-Image / Merge-Images (push) Successful in 23s
Build-Release-Image / Create-Release (push) Successful in 23s
Build-Release-Image / Notify (push) Successful in 17s
2024-10-01 12:00:06 +01:00
d666f5af3f 4.51.2
All checks were successful
Build-Release-Image / Build-Image (linux/arm64) (push) Successful in 3m33s
Build-Release-Image / Build-Image (linux/amd64) (push) Successful in 3m35s
Build-Release-Image / Merge-Images (push) Successful in 25s
Build-Release-Image / Create-Release (push) Successful in 10s
Build-Release-Image / Notify (push) Successful in 3s
2024-09-28 12:00:06 +01:00
556fae02d5 4.51.1
All checks were successful
Build-Release-Image / Build-Image (linux/amd64) (push) Successful in 3m21s
Build-Release-Image / Build-Image (linux/arm64) (push) Successful in 3m40s
Build-Release-Image / Merge-Images (push) Successful in 23s
Build-Release-Image / Create-Release (push) Successful in 9s
Build-Release-Image / Notify (push) Successful in 4s
2024-09-26 12:00:06 +01:00
fd4c67c3d1 4.51.0
All checks were successful
Build-Release-Image / Build-Image (linux/arm64) (push) Successful in 3m30s
Build-Release-Image / Build-Image (linux/amd64) (push) Successful in 3m31s
Build-Release-Image / Merge-Images (push) Successful in 11s
Build-Release-Image / Create-Release (push) Successful in 9s
Build-Release-Image / Notify (push) Successful in 2s
2024-09-25 12:00:07 +01:00
20 changed files with 496 additions and 179 deletions

View File

@ -424,7 +424,7 @@ def create_contact_route(alias_id):
contact_address = data.get("contact") contact_address = data.get("contact")
try: try:
contact = create_contact(g.user, alias, contact_address) contact = create_contact(alias, contact_address)
except ErrContactErrorUpgradeNeeded as err: except ErrContactErrorUpgradeNeeded as err:
return jsonify(error=err.error_for_user()), 403 return jsonify(error=err.error_for_user()), 403
except (ErrAddressInvalid, CannotCreateContactForReverseAlias) as err: except (ErrAddressInvalid, CannotCreateContactForReverseAlias) as err:

View File

@ -657,3 +657,7 @@ PARTNER_DOMAINS: dict[int, str] = read_partner_dict("PARTNER_DOMAINS")
PARTNER_DOMAIN_VALIDATION_PREFIXES: dict[int, str] = read_partner_dict( PARTNER_DOMAIN_VALIDATION_PREFIXES: dict[int, str] = read_partner_dict(
"PARTNER_DOMAIN_VALIDATION_PREFIXES" "PARTNER_DOMAIN_VALIDATION_PREFIXES"
) )
MAILBOX_VERIFICATION_OVERRIDE_CODE: Optional[str] = os.environ.get(
"MAILBOX_VERIFICATION_OVERRIDE_CODE", None
)

113
app/app/contact_utils.py Normal file
View File

@ -0,0 +1,113 @@
from dataclasses import dataclass
from enum import Enum
from typing import Optional
from sqlalchemy.exc import IntegrityError
from app.db import Session
from app.email_utils import generate_reply_email, parse_full_address
from app.email_validation import is_valid_email
from app.log import LOG
from app.models import Contact, Alias
from app.utils import sanitize_email
class ContactCreateError(Enum):
InvalidEmail = "Invalid email"
NotAllowed = "Your plan does not allow to create contacts"
@dataclass
class ContactCreateResult:
contact: Optional[Contact]
created: bool
error: Optional[ContactCreateError]
def __update_contact_if_needed(
contact: Contact, name: Optional[str], mail_from: Optional[str]
) -> ContactCreateResult:
if name and contact.name != name:
LOG.d(f"Setting {contact} name to {name}")
contact.name = name
Session.commit()
if mail_from and contact.mail_from is None:
LOG.d(f"Setting {contact} mail_from to {mail_from}")
contact.mail_from = mail_from
Session.commit()
return ContactCreateResult(contact, created=False, error=None)
def create_contact(
email: str,
alias: Alias,
name: Optional[str] = None,
mail_from: Optional[str] = None,
allow_empty_email: bool = False,
automatic_created: bool = False,
from_partner: bool = False,
) -> ContactCreateResult:
# If user cannot create contacts, they still need to be created when receiving an email for an alias
if not automatic_created and not alias.user.can_create_contacts():
return ContactCreateResult(
None, created=False, error=ContactCreateError.NotAllowed
)
# Parse emails with form 'name <email>'
try:
email_name, email = parse_full_address(email)
except ValueError:
email = ""
email_name = ""
# If no name is explicitly given try to get it from the parsed email
if name is None:
name = email_name[: Contact.MAX_NAME_LENGTH]
else:
name = name[: Contact.MAX_NAME_LENGTH]
# If still no name is there, make sure the name is None instead of empty string
if not name:
name = None
if name is not None and "\x00" in name:
LOG.w("Cannot use contact name because has \\x00")
name = ""
# Sanitize email and if it's not valid only allow to create a contact if it's explicitly allowed. Otherwise fail
email = sanitize_email(email, not_lower=True)
if not is_valid_email(email):
LOG.w(f"invalid contact email {email}")
if not allow_empty_email:
return ContactCreateResult(
None, created=False, error=ContactCreateError.InvalidEmail
)
LOG.d("Create a contact with invalid email for %s", alias)
# either reuse a contact with empty email or create a new contact with empty email
email = ""
# If contact exists, update name and mail_from if needed
contact = Contact.get_by(alias_id=alias.id, website_email=email)
if contact is not None:
return __update_contact_if_needed(contact, name, mail_from)
# Create the contact
reply_email = generate_reply_email(email, alias)
try:
flags = Contact.FLAG_PARTNER_CREATED if from_partner else 0
contact = Contact.create(
user_id=alias.user_id,
alias_id=alias.id,
website_email=email,
name=name,
reply_email=reply_email,
mail_from=mail_from,
automatic_created=automatic_created,
flags=flags,
invalid_email=email == "",
commit=True,
)
LOG.d(
f"Created contact {contact} for alias {alias} with email {email} invalid_email={contact.invalid_email}"
)
except IntegrityError:
Session.rollback()
LOG.info(
f"Contact with email {email} for alias_id {alias.id} already existed, fetching from DB"
)
contact = Contact.get_by(alias_id=alias.id, website_email=email)
return __update_contact_if_needed(contact, name, mail_from)
return ContactCreateResult(contact, created=True, error=None)

View File

@ -1,13 +1,15 @@
import arrow
import re import re
from dataclasses import dataclass from dataclasses import dataclass
from enum import Enum from enum import Enum
from typing import Optional from typing import Optional
from app.config import JOB_DELETE_DOMAIN
from app.db import Session from app.db import Session
from app.email_utils import get_email_domain_part from app.email_utils import get_email_domain_part
from app.log import LOG from app.log import LOG
from app.models import User, CustomDomain, SLDomain, Mailbox from app.models import User, CustomDomain, SLDomain, Mailbox, Job
_ALLOWED_DOMAIN_REGEX = re.compile(r"^(?!-)[A-Za-z0-9-]{1,63}(?<!-)$") _ALLOWED_DOMAIN_REGEX = re.compile(r"^(?!-)[A-Za-z0-9-]{1,63}(?<!-)$")
@ -126,3 +128,15 @@ def create_custom_domain(
success=True, success=True,
instance=new_custom_domain, instance=new_custom_domain,
) )
def delete_custom_domain(domain: CustomDomain):
# Schedule delete domain job
LOG.w("schedule delete domain job for %s", domain)
domain.pending_deletion = True
Job.create(
name=JOB_DELETE_DOMAIN,
payload={"custom_domain_id": domain.id},
run_at=arrow.now(),
commit=True,
)

View File

@ -9,13 +9,10 @@ from sqlalchemy import and_, func, case
from wtforms import StringField, validators, ValidationError from wtforms import StringField, validators, ValidationError
# Need to import directly from config to allow modification from the tests # Need to import directly from config to allow modification from the tests
from app import config, parallel_limiter from app import config, parallel_limiter, contact_utils
from app.contact_utils import ContactCreateError
from app.dashboard.base import dashboard_bp from app.dashboard.base import dashboard_bp
from app.db import Session from app.db import Session
from app.email_utils import (
generate_reply_email,
parse_full_address,
)
from app.email_validation import is_valid_email from app.email_validation import is_valid_email
from app.errors import ( from app.errors import (
CannotCreateContactForReverseAlias, CannotCreateContactForReverseAlias,
@ -24,8 +21,8 @@ from app.errors import (
ErrContactAlreadyExists, ErrContactAlreadyExists,
) )
from app.log import LOG from app.log import LOG
from app.models import Alias, Contact, EmailLog, User from app.models import Alias, Contact, EmailLog
from app.utils import sanitize_email, CSRFValidationForm from app.utils import CSRFValidationForm
def email_validator(): def email_validator():
@ -51,7 +48,7 @@ def email_validator():
return _check return _check
def create_contact(user: User, alias: Alias, contact_address: str) -> Contact: def create_contact(alias: Alias, contact_address: str) -> Contact:
""" """
Create a contact for a user. Can be restricted for new free users by enabling DISABLE_CREATE_CONTACTS_FOR_FREE_USERS. Create a contact for a user. Can be restricted for new free users by enabling DISABLE_CREATE_CONTACTS_FOR_FREE_USERS.
Can throw exceptions: Can throw exceptions:
@ -61,37 +58,23 @@ def create_contact(user: User, alias: Alias, contact_address: str) -> Contact:
""" """
if not contact_address: if not contact_address:
raise ErrAddressInvalid("Empty address") raise ErrAddressInvalid("Empty address")
try: output = contact_utils.create_contact(email=contact_address, alias=alias)
contact_name, contact_email = parse_full_address(contact_address) if output.error == ContactCreateError.InvalidEmail:
except ValueError:
raise ErrAddressInvalid(contact_address) raise ErrAddressInvalid(contact_address)
elif output.error == ContactCreateError.NotAllowed:
contact_email = sanitize_email(contact_email)
if not is_valid_email(contact_email):
raise ErrAddressInvalid(contact_email)
contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
if contact:
raise ErrContactAlreadyExists(contact)
if not user.can_create_contacts():
raise ErrContactErrorUpgradeNeeded() raise ErrContactErrorUpgradeNeeded()
elif output.error is not None:
raise ErrAddressInvalid("Invalid address")
elif not output.created:
raise ErrContactAlreadyExists(output.contact)
contact = Contact.create( contact = output.contact
user_id=alias.user_id,
alias_id=alias.id,
website_email=contact_email,
name=contact_name,
reply_email=generate_reply_email(contact_email, alias),
)
LOG.d( LOG.d(
"create reverse-alias for %s %s, reverse alias:%s", "create reverse-alias for %s %s, reverse alias:%s",
contact_address, contact_address,
alias, alias,
contact.reply_email, contact.reply_email,
) )
Session.commit()
return contact return contact
@ -261,7 +244,7 @@ def alias_contact_manager(alias_id):
if new_contact_form.validate(): if new_contact_form.validate():
contact_address = new_contact_form.email.data.strip() contact_address = new_contact_form.email.data.strip()
try: try:
contact = create_contact(current_user, alias, contact_address) contact = create_contact(alias, contact_address)
except ( except (
ErrContactErrorUpgradeNeeded, ErrContactErrorUpgradeNeeded,
ErrAddressInvalid, ErrAddressInvalid,

View File

@ -1,17 +1,16 @@
import re import re
import arrow
from flask import render_template, request, redirect, url_for, flash from flask import render_template, request, redirect, url_for, flash
from flask_login import login_required, current_user from flask_login import login_required, current_user
from flask_wtf import FlaskForm from flask_wtf import FlaskForm
from wtforms import StringField, validators, IntegerField from wtforms import StringField, validators, IntegerField
from app.constants import DMARC_RECORD from app.constants import DMARC_RECORD
from app.config import EMAIL_SERVERS_WITH_PRIORITY, EMAIL_DOMAIN, JOB_DELETE_DOMAIN from app.config import EMAIL_SERVERS_WITH_PRIORITY, EMAIL_DOMAIN
from app.custom_domain_utils import delete_custom_domain
from app.custom_domain_validation import CustomDomainValidation from app.custom_domain_validation import CustomDomainValidation
from app.dashboard.base import dashboard_bp from app.dashboard.base import dashboard_bp
from app.db import Session from app.db import Session
from app.log import LOG
from app.models import ( from app.models import (
CustomDomain, CustomDomain,
Alias, Alias,
@ -20,7 +19,6 @@ from app.models import (
DomainMailbox, DomainMailbox,
AutoCreateRule, AutoCreateRule,
AutoCreateRuleMailbox, AutoCreateRuleMailbox,
Job,
) )
from app.regex_utils import regex_match from app.regex_utils import regex_match
from app.utils import random_string, CSRFValidationForm from app.utils import random_string, CSRFValidationForm
@ -263,16 +261,8 @@ def domain_detail(custom_domain_id):
elif request.form.get("form-name") == "delete": elif request.form.get("form-name") == "delete":
name = custom_domain.domain name = custom_domain.domain
LOG.d("Schedule deleting %s", custom_domain)
# Schedule delete domain job delete_custom_domain(custom_domain)
LOG.w("schedule delete domain job for %s", custom_domain)
Job.create(
name=JOB_DELETE_DOMAIN,
payload={"custom_domain_id": custom_domain.id},
run_at=arrow.now(),
commit=True,
)
flash( flash(
f"{name} scheduled for deletion." f"{name} scheduled for deletion."

View File

@ -64,15 +64,9 @@ class EventDispatcher:
) )
return return
if config.EVENT_WEBHOOK_ENABLED_USER_IDS is not None:
if user.id not in config.EVENT_WEBHOOK_ENABLED_USER_IDS:
return
partner_user = EventDispatcher.__partner_user(user.id) partner_user = EventDispatcher.__partner_user(user.id)
if not partner_user: if not partner_user:
LOG.i( LOG.i(f"Not sending events because there's no partner user for user {user}")
f"Not sending events because there's no partner user for user {user}"
)
return return
event = event_pb2.Event( event = event_pb2.Event(
@ -84,7 +78,9 @@ class EventDispatcher:
serialized = event.SerializeToString() serialized = event.SerializeToString()
dispatcher.send(serialized) dispatcher.send(serialized)
newrelic.agent.record_custom_metric("Custom/events_stored", 1)
event_type = content.WhichOneof("content")
newrelic.agent.record_custom_event("EventStoredToDb", {"type": event_type})
LOG.i("Sent event to the dispatcher") LOG.i("Sent event to the dispatcher")
@staticmethod @staticmethod

View File

@ -213,7 +213,10 @@ def generate_activation_code(
) -> MailboxActivation: ) -> MailboxActivation:
clear_activation_codes_for_mailbox(mailbox) clear_activation_codes_for_mailbox(mailbox)
if use_digit_code: if use_digit_code:
code = "{:06d}".format(random.randint(1, 999999)) if config.MAILBOX_VERIFICATION_OVERRIDE_CODE:
code = config.MAILBOX_VERIFICATION_OVERRIDE_CODE
else:
code = "{:06d}".format(random.randint(1, 999999))
else: else:
code = secrets.token_urlsafe(16) code = secrets.token_urlsafe(16)
return MailboxActivation.create( return MailboxActivation.create(

View File

@ -336,7 +336,7 @@ class Fido(Base, ModelMixin):
class User(Base, ModelMixin, UserMixin, PasswordOracle): class User(Base, ModelMixin, UserMixin, PasswordOracle):
__tablename__ = "users" __tablename__ = "users"
FLAG_FREE_DISABLE_CREATE_ALIAS = 1 << 0 FLAG_DISABLE_CREATE_CONTACTS = 1 << 0
FLAG_CREATED_FROM_PARTNER = 1 << 1 FLAG_CREATED_FROM_PARTNER = 1 << 1
FLAG_FREE_OLD_ALIAS_LIMIT = 1 << 2 FLAG_FREE_OLD_ALIAS_LIMIT = 1 << 2
FLAG_CREATED_ALIAS_FROM_PARTNER = 1 << 3 FLAG_CREATED_ALIAS_FROM_PARTNER = 1 << 3
@ -543,7 +543,7 @@ class User(Base, ModelMixin, UserMixin, PasswordOracle):
# bitwise flags. Allow for future expansion # bitwise flags. Allow for future expansion
flags = sa.Column( flags = sa.Column(
sa.BigInteger, sa.BigInteger,
default=FLAG_FREE_DISABLE_CREATE_ALIAS, default=FLAG_DISABLE_CREATE_CONTACTS,
server_default="0", server_default="0",
nullable=False, nullable=False,
) )
@ -1168,7 +1168,7 @@ class User(Base, ModelMixin, UserMixin, PasswordOracle):
def can_create_contacts(self) -> bool: def can_create_contacts(self) -> bool:
if self.is_premium(): if self.is_premium():
return True return True
if self.flags & User.FLAG_FREE_DISABLE_CREATE_ALIAS == 0: if self.flags & User.FLAG_DISABLE_CREATE_CONTACTS == 0:
return True return True
return not config.DISABLE_CREATE_CONTACTS_FOR_FREE_USERS return not config.DISABLE_CREATE_CONTACTS_FOR_FREE_USERS
@ -1863,6 +1863,8 @@ class Contact(Base, ModelMixin):
MAX_NAME_LENGTH = 512 MAX_NAME_LENGTH = 512
FLAG_PARTNER_CREATED = 1 << 0
__tablename__ = "contact" __tablename__ = "contact"
__table_args__ = ( __table_args__ = (
@ -1921,6 +1923,9 @@ class Contact(Base, ModelMixin):
# whether contact is created automatically during the forward phase # whether contact is created automatically during the forward phase
automatic_created = sa.Column(sa.Boolean, nullable=True, default=False) automatic_created = sa.Column(sa.Boolean, nullable=True, default=False)
# contact flags
flags = sa.Column(sa.Integer, nullable=False, default=0, server_default="0")
@property @property
def email(self): def email(self):
return self.website_email return self.website_email
@ -2427,6 +2432,10 @@ class CustomDomain(Base, ModelMixin):
server_default=None, server_default=None,
) )
pending_deletion = sa.Column(
sa.Boolean, nullable=False, default=False, server_default="0"
)
__table_args__ = ( __table_args__ = (
Index( Index(
"ix_unique_domain", # Index name "ix_unique_domain", # Index name
@ -2434,6 +2443,8 @@ class CustomDomain(Base, ModelMixin):
unique=True, unique=True,
postgresql_where=Column("ownership_verified"), postgresql_where=Column("ownership_verified"),
), # The condition ), # The condition
Index("ix_custom_domain_user_id", "user_id"),
Index("ix_custom_domain_pending_deletion", "pending_deletion"),
) )
user = orm.relationship(User, foreign_keys=[user_id], backref="custom_domains") user = orm.relationship(User, foreign_keys=[user_id], backref="custom_domains")

View File

@ -52,7 +52,7 @@ from flanker.addresslib import address
from flanker.addresslib.address import EmailAddress from flanker.addresslib.address import EmailAddress
from sqlalchemy.exc import IntegrityError from sqlalchemy.exc import IntegrityError
from app import pgp_utils, s3, config from app import pgp_utils, s3, config, contact_utils
from app.alias_utils import try_auto_create, change_alias_status from app.alias_utils import try_auto_create, change_alias_status
from app.config import ( from app.config import (
EMAIL_DOMAIN, EMAIL_DOMAIN,
@ -195,79 +195,16 @@ def get_or_create_contact(from_header: str, mail_from: str, alias: Alias) -> Con
mail_from, mail_from,
) )
contact_email = mail_from contact_email = mail_from
contact_result = contact_utils.create_contact(
if not is_valid_email(contact_email): email=contact_email,
LOG.w( alias=alias,
"invalid contact email %s. Parse from %s %s", name=contact_name,
contact_email, mail_from=mail_from,
from_header, allow_empty_email=True,
mail_from, automatic_created=True,
) from_partner=False,
# either reuse a contact with empty email or create a new contact with empty email )
contact_email = "" return contact_result.contact
contact_email = sanitize_email(contact_email, not_lower=True)
if contact_name and "\x00" in contact_name:
LOG.w("issue with contact name %s", contact_name)
contact_name = ""
contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
if contact:
if contact.name != contact_name:
LOG.d(
"Update contact %s name %s to %s",
contact,
contact.name,
contact_name,
)
contact.name = contact_name
Session.commit()
# contact created in the past does not have mail_from and from_header field
if not contact.mail_from and mail_from:
LOG.d(
"Set contact mail_from %s: %s to %s",
contact,
contact.mail_from,
mail_from,
)
contact.mail_from = mail_from
Session.commit()
else:
alias_id = alias.id
try:
contact_email_for_reply = (
contact_email if is_valid_email(contact_email) else ""
)
contact = Contact.create(
user_id=alias.user_id,
alias_id=alias_id,
website_email=contact_email,
name=contact_name,
mail_from=mail_from,
reply_email=generate_reply_email(contact_email_for_reply, alias),
automatic_created=True,
)
if not contact_email:
LOG.d("Create a contact with invalid email for %s", alias)
contact.invalid_email = True
LOG.d(
"create contact %s for %s, reverse alias:%s",
contact_email,
alias,
contact.reply_email,
)
Session.commit()
except IntegrityError:
LOG.info(
f"Contact with email {contact_email} for alias_id {alias_id} already existed, fetching from DB"
)
contact = Contact.get_by(alias_id=alias_id, website_email=contact_email)
return contact
def get_or_create_reply_to_contact( def get_or_create_reply_to_contact(
@ -292,33 +229,7 @@ def get_or_create_reply_to_contact(
) )
return None return None
contact = Contact.get_by(alias_id=alias.id, website_email=contact_address) return contact_utils.create_contact(contact_address, alias, contact_name).contact
if contact:
return contact
else:
LOG.d(
"create contact %s for alias %s via reply-to header %s",
contact_address,
alias,
reply_to_header,
)
try:
contact = Contact.create(
user_id=alias.user_id,
alias_id=alias.id,
website_email=contact_address,
name=contact_name,
reply_email=generate_reply_email(contact_address, alias),
automatic_created=True,
)
Session.commit()
except IntegrityError:
LOG.w("Contact %s %s already exist", alias, contact_address)
Session.rollback()
contact = Contact.get_by(alias_id=alias.id, website_email=contact_address)
return contact
def replace_header_when_forward(msg: Message, alias: Alias, header: str): def replace_header_when_forward(msg: Message, alias: Alias, header: str):

View File

@ -9,7 +9,7 @@ from events.runner import Runner
from events.event_source import DeadLetterEventSource, PostgresEventSource from events.event_source import DeadLetterEventSource, PostgresEventSource
from events.event_sink import ConsoleEventSink, HttpEventSink from events.event_sink import ConsoleEventSink, HttpEventSink
_DEFAULT_MAX_RETRIES = 100 _DEFAULT_MAX_RETRIES = 10
class Mode(Enum): class Mode(Enum):

View File

@ -27,7 +27,9 @@ class HttpEventSink(EventSink):
headers={"Content-Type": "application/x-protobuf"}, headers={"Content-Type": "application/x-protobuf"},
verify=not EVENT_WEBHOOK_SKIP_VERIFY_SSL, verify=not EVENT_WEBHOOK_SKIP_VERIFY_SSL,
) )
newrelic.agent.record_custom_event("event_sent", {"http_code": res.status_code}) newrelic.agent.record_custom_event(
"EventSentToPartner", {"http_code": res.status_code}
)
if res.status_code != 200: if res.status_code != 200:
LOG.warning( LOG.warning(
f"Failed to send event to webhook: {res.status_code} {res.text}" f"Failed to send event to webhook: {res.status_code} {res.text}"

View File

@ -72,7 +72,9 @@ class PostgresEventSource(EventSource):
Session.close() # Ensure we get a new connection and we don't leave a dangling tx Session.close() # Ensure we get a new connection and we don't leave a dangling tx
def __connect(self): def __connect(self):
self.__connection = psycopg2.connect(self.__connection_string) self.__connection = psycopg2.connect(
self.__connection_string, application_name="sl-event-listen"
)
from app.db import Session from app.db import Session

View File

@ -3,7 +3,7 @@ Run scheduled jobs.
Not meant for running job at precise time (+- 1h) Not meant for running job at precise time (+- 1h)
""" """
import time import time
from typing import List from typing import List, Optional
import arrow import arrow
from sqlalchemy.sql.expression import or_, and_ from sqlalchemy.sql.expression import or_, and_
@ -240,7 +240,7 @@ def process_job(job: Job):
elif job.name == config.JOB_DELETE_DOMAIN: elif job.name == config.JOB_DELETE_DOMAIN:
custom_domain_id = job.payload.get("custom_domain_id") custom_domain_id = job.payload.get("custom_domain_id")
custom_domain = CustomDomain.get(custom_domain_id) custom_domain: Optional[CustomDomain] = CustomDomain.get(custom_domain_id)
if not custom_domain: if not custom_domain:
return return
@ -252,16 +252,17 @@ def process_job(job: Job):
LOG.d("Domain %s deleted", domain_name) LOG.d("Domain %s deleted", domain_name)
send_email( if custom_domain.partner_id is None:
user.email, send_email(
f"Your domain {domain_name} has been deleted", user.email,
f"""Domain {domain_name} along with its aliases are deleted successfully. f"Your domain {domain_name} has been deleted",
f"""Domain {domain_name} along with its aliases are deleted successfully.
Regards, Regards,
SimpleLogin team. SimpleLogin team.
""", """,
retries=3, retries=3,
) )
elif job.name == config.JOB_SEND_USER_REPORT: elif job.name == config.JOB_SEND_USER_REPORT:
export_job = ExportUserDataJob.create_from_job(job) export_job = ExportUserDataJob.create_from_job(job)
if export_job: if export_job:

View File

@ -0,0 +1,31 @@
"""contact.flags and custom_domain.pending_deletion
Revision ID: 88dd7a0abf54
Revises: 2441b7ff5da9
Create Date: 2024-09-19 15:41:20.910374
"""
import sqlalchemy_utils
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '88dd7a0abf54'
down_revision = '2441b7ff5da9'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('contact', sa.Column('flags', sa.Integer(), server_default='0', nullable=False))
op.add_column('custom_domain', sa.Column('pending_deletion', sa.Boolean(), server_default='0', nullable=False))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('custom_domain', 'pending_deletion')
op.drop_column('contact', 'flags')
# ### end Alembic commands ###

View File

@ -0,0 +1,27 @@
"""custom domain indices
Revision ID: 62afa3a10010
Revises: 88dd7a0abf54
Create Date: 2024-09-30 11:40:04.127791
"""
from alembic import op
# revision identifiers, used by Alembic.
revision = '62afa3a10010'
down_revision = '88dd7a0abf54'
branch_labels = None
depends_on = None
def upgrade():
with op.get_context().autocommit_block():
op.create_index('ix_custom_domain_pending_deletion', 'custom_domain', ['pending_deletion'], unique=False, postgresql_concurrently=True)
op.create_index('ix_custom_domain_user_id', 'custom_domain', ['user_id'], unique=False, postgresql_concurrently=True)
def downgrade():
with op.get_context().autocommit_block():
op.drop_index('ix_custom_domain_user_id', table_name='custom_domain', postgresql_concurrently=True)
op.drop_index('ix_custom_domain_pending_deletion', table_name='custom_domain', postgresql_concurrently=True)

View File

@ -94,6 +94,20 @@ def log_nb_db_connection():
newrelic.agent.record_custom_metric("Custom/nb_db_connections", nb_connection) newrelic.agent.record_custom_metric("Custom/nb_db_connections", nb_connection)
@newrelic.agent.background_task()
def log_nb_db_connection_by_app_name():
# get the number of connections to the DB
rows = Session.execute(
"SELECT application_name, count(datid) FROM pg_stat_activity group by application_name"
)
for row in rows:
if row[0].find("sl-") == 0:
LOG.d("number of db connections for app %s = %s", row[0], row[1])
newrelic.agent.record_custom_metric(
f"Custom/nb_db_app_connection/{row[0]}", row[1]
)
@newrelic.agent.background_task() @newrelic.agent.background_task()
def log_pending_to_process_events(): def log_pending_to_process_events():
r = Session.execute("select count(*) from sync_event WHERE taken_time IS NULL;") r = Session.execute("select count(*) from sync_event WHERE taken_time IS NULL;")
@ -125,6 +139,21 @@ def log_events_pending_dead_letter():
) )
@newrelic.agent.background_task()
def log_failed_events():
r = Session.execute(
"""
SELECT COUNT(*)
FROM sync_event
WHERE retry_count >= 10;
""",
)
failed_events = list(r)[0][0]
LOG.d("number of failed events %s", failed_events)
newrelic.agent.record_custom_metric("Custom/sync_events_failed", failed_events)
if __name__ == "__main__": if __name__ == "__main__":
exporter = MetricExporter(get_newrelic_license()) exporter = MetricExporter(get_newrelic_license())
while True: while True:
@ -132,6 +161,8 @@ if __name__ == "__main__":
log_nb_db_connection() log_nb_db_connection()
log_pending_to_process_events() log_pending_to_process_events()
log_events_pending_dead_letter() log_events_pending_dead_letter()
log_failed_events()
log_nb_db_connection_by_app_name()
Session.close() Session.close()
exporter.run() exporter.run()

View File

@ -536,7 +536,7 @@ def test_create_contact_route_free_users(flask_client):
assert r.status_code == 201 assert r.status_code == 201
# End trial and disallow for new free users. Config should allow it # End trial and disallow for new free users. Config should allow it
user.flags = User.FLAG_FREE_DISABLE_CREATE_ALIAS user.flags = User.FLAG_DISABLE_CREATE_CONTACTS
Session.commit() Session.commit()
r = flask_client.post( r = flask_client.post(
url_for("api.create_contact_route", alias_id=alias.id), url_for("api.create_contact_route", alias_id=alias.id),

View File

@ -4,7 +4,7 @@ from app.models import (
Alias, Alias,
Contact, Contact,
) )
from tests.utils import login from tests.utils import login, random_email
def test_add_contact_success(flask_client): def test_add_contact_success(flask_client):
@ -13,26 +13,28 @@ def test_add_contact_success(flask_client):
assert Contact.filter_by(user_id=user.id).count() == 0 assert Contact.filter_by(user_id=user.id).count() == 0
email = random_email()
# <<< Create a new contact >>> # <<< Create a new contact >>>
flask_client.post( flask_client.post(
url_for("dashboard.alias_contact_manager", alias_id=alias.id), url_for("dashboard.alias_contact_manager", alias_id=alias.id),
data={ data={
"form-name": "create", "form-name": "create",
"email": "abcd@gmail.com", "email": email,
}, },
follow_redirects=True, follow_redirects=True,
) )
# a new contact is added # a new contact is added
assert Contact.filter_by(user_id=user.id).count() == 1 assert Contact.filter_by(user_id=user.id).count() == 1
contact = Contact.filter_by(user_id=user.id).first() contact = Contact.filter_by(user_id=user.id).first()
assert contact.website_email == "abcd@gmail.com" assert contact.website_email == email
# <<< Create a new contact using a full email format >>> # <<< Create a new contact using a full email format >>>
email = random_email()
flask_client.post( flask_client.post(
url_for("dashboard.alias_contact_manager", alias_id=alias.id), url_for("dashboard.alias_contact_manager", alias_id=alias.id),
data={ data={
"form-name": "create", "form-name": "create",
"email": "First Last <another@gmail.com>", "email": f"First Last <{email}>",
}, },
follow_redirects=True, follow_redirects=True,
) )
@ -41,7 +43,7 @@ def test_add_contact_success(flask_client):
contact = ( contact = (
Contact.filter_by(user_id=user.id).filter(Contact.id != contact.id).first() Contact.filter_by(user_id=user.id).filter(Contact.id != contact.id).first()
) )
assert contact.website_email == "another@gmail.com" assert contact.website_email == email
assert contact.name == "First Last" assert contact.name == "First Last"
# <<< Create a new contact with invalid email address >>> # <<< Create a new contact with invalid email address >>>

View File

@ -0,0 +1,196 @@
from typing import Optional
import pytest
from app import config
from app.contact_utils import create_contact, ContactCreateError
from app.db import Session
from app.models import (
Alias,
Contact,
User,
)
from tests.utils import create_new_user, random_email, random_token
def setup_module(module):
config.DISABLE_CREATE_CONTACTS_FOR_FREE_USERS = True
def teardown_module(module):
config.DISABLE_CREATE_CONTACTS_FOR_FREE_USERS = False
def create_provider():
# name auto_created from_partner
yield ["name", "a@b.c", True, True]
yield [None, None, True, True]
yield [None, None, False, True]
yield [None, None, True, False]
yield [None, None, False, False]
@pytest.mark.parametrize(
"name, mail_from, automatic_created, from_partner", create_provider()
)
def test_create_contact(
name: Optional[str],
mail_from: Optional[str],
automatic_created: bool,
from_partner: bool,
):
user = create_new_user()
alias = Alias.create_new_random(user)
Session.commit()
email = random_email()
contact_result = create_contact(
email,
alias,
name=name,
mail_from=mail_from,
automatic_created=automatic_created,
from_partner=from_partner,
)
assert contact_result.error is None
contact = contact_result.contact
assert contact.user_id == user.id
assert contact.alias_id == alias.id
assert contact.website_email == email
assert contact.name == name
assert contact.mail_from == mail_from
assert contact.automatic_created == automatic_created
assert not contact.invalid_email
expected_flags = Contact.FLAG_PARTNER_CREATED if from_partner else 0
assert contact.flags == expected_flags
def test_create_contact_email_email_not_allowed():
user = create_new_user()
alias = Alias.create_new_random(user)
Session.commit()
contact_result = create_contact("", alias)
assert contact_result.contact is None
assert contact_result.error == ContactCreateError.InvalidEmail
def test_create_contact_email_email_allowed():
user = create_new_user()
alias = Alias.create_new_random(user)
Session.commit()
contact_result = create_contact("", alias, allow_empty_email=True)
assert contact_result.error is None
assert contact_result.contact is not None
assert contact_result.contact.website_email == ""
assert contact_result.contact.invalid_email
def test_create_contact_name_overrides_email_name():
user = create_new_user()
alias = Alias.create_new_random(user)
Session.commit()
email = random_email()
name = random_token()
contact_result = create_contact(f"superseeded <{email}>", alias, name=name)
assert contact_result.error is None
assert contact_result.contact is not None
assert contact_result.contact.website_email == email
assert contact_result.contact.name == name
def test_create_contact_name_taken_from_email():
user = create_new_user()
alias = Alias.create_new_random(user)
Session.commit()
email = random_email()
name = random_token()
contact_result = create_contact(f"{name} <{email}>", alias)
assert contact_result.error is None
assert contact_result.contact is not None
assert contact_result.contact.website_email == email
assert contact_result.contact.name == name
def test_create_contact_empty_name_is_none():
user = create_new_user()
alias = Alias.create_new_random(user)
Session.commit()
email = random_email()
contact_result = create_contact(email, alias, name="")
assert contact_result.error is None
assert contact_result.contact is not None
assert contact_result.contact.website_email == email
assert contact_result.contact.name is None
def test_create_contact_free_user():
user = create_new_user()
user.trial_end = None
user.flags = 0
alias = Alias.create_new_random(user)
Session.flush()
# Free users without the FREE_DISABLE_CREATE_CONTACTS
result = create_contact(random_email(), alias)
assert result.error is None
assert result.created
assert result.contact is not None
assert not result.contact.automatic_created
# Free users with the flag should be able to still create automatic emails
user.flags = User.FLAG_DISABLE_CREATE_CONTACTS
Session.flush()
result = create_contact(random_email(), alias, automatic_created=True)
assert result.error is None
assert result.created
assert result.contact is not None
assert result.contact.automatic_created
# Free users with the flag cannot create non-automatic emails
result = create_contact(random_email(), alias)
assert result.error == ContactCreateError.NotAllowed
def test_do_not_allow_invalid_email():
user = create_new_user()
alias = Alias.create_new_random(user)
Session.commit()
contact_result = create_contact("potato", alias)
assert contact_result.contact is None
assert contact_result.error == ContactCreateError.InvalidEmail
contact_result = create_contact("asdf\x00@gmail.com", alias)
assert contact_result.contact is None
assert contact_result.error == ContactCreateError.InvalidEmail
def test_update_name_for_existing():
user = create_new_user()
alias = Alias.create_new_random(user)
Session.commit()
email = random_email()
contact_result = create_contact(email, alias)
assert contact_result.error is None
assert contact_result.created
assert contact_result.contact is not None
assert contact_result.contact.name is None
name = random_token()
contact_result = create_contact(email, alias, name=name)
assert contact_result.error is None
assert not contact_result.created
assert contact_result.contact is not None
assert contact_result.contact.name == name
def test_update_mail_from_for_existing():
user = create_new_user()
alias = Alias.create_new_random(user)
Session.commit()
email = random_email()
contact_result = create_contact(email, alias)
assert contact_result.error is None
assert contact_result.created
assert contact_result.contact is not None
assert contact_result.contact is not None
assert contact_result.contact.mail_from is None
mail_from = random_email()
contact_result = create_contact(email, alias, mail_from=mail_from)
assert contact_result.error is None
assert not contact_result.created
assert contact_result.contact is not None
assert contact_result.contact.mail_from == mail_from