From 4f5564df16245eff71f0d54c2462df5358685703 Mon Sep 17 00:00:00 2001 From: MrMeeb <mrmeeb@noreply.git.mrmeeb.stream> Date: Fri, 29 Sep 2023 12:00:06 +0100 Subject: [PATCH] 4.35.0 --- app/app/admin_model.py | 11 ++++ app/app/alias_utils.py | 59 +++++++++++++++++ app/app/api/views/auth.py | 5 ++ app/app/auth/views/login.py | 6 ++ app/app/dashboard/views/alias_transfer.py | 64 +------------------ app/app/events/auth_event.py | 1 + app/app/models.py | 56 +++++++++++----- app/cron.py | 17 ++++- app/crontab.yml | 7 +- app/email_handler.py | 14 ++-- .../versions/2023_090715_0a5701a4f5e4_.py | 33 ++++++++++ .../versions/2023_092818_ec7fdde8da9f_.py | 34 ++++++++++ .../dashboard/domain_detail/dns.html | 2 +- app/tests/dashboard/test_alias_transfer.py | 4 +- app/tests/test_cron.py | 27 ++++++-- app/tests/test_domains.py | 28 ++++++++ app/tests/test_models.py | 10 +++ 17 files changed, 279 insertions(+), 99 deletions(-) create mode 100644 app/migrations/versions/2023_090715_0a5701a4f5e4_.py create mode 100644 app/migrations/versions/2023_092818_ec7fdde8da9f_.py diff --git a/app/app/admin_model.py b/app/app/admin_model.py index 0394fbe..4f7e6d0 100644 --- a/app/app/admin_model.py +++ b/app/app/admin_model.py @@ -256,6 +256,17 @@ class UserAdmin(SLModelView): Session.commit() + @action( + "clear_delete_on", + "Remove scheduled deletion of user", + "This will remove the scheduled deletion for this users", + ) + def clean_delete_on(self, ids): + for user in User.filter(User.id.in_(ids)): + user.delete_on = None + + Session.commit() + # @action( # "login_as", # "Login as this user", diff --git a/app/app/alias_utils.py b/app/app/alias_utils.py index 3034ee8..b02cb1f 100644 --- a/app/app/alias_utils.py +++ b/app/app/alias_utils.py @@ -21,6 +21,8 @@ from app.email_utils import ( send_cannot_create_directory_alias_disabled, get_email_local_part, send_cannot_create_domain_alias, + send_email, + render, ) from app.errors import AliasInTrashError from app.log import LOG @@ -36,6 +38,8 @@ from app.models import ( EmailLog, Contact, AutoCreateRule, + AliasUsedOn, + ClientUser, ) from app.regex_utils import regex_match @@ -399,3 +403,58 @@ def alias_export_csv(user, csv_direct_export=False): output.headers["Content-Disposition"] = "attachment; filename=aliases.csv" output.headers["Content-type"] = "text/csv" return output + + +def transfer_alias(alias, new_user, new_mailboxes: [Mailbox]): + # cannot transfer alias which is used for receiving newsletter + if User.get_by(newsletter_alias_id=alias.id): + raise Exception("Cannot transfer alias that's used to receive newsletter") + + # update user_id + Session.query(Contact).filter(Contact.alias_id == alias.id).update( + {"user_id": new_user.id} + ) + + Session.query(AliasUsedOn).filter(AliasUsedOn.alias_id == alias.id).update( + {"user_id": new_user.id} + ) + + Session.query(ClientUser).filter(ClientUser.alias_id == alias.id).update( + {"user_id": new_user.id} + ) + + # remove existing mailboxes from the alias + Session.query(AliasMailbox).filter(AliasMailbox.alias_id == alias.id).delete() + + # set mailboxes + alias.mailbox_id = new_mailboxes.pop().id + for mb in new_mailboxes: + AliasMailbox.create(alias_id=alias.id, mailbox_id=mb.id) + + # alias has never been transferred before + if not alias.original_owner_id: + alias.original_owner_id = alias.user_id + + # inform previous owner + old_user = alias.user + send_email( + old_user.email, + f"Alias {alias.email} has been received", + render( + "transactional/alias-transferred.txt", + alias=alias, + ), + render( + "transactional/alias-transferred.html", + alias=alias, + ), + ) + + # now the alias belongs to the new user + alias.user_id = new_user.id + + # set some fields back to default + alias.disable_pgp = False + alias.pinned = False + + Session.commit() diff --git a/app/app/api/views/auth.py b/app/app/api/views/auth.py index 79e019b..b77036a 100644 --- a/app/app/api/views/auth.py +++ b/app/app/api/views/auth.py @@ -63,6 +63,11 @@ def auth_login(): elif user.disabled: LoginEvent(LoginEvent.ActionType.disabled_login, LoginEvent.Source.api).send() return jsonify(error="Account disabled"), 400 + elif user.delete_on is not None: + LoginEvent( + LoginEvent.ActionType.scheduled_to_be_deleted, LoginEvent.Source.api + ).send() + return jsonify(error="Account scheduled for deletion"), 400 elif not user.activated: LoginEvent(LoginEvent.ActionType.not_activated, LoginEvent.Source.api).send() return jsonify(error="Account not activated"), 422 diff --git a/app/app/auth/views/login.py b/app/app/auth/views/login.py index 55cb0c6..56b2ac3 100644 --- a/app/app/auth/views/login.py +++ b/app/app/auth/views/login.py @@ -54,6 +54,12 @@ def login(): "error", ) LoginEvent(LoginEvent.ActionType.disabled_login).send() + elif user.delete_on is not None: + flash( + f"Your account is scheduled to be deleted on {user.delete_on}", + "error", + ) + LoginEvent(LoginEvent.ActionType.scheduled_to_be_deleted).send() elif not user.activated: show_resend_activation = True flash( diff --git a/app/app/dashboard/views/alias_transfer.py b/app/app/dashboard/views/alias_transfer.py index af8fab5..8b2e767 100644 --- a/app/app/dashboard/views/alias_transfer.py +++ b/app/app/dashboard/views/alias_transfer.py @@ -7,79 +7,19 @@ from flask import render_template, redirect, url_for, flash, request from flask_login import login_required, current_user from app import config +from app.alias_utils import transfer_alias from app.dashboard.base import dashboard_bp from app.dashboard.views.enter_sudo import sudo_required from app.db import Session -from app.email_utils import send_email, render from app.extensions import limiter from app.log import LOG from app.models import ( Alias, - Contact, - AliasUsedOn, - AliasMailbox, - User, - ClientUser, ) from app.models import Mailbox from app.utils import CSRFValidationForm -def transfer(alias, new_user, new_mailboxes: [Mailbox]): - # cannot transfer alias which is used for receiving newsletter - if User.get_by(newsletter_alias_id=alias.id): - raise Exception("Cannot transfer alias that's used to receive newsletter") - - # update user_id - Session.query(Contact).filter(Contact.alias_id == alias.id).update( - {"user_id": new_user.id} - ) - - Session.query(AliasUsedOn).filter(AliasUsedOn.alias_id == alias.id).update( - {"user_id": new_user.id} - ) - - Session.query(ClientUser).filter(ClientUser.alias_id == alias.id).update( - {"user_id": new_user.id} - ) - - # remove existing mailboxes from the alias - Session.query(AliasMailbox).filter(AliasMailbox.alias_id == alias.id).delete() - - # set mailboxes - alias.mailbox_id = new_mailboxes.pop().id - for mb in new_mailboxes: - AliasMailbox.create(alias_id=alias.id, mailbox_id=mb.id) - - # alias has never been transferred before - if not alias.original_owner_id: - alias.original_owner_id = alias.user_id - - # inform previous owner - old_user = alias.user - send_email( - old_user.email, - f"Alias {alias.email} has been received", - render( - "transactional/alias-transferred.txt", - alias=alias, - ), - render( - "transactional/alias-transferred.html", - alias=alias, - ), - ) - - # now the alias belongs to the new user - alias.user_id = new_user.id - - # set some fields back to default - alias.disable_pgp = False - alias.pinned = False - - Session.commit() - - def hmac_alias_transfer_token(transfer_token: str) -> str: alias_hmac = hmac.new( config.ALIAS_TRANSFER_TOKEN_SECRET.encode("utf-8"), @@ -214,7 +154,7 @@ def alias_transfer_receive_route(): mailboxes, token, ) - transfer(alias, current_user, mailboxes) + transfer_alias(alias, current_user, mailboxes) # reset transfer token alias.transfer_token = None diff --git a/app/app/events/auth_event.py b/app/app/events/auth_event.py index f675218..60b7b8c 100644 --- a/app/app/events/auth_event.py +++ b/app/app/events/auth_event.py @@ -9,6 +9,7 @@ class LoginEvent: failed = 1 disabled_login = 2 not_activated = 3 + scheduled_to_be_deleted = 4 class Source(EnumE): web = 0 diff --git a/app/app/models.py b/app/app/models.py index 036733b..8807dc0 100644 --- a/app/app/models.py +++ b/app/app/models.py @@ -280,6 +280,7 @@ class IntEnumType(sa.types.TypeDecorator): class AliasOptions: show_sl_domains: bool = True show_partner_domains: Optional[Partner] = None + show_partner_premium: Optional[bool] = None class Hibp(Base, ModelMixin): @@ -539,10 +540,14 @@ class User(Base, ModelMixin, UserMixin, PasswordOracle): nullable=False, ) + # Trigger hard deletion of the account at this time + delete_on = sa.Column(ArrowType, default=None) + __table_args__ = ( sa.Index( "ix_users_activated_trial_end_lifetime", activated, trial_end, lifetime ), + sa.Index("ix_users_delete_on", delete_on), ) @property @@ -833,6 +838,17 @@ class User(Base, ModelMixin, UserMixin, PasswordOracle): < self.max_alias_for_free_account() ) + def can_send_or_receive(self) -> bool: + if self.disabled: + LOG.i(f"User {self} is disabled. Cannot receive or send emails") + return False + if self.delete_on is not None: + LOG.i( + f"User {self} is scheduled to be deleted. Cannot receive or send emails" + ) + return False + return True + def profile_picture_url(self): if self.profile_picture_id: return self.profile_picture.get_url() @@ -1023,29 +1039,35 @@ class User(Base, ModelMixin, UserMixin, PasswordOracle): ) -> list["SLDomain"]: if alias_options is None: alias_options = AliasOptions() - conditions = [SLDomain.hidden == False] # noqa: E712 - if not self.is_premium(): - conditions.append(SLDomain.premium_only == False) # noqa: E712 - partner_domain_cond = [] # noqa:E711 + top_conds = [SLDomain.hidden == False] # noqa: E712 + or_conds = [] # noqa:E711 if self.default_alias_public_domain_id is not None: - partner_domain_cond.append( - SLDomain.id == self.default_alias_public_domain_id - ) + default_domain_conds = [SLDomain.id == self.default_alias_public_domain_id] + if not self.is_premium(): + default_domain_conds.append( + SLDomain.premium_only == False # noqa: E712 + ) + or_conds.append(and_(*default_domain_conds).self_group()) if alias_options.show_partner_domains is not None: partner_user = PartnerUser.filter_by( user_id=self.id, partner_id=alias_options.show_partner_domains.id ).first() if partner_user is not None: - partner_domain_cond.append( - SLDomain.partner_id == partner_user.partner_id - ) + partner_domain_cond = [SLDomain.partner_id == partner_user.partner_id] + if alias_options.show_partner_premium is None: + alias_options.show_partner_premium = self.is_premium() + if not alias_options.show_partner_premium: + partner_domain_cond.append( + SLDomain.premium_only == False # noqa: E712 + ) + or_conds.append(and_(*partner_domain_cond).self_group()) if alias_options.show_sl_domains: - partner_domain_cond.append(SLDomain.partner_id == None) # noqa:E711 - if len(partner_domain_cond) == 1: - conditions.append(partner_domain_cond[0]) - else: - conditions.append(or_(*partner_domain_cond)) - query = Session.query(SLDomain).filter(*conditions).order_by(SLDomain.order) + sl_conds = [SLDomain.partner_id == None] # noqa: E711 + if not self.is_premium(): + sl_conds.append(SLDomain.premium_only == False) # noqa: E712 + or_conds.append(and_(*sl_conds).self_group()) + top_conds.append(or_(*or_conds)) + query = Session.query(SLDomain).filter(*top_conds).order_by(SLDomain.order) return query.all() def available_alias_domains( @@ -1925,6 +1947,7 @@ class Contact(Base, ModelMixin): class EmailLog(Base, ModelMixin): __tablename__ = "email_log" + __table_args__ = (Index("ix_email_log_created_at", "created_at"),) user_id = sa.Column( sa.ForeignKey(User.id, ondelete="cascade"), nullable=False, index=True @@ -2576,6 +2599,7 @@ class Mailbox(Base, ModelMixin): self.email.endswith("@proton.me") or self.email.endswith("@protonmail.com") or self.email.endswith("@protonmail.ch") + or self.email.endswith("@proton.ch") or self.email.endswith("@pm.me") ): return True diff --git a/app/cron.py b/app/cron.py index 7c3cd2c..6c59343 100644 --- a/app/cron.py +++ b/app/cron.py @@ -5,7 +5,7 @@ from typing import List, Tuple import arrow import requests -from sqlalchemy import func, desc, or_ +from sqlalchemy import func, desc, or_, and_ from sqlalchemy.ext.compiler import compiles from sqlalchemy.orm import joinedload from sqlalchemy.orm.exc import ObjectDeletedError @@ -1106,6 +1106,18 @@ def notify_hibp(): Session.commit() +def clear_users_scheduled_to_be_deleted(): + users = User.filter( + and_(User.delete_on.isnot(None), User.delete_on < arrow.now()) + ).all() + for user in users: + LOG.i( + f"Scheduled deletion of user {user} with scheduled delete on {user.delete_on}" + ) + User.delete(user.id) + Session.commit() + + if __name__ == "__main__": LOG.d("Start running cronjob") parser = argparse.ArgumentParser() @@ -1172,3 +1184,6 @@ if __name__ == "__main__": elif args.job == "send_undelivered_mails": LOG.d("Sending undelivered emails") load_unsent_mails_from_fs_and_resend() + elif args.job == "delete_scheduled_users": + LOG.d("Deleting users scheduled to be deleted") + clear_users_scheduled_to_be_deleted() diff --git a/app/crontab.yml b/app/crontab.yml index ece8ac1..86877a5 100644 --- a/app/crontab.yml +++ b/app/crontab.yml @@ -61,7 +61,12 @@ jobs: schedule: "15 10 * * *" captureStderr: true - + - name: SimpleLogin delete users scheduled to be deleted + command: echo disabled_user_deletion #python /code/cron.py -j delete_scheduled_users + shell: /bin/bash + schedule: "15 11 * * *" + captureStderr: true + concurrencyPolicy: Forbid - name: SimpleLogin send unsent emails command: python /code/cron.py -j send_undelivered_mails diff --git a/app/email_handler.py b/app/email_handler.py index 4ccba49..c5be199 100644 --- a/app/email_handler.py +++ b/app/email_handler.py @@ -637,8 +637,8 @@ def handle_forward(envelope, msg: Message, rcpt_to: str) -> List[Tuple[bool, str user = alias.user - if user.disabled: - LOG.w("User %s disabled, disable forwarding emails for %s", user, alias) + if not user.can_send_or_receive(): + LOG.i(f"User {user} cannot receive emails") if should_ignore_bounce(envelope.mail_from): return [(True, status.E207)] else: @@ -1070,13 +1070,8 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str): user = alias.user mail_from = envelope.mail_from - if user.disabled: - LOG.e( - "User %s disabled, disable sending emails from %s to %s", - user, - alias, - contact, - ) + if not user.can_send_or_receive(): + LOG.i(f"User {user} cannot send emails") return False, status.E504 # Check if we need to reject or quarantine based on dmarc @@ -1257,7 +1252,6 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str): if str(msg[headers.TO]).lower() == "undisclosed-recipients:;": # no need to replace TO header LOG.d("email is sent in BCC mode") - del msg[headers.TO] else: replace_header_when_reply(msg, alias, headers.TO) diff --git a/app/migrations/versions/2023_090715_0a5701a4f5e4_.py b/app/migrations/versions/2023_090715_0a5701a4f5e4_.py new file mode 100644 index 0000000..08f15ae --- /dev/null +++ b/app/migrations/versions/2023_090715_0a5701a4f5e4_.py @@ -0,0 +1,33 @@ +"""empty message + +Revision ID: 0a5701a4f5e4 +Revises: 01827104004b +Create Date: 2023-09-07 15:28:10.122756 + +""" +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '0a5701a4f5e4' +down_revision = '01827104004b' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('users', sa.Column('delete_on', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True)) + with op.get_context().autocommit_block(): + op.create_index('ix_users_delete_on', 'users', ['delete_on'], unique=False, postgresql_concurrently=True) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.get_context().autocommit_block(): + op.drop_index('ix_users_delete_on', table_name='users', postgresql_concurrently=True) + op.drop_column('users', 'delete_on') + # ### end Alembic commands ### diff --git a/app/migrations/versions/2023_092818_ec7fdde8da9f_.py b/app/migrations/versions/2023_092818_ec7fdde8da9f_.py new file mode 100644 index 0000000..f1fae82 --- /dev/null +++ b/app/migrations/versions/2023_092818_ec7fdde8da9f_.py @@ -0,0 +1,34 @@ +"""empty message + +Revision ID: ec7fdde8da9f +Revises: 0a5701a4f5e4 +Create Date: 2023-09-28 18:09:48.016620 + +""" +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "ec7fdde8da9f" +down_revision = "0a5701a4f5e4" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.get_context().autocommit_block(): + op.create_index( + "ix_email_log_created_at", "email_log", ["created_at"], unique=False + ) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.get_context().autocommit_block(): + op.drop_index("ix_email_log_created_at", table_name="email_log") + # ### end Alembic commands ### diff --git a/app/templates/dashboard/domain_detail/dns.html b/app/templates/dashboard/domain_detail/dns.html index 75c2672..3b055e7 100644 --- a/app/templates/dashboard/domain_detail/dns.html +++ b/app/templates/dashboard/domain_detail/dns.html @@ -268,7 +268,7 @@ If you are using a subdomain, e.g. <i>subdomain.domain.com</i>, you need to use <i>dkim._domainkey.subdomain</i> as the domain instead. <br /> - That means, if your domain is <i>mail.domain.com</i> you should enter <i>dkim._domainkey.mail.domain.com</i> as the Domain. + That means, if your domain is <i>mail.domain.com</i> you should enter <i>dkim._domainkey.mail</i> as the Domain. <br /> </div> <div class="alert alert-info"> diff --git a/app/tests/dashboard/test_alias_transfer.py b/app/tests/dashboard/test_alias_transfer.py index 604c707..32a063f 100644 --- a/app/tests/dashboard/test_alias_transfer.py +++ b/app/tests/dashboard/test_alias_transfer.py @@ -1,4 +1,4 @@ -from app.dashboard.views import alias_transfer +import app.alias_utils from app.db import Session from app.models import ( Alias, @@ -29,7 +29,7 @@ def test_alias_transfer(flask_client): user_id=new_user.id, email="hey2@example.com", verified=True, commit=True ) - alias_transfer.transfer(alias, new_user, new_user.mailboxes()) + app.alias_utils.transfer_alias(alias, new_user, new_user.mailboxes()) # refresh from db alias = Alias.get(alias.id) diff --git a/app/tests/test_cron.py b/app/tests/test_cron.py index 99ca67d..834200f 100644 --- a/app/tests/test_cron.py +++ b/app/tests/test_cron.py @@ -1,18 +1,17 @@ import arrow -from app.models import CoinbaseSubscription, ApiToCookieToken, ApiKey -from cron import notify_manual_sub_end, delete_expired_tokens +import cron +from app.db import Session +from app.models import CoinbaseSubscription, ApiToCookieToken, ApiKey, User from tests.utils import create_new_user def test_notify_manual_sub_end(flask_client): user = create_new_user() - CoinbaseSubscription.create( user_id=user.id, end_at=arrow.now().shift(days=13, hours=2), commit=True ) - - notify_manual_sub_end() + cron.notify_manual_sub_end() def test_cleanup_tokens(flask_client): @@ -33,6 +32,22 @@ def test_cleanup_tokens(flask_client): api_key_id=api_key.id, commit=True, ).id - delete_expired_tokens() + cron.delete_expired_tokens() assert ApiToCookieToken.get(id_to_clean) is None assert ApiToCookieToken.get(id_to_keep) is not None + + +def test_cleanup_users(): + u_delete_none_id = create_new_user().id + u_delete_after = create_new_user() + u_delete_after_id = u_delete_after.id + u_delete_before = create_new_user() + u_delete_before_id = u_delete_before.id + now = arrow.now() + u_delete_after.delete_on = now.shift(minutes=1) + u_delete_before.delete_on = now.shift(minutes=-1) + Session.flush() + cron.clear_users_scheduled_to_be_deleted() + assert User.get(u_delete_none_id) is not None + assert User.get(u_delete_after_id) is not None + assert User.get(u_delete_before_id) is None diff --git a/app/tests/test_domains.py b/app/tests/test_domains.py index 298363a..5783aa0 100644 --- a/app/tests/test_domains.py +++ b/app/tests/test_domains.py @@ -199,3 +199,31 @@ def test_get_free_partner_and_hidden_default_domain(): assert [d.domain for d in domains] == user.available_sl_domains( alias_options=options ) + + +def test_get_free_partner_and_premium_partner(): + user = create_new_user() + user.trial_end = None + PartnerUser.create( + partner_id=get_proton_partner().id, + user_id=user.id, + external_user_id=random_token(10), + flush=True, + ) + user.default_alias_public_domain_id = ( + SLDomain.filter_by(hidden=False, premium_only=False).first().id + ) + Session.flush() + options = AliasOptions( + show_sl_domains=False, + show_partner_domains=get_proton_partner(), + show_partner_premium=True, + ) + domains = user.get_sl_domains(alias_options=options) + assert len(domains) == 3 + assert domains[0].domain == "premium_partner" + assert domains[1].domain == "free_partner" + assert domains[2].domain == "free_non_partner" + assert [d.domain for d in domains] == user.available_sl_domains( + alias_options=options + ) diff --git a/app/tests/test_models.py b/app/tests/test_models.py index 82da058..21b72c0 100644 --- a/app/tests/test_models.py +++ b/app/tests/test_models.py @@ -315,3 +315,13 @@ def test_create_contact_for_noreply(flask_client): reply_email=generate_reply_email(NOREPLY, alias), ) assert contact.website_email == NOREPLY + + +def test_user_can_send_receive(): + user = create_new_user() + assert user.can_send_or_receive() + user.disabled = True + assert not user.can_send_or_receive() + user.disabled = False + user.delete_on = arrow.now() + assert not user.can_send_or_receive()