From e36e9d3077fd7fd87ca3438bf9cde64e7a220f4a Mon Sep 17 00:00:00 2001 From: MrMeeb Date: Wed, 2 Aug 2023 16:49:54 +0100 Subject: [PATCH] 4.32.4 --- app/app/dashboard/views/api_key.py | 2 + app/app/dashboard/views/enter_sudo.py | 2 + app/app/models.py | 21 ++- app/cron.py | 23 +++- app/crontab.yml | 6 - app/docs/ssl.md | 123 +++++++++++++++++- .../versions/2023_072819_01827104004b_.py | 42 ++++++ app/templates/dashboard/support.html | 2 +- app/tests/dashboard/test_api_keys.py | 13 +- 9 files changed, 215 insertions(+), 19 deletions(-) create mode 100644 app/migrations/versions/2023_072819_01827104004b_.py diff --git a/app/app/dashboard/views/api_key.py b/app/app/dashboard/views/api_key.py index aa38075..5c19043 100644 --- a/app/app/dashboard/views/api_key.py +++ b/app/app/dashboard/views/api_key.py @@ -18,6 +18,8 @@ class NewApiKeyForm(FlaskForm): def clean_up_unused_or_old_api_keys(user_id: int): total_keys = ApiKey.filter_by(user_id=user_id).count() + if total_keys <= config.MAX_API_KEYS: + return # Remove oldest unused for api_key in ( ApiKey.filter_by(user_id=user_id, last_used=None) diff --git a/app/app/dashboard/views/enter_sudo.py b/app/app/dashboard/views/enter_sudo.py index 7246fce..d32deb8 100644 --- a/app/app/dashboard/views/enter_sudo.py +++ b/app/app/dashboard/views/enter_sudo.py @@ -8,6 +8,7 @@ from wtforms import PasswordField, validators from app.config import CONNECT_WITH_PROTON from app.dashboard.base import dashboard_bp +from app.extensions import limiter from app.log import LOG from app.models import PartnerUser from app.proton.utils import get_proton_partner @@ -21,6 +22,7 @@ class LoginForm(FlaskForm): @dashboard_bp.route("/enter_sudo", methods=["GET", "POST"]) +@limiter.limit("3/minute") @login_required def enter_sudo(): password_check_form = LoginForm() diff --git a/app/app/models.py b/app/app/models.py index 3b4df67..faec2e1 100644 --- a/app/app/models.py +++ b/app/app/models.py @@ -341,7 +341,7 @@ class User(Base, ModelMixin, UserMixin, PasswordOracle): sa.Boolean, default=True, nullable=False, server_default="1" ) - activated = sa.Column(sa.Boolean, default=False, nullable=False) + activated = sa.Column(sa.Boolean, default=False, nullable=False, index=True) # an account can be disabled if having harmful behavior disabled = sa.Column(sa.Boolean, default=False, nullable=False, server_default="0") @@ -411,7 +411,10 @@ class User(Base, ModelMixin, UserMixin, PasswordOracle): ) referral_id = sa.Column( - sa.ForeignKey("referral.id", ondelete="SET NULL"), nullable=True, default=None + sa.ForeignKey("referral.id", ondelete="SET NULL"), + nullable=True, + default=None, + index=True, ) referral = orm.relationship("Referral", foreign_keys=[referral_id]) @@ -534,6 +537,12 @@ class User(Base, ModelMixin, UserMixin, PasswordOracle): nullable=False, ) + __table_args__ = ( + sa.Index( + "ix_users_activated_trial_end_lifetime", activated, trial_end, lifetime + ), + ) + @property def directory_quota(self): return min( @@ -1445,7 +1454,7 @@ class Alias(Base, ModelMixin): ) # have I been pwned - hibp_last_check = sa.Column(ArrowType, default=None) + hibp_last_check = sa.Column(ArrowType, default=None, index=True) hibp_breaches = orm.relationship("Hibp", secondary="alias_hibp") # to use Postgres full text search. Only applied on "note" column for now @@ -2928,6 +2937,8 @@ class Monitoring(Base, ModelMixin): active_queue = sa.Column(sa.Integer, nullable=False) deferred_queue = sa.Column(sa.Integer, nullable=False) + __table_args__ = (Index("ix_monitoring_created_at", "created_at"),) + class BatchImport(Base, ModelMixin): __tablename__ = "batch_import" @@ -3053,6 +3064,8 @@ class Bounce(Base, ModelMixin): email = sa.Column(sa.String(256), nullable=False, index=True) info = sa.Column(sa.Text, nullable=True) + __table_args__ = (sa.Index("ix_bounce_created_at", "created_at"),) + class TransactionalEmail(Base, ModelMixin): """Storing all email addresses that receive transactional emails, including account email and mailboxes. @@ -3062,6 +3075,8 @@ class TransactionalEmail(Base, ModelMixin): __tablename__ = "transactional_email" email = sa.Column(sa.String(256), nullable=False, unique=False) + __table_args__ = (sa.Index("ix_transactional_email_created_at", "created_at"),) + class Payout(Base, ModelMixin): """Referral payouts""" diff --git a/app/cron.py b/app/cron.py index 75e8ed7..64c6532 100644 --- a/app/cron.py +++ b/app/cron.py @@ -66,12 +66,14 @@ from server import create_light_app def notify_trial_end(): for user in User.filter( - User.activated.is_(True), User.trial_end.isnot(None), User.lifetime.is_(False) + User.activated.is_(True), + User.trial_end.isnot(None), + User.trial_end >= arrow.now().shift(days=2), + User.trial_end < arrow.now().shift(days=3), + User.lifetime.is_(False), ).all(): try: - if user.in_trial() and arrow.now().shift( - days=3 - ) > user.trial_end >= arrow.now().shift(days=2): + if user.in_trial(): LOG.d("Send trial end email to user %s", user) send_trial_end_soon_email(user) # happens if user has been deleted in the meantime @@ -104,7 +106,9 @@ def delete_logs(): def delete_refused_emails(): - for refused_email in RefusedEmail.filter_by(deleted=False).all(): + for refused_email in ( + RefusedEmail.filter_by(deleted=False).order_by(RefusedEmail.id).all() + ): if arrow.now().shift(days=1) > refused_email.delete_at >= arrow.now(): LOG.d("Delete refused email %s", refused_email) if refused_email.path: @@ -272,7 +276,11 @@ def compute_metric2() -> Metric2: _24h_ago = now.shift(days=-1) nb_referred_user_paid = 0 - for user in User.filter(User.referral_id.isnot(None)): + for user in ( + User.filter(User.referral_id.isnot(None)) + .yield_per(500) + .enable_eagerloads(False) + ): if user.is_paid(): nb_referred_user_paid += 1 @@ -1020,7 +1028,8 @@ async def check_hibp(): ) .filter(Alias.enabled) .order_by(Alias.hibp_last_check.asc()) - .all() + .yield_per(500) + .enable_eagerloads(False) ): await queue.put(alias.id) diff --git a/app/crontab.yml b/app/crontab.yml index ec5a257..ba47303 100644 --- a/app/crontab.yml +++ b/app/crontab.yml @@ -35,12 +35,6 @@ jobs: schedule: "0 12 * * *" captureStderr: true - - name: SimpleLogin Sanity Check - command: python /code/cron.py -j sanity_check - shell: /bin/bash - schedule: "0 2 * * *" - captureStderr: true - - name: SimpleLogin Delete Old Monitoring records command: python /code/cron.py -j delete_old_monitoring shell: /bin/bash diff --git a/app/docs/ssl.md b/app/docs/ssl.md index 1114c4d..1832aef 100644 --- a/app/docs/ssl.md +++ b/app/docs/ssl.md @@ -1,4 +1,4 @@ -# SSL, HTTPS, and HSTS +# SSL, HTTPS, HSTS and additional security measures It's highly recommended to enable SSL/TLS on your server, both for the web app and email server. @@ -58,3 +58,124 @@ Now, reload Nginx: ```bash sudo systemctl reload nginx ``` + +## Additional security measures + +For additional security, we recommend you take some extra steps. + +### Enable Certificate Authority Authorization (CAA) + +[Certificate Authority Authorization](https://letsencrypt.org/docs/caa/) is a step you can take to restrict the list of certificate authorities that are allowed to issue certificates for your domains. + +Use [SSLMate’s CAA Record Generator](https://sslmate.com/caa/) to create a **CAA record** with the following configuration: + +- `flags`: `0` +- `tag`: `issue` +- `value`: `"letsencrypt.org"` + +To verify if the DNS works, the following command + +```bash +dig @1.1.1.1 mydomain.com caa +``` + +should return: + +``` +mydomain.com. 3600 IN CAA 0 issue "letsencrypt.org" +``` + +### SMTP MTA Strict Transport Security (MTA-STS) + +[MTA-STS](https://datatracker.ietf.org/doc/html/rfc8461) is an extra step you can take to broadcast the ability of your instance to receive and, optionally enforce, TSL-secure SMTP connections to protect email traffic. + +Enabling MTA-STS requires you serve a specific file from subdomain `mta-sts.domain.com` on a well-known route. + +Create a text file `/var/www/.well-known/mta-sts.txt` with the content: + +```txt +version: STSv1 +mode: testing +mx: app.mydomain.com +max_age: 86400 +``` + +It is recommended to start with `mode: testing` for starters to get time to review failure reports. Add as many `mx:` domain entries as you have matching **MX records** in your DNS configuration. + +Create a **TXT record** for `_mta-sts.mydomain.com.` with the following value: + +```txt +v=STSv1; id=UNIX_TIMESTAMP +``` + +With `UNIX_TIMESTAMP` being the current date/time. + +Use the following command to generate the record: + +```bash +echo "v=STSv1; id=$(date +%s)" +``` + +To verify if the DNS works, the following command + +```bash +dig @1.1.1.1 _mta-sts.mydomain.com txt +``` + +should return a result similar to this one: + +``` +_mta-sts.mydomain.com. 3600 IN TXT "v=STSv1; id=1689416399" +``` + +Create an additional Nginx configuration in `/etc/nginx/sites-enabled/mta-sts` with the following content: + +``` +server { + server_name mta-sts.mydomain.com; + root /var/www; + listen 80; + + location ^~ /.well-known {} +} +``` + +Restart Nginx with the following command: + +```sh +sudo service nginx restart +``` + +A correct configuration of MTA-STS, however, requires that the certificate used to host the `mta-sts` subdomain matches that of the subdomain referred to by the **MX record** from the DNS. In other words, both `mta-sts.mydomain.com` and `app.mydomain.com` must share the same certificate. + +The easiest way to do this is to _expand_ the certificate associated with `app.mydomain.com` to also support the `mta-sts` subdomain using the following command: + +```sh +certbot --expand --nginx -d app.mydomain.com,mta-sts.mydomain.com +``` + +## SMTP TLS Reporting + +[TLSRPT](https://datatracker.ietf.org/doc/html/rfc8460) is used by SMTP systems to report failures in establishing TLS-secure sessions as broadcast by the MTA-STS configuration. + +Configuring MTA-STS in `mode: testing` as shown in the previous section gives you time to review failures from some SMTP senders. + +Create a **TXT record** for `_smtp._tls.mydomain.com.` with the following value: + +```txt +v=TSLRPTv1; rua=mailto:YOUR_EMAIL +``` + +The TLSRPT configuration at the DNS level allows SMTP senders that fail to initiate TLS-secure sessions to send reports to a particular email address. We suggest creating a `tls-reports` alias in SimpleLogin for this purpose. + +To verify if the DNS works, the following command + +```bash +dig @1.1.1.1 _smtp._tls.mydomain.com txt +``` + +should return a result similar to this one: + +``` +_smtp._tls.mydomain.com. 3600 IN TXT "v=TSLRPTv1; rua=mailto:tls-reports@mydomain.com" +``` diff --git a/app/migrations/versions/2023_072819_01827104004b_.py b/app/migrations/versions/2023_072819_01827104004b_.py new file mode 100644 index 0000000..7030508 --- /dev/null +++ b/app/migrations/versions/2023_072819_01827104004b_.py @@ -0,0 +1,42 @@ +"""empty message + +Revision ID: 01827104004b +Revises: 2634b41f54db +Create Date: 2023-07-28 19:39:28.675490 + +""" +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '01827104004b' +down_revision = '2634b41f54db' +branch_labels = None +depends_on = None + + +def upgrade(): + with op.get_context().autocommit_block(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_index(op.f('ix_alias_hibp_last_check'), 'alias', ['hibp_last_check'], unique=False, postgresql_concurrently=True) + op.create_index('ix_bounce_created_at', 'bounce', ['created_at'], unique=False, postgresql_concurrently=True) + op.create_index('ix_monitoring_created_at', 'monitoring', ['created_at'], unique=False, postgresql_concurrently=True) + op.create_index('ix_transactional_email_created_at', 'transactional_email', ['created_at'], unique=False, postgresql_concurrently=True) + op.create_index(op.f('ix_users_activated'), 'users', ['activated'], unique=False, postgresql_concurrently=True) + op.create_index('ix_users_activated_trial_end_lifetime', 'users', ['activated', 'trial_end', 'lifetime'], unique=False, postgresql_concurrently=True) + op.create_index(op.f('ix_users_referral_id'), 'users', ['referral_id'], unique=False, postgresql_concurrently=True) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_users_referral_id'), table_name='users') + op.drop_index('ix_users_activated_trial_end_lifetime', table_name='users') + op.drop_index(op.f('ix_users_activated'), table_name='users') + op.drop_index('ix_transactional_email_created_at', table_name='transactional_email') + op.drop_index('ix_monitoring_created_at', table_name='monitoring') + op.drop_index('ix_bounce_created_at', table_name='bounce') + op.drop_index(op.f('ix_alias_hibp_last_check'), table_name='alias') + # ### end Alembic commands ### diff --git a/app/templates/dashboard/support.html b/app/templates/dashboard/support.html index 922342f..4009e6c 100644 --- a/app/templates/dashboard/support.html +++ b/app/templates/dashboard/support.html @@ -28,7 +28,7 @@
- +
Attach files to support request
Only images, text and emails are accepted
diff --git a/app/tests/dashboard/test_api_keys.py b/app/tests/dashboard/test_api_keys.py index c00cda4..80cdfeb 100644 --- a/app/tests/dashboard/test_api_keys.py +++ b/app/tests/dashboard/test_api_keys.py @@ -37,6 +37,17 @@ def test_create_delete_api_key(flask_client): assert ApiKey.filter(ApiKey.user_id == user.id).count() == 1 assert api_key.name == "for test" + # create second api_key + create_r = flask_client.post( + url_for("dashboard.api_key"), + data={"form-name": "create", "name": "for test 2"}, + follow_redirects=True, + ) + assert create_r.status_code == 200 + api_key_2 = ApiKey.filter_by(user_id=user.id).order_by(ApiKey.id.desc()).first() + assert ApiKey.filter(ApiKey.user_id == user.id).count() == 2 + assert api_key_2.name == "for test 2" + # delete api_key delete_r = flask_client.post( url_for("dashboard.api_key"), @@ -44,7 +55,7 @@ def test_create_delete_api_key(flask_client): follow_redirects=True, ) assert delete_r.status_code == 200 - assert ApiKey.count() == nb_api_key + assert ApiKey.count() == nb_api_key + 1 def test_delete_all_api_keys(flask_client):