4.29.3

app/app/alias_suffix.py

@@ -6,7 +6,7 @@ from typing import Optional
 import itsdangerous
 from app import config
 from app.log import LOG
-from app.models import User, AliasOptions
+from app.models import User, AliasOptions, SLDomain
 
 signer = itsdangerous.TimestampSigner(config.CUSTOM_ALIAS_SECRET)
 
@@ -105,10 +105,7 @@ def get_alias_suffixes(
     for custom_domain in user_custom_domains:
         if custom_domain.random_prefix_generation:
             suffix = (
-                "."
+                f".{user.get_random_alias_suffix(custom_domain)}@{custom_domain.domain}"
-                + user.get_random_alias_suffix(custom_domain)
-                + "@"
-                + custom_domain.domain
             )
             alias_suffix = AliasSuffix(
                 is_custom=True,
@@ -123,7 +120,7 @@ def get_alias_suffixes(
             else:
                 alias_suffixes.append(alias_suffix)
 
-        suffix = "@" + custom_domain.domain
+        suffix = f"@{custom_domain.domain}"
         alias_suffix = AliasSuffix(
             is_custom=True,
             suffix=suffix,
@@ -144,16 +141,13 @@ def get_alias_suffixes(
             alias_suffixes.append(alias_suffix)
 
     # then SimpleLogin domain
-    for sl_domain in user.get_sl_domains(alias_options=alias_options):
+    sl_domains = user.get_sl_domains(alias_options=alias_options)
-        suffix = (
+    default_domain_found = False
-            (
+    for sl_domain in sl_domains:
-                ""
+        prefix = (
-                if config.DISABLE_ALIAS_SUFFIX
+            "" if config.DISABLE_ALIAS_SUFFIX else f".{user.get_random_alias_suffix()}"
-                else "." + user.get_random_alias_suffix()
-            )
-            + "@"
-            + sl_domain.domain
         )
+        suffix = f"{prefix}@{sl_domain.domain}"
         alias_suffix = AliasSuffix(
             is_custom=False,
             suffix=suffix,
@@ -162,11 +156,38 @@ def get_alias_suffixes(
             domain=sl_domain.domain,
             mx_verified=True,
         )
-
+        # No default or this is not the default
-        # put the default domain to top
+        if (
-        if user.default_alias_public_domain_id == sl_domain.id:
+            user.default_alias_public_domain_id is None
-            alias_suffixes.insert(0, alias_suffix)
+            or user.default_alias_public_domain_id != sl_domain.id
-        else:
+        ):
             alias_suffixes.append(alias_suffix)
+            # If no default domain mark it as found
+            default_domain_found = user.default_alias_public_domain_id is None
+        else:
+            default_domain_found = True
+            alias_suffixes.insert(0, alias_suffix)
+
+    if not default_domain_found:
+        domain_conditions = {"id": user.default_alias_public_domain_id, "hidden": False}
+        if not user.is_premium():
+            domain_conditions["premium_only"] = False
+        sl_domain = SLDomain.get_by(**domain_conditions)
+        if sl_domain:
+            prefix = (
+                ""
+                if config.DISABLE_ALIAS_SUFFIX
+                else f".{user.get_random_alias_suffix()}"
+            )
+            suffix = f"{prefix}@{sl_domain.domain}"
+            alias_suffix = AliasSuffix(
+                is_custom=False,
+                suffix=suffix,
+                signed_suffix=signer.sign(suffix).decode(),
+                is_premium=sl_domain.premium_only,
+                domain=sl_domain.domain,
+                mx_verified=True,
+            )
+            alias_suffixes.insert(0, alias_suffix)
 
     return alias_suffixes

app/app/api/views/apple.py

@@ -9,6 +9,7 @@ from requests import RequestException
 
 from app.api.base import api_bp, require_api_auth
 from app.config import APPLE_API_SECRET, MACAPP_APPLE_API_SECRET
+from app.subscription_webhook import execute_subscription_webhook
 from app.db import Session
 from app.log import LOG
 from app.models import PlanEnum, AppleSubscription
@@ -50,6 +51,7 @@ def apple_process_payment():
 
     apple_sub = verify_receipt(receipt_data, user, password)
     if apple_sub:
+        execute_subscription_webhook(user)
         return jsonify(ok=True), 200
 
     return jsonify(error="Processing failed"), 400
@@ -282,6 +284,7 @@ def apple_update_notification():
             apple_sub.plan = plan
             apple_sub.product_id = transaction["product_id"]
             Session.commit()
+            execute_subscription_webhook(user)
             return jsonify(ok=True), 200
         else:
             LOG.w(
@@ -554,6 +557,7 @@ def verify_receipt(receipt_data, user, password) -> Optional[AppleSubscription]:
             product_id=latest_transaction["product_id"],
         )
 
+    execute_subscription_webhook(user)
     Session.commit()
 
     return apple_sub

app/app/config.py

@@ -532,3 +532,5 @@ if ENABLE_ALL_REVERSE_ALIAS_REPLACEMENT:
 SKIP_MX_LOOKUP_ON_CHECK = False
 
 DISABLE_RATE_LIMIT = "DISABLE_RATE_LIMIT" in os.environ
+
+SUBSCRIPTION_CHANGE_WEBHOOK = os.environ.get("SUBSCRIPTION_CHANGE_WEBHOOK", None)

app/app/dashboard/views/setting.py

@@ -206,7 +206,7 @@ def setting():
                             current_profile_file is not None
                             and current_profile_file.user_id == current_user.id
                         ):
-                            s3.delete(current_user.path)
+                            s3.delete(current_profile_file.path)
 
                     file_path = random_string(30)
                     file = File.create(user_id=current_user.id, path=file_path)

app/app/jobs/export_user_data_job.py

@@ -41,7 +41,7 @@ from app.models import (
 class ExportUserDataJob:
 
     REMOVE_FIELDS = {
-        "User": ("otp_secret",),
+        "User": ("otp_secret", "password"),
         "Alias": ("ts_vector", "transfer_token", "hibp_last_check"),
         "CustomDomain": ("ownership_txt_token",),
     }

app/app/mail_sender.py

@@ -32,6 +32,7 @@ class SendRequest:
     rcpt_options: Dict = {}
     is_forward: bool = False
     ignore_smtp_errors: bool = False
+    retries: int = 0
 
     def to_bytes(self) -> bytes:
         if not config.SAVE_UNSENT_DIR:
@@ -67,6 +68,30 @@ class SendRequest:
             is_forward=decoded_data["is_forward"],
         )
 
+    def save_request_to_unsent_dir(self, prefix: str = "DeliveryFail"):
+        file_name = (
+            f"{prefix}-{int(time.time())}-{uuid.uuid4()}.{SendRequest.SAVE_EXTENSION}"
+        )
+        file_path = os.path.join(config.SAVE_UNSENT_DIR, file_name)
+        self.save_request_to_file(file_path)
+
+    @staticmethod
+    def save_request_to_failed_dir(self, prefix: str = "DeliveryRetryFail"):
+        file_name = (
+            f"{prefix}-{int(time.time())}-{uuid.uuid4()}.{SendRequest.SAVE_EXTENSION}"
+        )
+        dir_name = os.path.join(config.SAVE_UNSENT_DIR, "failed")
+        if not os.path.isdir(dir_name):
+            os.makedirs(dir_name)
+        file_path = os.path.join(dir_name, file_name)
+        self.save_request_to_file(file_path)
+
+    def save_request_to_file(self, file_path: str):
+        file_contents = self.to_bytes()
+        with open(file_path, "wb") as fd:
+            fd.write(file_contents)
+        LOG.i(f"Saved unsent message {file_path}")
+
 
 class MailSender:
     def __init__(self):
@@ -171,21 +196,9 @@ class MailSender:
                     f"Could not send message to smtp server {config.POSTFIX_SERVER}:{config.POSTFIX_PORT}"
                 )
                 if config.SAVE_UNSENT_DIR:
-                    self._save_request_to_unsent_dir(send_request)
+                    send_request.save_request_to_unsent_dir()
                 return False
 
-    def _save_request_to_unsent_dir(
-        self, send_request: SendRequest, prefix: str = "DeliveryFail"
-    ):
-        file_name = (
-            f"{prefix}-{int(time.time())}-{uuid.uuid4()}.{SendRequest.SAVE_EXTENSION}"
-        )
-        file_path = os.path.join(config.SAVE_UNSENT_DIR, file_name)
-        file_contents = send_request.to_bytes()
-        with open(file_path, "wb") as fd:
-            fd.write(file_contents)
-        LOG.i(f"Saved unsent message {file_path}")
-
 
 mail_sender = MailSender()
 
@@ -219,6 +232,7 @@ def load_unsent_mails_from_fs_and_resend():
         LOG.i(f"Trying to re-deliver email {filename}")
         try:
             send_request = SendRequest.load_from_file(full_file_path)
+            send_request.retries += 1
         except Exception as e:
             LOG.e(f"Cannot load {filename}. Error {e}")
             continue
@@ -230,6 +244,11 @@ def load_unsent_mails_from_fs_and_resend():
                     "DeliverUnsentEmail", {"delivered": "true"}
                 )
             else:
+                if send_request.retries > 2:
+                    os.unlink(full_file_path)
+                    send_request.save_request_to_failed_dir()
+                else:
+                    send_request.save_request_to_file(full_file_path)
                 newrelic.agent.record_custom_event(
                     "DeliverUnsentEmail", {"delivered": "false"}
                 )

app/app/models.py

@@ -673,6 +673,22 @@ class User(Base, ModelMixin, UserMixin, PasswordOracle):
 
         return None
 
+    def get_active_subscription_end(
+        self, include_partner_subscription: bool = True
+    ) -> Optional[arrow.Arrow]:
+        sub = self.get_active_subscription(
+            include_partner_subscription=include_partner_subscription
+        )
+        if isinstance(sub, Subscription):
+            return arrow.get(sub.next_bill_date)
+        if isinstance(sub, AppleSubscription):
+            return sub.expires_date
+        if isinstance(sub, ManualSubscription):
+            return sub.end_at
+        if isinstance(sub, CoinbaseSubscription):
+            return sub.end_at
+        return None
+
     # region Billing
     def lifetime_or_active_subscription(
         self, include_partner_subscription: bool = True

app/app/subscription_webhook.py

@@ -0,0 +1,33 @@
+import requests
+from requests import RequestException
+
+from app import config
+from app.log import LOG
+from app.models import User
+
+
+def execute_subscription_webhook(user: User):
+    webhook_url = config.SUBSCRIPTION_CHANGE_WEBHOOK
+    if webhook_url is None:
+        return
+    subscription_end = user.get_active_subscription_end(
+        include_partner_subscription=False
+    )
+    sl_subscription_end = None
+    if subscription_end:
+        sl_subscription_end = subscription_end.timestamp
+    payload = {
+        "user_id": user.id,
+        "is_premium": user.is_premium(),
+        "active_subscription_end": sl_subscription_end,
+    }
+    try:
+        response = requests.post(webhook_url, json=payload, timeout=2)
+        if response.status_code == 200:
+            LOG.i("Sent request to subscription update webhook successfully")
+        else:
+            LOG.i(
+                f"Request to webhook failed with statue {response.status_code}: {response.text}"
+            )
+    except RequestException as e:
+        LOG.error(f"Subscription request exception: {e}")

app/server.py

@@ -79,6 +79,7 @@ from app.config import (
     MEM_STORE_URI,
 )
 from app.dashboard.base import dashboard_bp
+from app.subscription_webhook import execute_subscription_webhook
 from app.db import Session
 from app.developer.base import developer_bp
 from app.discover.base import discover_bp
@@ -491,6 +492,7 @@ def setup_paddle_callback(app: Flask):
                 # in case user cancels a plan and subscribes a new plan
                 sub.cancelled = False
 
+            execute_subscription_webhook(user)
             LOG.d("User %s upgrades!", user)
 
             Session.commit()
@@ -509,6 +511,7 @@ def setup_paddle_callback(app: Flask):
                 ).date()
 
                 Session.commit()
+                execute_subscription_webhook(sub.user)
 
         elif request.form.get("alert_name") == "subscription_cancelled":
             subscription_id = request.form.get("subscription_id")
@@ -538,6 +541,7 @@ def setup_paddle_callback(app: Flask):
                         end_date=request.form.get("cancellation_effective_date"),
                     ),
                 )
+                execute_subscription_webhook(sub.user)
 
             else:
                 # user might have deleted their account
@@ -580,6 +584,7 @@ def setup_paddle_callback(app: Flask):
                 sub.cancelled = False
 
                 Session.commit()
+                execute_subscription_webhook(sub.user)
             else:
                 LOG.w(
                     f"update non-exist subscription {subscription_id}. {request.form}"
@@ -596,6 +601,7 @@ def setup_paddle_callback(app: Flask):
                 Subscription.delete(sub.id)
                 Session.commit()
                 LOG.e("%s requests a refund", user)
+                execute_subscription_webhook(sub.user)
 
         elif request.form.get("alert_name") == "subscription_payment_refunded":
             subscription_id = request.form.get("subscription_id")
@@ -629,6 +635,7 @@ def setup_paddle_callback(app: Flask):
                     LOG.e("Unknown plan_id %s", plan_id)
             else:
                 LOG.w("partial subscription_payment_refunded, not handled")
+            execute_subscription_webhook(sub.user)
 
         return "OK"
 
@@ -742,6 +749,7 @@ def handle_coinbase_event(event) -> bool:
                 coinbase_subscription=coinbase_subscription,
             ),
         )
+    execute_subscription_webhook(user)
 
     return True
 

app/tests/test_alias_suffixes.py

@@ -0,0 +1,133 @@
+import re
+
+from app.alias_suffix import get_alias_suffixes
+from app.db import Session
+from app.models import SLDomain, PartnerUser, AliasOptions, CustomDomain
+from app.proton.utils import get_proton_partner
+from init_app import add_sl_domains
+from tests.utils import create_new_user, random_token
+
+
+def setup_module():
+    Session.query(SLDomain).delete()
+    SLDomain.create(
+        domain="hidden", premium_only=False, flush=True, order=5, hidden=True
+    )
+    SLDomain.create(domain="free_non_partner", premium_only=False, flush=True, order=4)
+    SLDomain.create(
+        domain="premium_non_partner", premium_only=True, flush=True, order=3
+    )
+    SLDomain.create(
+        domain="free_partner",
+        premium_only=False,
+        flush=True,
+        partner_id=get_proton_partner().id,
+        order=2,
+    )
+    SLDomain.create(
+        domain="premium_partner",
+        premium_only=True,
+        flush=True,
+        partner_id=get_proton_partner().id,
+        order=1,
+    )
+    Session.commit()
+
+
+def teardown_module():
+    Session.query(SLDomain).delete()
+    add_sl_domains()
+
+
+def test_get_default_domain_even_if_is_not_allowed():
+    user = create_new_user()
+    PartnerUser.create(
+        partner_id=get_proton_partner().id,
+        user_id=user.id,
+        external_user_id=random_token(10),
+        flush=True,
+    )
+    user.trial_end = None
+    default_domain = SLDomain.filter_by(
+        hidden=False, partner_id=None, premium_only=False
+    ).first()
+    user.default_alias_public_domain_id = default_domain.id
+    Session.flush()
+    options = AliasOptions(
+        show_sl_domains=False, show_partner_domains=get_proton_partner()
+    )
+    suffixes = get_alias_suffixes(user, alias_options=options)
+    assert suffixes[0].domain == default_domain.domain
+
+
+def test_get_default_domain_hidden():
+    user = create_new_user()
+    PartnerUser.create(
+        partner_id=get_proton_partner().id,
+        user_id=user.id,
+        external_user_id=random_token(10),
+        flush=True,
+    )
+    user.trial_end = None
+    default_domain = SLDomain.filter_by(
+        hidden=True, partner_id=None, premium_only=False
+    ).first()
+    user.default_alias_public_domain_id = default_domain.id
+    Session.flush()
+    options = AliasOptions(
+        show_sl_domains=False, show_partner_domains=get_proton_partner()
+    )
+    suffixes = get_alias_suffixes(user, alias_options=options)
+    for suffix in suffixes:
+        domain = SLDomain.get_by(domain=suffix.domain)
+        assert not domain.hidden
+    assert suffixes[0].domain != default_domain.domain
+
+
+def test_get_default_domain_is_premium_for_free_user():
+    user = create_new_user()
+    PartnerUser.create(
+        partner_id=get_proton_partner().id,
+        user_id=user.id,
+        external_user_id=random_token(10),
+        flush=True,
+    )
+    user.trial_end = None
+    default_domain = SLDomain.filter_by(partner_id=None, premium_only=True).first()
+    user.default_alias_public_domain_id = default_domain.id
+    Session.flush()
+    options = AliasOptions(
+        show_sl_domains=False, show_partner_domains=get_proton_partner()
+    )
+    suffixes = get_alias_suffixes(user, alias_options=options)
+    for suffix in suffixes:
+        domain = SLDomain.get_by(domain=suffix.domain)
+        assert not domain.premium_only
+    assert suffixes[0].domain != default_domain.domain
+
+
+def test_suffixes_are_valid():
+    user = create_new_user()
+    PartnerUser.create(
+        partner_id=get_proton_partner().id,
+        user_id=user.id,
+        external_user_id=random_token(10),
+        flush=True,
+    )
+    CustomDomain.create(
+        user_id=user.id, domain=f"{random_token(10)}.com", verified=True
+    )
+    user.trial_end = None
+    Session.flush()
+    options = AliasOptions(
+        show_sl_domains=True, show_partner_domains=get_proton_partner()
+    )
+    alias_suffixes = get_alias_suffixes(user, alias_options=options)
+    valid_re = re.compile(r"^(\.[\w_]+)?@[\.\w]+$")
+    has_prefix = 0
+    for suffix in alias_suffixes:
+        match = valid_re.match(suffix.suffix)
+        assert match is not None
+        if len(match.groups()) >= 1:
+            has_prefix += 1
+    assert has_prefix > 0

app/tests/test_subscription_webhook.py

@@ -0,0 +1,113 @@
+import http.server
+import json
+import threading
+
+import arrow
+
+from app import config
+from app.models import (
+    Subscription,
+    AppleSubscription,
+    CoinbaseSubscription,
+    ManualSubscription,
+)
+from tests.utils import create_new_user, random_token
+
+from app.subscription_webhook import execute_subscription_webhook
+
+http_server = None
+last_http_request = None
+
+
+def setup_module():
+    global http_server
+    http_server = http.server.ThreadingHTTPServer(("", 0), HTTPTestServer)
+    print(http_server.server_port)
+    threading.Thread(target=http_server.serve_forever, daemon=True).start()
+    config.SUBSCRIPTION_CHANGE_WEBHOOK = f"http://localhost:{http_server.server_port}"
+
+
+def teardown_module():
+    global http_server
+    config.SUBSCRIPTION_CHANGE_WEBHOOK = None
+    http_server.shutdown()
+
+
+class HTTPTestServer(http.server.BaseHTTPRequestHandler):
+    def do_POST(self):
+        global last_http_request
+        content_len = int(self.headers.get("Content-Length"))
+        body_data = self.rfile.read(content_len)
+        last_http_request = json.loads(body_data)
+        self.send_response(200)
+
+
+def test_webhook_with_trial():
+    user = create_new_user()
+    execute_subscription_webhook(user)
+    assert last_http_request["user_id"] == user.id
+    assert last_http_request["is_premium"]
+    assert last_http_request["active_subscription_end"] is None
+
+
+def test_webhook_with_subscription():
+    user = create_new_user()
+    end_at = arrow.utcnow().shift(days=1).replace(hour=0, minute=0, second=0)
+    Subscription.create(
+        user_id=user.id,
+        cancel_url="",
+        update_url="",
+        subscription_id=random_token(10),
+        event_time=arrow.now(),
+        next_bill_date=end_at.date(),
+        plan="yearly",
+        flush=True,
+    )
+    execute_subscription_webhook(user)
+    assert last_http_request["user_id"] == user.id
+    assert last_http_request["is_premium"]
+    assert last_http_request["active_subscription_end"] == end_at.timestamp
+
+
+def test_webhook_with_apple_subscription():
+    user = create_new_user()
+    end_at = arrow.utcnow().shift(days=2).replace(hour=0, minute=0, second=0)
+    AppleSubscription.create(
+        user_id=user.id,
+        receipt_data=arrow.now().date().strftime("%Y-%m-%d"),
+        expires_date=end_at.date().strftime("%Y-%m-%d"),
+        original_transaction_id=random_token(10),
+        plan="yearly",
+        product_id="",
+        flush=True,
+    )
+    execute_subscription_webhook(user)
+    assert last_http_request["user_id"] == user.id
+    assert last_http_request["is_premium"]
+    assert last_http_request["active_subscription_end"] == end_at.timestamp
+
+
+def test_webhook_with_coinbase_subscription():
+    user = create_new_user()
+    end_at = arrow.utcnow().shift(days=3).replace(hour=0, minute=0, second=0)
+    CoinbaseSubscription.create(
+        user_id=user.id, end_at=end_at.date().strftime("%Y-%m-%d"), flush=True
+    )
+
+    execute_subscription_webhook(user)
+    assert last_http_request["user_id"] == user.id
+    assert last_http_request["is_premium"]
+    assert last_http_request["active_subscription_end"] == end_at.timestamp
+
+
+def test_webhook_with_manual_subscription():
+    user = create_new_user()
+    end_at = arrow.utcnow().shift(days=3).replace(hour=0, minute=0, second=0)
+    ManualSubscription.create(
+        user_id=user.id, end_at=end_at.date().strftime("%Y-%m-%d"), flush=True
+    )
+
+    execute_subscription_webhook(user)
+    assert last_http_request["user_id"] == user.id
+    assert last_http_request["is_premium"]
+    assert last_http_request["active_subscription_end"] == end_at.timestamp