From a6f4995cb58713d53e13d95abd8432d35e4862c6 Mon Sep 17 00:00:00 2001 From: MrMeeb Date: Thu, 1 Jun 2023 11:00:05 +0000 Subject: [PATCH] 4.29.3 --- app/app/alias_suffix.py | 61 ++++++++---- app/app/api/views/apple.py | 4 + app/app/config.py | 2 + app/app/dashboard/views/setting.py | 2 +- app/app/jobs/export_user_data_job.py | 2 +- app/app/mail_sender.py | 45 ++++++--- app/app/models.py | 16 +++ app/app/subscription_webhook.py | 33 ++++++ app/server.py | 8 ++ app/tests/test_alias_suffixes.py | 133 +++++++++++++++++++++++++ app/tests/test_subscription_webhook.py | 113 +++++++++++++++++++++ 11 files changed, 384 insertions(+), 35 deletions(-) create mode 100644 app/app/subscription_webhook.py create mode 100644 app/tests/test_alias_suffixes.py create mode 100644 app/tests/test_subscription_webhook.py diff --git a/app/app/alias_suffix.py b/app/app/alias_suffix.py index 83d6392..40c5462 100644 --- a/app/app/alias_suffix.py +++ b/app/app/alias_suffix.py @@ -6,7 +6,7 @@ from typing import Optional import itsdangerous from app import config from app.log import LOG -from app.models import User, AliasOptions +from app.models import User, AliasOptions, SLDomain signer = itsdangerous.TimestampSigner(config.CUSTOM_ALIAS_SECRET) @@ -105,10 +105,7 @@ def get_alias_suffixes( for custom_domain in user_custom_domains: if custom_domain.random_prefix_generation: suffix = ( - "." - + user.get_random_alias_suffix(custom_domain) - + "@" - + custom_domain.domain + f".{user.get_random_alias_suffix(custom_domain)}@{custom_domain.domain}" ) alias_suffix = AliasSuffix( is_custom=True, @@ -123,7 +120,7 @@ def get_alias_suffixes( else: alias_suffixes.append(alias_suffix) - suffix = "@" + custom_domain.domain + suffix = f"@{custom_domain.domain}" alias_suffix = AliasSuffix( is_custom=True, suffix=suffix, @@ -144,16 +141,13 @@ def get_alias_suffixes( alias_suffixes.append(alias_suffix) # then SimpleLogin domain - for sl_domain in user.get_sl_domains(alias_options=alias_options): - suffix = ( - ( - "" - if config.DISABLE_ALIAS_SUFFIX - else "." + user.get_random_alias_suffix() - ) - + "@" - + sl_domain.domain + sl_domains = user.get_sl_domains(alias_options=alias_options) + default_domain_found = False + for sl_domain in sl_domains: + prefix = ( + "" if config.DISABLE_ALIAS_SUFFIX else f".{user.get_random_alias_suffix()}" ) + suffix = f"{prefix}@{sl_domain.domain}" alias_suffix = AliasSuffix( is_custom=False, suffix=suffix, @@ -162,11 +156,38 @@ def get_alias_suffixes( domain=sl_domain.domain, mx_verified=True, ) - - # put the default domain to top - if user.default_alias_public_domain_id == sl_domain.id: - alias_suffixes.insert(0, alias_suffix) - else: + # No default or this is not the default + if ( + user.default_alias_public_domain_id is None + or user.default_alias_public_domain_id != sl_domain.id + ): alias_suffixes.append(alias_suffix) + # If no default domain mark it as found + default_domain_found = user.default_alias_public_domain_id is None + else: + default_domain_found = True + alias_suffixes.insert(0, alias_suffix) + + if not default_domain_found: + domain_conditions = {"id": user.default_alias_public_domain_id, "hidden": False} + if not user.is_premium(): + domain_conditions["premium_only"] = False + sl_domain = SLDomain.get_by(**domain_conditions) + if sl_domain: + prefix = ( + "" + if config.DISABLE_ALIAS_SUFFIX + else f".{user.get_random_alias_suffix()}" + ) + suffix = f"{prefix}@{sl_domain.domain}" + alias_suffix = AliasSuffix( + is_custom=False, + suffix=suffix, + signed_suffix=signer.sign(suffix).decode(), + is_premium=sl_domain.premium_only, + domain=sl_domain.domain, + mx_verified=True, + ) + alias_suffixes.insert(0, alias_suffix) return alias_suffixes diff --git a/app/app/api/views/apple.py b/app/app/api/views/apple.py index 119bd29..8b5125b 100644 --- a/app/app/api/views/apple.py +++ b/app/app/api/views/apple.py @@ -9,6 +9,7 @@ from requests import RequestException from app.api.base import api_bp, require_api_auth from app.config import APPLE_API_SECRET, MACAPP_APPLE_API_SECRET +from app.subscription_webhook import execute_subscription_webhook from app.db import Session from app.log import LOG from app.models import PlanEnum, AppleSubscription @@ -50,6 +51,7 @@ def apple_process_payment(): apple_sub = verify_receipt(receipt_data, user, password) if apple_sub: + execute_subscription_webhook(user) return jsonify(ok=True), 200 return jsonify(error="Processing failed"), 400 @@ -282,6 +284,7 @@ def apple_update_notification(): apple_sub.plan = plan apple_sub.product_id = transaction["product_id"] Session.commit() + execute_subscription_webhook(user) return jsonify(ok=True), 200 else: LOG.w( @@ -554,6 +557,7 @@ def verify_receipt(receipt_data, user, password) -> Optional[AppleSubscription]: product_id=latest_transaction["product_id"], ) + execute_subscription_webhook(user) Session.commit() return apple_sub diff --git a/app/app/config.py b/app/app/config.py index 782f693..a33746b 100644 --- a/app/app/config.py +++ b/app/app/config.py @@ -532,3 +532,5 @@ if ENABLE_ALL_REVERSE_ALIAS_REPLACEMENT: SKIP_MX_LOOKUP_ON_CHECK = False DISABLE_RATE_LIMIT = "DISABLE_RATE_LIMIT" in os.environ + +SUBSCRIPTION_CHANGE_WEBHOOK = os.environ.get("SUBSCRIPTION_CHANGE_WEBHOOK", None) diff --git a/app/app/dashboard/views/setting.py b/app/app/dashboard/views/setting.py index 79ccb16..832ce83 100644 --- a/app/app/dashboard/views/setting.py +++ b/app/app/dashboard/views/setting.py @@ -206,7 +206,7 @@ def setting(): current_profile_file is not None and current_profile_file.user_id == current_user.id ): - s3.delete(current_user.path) + s3.delete(current_profile_file.path) file_path = random_string(30) file = File.create(user_id=current_user.id, path=file_path) diff --git a/app/app/jobs/export_user_data_job.py b/app/app/jobs/export_user_data_job.py index 831f840..8c3701b 100644 --- a/app/app/jobs/export_user_data_job.py +++ b/app/app/jobs/export_user_data_job.py @@ -41,7 +41,7 @@ from app.models import ( class ExportUserDataJob: REMOVE_FIELDS = { - "User": ("otp_secret",), + "User": ("otp_secret", "password"), "Alias": ("ts_vector", "transfer_token", "hibp_last_check"), "CustomDomain": ("ownership_txt_token",), } diff --git a/app/app/mail_sender.py b/app/app/mail_sender.py index 5f30002..7aba370 100644 --- a/app/app/mail_sender.py +++ b/app/app/mail_sender.py @@ -32,6 +32,7 @@ class SendRequest: rcpt_options: Dict = {} is_forward: bool = False ignore_smtp_errors: bool = False + retries: int = 0 def to_bytes(self) -> bytes: if not config.SAVE_UNSENT_DIR: @@ -67,6 +68,30 @@ class SendRequest: is_forward=decoded_data["is_forward"], ) + def save_request_to_unsent_dir(self, prefix: str = "DeliveryFail"): + file_name = ( + f"{prefix}-{int(time.time())}-{uuid.uuid4()}.{SendRequest.SAVE_EXTENSION}" + ) + file_path = os.path.join(config.SAVE_UNSENT_DIR, file_name) + self.save_request_to_file(file_path) + + @staticmethod + def save_request_to_failed_dir(self, prefix: str = "DeliveryRetryFail"): + file_name = ( + f"{prefix}-{int(time.time())}-{uuid.uuid4()}.{SendRequest.SAVE_EXTENSION}" + ) + dir_name = os.path.join(config.SAVE_UNSENT_DIR, "failed") + if not os.path.isdir(dir_name): + os.makedirs(dir_name) + file_path = os.path.join(dir_name, file_name) + self.save_request_to_file(file_path) + + def save_request_to_file(self, file_path: str): + file_contents = self.to_bytes() + with open(file_path, "wb") as fd: + fd.write(file_contents) + LOG.i(f"Saved unsent message {file_path}") + class MailSender: def __init__(self): @@ -171,21 +196,9 @@ class MailSender: f"Could not send message to smtp server {config.POSTFIX_SERVER}:{config.POSTFIX_PORT}" ) if config.SAVE_UNSENT_DIR: - self._save_request_to_unsent_dir(send_request) + send_request.save_request_to_unsent_dir() return False - def _save_request_to_unsent_dir( - self, send_request: SendRequest, prefix: str = "DeliveryFail" - ): - file_name = ( - f"{prefix}-{int(time.time())}-{uuid.uuid4()}.{SendRequest.SAVE_EXTENSION}" - ) - file_path = os.path.join(config.SAVE_UNSENT_DIR, file_name) - file_contents = send_request.to_bytes() - with open(file_path, "wb") as fd: - fd.write(file_contents) - LOG.i(f"Saved unsent message {file_path}") - mail_sender = MailSender() @@ -219,6 +232,7 @@ def load_unsent_mails_from_fs_and_resend(): LOG.i(f"Trying to re-deliver email {filename}") try: send_request = SendRequest.load_from_file(full_file_path) + send_request.retries += 1 except Exception as e: LOG.e(f"Cannot load {filename}. Error {e}") continue @@ -230,6 +244,11 @@ def load_unsent_mails_from_fs_and_resend(): "DeliverUnsentEmail", {"delivered": "true"} ) else: + if send_request.retries > 2: + os.unlink(full_file_path) + send_request.save_request_to_failed_dir() + else: + send_request.save_request_to_file(full_file_path) newrelic.agent.record_custom_event( "DeliverUnsentEmail", {"delivered": "false"} ) diff --git a/app/app/models.py b/app/app/models.py index 7a1ad0a..4b4d625 100644 --- a/app/app/models.py +++ b/app/app/models.py @@ -673,6 +673,22 @@ class User(Base, ModelMixin, UserMixin, PasswordOracle): return None + def get_active_subscription_end( + self, include_partner_subscription: bool = True + ) -> Optional[arrow.Arrow]: + sub = self.get_active_subscription( + include_partner_subscription=include_partner_subscription + ) + if isinstance(sub, Subscription): + return arrow.get(sub.next_bill_date) + if isinstance(sub, AppleSubscription): + return sub.expires_date + if isinstance(sub, ManualSubscription): + return sub.end_at + if isinstance(sub, CoinbaseSubscription): + return sub.end_at + return None + # region Billing def lifetime_or_active_subscription( self, include_partner_subscription: bool = True diff --git a/app/app/subscription_webhook.py b/app/app/subscription_webhook.py new file mode 100644 index 0000000..70a447e --- /dev/null +++ b/app/app/subscription_webhook.py @@ -0,0 +1,33 @@ +import requests +from requests import RequestException + +from app import config +from app.log import LOG +from app.models import User + + +def execute_subscription_webhook(user: User): + webhook_url = config.SUBSCRIPTION_CHANGE_WEBHOOK + if webhook_url is None: + return + subscription_end = user.get_active_subscription_end( + include_partner_subscription=False + ) + sl_subscription_end = None + if subscription_end: + sl_subscription_end = subscription_end.timestamp + payload = { + "user_id": user.id, + "is_premium": user.is_premium(), + "active_subscription_end": sl_subscription_end, + } + try: + response = requests.post(webhook_url, json=payload, timeout=2) + if response.status_code == 200: + LOG.i("Sent request to subscription update webhook successfully") + else: + LOG.i( + f"Request to webhook failed with statue {response.status_code}: {response.text}" + ) + except RequestException as e: + LOG.error(f"Subscription request exception: {e}") diff --git a/app/server.py b/app/server.py index c31fb5d..d6bb00e 100644 --- a/app/server.py +++ b/app/server.py @@ -79,6 +79,7 @@ from app.config import ( MEM_STORE_URI, ) from app.dashboard.base import dashboard_bp +from app.subscription_webhook import execute_subscription_webhook from app.db import Session from app.developer.base import developer_bp from app.discover.base import discover_bp @@ -491,6 +492,7 @@ def setup_paddle_callback(app: Flask): # in case user cancels a plan and subscribes a new plan sub.cancelled = False + execute_subscription_webhook(user) LOG.d("User %s upgrades!", user) Session.commit() @@ -509,6 +511,7 @@ def setup_paddle_callback(app: Flask): ).date() Session.commit() + execute_subscription_webhook(sub.user) elif request.form.get("alert_name") == "subscription_cancelled": subscription_id = request.form.get("subscription_id") @@ -538,6 +541,7 @@ def setup_paddle_callback(app: Flask): end_date=request.form.get("cancellation_effective_date"), ), ) + execute_subscription_webhook(sub.user) else: # user might have deleted their account @@ -580,6 +584,7 @@ def setup_paddle_callback(app: Flask): sub.cancelled = False Session.commit() + execute_subscription_webhook(sub.user) else: LOG.w( f"update non-exist subscription {subscription_id}. {request.form}" @@ -596,6 +601,7 @@ def setup_paddle_callback(app: Flask): Subscription.delete(sub.id) Session.commit() LOG.e("%s requests a refund", user) + execute_subscription_webhook(sub.user) elif request.form.get("alert_name") == "subscription_payment_refunded": subscription_id = request.form.get("subscription_id") @@ -629,6 +635,7 @@ def setup_paddle_callback(app: Flask): LOG.e("Unknown plan_id %s", plan_id) else: LOG.w("partial subscription_payment_refunded, not handled") + execute_subscription_webhook(sub.user) return "OK" @@ -742,6 +749,7 @@ def handle_coinbase_event(event) -> bool: coinbase_subscription=coinbase_subscription, ), ) + execute_subscription_webhook(user) return True diff --git a/app/tests/test_alias_suffixes.py b/app/tests/test_alias_suffixes.py new file mode 100644 index 0000000..9706a13 --- /dev/null +++ b/app/tests/test_alias_suffixes.py @@ -0,0 +1,133 @@ +import re + +from app.alias_suffix import get_alias_suffixes +from app.db import Session +from app.models import SLDomain, PartnerUser, AliasOptions, CustomDomain +from app.proton.utils import get_proton_partner +from init_app import add_sl_domains +from tests.utils import create_new_user, random_token + + +def setup_module(): + Session.query(SLDomain).delete() + SLDomain.create( + domain="hidden", premium_only=False, flush=True, order=5, hidden=True + ) + SLDomain.create(domain="free_non_partner", premium_only=False, flush=True, order=4) + SLDomain.create( + domain="premium_non_partner", premium_only=True, flush=True, order=3 + ) + SLDomain.create( + domain="free_partner", + premium_only=False, + flush=True, + partner_id=get_proton_partner().id, + order=2, + ) + SLDomain.create( + domain="premium_partner", + premium_only=True, + flush=True, + partner_id=get_proton_partner().id, + order=1, + ) + Session.commit() + + +def teardown_module(): + Session.query(SLDomain).delete() + add_sl_domains() + + +def test_get_default_domain_even_if_is_not_allowed(): + user = create_new_user() + PartnerUser.create( + partner_id=get_proton_partner().id, + user_id=user.id, + external_user_id=random_token(10), + flush=True, + ) + user.trial_end = None + default_domain = SLDomain.filter_by( + hidden=False, partner_id=None, premium_only=False + ).first() + user.default_alias_public_domain_id = default_domain.id + Session.flush() + options = AliasOptions( + show_sl_domains=False, show_partner_domains=get_proton_partner() + ) + suffixes = get_alias_suffixes(user, alias_options=options) + assert suffixes[0].domain == default_domain.domain + + +def test_get_default_domain_hidden(): + user = create_new_user() + PartnerUser.create( + partner_id=get_proton_partner().id, + user_id=user.id, + external_user_id=random_token(10), + flush=True, + ) + user.trial_end = None + default_domain = SLDomain.filter_by( + hidden=True, partner_id=None, premium_only=False + ).first() + user.default_alias_public_domain_id = default_domain.id + Session.flush() + options = AliasOptions( + show_sl_domains=False, show_partner_domains=get_proton_partner() + ) + suffixes = get_alias_suffixes(user, alias_options=options) + for suffix in suffixes: + domain = SLDomain.get_by(domain=suffix.domain) + assert not domain.hidden + assert suffixes[0].domain != default_domain.domain + + +def test_get_default_domain_is_premium_for_free_user(): + user = create_new_user() + PartnerUser.create( + partner_id=get_proton_partner().id, + user_id=user.id, + external_user_id=random_token(10), + flush=True, + ) + user.trial_end = None + default_domain = SLDomain.filter_by(partner_id=None, premium_only=True).first() + user.default_alias_public_domain_id = default_domain.id + Session.flush() + options = AliasOptions( + show_sl_domains=False, show_partner_domains=get_proton_partner() + ) + suffixes = get_alias_suffixes(user, alias_options=options) + for suffix in suffixes: + domain = SLDomain.get_by(domain=suffix.domain) + assert not domain.premium_only + assert suffixes[0].domain != default_domain.domain + + +def test_suffixes_are_valid(): + user = create_new_user() + PartnerUser.create( + partner_id=get_proton_partner().id, + user_id=user.id, + external_user_id=random_token(10), + flush=True, + ) + CustomDomain.create( + user_id=user.id, domain=f"{random_token(10)}.com", verified=True + ) + user.trial_end = None + Session.flush() + options = AliasOptions( + show_sl_domains=True, show_partner_domains=get_proton_partner() + ) + alias_suffixes = get_alias_suffixes(user, alias_options=options) + valid_re = re.compile(r"^(\.[\w_]+)?@[\.\w]+$") + has_prefix = 0 + for suffix in alias_suffixes: + match = valid_re.match(suffix.suffix) + assert match is not None + if len(match.groups()) >= 1: + has_prefix += 1 + assert has_prefix > 0 diff --git a/app/tests/test_subscription_webhook.py b/app/tests/test_subscription_webhook.py new file mode 100644 index 0000000..7e7ed07 --- /dev/null +++ b/app/tests/test_subscription_webhook.py @@ -0,0 +1,113 @@ +import http.server +import json +import threading + +import arrow + +from app import config +from app.models import ( + Subscription, + AppleSubscription, + CoinbaseSubscription, + ManualSubscription, +) +from tests.utils import create_new_user, random_token + +from app.subscription_webhook import execute_subscription_webhook + +http_server = None +last_http_request = None + + +def setup_module(): + global http_server + http_server = http.server.ThreadingHTTPServer(("", 0), HTTPTestServer) + print(http_server.server_port) + threading.Thread(target=http_server.serve_forever, daemon=True).start() + config.SUBSCRIPTION_CHANGE_WEBHOOK = f"http://localhost:{http_server.server_port}" + + +def teardown_module(): + global http_server + config.SUBSCRIPTION_CHANGE_WEBHOOK = None + http_server.shutdown() + + +class HTTPTestServer(http.server.BaseHTTPRequestHandler): + def do_POST(self): + global last_http_request + content_len = int(self.headers.get("Content-Length")) + body_data = self.rfile.read(content_len) + last_http_request = json.loads(body_data) + self.send_response(200) + + +def test_webhook_with_trial(): + user = create_new_user() + execute_subscription_webhook(user) + assert last_http_request["user_id"] == user.id + assert last_http_request["is_premium"] + assert last_http_request["active_subscription_end"] is None + + +def test_webhook_with_subscription(): + user = create_new_user() + end_at = arrow.utcnow().shift(days=1).replace(hour=0, minute=0, second=0) + Subscription.create( + user_id=user.id, + cancel_url="", + update_url="", + subscription_id=random_token(10), + event_time=arrow.now(), + next_bill_date=end_at.date(), + plan="yearly", + flush=True, + ) + execute_subscription_webhook(user) + assert last_http_request["user_id"] == user.id + assert last_http_request["is_premium"] + assert last_http_request["active_subscription_end"] == end_at.timestamp + + +def test_webhook_with_apple_subscription(): + user = create_new_user() + end_at = arrow.utcnow().shift(days=2).replace(hour=0, minute=0, second=0) + AppleSubscription.create( + user_id=user.id, + receipt_data=arrow.now().date().strftime("%Y-%m-%d"), + expires_date=end_at.date().strftime("%Y-%m-%d"), + original_transaction_id=random_token(10), + plan="yearly", + product_id="", + flush=True, + ) + execute_subscription_webhook(user) + assert last_http_request["user_id"] == user.id + assert last_http_request["is_premium"] + assert last_http_request["active_subscription_end"] == end_at.timestamp + + +def test_webhook_with_coinbase_subscription(): + user = create_new_user() + end_at = arrow.utcnow().shift(days=3).replace(hour=0, minute=0, second=0) + CoinbaseSubscription.create( + user_id=user.id, end_at=end_at.date().strftime("%Y-%m-%d"), flush=True + ) + + execute_subscription_webhook(user) + assert last_http_request["user_id"] == user.id + assert last_http_request["is_premium"] + assert last_http_request["active_subscription_end"] == end_at.timestamp + + +def test_webhook_with_manual_subscription(): + user = create_new_user() + end_at = arrow.utcnow().shift(days=3).replace(hour=0, minute=0, second=0) + ManualSubscription.create( + user_id=user.id, end_at=end_at.date().strftime("%Y-%m-%d"), flush=True + ) + + execute_subscription_webhook(user) + assert last_http_request["user_id"] == user.id + assert last_http_request["is_premium"] + assert last_http_request["active_subscription_end"] == end_at.timestamp