diff --git a/app/app/alias_suffix.py b/app/app/alias_suffix.py
index 83d6392..40c5462 100644
--- a/app/app/alias_suffix.py
+++ b/app/app/alias_suffix.py
@@ -6,7 +6,7 @@ from typing import Optional
import itsdangerous
from app import config
from app.log import LOG
-from app.models import User, AliasOptions
+from app.models import User, AliasOptions, SLDomain
signer = itsdangerous.TimestampSigner(config.CUSTOM_ALIAS_SECRET)
@@ -105,10 +105,7 @@ def get_alias_suffixes(
for custom_domain in user_custom_domains:
if custom_domain.random_prefix_generation:
suffix = (
- "."
- + user.get_random_alias_suffix(custom_domain)
- + "@"
- + custom_domain.domain
+ f".{user.get_random_alias_suffix(custom_domain)}@{custom_domain.domain}"
)
alias_suffix = AliasSuffix(
is_custom=True,
@@ -123,7 +120,7 @@ def get_alias_suffixes(
else:
alias_suffixes.append(alias_suffix)
- suffix = "@" + custom_domain.domain
+ suffix = f"@{custom_domain.domain}"
alias_suffix = AliasSuffix(
is_custom=True,
suffix=suffix,
@@ -144,16 +141,13 @@ def get_alias_suffixes(
alias_suffixes.append(alias_suffix)
# then SimpleLogin domain
- for sl_domain in user.get_sl_domains(alias_options=alias_options):
- suffix = (
- (
- ""
- if config.DISABLE_ALIAS_SUFFIX
- else "." + user.get_random_alias_suffix()
- )
- + "@"
- + sl_domain.domain
+ sl_domains = user.get_sl_domains(alias_options=alias_options)
+ default_domain_found = False
+ for sl_domain in sl_domains:
+ prefix = (
+ "" if config.DISABLE_ALIAS_SUFFIX else f".{user.get_random_alias_suffix()}"
)
+ suffix = f"{prefix}@{sl_domain.domain}"
alias_suffix = AliasSuffix(
is_custom=False,
suffix=suffix,
@@ -162,11 +156,38 @@ def get_alias_suffixes(
domain=sl_domain.domain,
mx_verified=True,
)
-
- # put the default domain to top
- if user.default_alias_public_domain_id == sl_domain.id:
- alias_suffixes.insert(0, alias_suffix)
- else:
+ # No default or this is not the default
+ if (
+ user.default_alias_public_domain_id is None
+ or user.default_alias_public_domain_id != sl_domain.id
+ ):
alias_suffixes.append(alias_suffix)
+ # If no default domain mark it as found
+ default_domain_found = user.default_alias_public_domain_id is None
+ else:
+ default_domain_found = True
+ alias_suffixes.insert(0, alias_suffix)
+
+ if not default_domain_found:
+ domain_conditions = {"id": user.default_alias_public_domain_id, "hidden": False}
+ if not user.is_premium():
+ domain_conditions["premium_only"] = False
+ sl_domain = SLDomain.get_by(**domain_conditions)
+ if sl_domain:
+ prefix = (
+ ""
+ if config.DISABLE_ALIAS_SUFFIX
+ else f".{user.get_random_alias_suffix()}"
+ )
+ suffix = f"{prefix}@{sl_domain.domain}"
+ alias_suffix = AliasSuffix(
+ is_custom=False,
+ suffix=suffix,
+ signed_suffix=signer.sign(suffix).decode(),
+ is_premium=sl_domain.premium_only,
+ domain=sl_domain.domain,
+ mx_verified=True,
+ )
+ alias_suffixes.insert(0, alias_suffix)
return alias_suffixes
diff --git a/app/app/api/views/apple.py b/app/app/api/views/apple.py
index 119bd29..8b5125b 100644
--- a/app/app/api/views/apple.py
+++ b/app/app/api/views/apple.py
@@ -9,6 +9,7 @@ from requests import RequestException
from app.api.base import api_bp, require_api_auth
from app.config import APPLE_API_SECRET, MACAPP_APPLE_API_SECRET
+from app.subscription_webhook import execute_subscription_webhook
from app.db import Session
from app.log import LOG
from app.models import PlanEnum, AppleSubscription
@@ -50,6 +51,7 @@ def apple_process_payment():
apple_sub = verify_receipt(receipt_data, user, password)
if apple_sub:
+ execute_subscription_webhook(user)
return jsonify(ok=True), 200
return jsonify(error="Processing failed"), 400
@@ -282,6 +284,7 @@ def apple_update_notification():
apple_sub.plan = plan
apple_sub.product_id = transaction["product_id"]
Session.commit()
+ execute_subscription_webhook(user)
return jsonify(ok=True), 200
else:
LOG.w(
@@ -554,6 +557,7 @@ def verify_receipt(receipt_data, user, password) -> Optional[AppleSubscription]:
product_id=latest_transaction["product_id"],
)
+ execute_subscription_webhook(user)
Session.commit()
return apple_sub
diff --git a/app/app/config.py b/app/app/config.py
index 782f693..a33746b 100644
--- a/app/app/config.py
+++ b/app/app/config.py
@@ -532,3 +532,5 @@ if ENABLE_ALL_REVERSE_ALIAS_REPLACEMENT:
SKIP_MX_LOOKUP_ON_CHECK = False
DISABLE_RATE_LIMIT = "DISABLE_RATE_LIMIT" in os.environ
+
+SUBSCRIPTION_CHANGE_WEBHOOK = os.environ.get("SUBSCRIPTION_CHANGE_WEBHOOK", None)
diff --git a/app/app/dashboard/views/setting.py b/app/app/dashboard/views/setting.py
index 79ccb16..832ce83 100644
--- a/app/app/dashboard/views/setting.py
+++ b/app/app/dashboard/views/setting.py
@@ -206,7 +206,7 @@ def setting():
current_profile_file is not None
and current_profile_file.user_id == current_user.id
):
- s3.delete(current_user.path)
+ s3.delete(current_profile_file.path)
file_path = random_string(30)
file = File.create(user_id=current_user.id, path=file_path)
diff --git a/app/app/jobs/export_user_data_job.py b/app/app/jobs/export_user_data_job.py
index 831f840..8c3701b 100644
--- a/app/app/jobs/export_user_data_job.py
+++ b/app/app/jobs/export_user_data_job.py
@@ -41,7 +41,7 @@ from app.models import (
class ExportUserDataJob:
REMOVE_FIELDS = {
- "User": ("otp_secret",),
+ "User": ("otp_secret", "password"),
"Alias": ("ts_vector", "transfer_token", "hibp_last_check"),
"CustomDomain": ("ownership_txt_token",),
}
diff --git a/app/app/mail_sender.py b/app/app/mail_sender.py
index 5f30002..7aba370 100644
--- a/app/app/mail_sender.py
+++ b/app/app/mail_sender.py
@@ -32,6 +32,7 @@ class SendRequest:
rcpt_options: Dict = {}
is_forward: bool = False
ignore_smtp_errors: bool = False
+ retries: int = 0
def to_bytes(self) -> bytes:
if not config.SAVE_UNSENT_DIR:
@@ -67,6 +68,30 @@ class SendRequest:
is_forward=decoded_data["is_forward"],
)
+ def save_request_to_unsent_dir(self, prefix: str = "DeliveryFail"):
+ file_name = (
+ f"{prefix}-{int(time.time())}-{uuid.uuid4()}.{SendRequest.SAVE_EXTENSION}"
+ )
+ file_path = os.path.join(config.SAVE_UNSENT_DIR, file_name)
+ self.save_request_to_file(file_path)
+
+ @staticmethod
+ def save_request_to_failed_dir(self, prefix: str = "DeliveryRetryFail"):
+ file_name = (
+ f"{prefix}-{int(time.time())}-{uuid.uuid4()}.{SendRequest.SAVE_EXTENSION}"
+ )
+ dir_name = os.path.join(config.SAVE_UNSENT_DIR, "failed")
+ if not os.path.isdir(dir_name):
+ os.makedirs(dir_name)
+ file_path = os.path.join(dir_name, file_name)
+ self.save_request_to_file(file_path)
+
+ def save_request_to_file(self, file_path: str):
+ file_contents = self.to_bytes()
+ with open(file_path, "wb") as fd:
+ fd.write(file_contents)
+ LOG.i(f"Saved unsent message {file_path}")
+
class MailSender:
def __init__(self):
@@ -171,21 +196,9 @@ class MailSender:
f"Could not send message to smtp server {config.POSTFIX_SERVER}:{config.POSTFIX_PORT}"
)
if config.SAVE_UNSENT_DIR:
- self._save_request_to_unsent_dir(send_request)
+ send_request.save_request_to_unsent_dir()
return False
- def _save_request_to_unsent_dir(
- self, send_request: SendRequest, prefix: str = "DeliveryFail"
- ):
- file_name = (
- f"{prefix}-{int(time.time())}-{uuid.uuid4()}.{SendRequest.SAVE_EXTENSION}"
- )
- file_path = os.path.join(config.SAVE_UNSENT_DIR, file_name)
- file_contents = send_request.to_bytes()
- with open(file_path, "wb") as fd:
- fd.write(file_contents)
- LOG.i(f"Saved unsent message {file_path}")
-
mail_sender = MailSender()
@@ -219,6 +232,7 @@ def load_unsent_mails_from_fs_and_resend():
LOG.i(f"Trying to re-deliver email {filename}")
try:
send_request = SendRequest.load_from_file(full_file_path)
+ send_request.retries += 1
except Exception as e:
LOG.e(f"Cannot load {filename}. Error {e}")
continue
@@ -230,6 +244,11 @@ def load_unsent_mails_from_fs_and_resend():
"DeliverUnsentEmail", {"delivered": "true"}
)
else:
+ if send_request.retries > 2:
+ os.unlink(full_file_path)
+ send_request.save_request_to_failed_dir()
+ else:
+ send_request.save_request_to_file(full_file_path)
newrelic.agent.record_custom_event(
"DeliverUnsentEmail", {"delivered": "false"}
)
diff --git a/app/app/models.py b/app/app/models.py
index 7a1ad0a..4b4d625 100644
--- a/app/app/models.py
+++ b/app/app/models.py
@@ -673,6 +673,22 @@ class User(Base, ModelMixin, UserMixin, PasswordOracle):
return None
+ def get_active_subscription_end(
+ self, include_partner_subscription: bool = True
+ ) -> Optional[arrow.Arrow]:
+ sub = self.get_active_subscription(
+ include_partner_subscription=include_partner_subscription
+ )
+ if isinstance(sub, Subscription):
+ return arrow.get(sub.next_bill_date)
+ if isinstance(sub, AppleSubscription):
+ return sub.expires_date
+ if isinstance(sub, ManualSubscription):
+ return sub.end_at
+ if isinstance(sub, CoinbaseSubscription):
+ return sub.end_at
+ return None
+
# region Billing
def lifetime_or_active_subscription(
self, include_partner_subscription: bool = True
diff --git a/app/app/subscription_webhook.py b/app/app/subscription_webhook.py
new file mode 100644
index 0000000..70a447e
--- /dev/null
+++ b/app/app/subscription_webhook.py
@@ -0,0 +1,33 @@
+import requests
+from requests import RequestException
+
+from app import config
+from app.log import LOG
+from app.models import User
+
+
+def execute_subscription_webhook(user: User):
+ webhook_url = config.SUBSCRIPTION_CHANGE_WEBHOOK
+ if webhook_url is None:
+ return
+ subscription_end = user.get_active_subscription_end(
+ include_partner_subscription=False
+ )
+ sl_subscription_end = None
+ if subscription_end:
+ sl_subscription_end = subscription_end.timestamp
+ payload = {
+ "user_id": user.id,
+ "is_premium": user.is_premium(),
+ "active_subscription_end": sl_subscription_end,
+ }
+ try:
+ response = requests.post(webhook_url, json=payload, timeout=2)
+ if response.status_code == 200:
+ LOG.i("Sent request to subscription update webhook successfully")
+ else:
+ LOG.i(
+ f"Request to webhook failed with statue {response.status_code}: {response.text}"
+ )
+ except RequestException as e:
+ LOG.error(f"Subscription request exception: {e}")
diff --git a/app/server.py b/app/server.py
index c31fb5d..d6bb00e 100644
--- a/app/server.py
+++ b/app/server.py
@@ -79,6 +79,7 @@ from app.config import (
MEM_STORE_URI,
)
from app.dashboard.base import dashboard_bp
+from app.subscription_webhook import execute_subscription_webhook
from app.db import Session
from app.developer.base import developer_bp
from app.discover.base import discover_bp
@@ -491,6 +492,7 @@ def setup_paddle_callback(app: Flask):
# in case user cancels a plan and subscribes a new plan
sub.cancelled = False
+ execute_subscription_webhook(user)
LOG.d("User %s upgrades!", user)
Session.commit()
@@ -509,6 +511,7 @@ def setup_paddle_callback(app: Flask):
).date()
Session.commit()
+ execute_subscription_webhook(sub.user)
elif request.form.get("alert_name") == "subscription_cancelled":
subscription_id = request.form.get("subscription_id")
@@ -538,6 +541,7 @@ def setup_paddle_callback(app: Flask):
end_date=request.form.get("cancellation_effective_date"),
),
)
+ execute_subscription_webhook(sub.user)
else:
# user might have deleted their account
@@ -580,6 +584,7 @@ def setup_paddle_callback(app: Flask):
sub.cancelled = False
Session.commit()
+ execute_subscription_webhook(sub.user)
else:
LOG.w(
f"update non-exist subscription {subscription_id}. {request.form}"
@@ -596,6 +601,7 @@ def setup_paddle_callback(app: Flask):
Subscription.delete(sub.id)
Session.commit()
LOG.e("%s requests a refund", user)
+ execute_subscription_webhook(sub.user)
elif request.form.get("alert_name") == "subscription_payment_refunded":
subscription_id = request.form.get("subscription_id")
@@ -629,6 +635,7 @@ def setup_paddle_callback(app: Flask):
LOG.e("Unknown plan_id %s", plan_id)
else:
LOG.w("partial subscription_payment_refunded, not handled")
+ execute_subscription_webhook(sub.user)
return "OK"
@@ -742,6 +749,7 @@ def handle_coinbase_event(event) -> bool:
coinbase_subscription=coinbase_subscription,
),
)
+ execute_subscription_webhook(user)
return True
diff --git a/app/tests/test_alias_suffixes.py b/app/tests/test_alias_suffixes.py
new file mode 100644
index 0000000..9706a13
--- /dev/null
+++ b/app/tests/test_alias_suffixes.py
@@ -0,0 +1,133 @@
+import re
+
+from app.alias_suffix import get_alias_suffixes
+from app.db import Session
+from app.models import SLDomain, PartnerUser, AliasOptions, CustomDomain
+from app.proton.utils import get_proton_partner
+from init_app import add_sl_domains
+from tests.utils import create_new_user, random_token
+
+
+def setup_module():
+ Session.query(SLDomain).delete()
+ SLDomain.create(
+ domain="hidden", premium_only=False, flush=True, order=5, hidden=True
+ )
+ SLDomain.create(domain="free_non_partner", premium_only=False, flush=True, order=4)
+ SLDomain.create(
+ domain="premium_non_partner", premium_only=True, flush=True, order=3
+ )
+ SLDomain.create(
+ domain="free_partner",
+ premium_only=False,
+ flush=True,
+ partner_id=get_proton_partner().id,
+ order=2,
+ )
+ SLDomain.create(
+ domain="premium_partner",
+ premium_only=True,
+ flush=True,
+ partner_id=get_proton_partner().id,
+ order=1,
+ )
+ Session.commit()
+
+
+def teardown_module():
+ Session.query(SLDomain).delete()
+ add_sl_domains()
+
+
+def test_get_default_domain_even_if_is_not_allowed():
+ user = create_new_user()
+ PartnerUser.create(
+ partner_id=get_proton_partner().id,
+ user_id=user.id,
+ external_user_id=random_token(10),
+ flush=True,
+ )
+ user.trial_end = None
+ default_domain = SLDomain.filter_by(
+ hidden=False, partner_id=None, premium_only=False
+ ).first()
+ user.default_alias_public_domain_id = default_domain.id
+ Session.flush()
+ options = AliasOptions(
+ show_sl_domains=False, show_partner_domains=get_proton_partner()
+ )
+ suffixes = get_alias_suffixes(user, alias_options=options)
+ assert suffixes[0].domain == default_domain.domain
+
+
+def test_get_default_domain_hidden():
+ user = create_new_user()
+ PartnerUser.create(
+ partner_id=get_proton_partner().id,
+ user_id=user.id,
+ external_user_id=random_token(10),
+ flush=True,
+ )
+ user.trial_end = None
+ default_domain = SLDomain.filter_by(
+ hidden=True, partner_id=None, premium_only=False
+ ).first()
+ user.default_alias_public_domain_id = default_domain.id
+ Session.flush()
+ options = AliasOptions(
+ show_sl_domains=False, show_partner_domains=get_proton_partner()
+ )
+ suffixes = get_alias_suffixes(user, alias_options=options)
+ for suffix in suffixes:
+ domain = SLDomain.get_by(domain=suffix.domain)
+ assert not domain.hidden
+ assert suffixes[0].domain != default_domain.domain
+
+
+def test_get_default_domain_is_premium_for_free_user():
+ user = create_new_user()
+ PartnerUser.create(
+ partner_id=get_proton_partner().id,
+ user_id=user.id,
+ external_user_id=random_token(10),
+ flush=True,
+ )
+ user.trial_end = None
+ default_domain = SLDomain.filter_by(partner_id=None, premium_only=True).first()
+ user.default_alias_public_domain_id = default_domain.id
+ Session.flush()
+ options = AliasOptions(
+ show_sl_domains=False, show_partner_domains=get_proton_partner()
+ )
+ suffixes = get_alias_suffixes(user, alias_options=options)
+ for suffix in suffixes:
+ domain = SLDomain.get_by(domain=suffix.domain)
+ assert not domain.premium_only
+ assert suffixes[0].domain != default_domain.domain
+
+
+def test_suffixes_are_valid():
+ user = create_new_user()
+ PartnerUser.create(
+ partner_id=get_proton_partner().id,
+ user_id=user.id,
+ external_user_id=random_token(10),
+ flush=True,
+ )
+ CustomDomain.create(
+ user_id=user.id, domain=f"{random_token(10)}.com", verified=True
+ )
+ user.trial_end = None
+ Session.flush()
+ options = AliasOptions(
+ show_sl_domains=True, show_partner_domains=get_proton_partner()
+ )
+ alias_suffixes = get_alias_suffixes(user, alias_options=options)
+ valid_re = re.compile(r"^(\.[\w_]+)?@[\.\w]+$")
+ has_prefix = 0
+ for suffix in alias_suffixes:
+ match = valid_re.match(suffix.suffix)
+ assert match is not None
+ if len(match.groups()) >= 1:
+ has_prefix += 1
+ assert has_prefix > 0
diff --git a/app/tests/test_subscription_webhook.py b/app/tests/test_subscription_webhook.py
new file mode 100644
index 0000000..7e7ed07
--- /dev/null
+++ b/app/tests/test_subscription_webhook.py
@@ -0,0 +1,113 @@
+import http.server
+import json
+import threading
+
+import arrow
+
+from app import config
+from app.models import (
+ Subscription,
+ AppleSubscription,
+ CoinbaseSubscription,
+ ManualSubscription,
+)
+from tests.utils import create_new_user, random_token
+
+from app.subscription_webhook import execute_subscription_webhook
+
+http_server = None
+last_http_request = None
+
+
+def setup_module():
+ global http_server
+ http_server = http.server.ThreadingHTTPServer(("", 0), HTTPTestServer)
+ print(http_server.server_port)
+ threading.Thread(target=http_server.serve_forever, daemon=True).start()
+ config.SUBSCRIPTION_CHANGE_WEBHOOK = f"http://localhost:{http_server.server_port}"
+
+
+def teardown_module():
+ global http_server
+ config.SUBSCRIPTION_CHANGE_WEBHOOK = None
+ http_server.shutdown()
+
+
+class HTTPTestServer(http.server.BaseHTTPRequestHandler):
+ def do_POST(self):
+ global last_http_request
+ content_len = int(self.headers.get("Content-Length"))
+ body_data = self.rfile.read(content_len)
+ last_http_request = json.loads(body_data)
+ self.send_response(200)
+
+
+def test_webhook_with_trial():
+ user = create_new_user()
+ execute_subscription_webhook(user)
+ assert last_http_request["user_id"] == user.id
+ assert last_http_request["is_premium"]
+ assert last_http_request["active_subscription_end"] is None
+
+
+def test_webhook_with_subscription():
+ user = create_new_user()
+ end_at = arrow.utcnow().shift(days=1).replace(hour=0, minute=0, second=0)
+ Subscription.create(
+ user_id=user.id,
+ cancel_url="",
+ update_url="",
+ subscription_id=random_token(10),
+ event_time=arrow.now(),
+ next_bill_date=end_at.date(),
+ plan="yearly",
+ flush=True,
+ )
+ execute_subscription_webhook(user)
+ assert last_http_request["user_id"] == user.id
+ assert last_http_request["is_premium"]
+ assert last_http_request["active_subscription_end"] == end_at.timestamp
+
+
+def test_webhook_with_apple_subscription():
+ user = create_new_user()
+ end_at = arrow.utcnow().shift(days=2).replace(hour=0, minute=0, second=0)
+ AppleSubscription.create(
+ user_id=user.id,
+ receipt_data=arrow.now().date().strftime("%Y-%m-%d"),
+ expires_date=end_at.date().strftime("%Y-%m-%d"),
+ original_transaction_id=random_token(10),
+ plan="yearly",
+ product_id="",
+ flush=True,
+ )
+ execute_subscription_webhook(user)
+ assert last_http_request["user_id"] == user.id
+ assert last_http_request["is_premium"]
+ assert last_http_request["active_subscription_end"] == end_at.timestamp
+
+
+def test_webhook_with_coinbase_subscription():
+ user = create_new_user()
+ end_at = arrow.utcnow().shift(days=3).replace(hour=0, minute=0, second=0)
+ CoinbaseSubscription.create(
+ user_id=user.id, end_at=end_at.date().strftime("%Y-%m-%d"), flush=True
+ )
+
+ execute_subscription_webhook(user)
+ assert last_http_request["user_id"] == user.id
+ assert last_http_request["is_premium"]
+ assert last_http_request["active_subscription_end"] == end_at.timestamp
+
+
+def test_webhook_with_manual_subscription():
+ user = create_new_user()
+ end_at = arrow.utcnow().shift(days=3).replace(hour=0, minute=0, second=0)
+ ManualSubscription.create(
+ user_id=user.id, end_at=end_at.date().strftime("%Y-%m-%d"), flush=True
+ )
+
+ execute_subscription_webhook(user)
+ assert last_http_request["user_id"] == user.id
+ assert last_http_request["is_premium"]
+ assert last_http_request["active_subscription_end"] == end_at.timestamp