4.49.6

app/CONTRIBUTING.md

@@ -20,7 +20,7 @@ SimpleLogin backend consists of 2 main components:
 ## Install dependencies
 
 The project requires:
-- Python 3.10 and [rye](https://github.com/astral-sh/rye) to manage dependencies
+- Python 3.10 and poetry to manage dependencies
 - Node v10 for front-end.
 - Postgres 13+
 
@@ -28,7 +28,7 @@ First, install all dependencies by running the following command.
 Feel free to use `virtualenv` or similar tools to isolate development environment.
 
 ```bash
-rye sync
+poetry sync
 ```
 
 On Mac, sometimes you might need to install some other packages via `brew`:
@@ -55,7 +55,7 @@ brew install -s re2 pybind11
 We use pre-commit to run all our linting and static analysis checks. Please run
 
 ```bash
-rye run pre-commit install
+poetry run pre-commit install
 ```
 
 To install it in your development environment.
@@ -160,25 +160,25 @@ Here are the small sum-ups of the directory structures and their roles:
 The code is formatted using [ruff](https://github.com/astral-sh/ruff), to format the code, simply run
 
 ```
-rye run ruff format .
+poetry run ruff format .
 ```
 
 The code is also checked with `flake8`, make sure to run `flake8` before creating the pull request by
 
 ```bash
-rye run flake8
+poetry run flake8
 ```
 
 For HTML templates, we use `djlint`. Before creating a pull request, please run
 
 ```bash
-rye run djlint --check templates
+poetry run djlint --check templates
 ```
 
 If some files aren't properly formatted, you can format all files with
 
 ```bash
-rye run djlint --reformat .
+poetry run djlint --reformat .
 ```
 
 ## Test sending email
@@ -226,3 +226,26 @@ Some features require a job handler (such as GDPR data export). To test such fea
 ```bash
 python job_runner.py
 ```
+
+# Setup for Mac
+
+There are several ways to setup Python and manage the project dependencies on Mac. For info we have successfully used this setup on a Mac silicon:
+
+```bash
+# we haven't managed to make python 3.12 work
+brew install python3.10
+
+# make sure to update the PATH so python, pip point to Python3
+# for us it can be done by adding "export PATH=/opt/homebrew/opt/python@3.10/libexec/bin:$PATH" to .zprofile
+
+# Although pipx is the recommended way to install poetry,
+# install pipx via brew will automatically install python 3.12
+# and poetry will then use python 3.12
+# so we recommend using poetry this way instead
+curl -sSL https://install.python-poetry.org | python3 -
+
+poetry install
+
+# activate the virtualenv and you should be good to go!
+source .venv/bin/activate
+```

app/app/alias_utils.py

@@ -330,7 +330,10 @@ def try_auto_create_via_domain(address: str) -> Optional[Alias]:
 
 
 def delete_alias(
-    alias: Alias, user: User, reason: AliasDeleteReason = AliasDeleteReason.Unspecified
+    alias: Alias,
+    user: User,
+    reason: AliasDeleteReason = AliasDeleteReason.Unspecified,
+    commit: bool = False,
 ):
     """
     Delete an alias and add it to either global or domain trash
@@ -366,6 +369,8 @@ def delete_alias(
     EventDispatcher.send_event(
         user, EventContent(alias_deleted=AliasDeleted(alias_id=alias.id))
     )
+    if commit:
+        Session.commit()
 
 
 def aliases_for_mailbox(mailbox: Mailbox) -> [Alias]:

app/app/config.py

@@ -3,7 +3,7 @@ import random
 import socket
 import string
 from ast import literal_eval
-from typing import Callable, List
+from typing import Callable, List, Optional
 from urllib.parse import urlparse
 
 from dotenv import load_dotenv
@@ -588,3 +588,24 @@ EVENT_WEBHOOK = os.environ.get("EVENT_WEBHOOK", None)
 # We want it disabled by default, so only skip if defined
 EVENT_WEBHOOK_SKIP_VERIFY_SSL = "EVENT_WEBHOOK_SKIP_VERIFY_SSL" in os.environ
 EVENT_WEBHOOK_DISABLE = "EVENT_WEBHOOK_DISABLE" in os.environ
+
+
+def read_webhook_enabled_user_ids() -> Optional[List[int]]:
+    user_ids = os.environ.get("EVENT_WEBHOOK_ENABLED_USER_IDS", None)
+    if user_ids is None:
+        return None
+
+    ids = []
+    for user_id in user_ids.split(","):
+        try:
+            ids.append(int(user_id.strip()))
+        except ValueError:
+            pass
+    return ids
+
+
+EVENT_WEBHOOK_ENABLED_USER_IDS: Optional[List[int]] = read_webhook_enabled_user_ids()
+
+# Allow to define a different DB_URI for the event listener, in case we want to skip the connection pool
+# It defaults to the regular DB_URI in case it's needed
+EVENT_LISTENER_DB_URI = os.environ.get("EVENT_LISTENER_DB_URI", DB_URI)

app/app/dashboard/views/index.py

@@ -145,7 +145,7 @@ def index():
                 LOG.i(f"User {current_user} requested deletion of alias {alias}")
                 email = alias.email
                 alias_utils.delete_alias(
-                    alias, current_user, AliasDeleteReason.ManualAction
+                    alias, current_user, AliasDeleteReason.ManualAction, commit=True
                 )
                 flash(f"Alias {email} has been deleted", "success")
             elif request.form.get("form-name") == "disable-alias":

app/app/events/event_dispatcher.py

@@ -40,6 +40,10 @@ class EventDispatcher:
         if not config.EVENT_WEBHOOK and skip_if_webhook_missing:
             return
 
+        if config.EVENT_WEBHOOK_ENABLED_USER_IDS is not None:
+            if user.id not in config.EVENT_WEBHOOK_ENABLED_USER_IDS:
+                return
+
         partner_user = EventDispatcher.__partner_user(user.id)
         if not partner_user:
             return

app/app/subscription_webhook.py

@@ -2,6 +2,7 @@ import requests
 from requests import RequestException
 
 from app import config
+from app.db import Session
 from app.events.event_dispatcher import EventDispatcher
 from app.events.generated.event_pb2 import EventContent, UserPlanChanged
 from app.log import LOG
@@ -29,10 +30,11 @@ def execute_subscription_webhook(user: User):
             LOG.i("Sent request to subscription update webhook successfully")
         else:
             LOG.i(
-                f"Request to webhook failed with statue {response.status_code}: {response.text}"
+                f"Request to webhook failed with status {response.status_code}: {response.text}"
             )
     except RequestException as e:
         LOG.error(f"Subscription request exception: {e}")
 
     event = UserPlanChanged(plan_end_time=sl_subscription_end)
     EventDispatcher.send_event(user, EventContent(user_plan_change=event))
+    Session.commit()

app/app/user_settings.py

@@ -16,12 +16,13 @@ class CannotSetMailbox(Exception):
 
 
 def set_default_alias_domain(user: User, domain_name: Optional[str]):
-    if domain_name is None:
+    if not domain_name:
         LOG.i(f"User {user} has set no domain as default domain")
         user.default_alias_public_domain_id = None
         user.default_alias_custom_domain_id = None
         Session.flush()
         return
+
     sl_domain: SLDomain = SLDomain.get_by(domain=domain_name)
     if sl_domain:
         if sl_domain.hidden:

app/event_listener.py

@@ -2,8 +2,9 @@ import argparse
 from enum import Enum
 from sys import argv, exit
 
-from app.config import DB_URI
+from app.config import EVENT_LISTENER_DB_URI
 from app.log import LOG
+from events import event_debugger
 from events.runner import Runner
 from events.event_source import DeadLetterEventSource, PostgresEventSource
 from events.event_sink import ConsoleEventSink, HttpEventSink
@@ -31,7 +32,7 @@ def main(mode: Mode, dry_run: bool, max_retries: int):
         source = DeadLetterEventSource(max_retries)
     elif mode == Mode.LISTENER:
         LOG.i("Using PostgresEventSource")
-        source = PostgresEventSource(DB_URI)
+        source = PostgresEventSource(EVENT_LISTENER_DB_URI)
     else:
         raise ValueError(f"Invalid mode: {mode}")
 
@@ -46,32 +47,67 @@ def main(mode: Mode, dry_run: bool, max_retries: int):
     runner.run()
 
 
+def debug_event(event_id: str):
+    LOG.i(f"Debugging event {event_id}")
+    try:
+        event_id_int = int(event_id)
+    except ValueError:
+        raise ValueError(f"Invalid event id: {event_id}")
+    event_debugger.debug_event(event_id_int)
+
+
+def run_event(event_id: str, delete_on_success: bool):
+    LOG.i(f"Running event {event_id}")
+    try:
+        event_id_int = int(event_id)
+    except ValueError:
+        raise ValueError(f"Invalid event id: {event_id}")
+    event_debugger.run_event(event_id_int, delete_on_success)
+
+
 def args():
     parser = argparse.ArgumentParser(description="Run event listener")
-    parser.add_argument(
-        "mode",
-        help="Mode to run",
-        choices=[Mode.DEAD_LETTER.value, Mode.LISTENER.value],
+    subparsers = parser.add_subparsers(dest="command")
+
+    listener_parser = subparsers.add_parser(Mode.LISTENER.value)
+    listener_parser.add_argument(
+        "--max-retries", type=int, default=_DEFAULT_MAX_RETRIES
     )
-    parser.add_argument(
-        "max_retries",
-        help="Max retries to consider an event as error and not try to process it again",
-        type=int,
-        nargs="?",
-        default=_DEFAULT_MAX_RETRIES,
+    listener_parser.add_argument("--dry-run", action="store_true")
+
+    dead_letter_parser = subparsers.add_parser(Mode.DEAD_LETTER.value)
+    dead_letter_parser.add_argument(
+        "--max-retries", type=int, default=_DEFAULT_MAX_RETRIES
     )
-    parser.add_argument("--dry-run", help="Dry run mode", action="store_true")
+    dead_letter_parser.add_argument("--dry-run", action="store_true")
+
+    debug_parser = subparsers.add_parser("debug")
+    debug_parser.add_argument("event_id", help="ID of the event to debug")
+
+    run_parser = subparsers.add_parser("run")
+    run_parser.add_argument("event_id", help="ID of the event to run")
+    run_parser.add_argument("--delete-on-success", action="store_true")
+
     return parser.parse_args()
 
 
 if __name__ == "__main__":
     if len(argv) < 2:
-        print("Invalid usage. Pass 'listener' or 'dead_letter' as argument")
+        print("Invalid usage. Pass a valid subcommand as argument")
         exit(1)
 
     args = args()
-    main(
-        mode=Mode.from_str(args.mode),
-        dry_run=args.dry_run,
-        max_retries=args.max_retries,
-    )
+
+    if args.command in [Mode.LISTENER.value, Mode.DEAD_LETTER.value]:
+        main(
+            mode=Mode.from_str(args.command),
+            dry_run=args.dry_run,
+            max_retries=args.max_retries,
+        )
+    elif args.command == "debug":
+        debug_event(args.event_id)
+    elif args.command == "run":
+        run_event(args.event_id, args.delete_on_success)
+    else:
+        print("Invalid command")
+        exit(1)

app/events/event_debugger.py

@@ -0,0 +1,43 @@
+from app.events.generated import event_pb2
+from app.models import SyncEvent
+from events.event_sink import HttpEventSink
+
+
+def debug_event(event_id: int):
+    event = SyncEvent.get_by(id=event_id)
+    if not event:
+        print("Event not found")
+        return
+
+    print(f"Info for event {event_id}")
+    print(f"- Created at: {event.created_at}")
+    print(f"- Updated at: {event.updated_at}")
+    print(f"- Taken time: {event.taken_time}")
+    print(f"- Retry count: {event.retry_count}")
+
+    print()
+    print("Event contents")
+    event_contents = event.content
+    parsed = event_pb2.Event.FromString(event_contents)
+
+    print(f"- UserID: {parsed.user_id}")
+    print(f"- ExternalUserID: {parsed.external_user_id}")
+    print(f"- PartnerID: {parsed.partner_id}")
+
+    content = parsed.content
+    print(f"Content: {content}")
+
+
+def run_event(event_id: int, delete_on_success: bool = True):
+    event = SyncEvent.get_by(id=event_id)
+    if not event:
+        print("Event not found")
+        return
+
+    print(f"Processing event {event_id}")
+    sink = HttpEventSink()
+    res = sink.process(event)
+    if res:
+        print(f"Processed event {event_id}")
+        if delete_on_success:
+            SyncEvent.delete(event_id, commit=True)

app/events/event_source.py

@@ -46,6 +46,7 @@ class PostgresEventSource(EventSource):
         cursor = self.__connection.cursor()
         cursor.execute(f"LISTEN {NOTIFICATION_CHANNEL};")
 
+        LOG.info("Starting to listen to events")
         while True:
             if select.select([self.__connection], [], [], 5) != ([], [], []):
                 self.__connection.poll()

app/job_runner.py

@@ -14,6 +14,7 @@ from app.email_utils import (
     send_email,
     render,
 )
+from app.events.event_dispatcher import PostgresDispatcher
 from app.import_utils import handle_batch_import
 from app.jobs.event_jobs import send_alias_creation_events_for_user
 from app.jobs.export_user_data_job import ExportUserDataJob
@@ -276,7 +277,9 @@ SimpleLogin team.
         user = User.get(user_id)
         if user and user.activated:
             LOG.d(f"Sending alias creation events for {user}")
-            send_alias_creation_events_for_user(user)
+            send_alias_creation_events_for_user(
+                user, dispatcher=PostgresDispatcher.get()
+            )
     else:
         LOG.e("Unknown job name %s", job.name)
 

app/templates/dashboard/support.html

@@ -18,7 +18,7 @@
           <br />
           For generic questions, i.e. not related to your account, we recommend to post the question on
           our
-          <a href="https://www.reddit.com/r/Simplelogin/">Reddit</a> or <a href="https://forum.simplelogin.io/">our official forum</a>
+          <a href="https://www.reddit.com/r/Simplelogin/">Reddit</a> or <a href="https://github.com/simple-login/app/discussions">forum</a>
           where our community can help answer the question
           and other people with the same question can find the answer there.
         </div>

app/templates/emails/com/onboarding/welcome-proton-user.html

@@ -43,8 +43,7 @@ Note, if you are a paying Proton Mail user, you automatically receive the premiu
 {% endcall %}
 
 {% call text() %}
-For any question or feedback, please join our <a href="https://forum.simplelogin.io/">official forum</a>.
-If you want to request a feature, please submit it on our <a href="https://github.com/simple-login/app/discussions">GitHub repo</a>.
+For any question or if you want to request a feature, please submit it on our <a href="https://github.com/simple-login/app/discussions">forum</a>.
 You can also join our
 <a href="https://www.reddit.com/r/Simplelogin/">Reddit</a>
 or follow our

app/templates/emails/com/onboarding/welcome-proton-user.txt.jinja2

@@ -27,8 +27,7 @@ Firefox: https://addons.mozilla.org/firefox/addon/simplelogin/
 Edge: https://microsoftedge.microsoft.com/addons/detail/simpleloginreceive-sen/diacfpipniklenphgljfkmhinphjlfff
 Android: https://play.google.com/store/apps/details?id=io.simplelogin.android
 iOS: https://apps.apple.com/app/id1494359858
-Github repo: https://github.com/simple-login/app/discussions
-Official forum: https://forum.simplelogin.io/
+Forum: https://github.com/simple-login/app/discussions
 Reddit: https://www.reddit.com/r/Simplelogin/
 Twitter: https://twitter.com/simple_login
 

app/templates/emails/com/welcome.html

@@ -71,10 +71,8 @@ Please note that you can't create more than {{ MAX_NB_EMAIL_FREE_PLAN }} aliases
 
 {% endif %}
 {% call text() %}
-For any question or feedback,
-please join our <a href="https://forum.simplelogin.io/">official forum</a>.
-If you want to request a feature,
-please submit it on our <a href="https://github.com/simple-login/app/discussions">GitHub repo</a>.
+For any question or if you want to request a feature,
+please submit it on our <a href="https://github.com/simple-login/app/discussions">forum</a>.
 You can also join our
 <a href="https://www.reddit.com/r/Simplelogin/">Reddit</a>
 or follow our

app/templates/emails/com/welcome.txt

@@ -26,8 +26,6 @@ No worries: all aliases you create during this period will continue to work norm
 
 At any time, you can reach out to us by simply replying to this email.
 
-For any question or feedback, please join our official forum at https://forum.simplelogin.io/
-
-If you want to request a feature, please submit it on our GitHub repo at https://github.com/simple-login/app/discussions
+For any question or if you want to request a feature, please submit it on our forum at https://github.com/simple-login/app/discussions
 
 You can also join our Reddit at https://www.reddit.com/r/Simplelogin/ follow our Twitter at https://twitter.com/simplelogin

app/templates/footer.html

@@ -93,7 +93,7 @@
           </li>
           <li>
             <a class="list-group-item text-white footer-item "
-               href="https://forum.simplelogin.io">Forum</a>
+               href="https://github.com/simple-login/app/discussions">Forum</a>
           </li>
         </ul>
       </div>

app/templates/header.html

@@ -91,7 +91,7 @@
                 </a>
               </div>
               <div class="dropdown-item">
-                <a href="https://forum.simplelogin.io"
+                <a href="https://github.com/simple-login/app/discussions"
                    target="_blank"
                    rel="noopener noreferrer">
                   Forum

app/templates/menu.html

@@ -107,7 +107,7 @@
             </a>
           </div>
           <div class="dropdown-item">
-            <a href="https://forum.simplelogin.io/"
+            <a href="https://github.com/simple-login/app/discussions"
                target="_blank"
                rel="noopener noreferrer">
               Forum