4.33.3

app/app/api/views/mailbox.py

@@ -13,8 +13,8 @@ from app.db import Session
 from app.email_utils import (
     mailbox_already_used,
     email_can_be_used_as_mailbox,
-    is_valid_email,
 )
+from app.email_validation import is_valid_email
 from app.log import LOG
 from app.models import Mailbox, Job
 from app.utils import sanitize_email

app/app/config.py

@@ -535,3 +535,7 @@ DISABLE_RATE_LIMIT = "DISABLE_RATE_LIMIT" in os.environ
 
 SUBSCRIPTION_CHANGE_WEBHOOK = os.environ.get("SUBSCRIPTION_CHANGE_WEBHOOK", None)
 MAX_API_KEYS = int(os.environ.get("MAX_API_KEYS", 30))
+
+UPCLOUD_USERNAME = os.environ.get("UPCLOUD_USERNAME", None)
+UPCLOUD_PASSWORD = os.environ.get("UPCLOUD_PASSWORD", None)
+UPCLOUD_DB_ID = os.environ.get("UPCLOUD_DB_ID", None)

app/app/dashboard/views/alias_contact_manager.py

@@ -13,10 +13,10 @@ from app import config, parallel_limiter
 from app.dashboard.base import dashboard_bp
 from app.db import Session
 from app.email_utils import (
-    is_valid_email,
     generate_reply_email,
     parse_full_address,
 )
+from app.email_validation import is_valid_email
 from app.errors import (
     CannotCreateContactForReverseAlias,
     ErrContactErrorUpgradeNeeded,

app/app/dashboard/views/mailbox.py

@@ -19,8 +19,8 @@ from app.email_utils import (
     mailbox_already_used,
     render,
     send_email,
-    is_valid_email,
 )
+from app.email_validation import is_valid_email
 from app.log import LOG
 from app.models import Mailbox, Job
 from app.utils import CSRFValidationForm

app/app/email_utils.py

@@ -828,19 +828,6 @@ def should_add_dkim_signature(domain: str) -> bool:
     return False
 
 
-def is_valid_email(email_address: str) -> bool:
-    """
-    Used to check whether an email address is valid
-    NOT run MX check.
-    NOT allow unicode.
-    """
-    try:
-        validate_email(email_address, check_deliverability=False, allow_smtputf8=False)
-        return True
-    except EmailNotValidError:
-        return False
-
-
 class EmailEncoding(enum.Enum):
     BASE64 = "base64"
     QUOTED = "quoted-printable"
@@ -1116,26 +1103,6 @@ def is_reverse_alias(address: str) -> bool:
     )
 
 
-# allow also + and @ that are present in a reply address
-_ALLOWED_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-.+@"
-
-
-def normalize_reply_email(reply_email: str) -> str:
-    """Handle the case where reply email contains *strange* char that was wrongly generated in the past"""
-    if not reply_email.isascii():
-        reply_email = convert_to_id(reply_email)
-
-    ret = []
-    # drop all control characters like shift, separator, etc
-    for c in reply_email:
-        if c not in _ALLOWED_CHARS:
-            ret.append("_")
-        else:
-            ret.append(c)
-
-    return "".join(ret)
-
-
 def should_disable(alias: Alias) -> (bool, str):
     """
     Return whether an alias should be disabled and if yes, the reason why

app/app/email_validation.py

@@ -0,0 +1,38 @@
+from email_validator import (
+    validate_email,
+    EmailNotValidError,
+)
+
+from app.utils import convert_to_id
+
+# allow also + and @ that are present in a reply address
+_ALLOWED_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-.+@"
+
+
+def is_valid_email(email_address: str) -> bool:
+    """
+    Used to check whether an email address is valid
+    NOT run MX check.
+    NOT allow unicode.
+    """
+    try:
+        validate_email(email_address, check_deliverability=False, allow_smtputf8=False)
+        return True
+    except EmailNotValidError:
+        return False
+
+
+def normalize_reply_email(reply_email: str) -> str:
+    """Handle the case where reply email contains *strange* char that was wrongly generated in the past"""
+    if not reply_email.isascii():
+        reply_email = convert_to_id(reply_email)
+
+    ret = []
+    # drop all control characters like shift, separator, etc
+    for c in reply_email:
+        if c not in _ALLOWED_CHARS:
+            ret.append("_")
+        else:
+            ret.append(c)
+
+    return "".join(ret)

app/app/errors.py

@@ -84,6 +84,14 @@ class ErrAddressInvalid(SLException):
         return f"{self.address} is not a valid email address"
 
 
+class InvalidContactEmailError(SLException):
+    def __init__(self, website_email: str):  # noqa: F821
+        self.website_email = website_email
+
+    def error_for_user(self) -> str:
+        return f"Cannot create contact with invalid email {self.website_email}"
+
+
 class ErrContactAlreadyExists(SLException):
     """raised when a contact already exists"""
 

app/app/models.py

@@ -577,6 +577,7 @@ class User(Base, ModelMixin, UserMixin, PasswordOracle):
 
     @classmethod
     def create(cls, email, name="", password=None, from_partner=False, **kwargs):
+        email = sanitize_email(email)
         user: User = super(User, cls).create(email=email, name=name[:100], **kwargs)
 
         if password:
@@ -2300,6 +2301,7 @@ class CustomDomain(Base, ModelMixin):
     @classmethod
     def create(cls, **kwargs):
         domain = kwargs.get("domain")
+        kwargs["domain"] = domain.replace("\n", "")
         if DeletedSubdomain.get_by(domain=domain):
             raise SubdomainInTrashError
 
@@ -2599,6 +2601,12 @@ class Mailbox(Base, ModelMixin):
 
         return ret
 
+    @classmethod
+    def create(cls, **kw):
+        if "email" in kw:
+            kw["email"] = sanitize_email(kw["email"])
+        return super().create(**kw)
+
     def __repr__(self):
         return f"<Mailbox {self.id} {self.email}>"
 

app/app/utils.py

@@ -99,7 +99,7 @@ def sanitize_email(email_address: str, not_lower=False) -> str:
         email_address = email_address.strip().replace(" ", "").replace("\n", " ")
         if not not_lower:
             email_address = email_address.lower()
-    return email_address
+    return email_address.replace("\u200f", "")
 
 
 class NextUrlSanitizer:

app/cron.py

@@ -22,10 +22,9 @@ from app.email_utils import (
     render,
     email_can_be_used_as_mailbox,
     send_email_with_rate_control,
-    normalize_reply_email,
-    is_valid_email,
     get_email_domain_part,
 )
+from app.email_validation import is_valid_email, normalize_reply_email
 from app.errors import ProtonPartnerNotSetUp
 from app.log import LOG
 from app.mail_sender import load_unsent_mails_from_fs_and_resend

app/crontab.yml

@@ -5,62 +5,64 @@ jobs:
     schedule: "0 0 * * *"
     captureStderr: true
 
-  - name: SimpleLogin Notify Trial Ends
-    command: python /code/cron.py -j notify_trial_end
-    shell: /bin/bash
-    schedule: "0 8 * * *"
-    captureStderr: true
-
-  - name: SimpleLogin Notify Manual Subscription Ends
-    command: python /code/cron.py -j notify_manual_subscription_end
-    shell: /bin/bash
-    schedule: "0 9 * * *"
-    captureStderr: true
-
-  - name: SimpleLogin Notify Premium Ends
-    command: python /code/cron.py -j notify_premium_end
-    shell: /bin/bash
-    schedule: "0 10 * * *"
-    captureStderr: true
-
-  - name: SimpleLogin Delete Logs
-    command: python /code/cron.py -j delete_logs
-    shell: /bin/bash
-    schedule: "0 11 * * *"
-    captureStderr: true
-
-  - name: SimpleLogin Poll Apple Subscriptions
-    command: python /code/cron.py -j poll_apple_subscription
-    shell: /bin/bash
-    schedule: "0 12 * * *"
-    captureStderr: true
-
   - name: SimpleLogin Delete Old Monitoring records
     command: python /code/cron.py -j delete_old_monitoring
     shell: /bin/bash
-    schedule: "0 14 * * *"
+    schedule: "15 1 * * *"
     captureStderr: true
 
   - name: SimpleLogin Custom Domain check
     command: python /code/cron.py -j check_custom_domain
     shell: /bin/bash
-    schedule: "0 15 * * *"
+    schedule: "15 2 * * *"
     captureStderr: true
 
   - name: SimpleLogin HIBP check
     command: python /code/cron.py -j check_hibp
     shell: /bin/bash
-    schedule: "0 18 * * *"
+    schedule: "15 3 * * *"
     captureStderr: true
     concurrencyPolicy: Forbid
 
   - name: SimpleLogin Notify HIBP breaches
     command: python /code/cron.py -j notify_hibp
     shell: /bin/bash
-    schedule: "0 19 * * *"
+    schedule: "15 4 * * *"
     captureStderr: true
     concurrencyPolicy: Forbid
 
+  - name: SimpleLogin Delete Logs
+    command: python /code/cron.py -j delete_logs
+    shell: /bin/bash
+    schedule: "15 5 * * *"
+    captureStderr: true
+
+  - name: SimpleLogin Poll Apple Subscriptions
+    command: python /code/cron.py -j poll_apple_subscription
+    shell: /bin/bash
+    schedule: "15 6 * * *"
+    captureStderr: true
+
+  - name: SimpleLogin Notify Trial Ends
+    command: python /code/cron.py -j notify_trial_end
+    shell: /bin/bash
+    schedule: "15 8 * * *"
+    captureStderr: true
+
+  - name: SimpleLogin Notify Manual Subscription Ends
+    command: python /code/cron.py -j notify_manual_subscription_end
+    shell: /bin/bash
+    schedule: "15 9 * * *"
+    captureStderr: true
+
+  - name: SimpleLogin Notify Premium Ends
+    command: python /code/cron.py -j notify_premium_end
+    shell: /bin/bash
+    schedule: "15 10 * * *"
+    captureStderr: true
+
+
+
   - name: SimpleLogin send unsent emails
     command: python /code/cron.py -j send_undelivered_mails
     shell: /bin/bash

app/email_handler.py

@@ -106,8 +106,6 @@ from app.email_utils import (
     get_header_unicode,
     generate_reply_email,
     is_reverse_alias,
-    normalize_reply_email,
-    is_valid_email,
     replace,
     should_disable,
     parse_id_from_bounce,
@@ -123,6 +121,7 @@ from app.email_utils import (
     generate_verp_email,
     sl_formataddr,
 )
+from app.email_validation import is_valid_email, normalize_reply_email
 from app.errors import (
     NonReverseAliasInReplyPhase,
     VERPTransactional,
@@ -262,7 +261,7 @@ def get_or_create_contact(from_header: str, mail_from: str, alias: Alias) -> Con
 
             Session.commit()
         except IntegrityError:
-            LOG.w("Contact %s %s already exist", alias, contact_email)
+            LOG.w(f"Contact with email {contact_email} for alias {alias} already exist")
             Session.rollback()
             contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
 
@@ -280,6 +279,9 @@ def get_or_create_reply_to_contact(
     except ValueError:
         return
 
+    if len(contact_name) >= Contact.MAX_NAME_LENGTH:
+        contact_name = contact_name[0 : Contact.MAX_NAME_LENGTH]
+
     if not is_valid_email(contact_address):
         LOG.w(
             "invalid reply-to address %s. Parse from %s",
@@ -348,6 +350,10 @@ def replace_header_when_forward(msg: Message, alias: Alias, header: str):
             continue
 
         contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
+        contact_name = full_address.display_name
+        if len(contact_name) >= Contact.MAX_NAME_LENGTH:
+            contact_name = contact_name[0 : Contact.MAX_NAME_LENGTH]
+
         if contact:
             # update the contact name if needed
             if contact.name != full_address.display_name:
@@ -355,9 +361,9 @@ def replace_header_when_forward(msg: Message, alias: Alias, header: str):
                     "Update contact %s name %s to %s",
                     contact,
                     contact.name,
-                    full_address.display_name,
+                    contact_name,
                 )
-                contact.name = full_address.display_name
+                contact.name = contact_name
                 Session.commit()
         else:
             LOG.d(
@@ -372,7 +378,7 @@ def replace_header_when_forward(msg: Message, alias: Alias, header: str):
                     user_id=alias.user_id,
                     alias_id=alias.id,
                     website_email=contact_email,
-                    name=full_address.display_name,
+                    name=contact_name,
                     reply_email=generate_reply_email(contact_email, alias),
                     is_cc=header.lower() == "cc",
                     automatic_created=True,
@@ -541,12 +547,20 @@ def sign_msg(msg: Message) -> Message:
     signature.add_header("Content-Disposition", 'attachment; filename="signature.asc"')
 
     try:
-        signature.set_payload(sign_data(message_to_bytes(msg).replace(b"\n", b"\r\n")))
+        payload = sign_data(message_to_bytes(msg).replace(b"\n", b"\r\n"))
+
+        if not payload:
+            raise PGPException("Empty signature by gnupg")
+
+        signature.set_payload(payload)
     except Exception:
         LOG.e("Cannot sign, try using pgpy")
-        signature.set_payload(
+        payload = sign_data_with_pgpy(message_to_bytes(msg).replace(b"\n", b"\r\n"))
-            sign_data_with_pgpy(message_to_bytes(msg).replace(b"\n", b"\r\n"))
+
-        )
+        if not payload:
+            raise PGPException("Empty signature by pgpy")
+
+        signature.set_payload(payload)
 
     container.attach(signature)
 

app/monitor/metric.py

@@ -0,0 +1,21 @@
+from dataclasses import dataclass
+from typing import List
+
+
+@dataclass
+class UpcloudRecord:
+    db_role: str
+    label: str
+    time: str
+    value: float
+
+
+@dataclass
+class UpcloudMetric:
+    metric_name: str
+    records: List[UpcloudRecord]
+
+
+@dataclass
+class UpcloudMetrics:
+    metrics: List[UpcloudMetric]

app/monitor/metric_exporter.py

@@ -0,0 +1,20 @@
+from app.config import UPCLOUD_DB_ID, UPCLOUD_PASSWORD, UPCLOUD_USERNAME
+from app.log import LOG
+from monitor.newrelic import NewRelicClient
+from monitor.upcloud import UpcloudClient
+
+
+class MetricExporter:
+    def __init__(self, newrelic_license: str):
+        self.__upcloud = UpcloudClient(
+            username=UPCLOUD_USERNAME, password=UPCLOUD_PASSWORD
+        )
+        self.__newrelic = NewRelicClient(newrelic_license)
+
+    def run(self):
+        try:
+            metrics = self.__upcloud.get_metrics(UPCLOUD_DB_ID)
+            self.__newrelic.send(metrics)
+            LOG.info("Upcloud metrics sent to NewRelic")
+        except Exception as e:
+            LOG.warn(f"Could not export metrics: {e}")

app/monitor/newrelic.py

@@ -0,0 +1,26 @@
+from monitor.metric import UpcloudMetrics
+
+from newrelic_telemetry_sdk import GaugeMetric, MetricClient
+
+_NEWRELIC_BASE_HOST = "metric-api.eu.newrelic.com"
+
+
+class NewRelicClient:
+    def __init__(self, license_key: str):
+        self.__client = MetricClient(license_key=license_key, host=_NEWRELIC_BASE_HOST)
+
+    def send(self, metrics: UpcloudMetrics):
+        batch = []
+
+        for metric in metrics.metrics:
+            for record in metric.records:
+                batch.append(
+                    GaugeMetric(
+                        name=f"upcloud.db.{metric.metric_name}",
+                        value=record.value,
+                        tags={"host": record.label, "db_role": record.db_role},
+                    )
+                )
+
+        response = self.__client.send_batch(batch)
+        response.raise_for_status()

app/monitor/upcloud.py

@@ -0,0 +1,82 @@
+from app.log import LOG
+from monitor.metric import UpcloudMetric, UpcloudMetrics, UpcloudRecord
+
+import base64
+import requests
+from typing import Any
+
+
+BASE_URL = "https://api.upcloud.com"
+
+
+def get_metric(json: Any, metric: str) -> UpcloudMetric:
+    records = []
+
+    if metric in json:
+        metric_data = json[metric]
+        data = metric_data["data"]
+        cols = list(map(lambda x: x["label"], data["cols"][1:]))
+        latest = data["rows"][-1]
+        time = latest[0]
+        for column_idx in range(len(cols)):
+            value = latest[1 + column_idx]
+
+            # If the latest value is None, try to fetch the second to last
+            if value is None:
+                value = data["rows"][-2][1 + column_idx]
+
+            if value is not None:
+                label = cols[column_idx]
+                if "(master)" in label:
+                    db_role = "master"
+                else:
+                    db_role = "standby"
+                records.append(
+                    UpcloudRecord(time=time, db_role=db_role, label=label, value=value)
+                )
+            else:
+                LOG.warn(f"Could not get value for metric {metric}")
+
+    return UpcloudMetric(metric_name=metric, records=records)
+
+
+def get_metrics(json: Any) -> UpcloudMetrics:
+    return UpcloudMetrics(
+        metrics=[
+            get_metric(json, "cpu_usage"),
+            get_metric(json, "disk_usage"),
+            get_metric(json, "diskio_reads"),
+            get_metric(json, "diskio_writes"),
+            get_metric(json, "load_average"),
+            get_metric(json, "mem_usage"),
+            get_metric(json, "net_receive"),
+            get_metric(json, "net_send"),
+        ]
+    )
+
+
+class UpcloudClient:
+    def __init__(self, username: str, password: str):
+        if not username:
+            raise Exception("UpcloudClient username must be set")
+        if not password:
+            raise Exception("UpcloudClient password must be set")
+
+        client = requests.Session()
+        encoded_auth = base64.b64encode(
+            f"{username}:{password}".encode("utf-8")
+        ).decode("utf-8")
+        client.headers = {"Authorization": f"Basic {encoded_auth}"}
+        self.__client = client
+
+    def get_metrics(self, db_uuid: str) -> UpcloudMetrics:
+        url = f"{BASE_URL}/1.3/database/{db_uuid}/metrics?period=hour"
+        LOG.d(f"Performing request to {url}")
+        response = self.__client.get(url)
+        LOG.d(f"Status code: {response.status_code}")
+        if response.status_code != 200:
+            return UpcloudMetrics(metrics=[])
+
+        as_json = response.json()
+
+        return get_metrics(as_json)

app/monitoring.py

@@ -1,3 +1,4 @@
+import configparser
 import os
 import subprocess
 from time import sleep
@@ -7,6 +8,7 @@ import newrelic.agent
 
 from app.db import Session
 from app.log import LOG
+from monitor.metric_exporter import MetricExporter
 
 # the number of consecutive fails
 # if more than _max_nb_fails, alert
@@ -19,6 +21,18 @@ _max_nb_fails = 10
 # the maximum number of emails in incoming & active queue
 _max_incoming = 50
 
+_NR_CONFIG_FILE_LOCATION_VAR = "NEW_RELIC_CONFIG_FILE"
+
+
+def get_newrelic_license() -> str:
+    nr_file = os.environ.get(_NR_CONFIG_FILE_LOCATION_VAR, None)
+    if nr_file is None:
+        raise Exception(f"{_NR_CONFIG_FILE_LOCATION_VAR} not defined")
+
+    config = configparser.ConfigParser()
+    config.read(nr_file)
+    return config["newrelic"]["license_key"]
+
 
 @newrelic.agent.background_task()
 def log_postfix_metrics():
@@ -80,10 +94,13 @@ def log_nb_db_connection():
 
 
 if __name__ == "__main__":
+    exporter = MetricExporter(get_newrelic_license())
     while True:
         log_postfix_metrics()
         log_nb_db_connection()
         Session.close()
 
+        exporter.run()
+
         # 1 min
         sleep(60)

app/pyproject.toml

@@ -111,6 +111,7 @@ Deprecated = "^1.2.13"
 cryptography = "37.0.1"
 SQLAlchemy = "1.3.24"
 redis = "^4.5.3"
+newrelic-telemetry-sdk = "^0.5.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^7.0.0"

app/tests/monitor/test_upcloud_get_metric.py

@@ -0,0 +1,350 @@
+from monitor.upcloud import get_metric, get_metrics
+from monitor.metric import UpcloudMetrics, UpcloudMetric, UpcloudRecord
+
+import json
+
+MOCK_RESPONSE = """
+{
+  "cpu_usage": {
+    "data": {
+      "cols": [
+        { "label": "time", "type": "date" },
+        { "label": "test-1 (master)", "type": "number" },
+        { "label": "test-2 (standby)", "type": "number" }
+      ],
+      "rows": [
+        ["2022-01-21T13:10:30Z", 2.744682398273781, 3.054323473090861],
+        ["2022-01-21T13:11:00Z", 3.0735645433218366, 2.972423595745795],
+        ["2022-01-21T13:11:30Z", 2.61619694060839, 3.1358378052207883],
+        ["2022-01-21T13:12:00Z", 3.275132296130991, 4.196249043309251]
+      ]
+    },
+    "hints": { "title": "CPU usage %" }
+  },
+  "disk_usage": {
+    "data": {
+      "cols": [
+        { "label": "time", "type": "date" },
+        { "label": "test-1 (master)", "type": "number" },
+        { "label": "test-2 (standby)", "type": "number" }
+      ],
+      "rows": [
+        ["2022-01-21T13:10:30Z", 5.654416415900109, 5.58959125727556],
+        ["2022-01-21T13:11:00Z", 5.654416415900109, 5.58959125727556],
+        ["2022-01-21T13:11:30Z", 5.654416415900109, 5.58959125727556]
+      ]
+    },
+    "hints": { "title": "Disk space usage %" }
+  },
+  "diskio_reads": {
+    "data": {
+      "cols": [
+        { "label": "time", "type": "date" },
+        { "label": "test-1 (master)", "type": "number" },
+        { "label": "test-2 (standby)", "type": "number" }
+      ],
+      "rows": [
+        ["2022-01-21T13:10:30Z", 0, 0],
+        ["2022-01-21T13:11:00Z", 0, 0],
+        ["2022-01-21T13:11:30Z", 0, 0]
+      ]
+    },
+    "hints": { "title": "Disk iops (reads)" }
+  },
+  "diskio_writes": {
+    "data": {
+      "cols": [
+        { "label": "time", "type": "date" },
+        { "label": "test-1 (master)", "type": "number" },
+        { "label": "test-2 (standby)", "type": "number" }
+      ],
+      "rows": [
+        ["2022-01-21T13:10:30Z", 3, 2],
+        ["2022-01-21T13:11:00Z", 2, 3],
+        ["2022-01-21T13:11:30Z", 4, 3]
+      ]
+    },
+    "hints": { "title": "Disk iops (writes)" }
+  },
+  "load_average": {
+    "data": {
+      "cols": [
+        { "label": "time", "type": "date" },
+        { "label": "test-1 (master)", "type": "number" },
+        { "label": "test-2 (standby)", "type": "number" }
+      ],
+      "rows": [
+        ["2022-01-21T13:10:30Z", 0.11, 0.11],
+        ["2022-01-21T13:11:00Z", 0.14, 0.1],
+        ["2022-01-21T13:11:30Z", 0.14, 0.09]
+      ]
+    },
+    "hints": { "title": "Load average (5 min)" }
+  },
+  "mem_usage": {
+    "data": {
+      "cols": [
+        { "label": "time", "type": "date" },
+        { "label": "test-1 (master)", "type": "number" },
+        { "label": "test-2 (standby)", "type": "number" }
+      ],
+      "rows": [
+        ["2022-01-21T13:10:30Z", 11.491766148261078, 12.318932883261219],
+        ["2022-01-21T13:11:00Z", 11.511967645759277, 12.304403727425075],
+        ["2022-01-21T13:11:30Z", 11.488581675749048, 12.272260458006759]
+      ]
+    },
+    "hints": { "title": "Memory usage %" }
+  },
+  "net_receive": {
+    "data": {
+      "cols": [
+        { "label": "time", "type": "date" },
+        { "label": "test-1 (master)", "type": "number" },
+        { "label": "test-2 (standby)", "type": "number" }
+      ],
+      "rows": [
+        ["2022-01-21T13:10:30Z", 442, 470],
+        ["2022-01-21T13:11:00Z", 439, 384],
+        ["2022-01-21T13:11:30Z", 466, 458]
+      ]
+    },
+    "hints": { "title": "Network receive (bytes/s)" }
+  },
+  "net_send": {
+    "data": {
+      "cols": [
+        { "label": "time", "type": "date" },
+        { "label": "test-1 (master)", "type": "number" },
+        { "label": "test-2 (standby)", "type": "number" }
+      ],
+      "rows": [
+        ["2022-01-21T13:10:30Z", 672, 581],
+        ["2022-01-21T13:11:00Z", 660, 555],
+        ["2022-01-21T13:11:30Z", 694, 573]
+      ]
+    },
+    "hints": { "title": "Network transmit (bytes/s)" }
+  }
+}
+"""
+
+
+def test_get_metrics():
+    response = json.loads(MOCK_RESPONSE)
+    metrics = get_metrics(response)
+    assert metrics == UpcloudMetrics(
+        metrics=[
+            UpcloudMetric(
+                metric_name="cpu_usage",
+                records=[
+                    UpcloudRecord(
+                        db_role="master",
+                        label="test-1 " "(master)",
+                        time="2022-01-21T13:12:00Z",
+                        value=3.275132296130991,
+                    ),
+                    UpcloudRecord(
+                        db_role="standby",
+                        label="test-2 " "(standby)",
+                        time="2022-01-21T13:12:00Z",
+                        value=4.196249043309251,
+                    ),
+                ],
+            ),
+            UpcloudMetric(
+                metric_name="disk_usage",
+                records=[
+                    UpcloudRecord(
+                        db_role="master",
+                        label="test-1 " "(master)",
+                        time="2022-01-21T13:11:30Z",
+                        value=5.654416415900109,
+                    ),
+                    UpcloudRecord(
+                        db_role="standby",
+                        label="test-2 " "(standby)",
+                        time="2022-01-21T13:11:30Z",
+                        value=5.58959125727556,
+                    ),
+                ],
+            ),
+            UpcloudMetric(
+                metric_name="diskio_reads",
+                records=[
+                    UpcloudRecord(
+                        db_role="master",
+                        label="test-1 " "(master)",
+                        time="2022-01-21T13:11:30Z",
+                        value=0,
+                    ),
+                    UpcloudRecord(
+                        db_role="standby",
+                        label="test-2 " "(standby)",
+                        time="2022-01-21T13:11:30Z",
+                        value=0,
+                    ),
+                ],
+            ),
+            UpcloudMetric(
+                metric_name="diskio_writes",
+                records=[
+                    UpcloudRecord(
+                        db_role="master",
+                        label="test-1 " "(master)",
+                        time="2022-01-21T13:11:30Z",
+                        value=4,
+                    ),
+                    UpcloudRecord(
+                        db_role="standby",
+                        label="test-2 " "(standby)",
+                        time="2022-01-21T13:11:30Z",
+                        value=3,
+                    ),
+                ],
+            ),
+            UpcloudMetric(
+                metric_name="load_average",
+                records=[
+                    UpcloudRecord(
+                        db_role="master",
+                        label="test-1 " "(master)",
+                        time="2022-01-21T13:11:30Z",
+                        value=0.14,
+                    ),
+                    UpcloudRecord(
+                        db_role="standby",
+                        label="test-2 " "(standby)",
+                        time="2022-01-21T13:11:30Z",
+                        value=0.09,
+                    ),
+                ],
+            ),
+            UpcloudMetric(
+                metric_name="mem_usage",
+                records=[
+                    UpcloudRecord(
+                        db_role="master",
+                        label="test-1 " "(master)",
+                        time="2022-01-21T13:11:30Z",
+                        value=11.488581675749048,
+                    ),
+                    UpcloudRecord(
+                        db_role="standby",
+                        label="test-2 " "(standby)",
+                        time="2022-01-21T13:11:30Z",
+                        value=12.272260458006759,
+                    ),
+                ],
+            ),
+            UpcloudMetric(
+                metric_name="net_receive",
+                records=[
+                    UpcloudRecord(
+                        db_role="master",
+                        label="test-1 " "(master)",
+                        time="2022-01-21T13:11:30Z",
+                        value=466,
+                    ),
+                    UpcloudRecord(
+                        db_role="standby",
+                        label="test-2 " "(standby)",
+                        time="2022-01-21T13:11:30Z",
+                        value=458,
+                    ),
+                ],
+            ),
+            UpcloudMetric(
+                metric_name="net_send",
+                records=[
+                    UpcloudRecord(
+                        db_role="master",
+                        label="test-1 " "(master)",
+                        time="2022-01-21T13:11:30Z",
+                        value=694,
+                    ),
+                    UpcloudRecord(
+                        db_role="standby",
+                        label="test-2 " "(standby)",
+                        time="2022-01-21T13:11:30Z",
+                        value=573,
+                    ),
+                ],
+            ),
+        ]
+    )
+
+
+def test_get_metric():
+    response = json.loads(MOCK_RESPONSE)
+    metric_name = "cpu_usage"
+    metric = get_metric(response, metric_name)
+
+    assert metric.metric_name == metric_name
+    assert len(metric.records) == 2
+    assert metric.records[0].label == "test-1 (master)"
+    assert metric.records[0].time == "2022-01-21T13:12:00Z"
+    assert metric.records[0].value == 3.275132296130991
+
+    assert metric.records[1].label == "test-2 (standby)"
+    assert metric.records[1].time == "2022-01-21T13:12:00Z"
+    assert metric.records[1].value == 4.196249043309251
+
+
+def test_get_metric_with_none_value():
+    response_str = """
+{
+  "cpu_usage": {
+    "data": {
+      "cols": [
+        { "label": "time", "type": "date" },
+        { "label": "test-1 (master)", "type": "number" },
+        { "label": "test-2 (standby)", "type": "number" }
+      ],
+      "rows": [
+        ["2022-01-21T13:10:30Z", 2.744682398273781, 3.054323473090861],
+        ["2022-01-21T13:11:00Z", 3.0735645433218366, 2.972423595745795],
+        ["2022-01-21T13:11:30Z", null, 3.1358378052207883],
+        ["2022-01-21T13:12:00Z", 3.275132296130991, null]
+      ]
+    },
+    "hints": { "title": "CPU usage %" }
+  }
+}
+"""
+    response = json.loads(response_str)
+    metric = get_metric(response, "cpu_usage")
+
+    assert metric.records[0].label == "test-1 (master)"
+    assert metric.records[0].value == 3.275132296130991
+    assert metric.records[1].label == "test-2 (standby)"
+    assert metric.records[1].value == 3.1358378052207883
+
+
+def test_get_metric_with_none_value_in_last_two_positions():
+    response_str = """
+{
+  "cpu_usage": {
+    "data": {
+      "cols": [
+        { "label": "time", "type": "date" },
+        { "label": "test-1 (master)", "type": "number" },
+        { "label": "test-2 (standby)", "type": "number" }
+      ],
+      "rows": [
+        ["2022-01-21T13:10:30Z", 2.744682398273781, 3.054323473090861],
+        ["2022-01-21T13:11:00Z", 3.0735645433218366, 2.972423595745795],
+        ["2022-01-21T13:11:30Z", null, null],
+        ["2022-01-21T13:12:00Z", 3.275132296130991, null]
+      ]
+    },
+    "hints": { "title": "CPU usage %" }
+  }
+}
+"""
+    response = json.loads(response_str)
+    metric = get_metric(response, "cpu_usage")
+
+    assert len(metric.records) == 1
+    assert metric.records[0].label == "test-1 (master)"
+    assert metric.records[0].value == 3.275132296130991

app/tests/test_email_utils.py

@@ -19,10 +19,8 @@ from app.email_utils import (
     copy,
     get_spam_from_header,
     get_header_from_bounce,
-    is_valid_email,
     add_header,
     generate_reply_email,
-    normalize_reply_email,
     get_encoding,
     encode_text,
     EmailEncoding,
@@ -41,6 +39,7 @@ from app.email_utils import (
     get_verp_info_from_email,
     sl_formataddr,
 )
+from app.email_validation import is_valid_email, normalize_reply_email
 from app.models import (
     CustomDomain,
     Alias,

app/tests/test_models.py

@@ -78,20 +78,20 @@ def test_website_send_to(flask_client):
         user_id=user.id,
         alias_id=alias.id,
         website_email=f"{prefix}@example.com",
-        reply_email="rep@SL",
+        reply_email="rep@sl",
         name="First Last",
     )
-    assert c1.website_send_to() == f'"First Last | {prefix} at example.com" <rep@SL>'
+    assert c1.website_send_to() == f'"First Last | {prefix} at example.com" <rep@sl>'
 
     # empty name, ascii website_from, easy case
     c1.name = None
     c1.website_from = f"First Last <{prefix}@example.com>"
-    assert c1.website_send_to() == f'"First Last | {prefix} at example.com" <rep@SL>'
+    assert c1.website_send_to() == f'"First Last | {prefix} at example.com" <rep@sl>'
 
     # empty name, RFC 2047 website_from
     c1.name = None
     c1.website_from = f"=?UTF-8?B?TmjGoW4gTmd1eeG7hW4=?= <{prefix}@example.com>"
-    assert c1.website_send_to() == f'"Nhơn Nguyễn | {prefix} at example.com" <rep@SL>'
+    assert c1.website_send_to() == f'"Nhơn Nguyễn | {prefix} at example.com" <rep@sl>'
 
 
 def test_new_addr_default_sender_format(flask_client):
@@ -103,16 +103,16 @@ def test_new_addr_default_sender_format(flask_client):
         user_id=user.id,
         alias_id=alias.id,
         website_email=f"{prefix}@example.com",
-        reply_email="rep@SL",
+        reply_email="rep@sl",
         name="First Last",
         commit=True,
     )
 
-    assert contact.new_addr() == f'"First Last - {prefix} at example.com" <rep@SL>'
+    assert contact.new_addr() == f'"First Last - {prefix} at example.com" <rep@sl>'
 
     # Make sure email isn't duplicated if sender name equals email
     contact.name = f"{prefix}@example.com"
-    assert contact.new_addr() == f'"{prefix} at example.com" <rep@SL>'
+    assert contact.new_addr() == f'"{prefix} at example.com" <rep@sl>'
 
 
 def test_new_addr_a_sender_format(flask_client):
@@ -126,12 +126,12 @@ def test_new_addr_a_sender_format(flask_client):
         user_id=user.id,
         alias_id=alias.id,
         website_email=f"{prefix}@example.com",
-        reply_email="rep@SL",
+        reply_email="rep@sl",
         name="First Last",
         commit=True,
     )
 
-    assert contact.new_addr() == f'"First Last - {prefix}(a)example.com" <rep@SL>'
+    assert contact.new_addr() == f'"First Last - {prefix}(a)example.com" <rep@sl>'
 
 
 def test_new_addr_no_name_sender_format(flask_client):
@@ -145,12 +145,12 @@ def test_new_addr_no_name_sender_format(flask_client):
         user_id=user.id,
         alias_id=alias.id,
         website_email=f"{prefix}@example.com",
-        reply_email="rep@SL",
+        reply_email="rep@sl",
         name="First Last",
         commit=True,
     )
 
-    assert contact.new_addr() == "rep@SL"
+    assert contact.new_addr() == "rep@sl"
 
 
 def test_new_addr_name_only_sender_format(flask_client):
@@ -164,12 +164,12 @@ def test_new_addr_name_only_sender_format(flask_client):
         user_id=user.id,
         alias_id=alias.id,
         website_email=f"{prefix}@example.com",
-        reply_email="rep@SL",
+        reply_email="rep@sl",
         name="First Last",
         commit=True,
     )
 
-    assert contact.new_addr() == "First Last <rep@SL>"
+    assert contact.new_addr() == "First Last <rep@sl>"
 
 
 def test_new_addr_at_only_sender_format(flask_client):
@@ -183,12 +183,12 @@ def test_new_addr_at_only_sender_format(flask_client):
         user_id=user.id,
         alias_id=alias.id,
         website_email=f"{prefix}@example.com",
-        reply_email="rep@SL",
+        reply_email="rep@sl",
         name="First Last",
         commit=True,
     )
 
-    assert contact.new_addr() == f'"{prefix} at example.com" <rep@SL>'
+    assert contact.new_addr() == f'"{prefix} at example.com" <rep@sl>'
 
 
 def test_new_addr_unicode(flask_client):
@@ -200,14 +200,14 @@ def test_new_addr_unicode(flask_client):
         user_id=user.id,
         alias_id=alias.id,
         website_email=f"{random_prefix}@example.com",
-        reply_email="rep@SL",
+        reply_email="rep@sl",
         name="Nhơn Nguyễn",
         commit=True,
     )
 
     assert (
         contact.new_addr()
-        == f"=?utf-8?q?Nh=C6=A1n_Nguy=E1=BB=85n_-_{random_prefix}_at_example=2Ecom?= <rep@SL>"
+        == f"=?utf-8?q?Nh=C6=A1n_Nguy=E1=BB=85n_-_{random_prefix}_at_example=2Ecom?= <rep@sl>"
     )
 
     # sanity check