4.44.0

app/events/event_sink.py

@@ -0,0 +1,19 @@
+from abc import ABC, abstractmethod
+from app.log import LOG
+from app.models import SyncEvent
+
+
+class EventSink(ABC):
+    @abstractmethod
+    def process(self, event: SyncEvent):
+        pass
+
+
+class HttpEventSink(EventSink):
+    def process(self, event: SyncEvent):
+        pass
+
+
+class ConsoleEventSink(EventSink):
+    def process(self, event: SyncEvent):
+        LOG.info(f"Handling event {event.id}")

app/events/event_source.py

@@ -0,0 +1,75 @@
+import arrow
+import newrelic.agent
+import psycopg2
+import select
+
+from abc import ABC, abstractmethod
+from app.log import LOG
+from app.models import SyncEvent
+from app.events.event_dispatcher import NOTIFICATION_CHANNEL
+from time import sleep
+from typing import Callable, NoReturn
+
+_DEAD_LETTER_THRESHOLD_MINUTES = 10
+_DEAD_LETTER_INTERVAL_SECONDS = 30
+
+
+class EventSource(ABC):
+    @abstractmethod
+    def run(self, on_event: Callable[[SyncEvent], NoReturn]):
+        pass
+
+
+class PostgresEventSource(EventSource):
+    def __init__(self, connection_string: str):
+        self.__connection = psycopg2.connect(connection_string)
+
+    def run(self, on_event: Callable[[SyncEvent], NoReturn]):
+        self.__connection.set_isolation_level(
+            psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT
+        )
+
+        cursor = self.__connection.cursor()
+        cursor.execute(f"LISTEN {NOTIFICATION_CHANNEL};")
+
+        while True:
+            if select.select([self.__connection], [], [], 5) != ([], [], []):
+                self.__connection.poll()
+                while self.__connection.notifies:
+                    notify = self.__connection.notifies.pop(0)
+                    LOG.debug(
+                        f"Got NOTIFY: pid={notify.pid} channel={notify.channel} payload={notify.payload}"
+                    )
+                    try:
+                        webhook_id = int(notify.payload)
+                        event = SyncEvent.get_by(id=webhook_id)
+                        if event is not None:
+                            on_event(event)
+                        else:
+                            LOG.info(f"Could not find event with id={notify.payload}")
+                    except Exception as e:
+                        LOG.warn(f"Error getting event: {e}")
+
+
+class DeadLetterEventSource(EventSource):
+    @newrelic.agent.background_task()
+    def run(self, on_event: Callable[[SyncEvent], NoReturn]):
+        while True:
+            try:
+                threshold = arrow.utcnow().shift(
+                    minutes=-_DEAD_LETTER_THRESHOLD_MINUTES
+                )
+                events = SyncEvent.get_dead_letter(older_than=threshold)
+                if events:
+                    LOG.info(f"Got {len(events)} dead letter events")
+                    if events:
+                        newrelic.agent.record_custom_metric(
+                            "Custom/dead_letter_events_to_process", len(events)
+                        )
+                    for event in events:
+                        on_event(event)
+                else:
+                    LOG.debug("No dead letter events")
+                    sleep(_DEAD_LETTER_INTERVAL_SECONDS)
+            except Exception as e:
+                LOG.warn(f"Error getting dead letter event: {e}")

app/events/runner.py

@@ -0,0 +1,45 @@
+import arrow
+import newrelic.agent
+
+from app.log import LOG
+from app.models import SyncEvent
+from events.event_sink import EventSink
+from events.event_source import EventSource
+
+
+class Runner:
+    def __init__(self, source: EventSource, sink: EventSink):
+        self.__source = source
+        self.__sink = sink
+
+    def run(self):
+        self.__source.run(self.__on_event)
+
+    @newrelic.agent.background_task()
+    def __on_event(self, event: SyncEvent):
+        try:
+            can_process = event.mark_as_taken()
+            if can_process:
+                event_created_at = event.created_at
+                start_time = arrow.now()
+                self.__sink.process(event)
+                event_id = event.id
+                SyncEvent.delete(event.id, commit=True)
+                LOG.info(f"Marked {event_id} as done")
+
+                end_time = arrow.now() - start_time
+                time_between_taken_and_created = start_time - event_created_at
+
+                newrelic.agent.record_custom_metric("Custom/sync_event_processed", 1)
+                newrelic.agent.record_custom_metric(
+                    "Custom/sync_event_process_time", end_time.total_seconds()
+                )
+                newrelic.agent.record_custom_metric(
+                    "Custom/sync_event_elapsed_time",
+                    time_between_taken_and_created.total_seconds(),
+                )
+            else:
+                LOG.info(f"{event.id} was handled by another runner")
+        except Exception as e:
+            LOG.warn(f"Exception processing event [id={event.id}]: {e}")
+            newrelic.agent.record_custom_metric("Custom/sync_event_failed", 1)