4.42.0

app/tasks/cleanup_old_imports.py

@@ -0,0 +1,19 @@
+import arrow
+
+from app import s3
+from app.log import LOG
+from app.models import BatchImport
+
+
+def cleanup_old_imports(oldest_allowed: arrow.Arrow):
+    LOG.i(f"Deleting imports older than {oldest_allowed}")
+    for batch_import in (
+        BatchImport.filter(BatchImport.created_at < oldest_allowed).yield_per(500).all()
+    ):
+        LOG.i(
+            f"Deleting batch import {batch_import} with file {batch_import.file.path}"
+        )
+        file = batch_import.file
+        if file is not None:
+            s3.delete(file.path)
+        BatchImport.delete(batch_import.id, commit=True)

app/tasks/cleanup_old_jobs.py

@@ -0,0 +1,24 @@
+import arrow
+from sqlalchemy import or_, and_
+
+from app import config
+from app.db import Session
+from app.log import LOG
+from app.models import Job, JobState
+
+
+def cleanup_old_jobs(oldest_allowed: arrow.Arrow):
+    LOG.i(f"Deleting jobs older than {oldest_allowed}")
+    count = Job.filter(
+        or_(
+            Job.state == JobState.done.value,
+            Job.state == JobState.error.value,
+            and_(
+                Job.state == JobState.taken.value,
+                Job.attempts >= config.JOB_MAX_ATTEMPTS,
+            ),
+        ),
+        Job.updated_at < oldest_allowed,
+    ).delete()
+    Session.commit()
+    LOG.i(f"Deleted {count} jobs")

app/tasks/cleanup_old_notifications.py

@@ -0,0 +1,12 @@
+import arrow
+
+from app.db import Session
+from app.log import LOG
+from app.models import Notification
+
+
+def cleanup_old_notifications(oldest_allowed: arrow.Arrow):
+    LOG.i(f"Deleting notifications older than {oldest_allowed}")
+    count = Notification.filter(Notification.created_at < oldest_allowed).delete()
+    Session.commit()
+    LOG.i(f"Deleted {count} notifications")