Add postgres-exporter alerts

Author: DTLP

postgres/system-alerts.yaml.tmpl

@@ -0,0 +1,20 @@
+# PROMETHEUS RULES
+# DO NOT REMOVE line above, used in `pre-commit` hook
+
+groups:
+  - name: postgres-exporter
+    rules:
+      - alert: "PostgresExporterDown"
+        expr: |
+          job="postgres-exporter" == 0
+        for: 15m
+        labels:
+          team: infra
+        annotations:
+          summary: "Postgres Exporter is down"
+          impact: "Postgres instances are not monitored"
+          qonto_runbook: https://qonto.github.io/database-monitoring-framework/latest/runbooks/postgresql/SQLExporterDown
+          action: |
+            Check if postgres-exporter is running in sys-prom namespace.
+            Check the logs.
+            Restart pods.

postgres/team-alerts.yaml.tmpl

@@ -0,0 +1,91 @@
+# PROMETHEUS RULES
+# DO NOT REMOVE line above, used in `pre-commit` hook
+
+groups:
+  - name: Postgres
+    rules:
+      - alert: "PostgresExporterMissingTarget"
+        expr: |
+          min(up{job="postgres-exporter", instance!=""}) by (instance) == 0
+        for: 15m
+        labels:
+          team: infra
+        annotations:
+          summary: Postgres Exporter scrape for {{ $labels.target }} failed
+          impact: "{{ $labels.target }} instance is not monitored"
+          qonto_runbook: https://qonto.github.io/database-monitoring-framework/latest/runbooks/postgresql/SQLExporterMissingTarget
+          action: |
+            Check if the postgres instance is down.
+            Check if there's an issue with the prometheus_postgres_exporter user.
+
+      - alert: "PostgresExporterScrapingLimit"
+        expr: |
+          avg_over_time(pg_exporter_last_scrape_duration_seconds{job="postgres-exporter", instance!=""}[10m]) > 30
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          description: Postgres Exporter scraping is taking a long time
+          runbook_url: https://qonto.github.io/database-monitoring-framework/latest/runbooks/postgresql/SQLExporterScrapingLimit
+          summary: Exporter scraping takes long time
+
+      - alert: "PostgreSQLLongRunningQueries"
+        expr: |
+          pg_long_running_transactions_oldest_timestamp_seconds > 1800
+        for: 1m
+        labels:
+          severity: warning
+        annotations:
+          description: "PostgreSQL has a long-running transaction (>30 mins)"
+          runbook_url: https://qonto.github.io/database-monitoring-framework/latest/runbooks/postgresql/PostgreSQLLongRunningQueries
+          summary: Long running query on {{ $labels.instance }}
+
+      - alert: "PostgreSQLMaxConnections"
+        expr: |
+          sum(pg_stat_activity_count) by (server)
+          /
+          on(server)
+          pg_settings_max_connections
+          > 0.8
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          description: "{{ $labels.target }} uses >80% of the maximum database connections"
+          runbook_url: https://qonto.github.io/database-monitoring-framework/latest/runbooks/postgresql/PostgreSQLMaxConnections
+          summary: "{{ $labels.target }} is close from the maximum database connections"
+
+      - alert: "PostgresExporterDown"
+        expr: |
+          job="postgres-exporter" == 0
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          description: Postgres Exporter is down
+          runbook_url: https://qonto.github.io/database-monitoring-framework/latest/runbooks/postgresql/SQLExporterDown
+          summary: Exporter is down
+
+      - alert: "PostgresExporterMissingTarget"
+        expr: |
+          min(up{job="postgres-exporter", instance!=""}) by (instance,exported_job) == 0
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          description:
+            Postgres Exporter cannot collect metrics for this target, it's either because
+            the target is down, or the sql-exporter user cannot connect.
+          runbook_url: https://qonto.github.io/database-monitoring-framework/latest/runbooks/postgresql/SQLExporterMissingTarget
+          summary: Exporter scrape for {{ $labels.target }} ({{ $labels.exported_job }}) failed
+
+      - alert: "PostgresExporterScrapingLimit"
+        expr: |
+          avg_over_time(pg_exporter_last_scrape_duration_seconds{job="postgres-exporter", instance!=""}[10m]) > 30
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          description: Postgres Exporter scraping is taking a long time
+          runbook_url: https://qonto.github.io/database-monitoring-framework/latest/runbooks/postgresql/SQLExporterScrapingLimit
+          summary: Exporter scraping takes long time