feat: Implement Phase 2 alerting system with anomaly detection
All checks were successful
Test Asgard Runner / test (push) Successful in 2s

Proactive monitoring infrastructure for server health:

**Alert Service:**
- Population drop detection (configurable % threshold)
- FPS degradation monitoring (configurable FPS threshold)
- Multi-channel notifications (Discord, Pushbullet, Email)
- Spam prevention (30-min duplicate suppression)
- Severity levels (Info, Warning, Critical)

**Database:**
- alert_config table (thresholds per license)
- alert_history table (event log with metadata)
- 90-day retention with cleanup job

**Integration:**
- Discord/Pushbullet service integration
- Notification config retrieval from public_site_config
- Ready for stats pipeline integration

Purpose: Server admins get alerted when anomalies occur
(population crashes, performance degradation). Configurable
thresholds enable proactive server management.

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Vantz Stockwell
2026-02-15 14:28:51 -05:00
parent 8790072609
commit 3e8b29f2ee
7 changed files with 606 additions and 9 deletions

222
backend/src/db/alerts.rs Normal file
View File

@@ -0,0 +1,222 @@
use anyhow::{Context, Result};
use sqlx::PgPool;
use uuid::Uuid;
use serde::{Deserialize, Serialize};
/// Alert configuration for a license
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AlertConfig {
pub id: Uuid,
pub license_id: Uuid,
pub population_drop_enabled: bool,
pub population_drop_threshold_percent: i32,
pub fps_degradation_enabled: bool,
pub fps_threshold: i32,
pub notify_discord: bool,
pub notify_pushbullet: bool,
pub notify_email: bool,
}
/// Alert history entry
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AlertHistoryEntry {
pub id: Uuid,
pub license_id: Uuid,
pub alert_type: String,
pub severity: String,
pub title: String,
pub message: String,
pub metadata: serde_json::Value,
pub notified_discord: bool,
pub notified_pushbullet: bool,
pub notified_email: bool,
pub triggered_at: chrono::DateTime<chrono::Utc>,
}
/// Get alert configuration for a license
pub async fn get_alert_config(pool: &PgPool, license_id: Uuid) -> Result<AlertConfig> {
sqlx::query_as!(
AlertConfig,
r#"
SELECT id, license_id, population_drop_enabled, population_drop_threshold_percent,
fps_degradation_enabled, fps_threshold, notify_discord, notify_pushbullet, notify_email
FROM alert_config
WHERE license_id = $1
"#,
license_id
)
.fetch_one(pool)
.await
.context("Failed to fetch alert config")
}
/// Update alert configuration
pub async fn update_alert_config(
pool: &PgPool,
license_id: Uuid,
population_drop_enabled: bool,
population_drop_threshold: i32,
fps_degradation_enabled: bool,
fps_threshold: i32,
notify_discord: bool,
notify_pushbullet: bool,
notify_email: bool,
) -> Result<()> {
sqlx::query!(
r#"
INSERT INTO alert_config (
license_id, population_drop_enabled, population_drop_threshold_percent,
fps_degradation_enabled, fps_threshold, notify_discord, notify_pushbullet, notify_email
)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (license_id)
DO UPDATE SET
population_drop_enabled = $2,
population_drop_threshold_percent = $3,
fps_degradation_enabled = $4,
fps_threshold = $5,
notify_discord = $6,
notify_pushbullet = $7,
notify_email = $8,
updated_at = NOW()
"#,
license_id,
population_drop_enabled,
population_drop_threshold,
fps_degradation_enabled,
fps_threshold,
notify_discord,
notify_pushbullet,
notify_email
)
.execute(pool)
.await
.context("Failed to update alert config")?;
Ok(())
}
/// Insert alert into history
pub async fn insert_alert(
pool: &PgPool,
license_id: Uuid,
alert_type: &str,
severity: &str,
title: &str,
message: &str,
metadata: serde_json::Value,
) -> Result<Uuid> {
let id = Uuid::new_v4();
sqlx::query!(
r#"
INSERT INTO alert_history (id, license_id, alert_type, severity, title, message, metadata)
VALUES ($1, $2, $3, $4, $5, $6, $7)
"#,
id,
license_id,
alert_type,
severity,
title,
message,
metadata
)
.execute(pool)
.await
.context("Failed to insert alert")?;
Ok(id)
}
/// Mark alert as notified via specific channel
pub async fn mark_alert_notified(pool: &PgPool, alert_id: Uuid, channel: &str) -> Result<()> {
match channel {
"discord" => {
sqlx::query!(
"UPDATE alert_history SET notified_discord = true WHERE id = $1",
alert_id
)
.execute(pool)
.await?;
}
"pushbullet" => {
sqlx::query!(
"UPDATE alert_history SET notified_pushbullet = true WHERE id = $1",
alert_id
)
.execute(pool)
.await?;
}
"email" => {
sqlx::query!(
"UPDATE alert_history SET notified_email = true WHERE id = $1",
alert_id
)
.execute(pool)
.await?;
}
_ => {}
}
Ok(())
}
/// Check if alert of same type was triggered recently (spam prevention)
pub async fn check_recent_alert(
pool: &PgPool,
license_id: Uuid,
alert_type: &str,
within_minutes: i32,
) -> Result<bool> {
let result = sqlx::query!(
r#"
SELECT COUNT(*) as count FROM alert_history
WHERE license_id = $1
AND alert_type = $2
AND triggered_at > NOW() - INTERVAL '1 minute' * $3
"#,
license_id,
alert_type,
within_minutes
)
.fetch_one(pool)
.await
.context("Failed to check recent alerts")?;
Ok(result.count.unwrap_or(0) > 0)
}
/// Get alert history for a license
pub async fn get_alert_history(
pool: &PgPool,
license_id: Uuid,
limit: i64,
) -> Result<Vec<AlertHistoryEntry>> {
sqlx::query_as!(
AlertHistoryEntry,
r#"
SELECT id, license_id, alert_type, severity, title, message, metadata,
notified_discord, notified_pushbullet, notified_email,
triggered_at
FROM alert_history
WHERE license_id = $1
ORDER BY triggered_at DESC
LIMIT $2
"#,
license_id,
limit
)
.fetch_all(pool)
.await
.context("Failed to fetch alert history")
}
/// Cleanup old alert history (retain 90 days)
pub async fn cleanup_old_alerts(pool: &PgPool) -> Result<u64> {
let result = sqlx::query!(
"DELETE FROM alert_history WHERE triggered_at < NOW() - INTERVAL '90 days'"
)
.execute(pool)
.await
.context("Failed to cleanup old alerts")?;
Ok(result.rows_affected())
}

View File

@@ -14,3 +14,4 @@ pub mod stats;
pub mod store;
pub mod player_sessions;
pub mod public;
pub mod alerts;

View File

@@ -1,15 +1,40 @@
use sqlx::PgPool;
use uuid::Uuid;
use anyhow::Result;
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
// TODO: Define NotificationConfig struct (id, server_id, discord_webhook_url, events jsonb, enabled, created_at, updated_at)
/// Fetch the notification configuration for a server.
pub async fn get_notification_config(pool: &PgPool, server_id: Uuid) -> Result<()> {
todo!()
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NotificationConfig {
pub license_id: Uuid,
pub server_name: String,
pub discord_webhook_url: Option<String>,
pub pushbullet_api_key: Option<String>,
}
/// Insert or update the notification configuration for a server.
pub async fn upsert_notification_config(pool: &PgPool, server_id: Uuid, discord_webhook_url: Option<&str>, events: &str, enabled: bool) -> Result<()> {
todo!()
/// Fetch the notification configuration for a license
pub async fn get_notification_config(pool: &PgPool, license_id: Uuid) -> Result<NotificationConfig> {
// Join with licenses to get server_name and public_site_config to get webhook/pushbullet
let row = sqlx::query!(
r#"
SELECT
l.id as license_id,
l.server_name,
psc.discord_webhook_url,
psc.pushbullet_api_key
FROM licenses l
LEFT JOIN public_site_config psc ON psc.license_id = l.id
WHERE l.id = $1
"#,
license_id
)
.fetch_one(pool)
.await
.context("Failed to fetch notification config")?;
Ok(NotificationConfig {
license_id: row.license_id,
server_name: row.server_name,
discord_webhook_url: row.discord_webhook_url,
pushbullet_api_key: row.pushbullet_api_key,
})
}