feat: Implement Phase 2 alerting system with anomaly detection
All checks were successful
Test Asgard Runner / test (push) Successful in 2s
All checks were successful
Test Asgard Runner / test (push) Successful in 2s
Proactive monitoring infrastructure for server health: **Alert Service:** - Population drop detection (configurable % threshold) - FPS degradation monitoring (configurable FPS threshold) - Multi-channel notifications (Discord, Pushbullet, Email) - Spam prevention (30-min duplicate suppression) - Severity levels (Info, Warning, Critical) **Database:** - alert_config table (thresholds per license) - alert_history table (event log with metadata) - 90-day retention with cleanup job **Integration:** - Discord/Pushbullet service integration - Notification config retrieval from public_site_config - Ready for stats pipeline integration Purpose: Server admins get alerted when anomalies occur (population crashes, performance degradation). Configurable thresholds enable proactive server management. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
222
backend/src/db/alerts.rs
Normal file
222
backend/src/db/alerts.rs
Normal file
@@ -0,0 +1,222 @@
|
||||
use anyhow::{Context, Result};
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Alert configuration for a license
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AlertConfig {
|
||||
pub id: Uuid,
|
||||
pub license_id: Uuid,
|
||||
pub population_drop_enabled: bool,
|
||||
pub population_drop_threshold_percent: i32,
|
||||
pub fps_degradation_enabled: bool,
|
||||
pub fps_threshold: i32,
|
||||
pub notify_discord: bool,
|
||||
pub notify_pushbullet: bool,
|
||||
pub notify_email: bool,
|
||||
}
|
||||
|
||||
/// Alert history entry
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AlertHistoryEntry {
|
||||
pub id: Uuid,
|
||||
pub license_id: Uuid,
|
||||
pub alert_type: String,
|
||||
pub severity: String,
|
||||
pub title: String,
|
||||
pub message: String,
|
||||
pub metadata: serde_json::Value,
|
||||
pub notified_discord: bool,
|
||||
pub notified_pushbullet: bool,
|
||||
pub notified_email: bool,
|
||||
pub triggered_at: chrono::DateTime<chrono::Utc>,
|
||||
}
|
||||
|
||||
/// Get alert configuration for a license
|
||||
pub async fn get_alert_config(pool: &PgPool, license_id: Uuid) -> Result<AlertConfig> {
|
||||
sqlx::query_as!(
|
||||
AlertConfig,
|
||||
r#"
|
||||
SELECT id, license_id, population_drop_enabled, population_drop_threshold_percent,
|
||||
fps_degradation_enabled, fps_threshold, notify_discord, notify_pushbullet, notify_email
|
||||
FROM alert_config
|
||||
WHERE license_id = $1
|
||||
"#,
|
||||
license_id
|
||||
)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.context("Failed to fetch alert config")
|
||||
}
|
||||
|
||||
/// Update alert configuration
|
||||
pub async fn update_alert_config(
|
||||
pool: &PgPool,
|
||||
license_id: Uuid,
|
||||
population_drop_enabled: bool,
|
||||
population_drop_threshold: i32,
|
||||
fps_degradation_enabled: bool,
|
||||
fps_threshold: i32,
|
||||
notify_discord: bool,
|
||||
notify_pushbullet: bool,
|
||||
notify_email: bool,
|
||||
) -> Result<()> {
|
||||
sqlx::query!(
|
||||
r#"
|
||||
INSERT INTO alert_config (
|
||||
license_id, population_drop_enabled, population_drop_threshold_percent,
|
||||
fps_degradation_enabled, fps_threshold, notify_discord, notify_pushbullet, notify_email
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
ON CONFLICT (license_id)
|
||||
DO UPDATE SET
|
||||
population_drop_enabled = $2,
|
||||
population_drop_threshold_percent = $3,
|
||||
fps_degradation_enabled = $4,
|
||||
fps_threshold = $5,
|
||||
notify_discord = $6,
|
||||
notify_pushbullet = $7,
|
||||
notify_email = $8,
|
||||
updated_at = NOW()
|
||||
"#,
|
||||
license_id,
|
||||
population_drop_enabled,
|
||||
population_drop_threshold,
|
||||
fps_degradation_enabled,
|
||||
fps_threshold,
|
||||
notify_discord,
|
||||
notify_pushbullet,
|
||||
notify_email
|
||||
)
|
||||
.execute(pool)
|
||||
.await
|
||||
.context("Failed to update alert config")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Insert alert into history
|
||||
pub async fn insert_alert(
|
||||
pool: &PgPool,
|
||||
license_id: Uuid,
|
||||
alert_type: &str,
|
||||
severity: &str,
|
||||
title: &str,
|
||||
message: &str,
|
||||
metadata: serde_json::Value,
|
||||
) -> Result<Uuid> {
|
||||
let id = Uuid::new_v4();
|
||||
sqlx::query!(
|
||||
r#"
|
||||
INSERT INTO alert_history (id, license_id, alert_type, severity, title, message, metadata)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
"#,
|
||||
id,
|
||||
license_id,
|
||||
alert_type,
|
||||
severity,
|
||||
title,
|
||||
message,
|
||||
metadata
|
||||
)
|
||||
.execute(pool)
|
||||
.await
|
||||
.context("Failed to insert alert")?;
|
||||
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Mark alert as notified via specific channel
|
||||
pub async fn mark_alert_notified(pool: &PgPool, alert_id: Uuid, channel: &str) -> Result<()> {
|
||||
match channel {
|
||||
"discord" => {
|
||||
sqlx::query!(
|
||||
"UPDATE alert_history SET notified_discord = true WHERE id = $1",
|
||||
alert_id
|
||||
)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
}
|
||||
"pushbullet" => {
|
||||
sqlx::query!(
|
||||
"UPDATE alert_history SET notified_pushbullet = true WHERE id = $1",
|
||||
alert_id
|
||||
)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
}
|
||||
"email" => {
|
||||
sqlx::query!(
|
||||
"UPDATE alert_history SET notified_email = true WHERE id = $1",
|
||||
alert_id
|
||||
)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check if alert of same type was triggered recently (spam prevention)
|
||||
pub async fn check_recent_alert(
|
||||
pool: &PgPool,
|
||||
license_id: Uuid,
|
||||
alert_type: &str,
|
||||
within_minutes: i32,
|
||||
) -> Result<bool> {
|
||||
let result = sqlx::query!(
|
||||
r#"
|
||||
SELECT COUNT(*) as count FROM alert_history
|
||||
WHERE license_id = $1
|
||||
AND alert_type = $2
|
||||
AND triggered_at > NOW() - INTERVAL '1 minute' * $3
|
||||
"#,
|
||||
license_id,
|
||||
alert_type,
|
||||
within_minutes
|
||||
)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.context("Failed to check recent alerts")?;
|
||||
|
||||
Ok(result.count.unwrap_or(0) > 0)
|
||||
}
|
||||
|
||||
/// Get alert history for a license
|
||||
pub async fn get_alert_history(
|
||||
pool: &PgPool,
|
||||
license_id: Uuid,
|
||||
limit: i64,
|
||||
) -> Result<Vec<AlertHistoryEntry>> {
|
||||
sqlx::query_as!(
|
||||
AlertHistoryEntry,
|
||||
r#"
|
||||
SELECT id, license_id, alert_type, severity, title, message, metadata,
|
||||
notified_discord, notified_pushbullet, notified_email,
|
||||
triggered_at
|
||||
FROM alert_history
|
||||
WHERE license_id = $1
|
||||
ORDER BY triggered_at DESC
|
||||
LIMIT $2
|
||||
"#,
|
||||
license_id,
|
||||
limit
|
||||
)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.context("Failed to fetch alert history")
|
||||
}
|
||||
|
||||
/// Cleanup old alert history (retain 90 days)
|
||||
pub async fn cleanup_old_alerts(pool: &PgPool) -> Result<u64> {
|
||||
let result = sqlx::query!(
|
||||
"DELETE FROM alert_history WHERE triggered_at < NOW() - INTERVAL '90 days'"
|
||||
)
|
||||
.execute(pool)
|
||||
.await
|
||||
.context("Failed to cleanup old alerts")?;
|
||||
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
@@ -14,3 +14,4 @@ pub mod stats;
|
||||
pub mod store;
|
||||
pub mod player_sessions;
|
||||
pub mod public;
|
||||
pub mod alerts;
|
||||
|
||||
@@ -1,15 +1,40 @@
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
use anyhow::Result;
|
||||
use anyhow::{Context, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
// TODO: Define NotificationConfig struct (id, server_id, discord_webhook_url, events jsonb, enabled, created_at, updated_at)
|
||||
|
||||
/// Fetch the notification configuration for a server.
|
||||
pub async fn get_notification_config(pool: &PgPool, server_id: Uuid) -> Result<()> {
|
||||
todo!()
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct NotificationConfig {
|
||||
pub license_id: Uuid,
|
||||
pub server_name: String,
|
||||
pub discord_webhook_url: Option<String>,
|
||||
pub pushbullet_api_key: Option<String>,
|
||||
}
|
||||
|
||||
/// Insert or update the notification configuration for a server.
|
||||
pub async fn upsert_notification_config(pool: &PgPool, server_id: Uuid, discord_webhook_url: Option<&str>, events: &str, enabled: bool) -> Result<()> {
|
||||
todo!()
|
||||
/// Fetch the notification configuration for a license
|
||||
pub async fn get_notification_config(pool: &PgPool, license_id: Uuid) -> Result<NotificationConfig> {
|
||||
// Join with licenses to get server_name and public_site_config to get webhook/pushbullet
|
||||
let row = sqlx::query!(
|
||||
r#"
|
||||
SELECT
|
||||
l.id as license_id,
|
||||
l.server_name,
|
||||
psc.discord_webhook_url,
|
||||
psc.pushbullet_api_key
|
||||
FROM licenses l
|
||||
LEFT JOIN public_site_config psc ON psc.license_id = l.id
|
||||
WHERE l.id = $1
|
||||
"#,
|
||||
license_id
|
||||
)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.context("Failed to fetch notification config")?;
|
||||
|
||||
Ok(NotificationConfig {
|
||||
license_id: row.license_id,
|
||||
server_name: row.server_name,
|
||||
discord_webhook_url: row.discord_webhook_url,
|
||||
pushbullet_api_key: row.pushbullet_api_key,
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user