|
|
|
|
@ -0,0 +1,516 @@
|
|
|
|
|
"""告警通道模块
|
|
|
|
|
|
|
|
|
|
支持多种告警方式:
|
|
|
|
|
- 日志告警（默认）
|
|
|
|
|
- 钉钉机器人
|
|
|
|
|
- 邮件
|
|
|
|
|
- Webhook
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
import json
|
|
|
|
|
import smtplib
|
|
|
|
|
from abc import ABC, abstractmethod
|
|
|
|
|
from dataclasses import dataclass
|
|
|
|
|
from datetime import datetime
|
|
|
|
|
from email.mime.text import MIMEText
|
|
|
|
|
from email.mime.multipart import MIMEMultipart
|
|
|
|
|
from typing import Dict, List, Optional
|
|
|
|
|
|
|
|
|
|
import httpx
|
|
|
|
|
from app.core.logger import info, error, warning
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
|
class AlertMessage:
|
|
|
|
|
"""告警消息"""
|
|
|
|
|
title: str
|
|
|
|
|
content: str
|
|
|
|
|
level: str = "warning" # info, warning, error, critical
|
|
|
|
|
timestamp: datetime = None
|
|
|
|
|
metadata: Dict = None
|
|
|
|
|
|
|
|
|
|
def __post_init__(self):
|
|
|
|
|
if self.timestamp is None:
|
|
|
|
|
self.timestamp = datetime.now()
|
|
|
|
|
if self.metadata is None:
|
|
|
|
|
self.metadata = {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class AlertChannel(ABC):
|
|
|
|
|
"""告警通道基类"""
|
|
|
|
|
|
|
|
|
|
def __init__(self, name: str, enabled: bool = True):
|
|
|
|
|
self.name = name
|
|
|
|
|
self.enabled = enabled
|
|
|
|
|
|
|
|
|
|
@abstractmethod
|
|
|
|
|
async def send(self, message: AlertMessage) -> bool:
|
|
|
|
|
"""发送告警消息"""
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
async def send_batch(self, messages: List[AlertMessage]) -> List[bool]:
|
|
|
|
|
"""批量发送告警"""
|
|
|
|
|
results = []
|
|
|
|
|
for msg in messages:
|
|
|
|
|
result = await self.send(msg)
|
|
|
|
|
results.append(result)
|
|
|
|
|
return results
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class LogAlertChannel(AlertChannel):
|
|
|
|
|
"""日志告警通道"""
|
|
|
|
|
|
|
|
|
|
def __init__(self, enabled: bool = True):
|
|
|
|
|
super().__init__("log", enabled)
|
|
|
|
|
|
|
|
|
|
async def send(self, message: AlertMessage) -> bool:
|
|
|
|
|
"""发送日志告警"""
|
|
|
|
|
if not self.enabled:
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
log_msg = f"[{message.level.upper()}] {message.title}: {message.content}"
|
|
|
|
|
|
|
|
|
|
if message.level == "info":
|
|
|
|
|
info(log_msg)
|
|
|
|
|
elif message.level == "warning":
|
|
|
|
|
warning(log_msg)
|
|
|
|
|
else:
|
|
|
|
|
error(log_msg)
|
|
|
|
|
|
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class DingTalkAlertChannel(AlertChannel):
|
|
|
|
|
"""钉钉机器人告警通道"""
|
|
|
|
|
|
|
|
|
|
def __init__(
|
|
|
|
|
self,
|
|
|
|
|
webhook_url: str,
|
|
|
|
|
secret: Optional[str] = None,
|
|
|
|
|
at_mobiles: Optional[List[str]] = None,
|
|
|
|
|
at_all: bool = False,
|
|
|
|
|
enabled: bool = True
|
|
|
|
|
):
|
|
|
|
|
super().__init__("dingtalk", enabled)
|
|
|
|
|
self.webhook_url = webhook_url
|
|
|
|
|
self.secret = secret
|
|
|
|
|
self.at_mobiles = at_mobiles or []
|
|
|
|
|
self.at_all = at_all
|
|
|
|
|
|
|
|
|
|
def _generate_sign(self, timestamp: str) -> str:
|
|
|
|
|
"""生成钉钉签名"""
|
|
|
|
|
import hmac
|
|
|
|
|
import hashlib
|
|
|
|
|
import urllib.parse
|
|
|
|
|
|
|
|
|
|
if not self.secret:
|
|
|
|
|
return ""
|
|
|
|
|
|
|
|
|
|
string_to_sign = f"{timestamp}\n{self.secret}"
|
|
|
|
|
hmac_code = hmac.new(
|
|
|
|
|
self.secret.encode('utf-8'),
|
|
|
|
|
string_to_sign.encode('utf-8'),
|
|
|
|
|
digestmod=hashlib.sha256
|
|
|
|
|
).digest()
|
|
|
|
|
sign = urllib.parse.quote_plus(base64.b64encode(hmac_code))
|
|
|
|
|
return sign
|
|
|
|
|
|
|
|
|
|
def _build_markdown_message(self, message: AlertMessage) -> Dict:
|
|
|
|
|
"""构建Markdown格式的消息"""
|
|
|
|
|
# 根据级别选择颜色
|
|
|
|
|
color_map = {
|
|
|
|
|
"info": "#007bff",
|
|
|
|
|
"warning": "#ffc107",
|
|
|
|
|
"error": "#dc3545",
|
|
|
|
|
"critical": "#6f42c1"
|
|
|
|
|
}
|
|
|
|
|
color = color_map.get(message.level, "#6c757d")
|
|
|
|
|
|
|
|
|
|
# 构建@信息
|
|
|
|
|
at_text = ""
|
|
|
|
|
if self.at_all:
|
|
|
|
|
at_text = "@所有人 "
|
|
|
|
|
elif self.at_mobiles:
|
|
|
|
|
at_text = " ".join([f"@{mobile}" for mobile in self.at_mobiles])
|
|
|
|
|
|
|
|
|
|
content = f"""### {message.title} {at_text}
|
|
|
|
|
|
|
|
|
|
**告警级别:** <font color='{color}'>{message.level.upper()}</font>
|
|
|
|
|
**告警时间:** {message.timestamp.strftime('%Y-%m-%d %H:%M:%S')}
|
|
|
|
|
|
|
|
|
|
---
|
|
|
|
|
|
|
|
|
|
{message.content}
|
|
|
|
|
|
|
|
|
|
---
|
|
|
|
|
|
|
|
|
|
**详细信息:**
|
|
|
|
|
```json
|
|
|
|
|
{json.dumps(message.metadata, indent=2, ensure_ascii=False, default=str)}
|
|
|
|
|
```
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
"msgtype": "markdown",
|
|
|
|
|
"markdown": {
|
|
|
|
|
"title": message.title,
|
|
|
|
|
"text": content
|
|
|
|
|
},
|
|
|
|
|
"at": {
|
|
|
|
|
"atMobiles": self.at_mobiles,
|
|
|
|
|
"isAtAll": self.at_all
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
def _build_text_message(self, message: AlertMessage) -> Dict:
|
|
|
|
|
"""构建文本格式的消息"""
|
|
|
|
|
return {
|
|
|
|
|
"msgtype": "text",
|
|
|
|
|
"text": {
|
|
|
|
|
"content": f"[{message.level.upper()}] {message.title}\n\n{message.content}"
|
|
|
|
|
},
|
|
|
|
|
"at": {
|
|
|
|
|
"atMobiles": self.at_mobiles,
|
|
|
|
|
"isAtAll": self.at_all
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async def send(self, message: AlertMessage, msg_type: str = "markdown") -> bool:
|
|
|
|
|
"""发送钉钉告警
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
message: 告警消息
|
|
|
|
|
msg_type: 消息类型 (markdown 或 text)
|
|
|
|
|
"""
|
|
|
|
|
if not self.enabled or not self.webhook_url:
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
import base64
|
|
|
|
|
import time as time_module
|
|
|
|
|
|
|
|
|
|
timestamp = str(int(round(time_module.time() * 1000)))
|
|
|
|
|
sign = self._generate_sign(timestamp)
|
|
|
|
|
|
|
|
|
|
# 构建URL
|
|
|
|
|
url = self.webhook_url
|
|
|
|
|
if self.secret:
|
|
|
|
|
url = f"{self.webhook_url}×tamp={timestamp}&sign={sign}"
|
|
|
|
|
|
|
|
|
|
# 构建消息
|
|
|
|
|
if msg_type == "markdown":
|
|
|
|
|
payload = self._build_markdown_message(message)
|
|
|
|
|
else:
|
|
|
|
|
payload = self._build_text_message(message)
|
|
|
|
|
|
|
|
|
|
# 发送请求
|
|
|
|
|
async with httpx.AsyncClient() as client:
|
|
|
|
|
response = await client.post(
|
|
|
|
|
url,
|
|
|
|
|
json=payload,
|
|
|
|
|
headers={"Content-Type": "application/json"},
|
|
|
|
|
timeout=10.0
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
if response.status_code == 200:
|
|
|
|
|
result = response.json()
|
|
|
|
|
if result.get("errcode") == 0:
|
|
|
|
|
info(f"DingTalk alert sent: {message.title}")
|
|
|
|
|
return True
|
|
|
|
|
else:
|
|
|
|
|
error(f"DingTalk API error: {result}")
|
|
|
|
|
return False
|
|
|
|
|
else:
|
|
|
|
|
error(f"DingTalk HTTP error: {response.status_code}")
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
except Exception as e:
|
|
|
|
|
error(f"Failed to send DingTalk alert: {e}")
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class EmailAlertChannel(AlertChannel):
|
|
|
|
|
"""邮件告警通道"""
|
|
|
|
|
|
|
|
|
|
def __init__(
|
|
|
|
|
self,
|
|
|
|
|
smtp_host: str,
|
|
|
|
|
smtp_port: int,
|
|
|
|
|
username: str,
|
|
|
|
|
password: str,
|
|
|
|
|
from_addr: str,
|
|
|
|
|
to_addrs: List[str],
|
|
|
|
|
use_tls: bool = True,
|
|
|
|
|
enabled: bool = True
|
|
|
|
|
):
|
|
|
|
|
super().__init__("email", enabled)
|
|
|
|
|
self.smtp_host = smtp_host
|
|
|
|
|
self.smtp_port = smtp_port
|
|
|
|
|
self.username = username
|
|
|
|
|
self.password = password
|
|
|
|
|
self.from_addr = from_addr
|
|
|
|
|
self.to_addrs = to_addrs
|
|
|
|
|
self.use_tls = use_tls
|
|
|
|
|
|
|
|
|
|
def _build_html_content(self, message: AlertMessage) -> str:
|
|
|
|
|
"""构建HTML格式的邮件内容"""
|
|
|
|
|
# 根据级别选择颜色
|
|
|
|
|
color_map = {
|
|
|
|
|
"info": "#007bff",
|
|
|
|
|
"warning": "#ffc107",
|
|
|
|
|
"error": "#dc3545",
|
|
|
|
|
"critical": "#6f42c1"
|
|
|
|
|
}
|
|
|
|
|
color = color_map.get(message.level, "#6c757d")
|
|
|
|
|
|
|
|
|
|
metadata_html = ""
|
|
|
|
|
if message.metadata:
|
|
|
|
|
rows = ""
|
|
|
|
|
for key, value in message.metadata.items():
|
|
|
|
|
rows += f"<tr><td><strong>{key}</strong></td><td>{value}</td></tr>"
|
|
|
|
|
metadata_html = f"""
|
|
|
|
|
<h3>详细信息</h3>
|
|
|
|
|
<table border="1" cellpadding="5" cellspacing="0">
|
|
|
|
|
{rows}
|
|
|
|
|
</table>
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
return f"""
|
|
|
|
|
<html>
|
|
|
|
|
<body>
|
|
|
|
|
<h2 style="color: {color};">[{message.level.upper()}] {message.title}</h2>
|
|
|
|
|
<p><strong>告警时间:</strong> {message.timestamp.strftime('%Y-%m-%d %H:%M:%S')}</p>
|
|
|
|
|
<hr>
|
|
|
|
|
<p>{message.content.replace(chr(10), '<br>')}</p>
|
|
|
|
|
<hr>
|
|
|
|
|
{metadata_html}
|
|
|
|
|
<p style="color: #666; font-size: 12px;">
|
|
|
|
|
本邮件由行情数据服务自动发送，请勿回复。
|
|
|
|
|
</p>
|
|
|
|
|
</body>
|
|
|
|
|
</html>
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
async def send(self, message: AlertMessage) -> bool:
|
|
|
|
|
"""发送邮件告警"""
|
|
|
|
|
if not self.enabled or not self.to_addrs:
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
# 构建邮件
|
|
|
|
|
msg = MIMEMultipart('alternative')
|
|
|
|
|
msg['Subject'] = f"[{message.level.upper()}] {message.title}"
|
|
|
|
|
msg['From'] = self.from_addr
|
|
|
|
|
msg['To'] = ', '.join(self.to_addrs)
|
|
|
|
|
|
|
|
|
|
# 添加HTML内容
|
|
|
|
|
html_content = self._build_html_content(message)
|
|
|
|
|
msg.attach(MIMEText(html_content, 'html', 'utf-8'))
|
|
|
|
|
|
|
|
|
|
# 发送邮件（在executor中执行同步操作）
|
|
|
|
|
import asyncio
|
|
|
|
|
loop = asyncio.get_event_loop()
|
|
|
|
|
|
|
|
|
|
def send_email():
|
|
|
|
|
server = smtplib.SMTP(self.smtp_host, self.smtp_port)
|
|
|
|
|
if self.use_tls:
|
|
|
|
|
server.starttls()
|
|
|
|
|
server.login(self.username, self.password)
|
|
|
|
|
server.sendmail(self.from_addr, self.to_addrs, msg.as_string())
|
|
|
|
|
server.quit()
|
|
|
|
|
|
|
|
|
|
await loop.run_in_executor(None, send_email)
|
|
|
|
|
|
|
|
|
|
info(f"Email alert sent: {message.title}")
|
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
except Exception as e:
|
|
|
|
|
error(f"Failed to send email alert: {e}")
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class WebhookAlertChannel(AlertChannel):
|
|
|
|
|
"""Webhook告警通道"""
|
|
|
|
|
|
|
|
|
|
def __init__(
|
|
|
|
|
self,
|
|
|
|
|
webhook_url: str,
|
|
|
|
|
headers: Optional[Dict[str, str]] = None,
|
|
|
|
|
timeout: float = 10.0,
|
|
|
|
|
enabled: bool = True
|
|
|
|
|
):
|
|
|
|
|
super().__init__("webhook", enabled)
|
|
|
|
|
self.webhook_url = webhook_url
|
|
|
|
|
self.headers = headers or {"Content-Type": "application/json"}
|
|
|
|
|
self.timeout = timeout
|
|
|
|
|
|
|
|
|
|
async def send(self, message: AlertMessage) -> bool:
|
|
|
|
|
"""发送Webhook告警"""
|
|
|
|
|
if not self.enabled or not self.webhook_url:
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
payload = {
|
|
|
|
|
"title": message.title,
|
|
|
|
|
"content": message.content,
|
|
|
|
|
"level": message.level,
|
|
|
|
|
"timestamp": message.timestamp.isoformat(),
|
|
|
|
|
"metadata": message.metadata
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async with httpx.AsyncClient() as client:
|
|
|
|
|
response = await client.post(
|
|
|
|
|
self.webhook_url,
|
|
|
|
|
json=payload,
|
|
|
|
|
headers=self.headers,
|
|
|
|
|
timeout=self.timeout
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
if response.status_code < 400:
|
|
|
|
|
info(f"Webhook alert sent: {message.title}")
|
|
|
|
|
return True
|
|
|
|
|
else:
|
|
|
|
|
error(f"Webhook error: {response.status_code}")
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
except Exception as e:
|
|
|
|
|
error(f"Failed to send webhook alert: {e}")
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class AlertManager:
|
|
|
|
|
"""告警管理器
|
|
|
|
|
|
|
|
|
|
管理多个告警通道，支持消息路由和批量发送。
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
def __init__(self):
|
|
|
|
|
self.channels: Dict[str, AlertChannel] = {}
|
|
|
|
|
self.level_routing = {
|
|
|
|
|
"info": ["log"],
|
|
|
|
|
"warning": ["log", "dingtalk"],
|
|
|
|
|
"error": ["log", "dingtalk", "email"],
|
|
|
|
|
"critical": ["log", "dingtalk", "email", "webhook"]
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
def register_channel(self, channel: AlertChannel):
|
|
|
|
|
"""注册告警通道"""
|
|
|
|
|
self.channels[channel.name] = channel
|
|
|
|
|
info(f"Alert channel registered: {channel.name}")
|
|
|
|
|
|
|
|
|
|
def configure_routing(self, level_routing: Dict[str, List[str]]):
|
|
|
|
|
"""配置告警路由规则"""
|
|
|
|
|
self.level_routing = level_routing
|
|
|
|
|
|
|
|
|
|
async def send(
|
|
|
|
|
self,
|
|
|
|
|
message: AlertMessage,
|
|
|
|
|
channels: Optional[List[str]] = None
|
|
|
|
|
) -> Dict[str, bool]:
|
|
|
|
|
"""发送告警
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
message: 告警消息
|
|
|
|
|
channels: 指定通道列表，None则根据级别路由
|
|
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
|
各通道发送结果
|
|
|
|
|
"""
|
|
|
|
|
# 确定目标通道
|
|
|
|
|
target_channels = channels
|
|
|
|
|
if target_channels is None:
|
|
|
|
|
target_channels = self.level_routing.get(message.level, ["log"])
|
|
|
|
|
|
|
|
|
|
# 发送到各通道
|
|
|
|
|
results = {}
|
|
|
|
|
for channel_name in target_channels:
|
|
|
|
|
channel = self.channels.get(channel_name)
|
|
|
|
|
if channel:
|
|
|
|
|
results[channel_name] = await channel.send(message)
|
|
|
|
|
else:
|
|
|
|
|
warning(f"Alert channel not found: {channel_name}")
|
|
|
|
|
results[channel_name] = False
|
|
|
|
|
|
|
|
|
|
return results
|
|
|
|
|
|
|
|
|
|
async def send_simple(
|
|
|
|
|
self,
|
|
|
|
|
title: str,
|
|
|
|
|
content: str,
|
|
|
|
|
level: str = "warning",
|
|
|
|
|
**kwargs
|
|
|
|
|
) -> Dict[str, bool]:
|
|
|
|
|
"""发送简单告警"""
|
|
|
|
|
message = AlertMessage(
|
|
|
|
|
title=title,
|
|
|
|
|
content=content,
|
|
|
|
|
level=level,
|
|
|
|
|
metadata=kwargs
|
|
|
|
|
)
|
|
|
|
|
return await self.send(message)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# 全局告警管理器实例
|
|
|
|
|
_alert_manager: Optional[AlertManager] = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_alert_manager() -> AlertManager:
|
|
|
|
|
"""获取全局告警管理器"""
|
|
|
|
|
global _alert_manager
|
|
|
|
|
if _alert_manager is None:
|
|
|
|
|
_alert_manager = AlertManager()
|
|
|
|
|
# 默认注册日志通道
|
|
|
|
|
_alert_manager.register_channel(LogAlertChannel())
|
|
|
|
|
return _alert_manager
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def init_alert_manager(config: Dict):
|
|
|
|
|
"""从配置初始化告警管理器"""
|
|
|
|
|
global _alert_manager
|
|
|
|
|
_alert_manager = AlertManager()
|
|
|
|
|
|
|
|
|
|
# 注册日志通道
|
|
|
|
|
_alert_manager.register_channel(LogAlertChannel(
|
|
|
|
|
enabled=config.get("log", {}).get("enabled", True)
|
|
|
|
|
))
|
|
|
|
|
|
|
|
|
|
# 注册钉钉通道
|
|
|
|
|
dingtalk_config = config.get("dingtalk", {})
|
|
|
|
|
if dingtalk_config.get("enabled"):
|
|
|
|
|
_alert_manager.register_channel(DingTalkAlertChannel(
|
|
|
|
|
webhook_url=dingtalk_config["webhook_url"],
|
|
|
|
|
secret=dingtalk_config.get("secret"),
|
|
|
|
|
at_mobiles=dingtalk_config.get("at_mobiles", []),
|
|
|
|
|
at_all=dingtalk_config.get("at_all", False),
|
|
|
|
|
enabled=True
|
|
|
|
|
))
|
|
|
|
|
|
|
|
|
|
# 注册邮件通道
|
|
|
|
|
email_config = config.get("email", {})
|
|
|
|
|
if email_config.get("enabled"):
|
|
|
|
|
_alert_manager.register_channel(EmailAlertChannel(
|
|
|
|
|
smtp_host=email_config["smtp_host"],
|
|
|
|
|
smtp_port=email_config["smtp_port"],
|
|
|
|
|
username=email_config["username"],
|
|
|
|
|
password=email_config["password"],
|
|
|
|
|
from_addr=email_config["from_addr"],
|
|
|
|
|
to_addrs=email_config["to_addrs"],
|
|
|
|
|
use_tls=email_config.get("use_tls", True),
|
|
|
|
|
enabled=True
|
|
|
|
|
))
|
|
|
|
|
|
|
|
|
|
# 注册Webhook通道
|
|
|
|
|
webhook_config = config.get("webhook", {})
|
|
|
|
|
if webhook_config.get("enabled"):
|
|
|
|
|
_alert_manager.register_channel(WebhookAlertChannel(
|
|
|
|
|
webhook_url=webhook_config["webhook_url"],
|
|
|
|
|
headers=webhook_config.get("headers"),
|
|
|
|
|
timeout=webhook_config.get("timeout", 10.0),
|
|
|
|
|
enabled=True
|
|
|
|
|
))
|
|
|
|
|
|
|
|
|
|
# 配置路由规则
|
|
|
|
|
if "routing" in config:
|
|
|
|
|
_alert_manager.configure_routing(config["routing"])
|
|
|
|
|
|
|
|
|
|
return _alert_manager
|
