# 数据流量分配查询应用
我将设计一个基于Python和Go开发的数据流量分配查询应用,帮助用户监控和分析网络流量分配情况。
## 设计思路
这个应用将实现以下核心功能:
- 实时监控网络流量分配情况
- 多维度流量数据分析(设备、应用、时间段)
- 流量分配策略设置与管理
- 异常流量告警系统
- 直观的数据可视化展示
## 技术架构
```
前端 (Python + Streamlit) 后端 (Go)
┌───────────────────────┐ ┌───────────────────────┐
│ 用户界面 │ │ 流量数据收集与处理 │
│ - 实时监控仪表盘 │◄───►│ - 数据采集API │
│ - 流量分配策略配置 │ │ - 流量分析引擎 │
│ - 历史数据查询 │ │ - 告警系统 │
│ - 告警通知中心 │ │ - 数据存储 │
└───────────────────────┘ └───────────────────────┘
▲ ▲
│ │
▼ ▼
┌───────────────────────┐ ┌───────────────────────┐
│ 数据可视化 │ │ 数据存储 │
│ - 图表库 │ │ - PostgreSQL │
│ - 交互式控件 │ │ - Redis缓存 │
└───────────────────────┘ └───────────────────────┘
```
## 完整实现
### 后端服务 (Go) - `backend/main.go`
```go
package main
import (
"database/sql"
"encoding/json"
"fmt"
"log"
"math/rand"
"net/http"
"os"
"time"
"github.com/gin-gonic/gin"
_ "github.com/lib/pq"
"github.com/patrickmn/go-cache"
)
// 流量数据结构
type TrafficData struct {
ID int `json:"id"`
DeviceID string `json:"device_id"`
AppID string `json:"app_id"`
BytesSent int `json:"bytes_sent"`
BytesRecv int `json:"bytes_recv"`
Timestamp time.Time `json:"timestamp"`
}
// 流量分配策略
type AllocationPolicy struct {
ID int `json:"id"`
Name string `json:"name"`
PolicyType string `json:"policy_type"` // equal, weighted, priority
Rules string `json:"rules"` // JSON规则
}
// 告警规则
type AlertRule struct {
ID int `json:"id"`
Name string `json:"name"`
Condition string `json:"condition"` // SQL条件
Threshold int `json:"threshold"`
}
// 数据库连接
var db *sql.DB
var memoryCache *cache.Cache
func main() {
// 初始化缓存
memoryCache = cache.New(5*time.Minute, 10*time.Minute)
// 连接数据库
initDB()
defer db.Close()
// 创建Gin路由
router := gin.Default()
// API路由
api := router.Group("/api")
{
api.GET("/traffic", getTrafficData)
api.GET("/traffic/summary", getTrafficSummary)
api.GET("/policies", getAllocationPolicies)
api.POST("/policies", createAllocationPolicy)
api.GET("/alerts", getAlertRules)
api.POST("/alerts", createAlertRule)
api.GET("/devices", getDevices)
api.GET("/applications", getApplications)
}
// 模拟数据生成器
go generateTrafficData()
// 启动服务
port := os.Getenv("PORT")
if port == "" {
port = "8080"
}
log.Printf("Server running on port %s", port)
log.Fatal(router.Run(":" + port))
}
func initDB() {
connStr := "user=postgres dbname=trafficdb password=postgres host=localhost sslmode=disable"
var err error
db, err = sql.Open("postgres", connStr)
if err != nil {
log.Fatal(err)
}
// 创建表
_, err = db.Exec(`
CREATE TABLE IF NOT EXISTS traffic_data (
id SERIAL PRIMARY KEY,
device_id VARCHAR(50) NOT NULL,
app_id VARCHAR(50) NOT NULL,
bytes_sent INT NOT NULL,
bytes_recv INT NOT NULL,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS allocation_policies (
id SERIAL PRIMARY KEY,
name VARCHAR(100) NOT NULL,
policy_type VARCHAR(20) NOT NULL,
rules TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS alert_rules (
id SERIAL PRIMARY KEY,
name VARCHAR(100) NOT NULL,
condition TEXT NOT NULL,
threshold INT NOT NULL
);
`)
if err != nil {
log.Fatal("Failed to create tables:", err)
}
}
// 获取流量数据
func getTrafficData(c *gin.Context) {
// 尝试从缓存获取
if cached, found := memoryCache.Get("traffic_data"); found {
c.JSON(http.StatusOK, cached)
return
}
query := `SELECT id, device_id, app_id, bytes_sent, bytes_recv, timestamp
FROM traffic_data
ORDER BY timestamp DESC
LIMIT 1000`
rows, err := db.Query(query)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
defer rows.Close()
var traffic []TrafficData
for rows.Next() {
var t TrafficData
err := rows.Scan(&t.ID, &t.DeviceID, &t.AppID, &t.BytesSent, &t.BytesRecv, &t.Timestamp)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
traffic = append(traffic, t)
}
// 存入缓存
memoryCache.Set("traffic_data", traffic, cache.DefaultExpiration)
c.JSON(http.StatusOK, traffic)
}
// 获取流量摘要
func getTrafficSummary(c *gin.Context) {
// 尝试从缓存获取
if cached, found := memoryCache.Get("traffic_summary"); found {
c.JSON(http.StatusOK, cached)
return
}
query := `
SELECT
device_id,
app_id,
SUM(bytes_sent) AS total_sent,
SUM(bytes_recv) AS total_recv,
COUNT(*) AS records
FROM traffic_data
WHERE timestamp > NOW() - INTERVAL '1 hour'
GROUP BY device_id, app_id
ORDER BY total_sent DESC, total_recv DESC
`
rows, err := db.Query(query)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
defer rows.Close()
type Summary struct {
DeviceID string `json:"device_id"`
AppID string `json:"app_id"`
TotalSent int `json:"total_sent"`
TotalRecv int `json:"total_recv"`
Records int `json:"records"`
}
var summaries []Summary
for rows.Next() {
var s Summary
err := rows.Scan(&s.DeviceID, &s.AppID, &s.TotalSent, &s.TotalRecv, &s.Records)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
summaries = append(summaries, s)
}
// 存入缓存
memoryCache.Set("traffic_summary", summaries, cache.DefaultExpiration)
c.JSON(http.StatusOK, summaries)
}
// 获取所有分配策略
func getAllocationPolicies(c *gin.Context) {
rows, err := db.Query("SELECT id, name, policy_type, rules FROM allocation_policies")
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
defer rows.Close()
var policies []AllocationPolicy
for rows.Next() {
var p AllocationPolicy
err := rows.Scan(&p.ID, &p.Name, &p.PolicyType, &p.Rules)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
policies = append(policies, p)
}
c.JSON(http.StatusOK, policies)
}
// 创建分配策略
func createAllocationPolicy(c *gin.Context) {
var policy AllocationPolicy
if err := c.ShouldBindJSON(&policy); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
query := `INSERT INTO allocation_policies (name, policy_type, rules)
VALUES ($1, $2, $3) RETURNING id`
err := db.QueryRow(query, policy.Name, policy.PolicyType, policy.Rules).Scan(&policy.ID)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusCreated, policy)
}
// 获取告警规则
func getAlertRules(c *gin.Context) {
rows, err := db.Query("SELECT id, name, condition, threshold FROM alert_rules")
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
defer rows.Close()
var alerts []AlertRule
for rows.Next() {
var a AlertRule
err := rows.Scan(&a.ID, &a.Name, &a.Condition, &a.Threshold)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
alerts = append(alerts, a)
}
c.JSON(http.StatusOK, alerts)
}
// 创建告警规则
func createAlertRule(c *gin.Context) {
var alert AlertRule
if err := c.ShouldBindJSON(&alert); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
query := `INSERT INTO alert_rules (name, condition, threshold)
VALUES ($1, $2, $3) RETURNING id`
err := db.QueryRow(query, alert.Name, alert.Condition, alert.Threshold).Scan(&alert.ID)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusCreated, alert)
}
// 获取设备列表
func getDevices(c *gin.Context) {
rows, err := db.Query("SELECT DISTINCT device_id FROM traffic_data")
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
defer rows.Close()
var devices []string
for rows.Next() {
var device string
err := rows.Scan(&device)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
devices = append(devices, device)
}
c.JSON(http.StatusOK, devices)
}
// 获取应用列表
func getApplications(c *gin.Context) {
rows, err := db.Query("SELECT DISTINCT app_id FROM traffic_data")
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
defer rows.Close()
var apps []string
for rows.Next() {
var app string
err := rows.Scan(&app)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
apps = append(apps, app)
}
c.JSON(http.StatusOK, apps)
}
// 生成模拟流量数据
func generateTrafficData() {
devices := []string{"device-001", "device-002", "device-003", "device-004", "device-005"}
apps := []string{"web-browsing", "video-streaming", "file-transfer", "gaming", "voip"}
for {
// 随机生成流量记录
device := devices[rand.Intn(len(devices))]
app := apps[rand.Intn(len(apps))]
bytesSent := rand.Intn(10000) + 100
bytesRecv := rand.Intn(5000) + 50
_, err := db.Exec(
"INSERT INTO traffic_data (device_id, app_id, bytes_sent, bytes_recv) VALUES ($1, $2, $3, $4)",
device, app, bytesSent, bytesRecv,
)
if err != nil {
log.Printf("Failed to insert traffic data: %v", err)
}
// 每隔1-5秒生成一条新记录
time.Sleep(time.Duration(rand.Intn(4)+1) * time.Second)
}
}
```
### 前端界面 (Python + Streamlit) - `frontend/app.py`
```python
import streamlit as st
import requests
import pandas as pd
import plotly.express as px
from datetime import datetime, timedelta
# 配置
BACKEND_URL = "http://localhost:8080/api"
PAGE_TITLE = "数据流量分配查询系统"
# 初始化Session State
if 'page' not in st.session_state:
st.session_state.page = 'dashboard'
if 'selected_device' not in st.session_state:
st.session_state.selected_device = 'All'
if 'selected_app' not in st.session_state:
st.session_state.selected_app = 'All'
# 页面标题
st.set_page_config(page_title=PAGE_TITLE, layout="wide")
st.title(PAGE_TITLE)
# 侧边栏导航
with st.sidebar:
st.header("导航菜单")
if st.button(" 实时监控"):
st.session_state.page = 'dashboard'
if st.button(" 流量分析"):
st.session_state.page = 'analysis'
if st.button("⚙️ 分配策略"):
st.session_state.page = 'policies'
if st.button("⚠️ 告警设置"):
st.session_state.page = 'alerts'
if st.button(" 历史查询"):
st.session_state.page = 'history'
st.divider()
st.header("筛选条件")
# 获取设备列表
devices_response = requests.get(f"{BACKEND_URL}/devices")
devices = ["All"] + [d for d in devices_response.json()] if devices_response.status_code == 200 else ["All"]
st.session_state.selected_device = st.selectbox("选择设备", devices, index=devices.index(st.session_state.selected_device))
# 获取应用列表
apps_response = requests.get(f"{BACKEND_URL}/applications")
apps = ["All"] + [a for a in apps_response.json()] if apps_response.status_code == 200 else ["All"]
st.session_state.selected_app = st.selectbox("选择应用", apps, index=apps.index(st.session_state.selected_app))
# 时间范围选择
time_range = st.selectbox("时间范围", ["实时", "过去1小时", "过去24小时", "自定义"])
# 自定义时间范围
if time_range == "自定义":
col1, col2 = st.columns(2)
start_date = col1.date_input("开始日期", datetime.now() - timedelta(days=1))
start_time = col1.time_input("开始时间", datetime.now().time())
end_date = col2.date_input("结束日期", datetime.now())
end_time = col2.time_input("结束时间", datetime.now().time())
start_datetime = datetime.combine(start_date, start_time)
end_datetime = datetime.combine(end_date, end_time)
else:
end_datetime = datetime.now()
if time_range == "实时":
start_datetime = end_datetime - timedelta(minutes=5)
elif time_range == "过去1小时":
start_datetime = end_datetime - timedelta(hours=1)
else: # 过去24小时
start_datetime = end_datetime - timedelta(days=1)
# 获取流量数据
def fetch_traffic_data():
response = requests.get(f"{BACKEND_URL}/traffic")
if response.status_code == 200:
return pd.DataFrame(response.json())
return pd.DataFrame()
# 获取流量摘要
def fetch_traffic_summary():
response = requests.get(f"{BACKEND_URL}/traffic/summary")
if response.status_code == 200:
return pd.DataFrame(response.json())
return pd.DataFrame()
# 获取分配策略
def fetch_allocation_policies():
response = requests.get(f"{BACKEND_URL}/policies")
if response.status_code == 200:
return response.json()
return []
# 获取告警规则
def fetch_alert_rules():
response = requests.get(f"{BACKEND_URL}/alerts")
if response.status_code == 200:
return response.json()
return []
# 仪表盘页面
if st.session_state.page == 'dashboard':
st.header("实时流量监控")
# 实时流量数据
traffic_df = fetch_traffic_data()
if not traffic_df.empty:
# 应用筛选条件
if st.session_state.selected_device != 'All':
traffic_df = traffic_df[traffic_df['device_id'] == st.session_state.selected_device]
if st.session_state.selected_app != 'All':
traffic_df = traffic_df[traffic_df['app_id'] == st.session_state.selected_app]
# 转换为时间序列
traffic_df['timestamp'] = pd.to_datetime(traffic_df['timestamp'])
traffic_df.set_index('timestamp', inplace=True)
# 计算总流量
traffic_df['total_bytes'] = traffic_df['bytes_sent'] + traffic_df['bytes_recv']
# 创建布局
col1, col2, col3 = st.columns(3)
# 关键指标
total_sent = traffic_df['bytes_sent'].sum() / (1024 * 1024) # 转换为MB
total_recv = traffic_df['bytes_recv'].sum() / (1024 * 1024) # 转换为MB
avg_throughput = traffic_df['total_bytes'].mean() / 1024 # 转换为KB
col1.metric("发送数据总量", f"{total_sent:.2f} MB")
col2.metric("接收数据总量", f"{total_recv:.2f} MB")
col3.metric("平均吞吐量", f"{avg_throughput:.2f} KB/s")
# 流量趋势图
st.subheader("流量趋势")
fig = px.line(traffic_df.resample('1T').sum(),
y=['bytes_sent', 'bytes_recv'],
labels={'value': '字节数', 'timestamp': '时间'},
title='发送与接收流量趋势')
st.plotly_chart(fig, use_container_width=True)
# 应用流量分布
st.subheader("应用流量分布")
app_traffic = traffic_df.groupby('app_id').sum()[['bytes_sent', 'bytes_recv']].reset_index()
fig = px.pie(app_traffic, names='app_id', values='bytes_sent',
title='应用发送流量分布')
st.plotly_chart(fig, use_container_width=True)
# 设备流量排行
st.subheader("设备流量排行")
device_traffic = traffic_df.groupby('device_id').sum()[['bytes_sent', 'bytes_recv']].reset_index()
device_traffic = device_traffic.sort_values('bytes_sent', ascending=False).head(10)
fig = px.bar(device_traffic, x='device_id', y='bytes_sent',
title='设备发送流量排行')
st.plotly_chart(fig, use_container_width=True)
else:
st.warning("没有可用的流量数据")
# 流量分析页面
elif st.session_state.page == 'analysis':
st.header("流量分析")
traffic_summary = fetch_traffic_summary()
if not traffic_summary.empty:
# 应用筛选条件
if st.session_state.selected_device != 'All':
traffic_summary = traffic_summary[traffic_summary['device_id'] == st.session_state.selected_device]
if st.session_state.selected_app != 'All':
traffic_summary = traffic_summary[traffic_summary['app_id'] == st.session_state.selected_app]
# 显示摘要表格
st.subheader("流量摘要")
st.dataframe(traffic_summary)
# 设备流量分析
st.subheader("设备流量分析")
device_traffic = traffic_summary.groupby('device_id').sum()[['total_sent', 'total_recv']].reset_index()
device_traffic = device_traffic.sort_values('total_sent', ascending=False)
col1, col2 = st.columns(2)
with col1:
fig = px.bar(device_traffic, x='device_id', y='total_sent',
title='设备发送流量')
st.plotly_chart(fig, use_container_width=True)
with col2:
fig = px.bar(device_traffic, x='device_id', y='total_recv',
title='设备接收流量')
st.plotly_chart(fig, use_container_width=True)
# 应用流量分析
st.subheader("应用流量分析")
app_traffic = traffic_summary.groupby('app_id').sum()[['total_sent', 'total_recv']].reset_index()
app_traffic = app_traffic.sort_values('total_sent', ascending=False)
col1, col2 = st.columns(2)
with col1:
fig = px.pie(app_traffic, names='app_id', values='total_sent',
title='应用发送流量分布')
st.plotly_chart(fig, use_container_width=True)
with col2:
fig = px.pie(app_traffic, names='app_id', values='total_recv',
title='应用接收流量分布')
st.plotly_chart(fig, use_container_width=True)
else:
st.warning("没有可用的流量摘要数据")
# 分配策略页面
elif st.session_state.page == 'policies':
st.header("流量分配策略管理")
policies = fetch_allocation_policies()
# 显示现有策略
st.subheader("现有分配策略")
if policies:
policy_df = pd.DataFrame(policies)
st.dataframe(policy_df[['id', 'name', 'policy_type']])
else:
st.info("没有配置任何分配策略")
# 添加新策略
st.subheader("添加新策略")
with st.form("policy_form"):
name = st.text_input("策略名称")
policy_type = st.selectbox("策略类型", ["equal", "weighted", "priority"])
# 根据策略类型显示不同的规则输入
if policy_type == "equal":
st.info("均等分配策略不需要额外规则")
rules = "{}"
elif policy_type == "weighted":
st.info("加权分配策略需要指定权重规则")
rules = st.text_area("权重规则 (JSON格式)", value='{"device-001": 0.5, "device-002": 0.3, "device-003": 0.2}')
else: # priority
st.info("优先级分配策略需要指定优先级顺序")
rules = st.text_area("优先级规则 (JSON格式)", value='["device-001", "device-002", "device-003"]')
submitted = st.form_submit_button("添加策略")
if submitted:
response = requests.post(f"{BACKEND_URL}/policies", json={
"name": name,
"policy_type": policy_type,
"rules": rules
})
if response.status_code == 201:
st.success("策略添加成功!")
st.experimental_rerun()
else:
st.error(f"添加策略失败: {response.text}")
# 告警设置页面
elif st.session_state.page == 'alerts':
st.header("告警规则设置")
alerts = fetch_alert_rules()
# 显示现有告警规则
st.subheader("现有告警规则")
if alerts:
alert_df = pd.DataFrame(alerts)
st.dataframe(alert_df[['id', 'name', 'threshold']])
else:
st.info("没有配置任何告警规则")
# 添加新告警规则
st.subheader("添加新告警规则")
with st.form("alert_form"):
name = st.text_input("规则名称")
condition = st.selectbox("告警条件",
["bytes_sent >", "bytes_recv >", "total_bytes >"])
threshold = st.number_input("阈值 (字节)", min_value=0, value=1000000)
submitted = st.form_submit_button("添加规则")
if submitted:
response = requests.post(f"{BACKEND_URL}/alerts", json={
"name": name,
"condition": f"{condition} {threshold}",
"threshold": threshold
})
if response.status_code == 201:
st.success("告警规则添加成功!")
st.experimental_rerun()
else:
st.error(f"添加告警规则失败: {response.text}")
# 历史查询页面
elif st.session_state.page == 'history':
st.header("历史流量查询")
# 日期选择器
col1, col2 = st.columns(2)
start_date = col1.date_input("开始日期", datetime.now() - timedelta(days=7))
end_date = col2.date_input("结束日期", datetime.now())
# 获取历史数据
if st.button("查询历史数据"):
# 在实际应用中,这里应该调用后端API查询指定时间范围的数据
st.info("历史数据查询功能将在完整版本中实现")
# 模拟历史数据
dates = pd.date_range(start_date, end_date, freq='D')
history_data = pd.DataFrame({
'date': dates,
'bytes_sent': [randint(1000000, 5000000) for _ in range(len(dates))],
'bytes_recv': [randint(500000, 2500000) for _ in range(len(dates))]
})
# 显示历史数据图表
st.subheader("历史流量趋势")
fig = px.line(history_data, x='date', y=['bytes_sent', 'bytes_recv'],
labels={'value': '字节数', 'date': '日期'},
title='历史流量趋势')
st.plotly_chart(fig, use_container_width=True)
# 显示数据表格
st.subheader("详细数据")
st.dataframe(history_data)
else:
st.info("请选择日期范围并点击查询按钮")
# 页面底部状态栏
st.divider()
st.caption(f"最后更新时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} | 系统状态: 运行正常")
```
## 应用功能说明
这个数据流量分配查询应用提供以下核心功能:
1. **实时流量监控**
- 实时显示网络流量数据
- 流量趋势图表可视化
- 设备与应用流量分布分析
- 关键性能指标(KPI)展示
2. **流量分析**
- 多维度流量数据分析(设备、应用)
- 流量分布饼图
- 设备流量排行
- 流量摘要表格
3. **分配策略管理**
- 创建和管理流量分配策略
- 支持均等分配、加权分配和优先级分配
- 可视化策略配置界面
4. **告警系统**
- 设置流量异常告警规则
- 监控流量阈值
- 告警规则管理界面
5. **历史查询**
- 查询历史流量数据
- 时间范围筛选功能
- 历史流量趋势图表
## 系统架构优势
1. **高性能后端**:
- 使用Go语言开发,处理高并发流量数据
- 缓存机制减少数据库访问
- 高效的数据处理能力
2. **直观的前端界面**:
- 使用Python Streamlit构建响应式UI
- 交互式数据可视化
- 实时数据更新
3. **灵活的数据分析**:
- 多维度流量分析
- 自定义筛选条件
- 丰富的图表展示
4. **可扩展性**:
- 模块化设计便于功能扩展
- 支持多种数据库后端
- 可集成到现有网络监控系统
## 使用方法
### 后端服务
1. 确保已安装Go和PostgreSQL
2. 创建数据库:`createdb trafficdb`
3. 安装依赖:`go get -u github.com/gin-gonic/gin github.com/lib/pq github.com/patrickmn/go-cache`
4. 运行后端:`go run main.go`
### 前端界面
1. 确保已安装Python 3.7+
2. 安装依赖:`pip install streamlit requests pandas plotly`
3. 运行前端:`streamlit run app.py`
## 扩展可能性
1. **用户认证系统**:
- 添加用户登录和权限管理
- 基于角色的访问控制
2. **告警通知集成**:
- 集成邮件/SMS通知
- 支持Webhook告警
3. **流量控制功能**:
- 基于策略的流量分配执行
- 动态流量调整
4. **高级分析**:
- 流量预测模型
- 异常检测算法
- 网络性能优化建议
这个应用为网络管理员提供了一个强大的工具,用于监控、分析和优化网络流量分配,确保网络资源的高效利用。