Python实现数据自动生成表格：从数据源到可视化表格的完整解决方案
在现代数据处理和报告生成中，将原始数据转换为结构化、美观的表格是一个常见且重要的需求。无论是生成Excel报表、Word文档中的表格，还是HTML网页表格，自动化的表格生成能够大大提高工作效率，减少人工错误，并确保数据展示的一致性。本文将深入探讨如何使用Python实现数据自动生成表格的完整解决方案，涵盖多种数据源、多种输出格式，以及高级的表格样式和交互功能。
目录
- 数据自动生成表格概述
- 技术栈与环境准备
- 数据源处理与标准化
- Excel表格自动生成
- Word文档表格生成
- HTML表格生成与Web展示
- PDF表格生成
- 高级表格样式与格式化
- 动态表格与交互功能
- 批量处理与模板系统
- 性能优化与大数据处理
- 实际应用案例
- 最佳实践与注意事项
- 总结与展望
数据自动生成表格概述
什么是数据自动生成表格
数据自动生成表格是指通过编程方式，将各种格式的原始数据（如CSV、JSON、数据库查询结果等）自动转换为格式化的表格文档。这个过程包括数据读取、处理、格式化、样式应用和输出等多个步骤。
核心优势
- 效率提升：自动化处理大量数据，避免手工制表的繁琐工作
- 一致性保证：确保所有表格遵循统一的格式和样式标准
- 错误减少：消除人工操作中的计算错误和格式错误
- 可重复性：相同的数据处理逻辑可以重复应用于不同的数据集
- 灵活性：支持多种数据源和输出格式，适应不同的业务需求
应用场景
- 财务报表生成：自动生成月度、季度、年度财务报表
- 销售数据分析：生成销售业绩表格和趋势分析
- 库存管理报告：自动更新库存状态表格
- 学生成绩统计：批量生成成绩单和统计报表
- 项目进度跟踪：生成项目状态和里程碑表格
- 数据分析报告：将分析结果以表格形式展示
技术栈与环境准备
核心Python库
我们将使用以下Python库来实现完整的表格生成功能：
# 数据处理
pip install pandas numpy
# Excel处理
pip install openpyxl xlsxwriter
# Word文档处理
pip install python-docx
# PDF生成
pip install reportlab
# HTML生成
pip install jinja2 beautifulsoup4
# 数据库连接
pip install sqlalchemy pymysql psycopg2-binary
# 图表生成
pip install matplotlib seaborn plotly
# 进度条和日志
pip install tqdm loguru
# 配置文件处理
pip install pyyaml configparser
项目结构设计
auto_table_generator/
│
├── core/
│ ├── __init__.py
│ ├── data_processor.py # 数据处理核心
│ ├── table_generator.py # 表格生成器基类
│ └── style_manager.py # 样式管理器
│
├── generators/
│ ├── __init__.py
│ ├── excel_generator.py # Excel表格生成器
│ ├── word_generator.py # Word表格生成器
│ ├── html_generator.py # HTML表格生成器
│ └── pdf_generator.py # PDF表格生成器
│
├── data_sources/
│ ├── __init__.py
│ ├── csv_source.py # CSV数据源
│ ├── json_source.py # JSON数据源
│ ├── database_source.py # 数据库数据源
│ └── api_source.py # API数据源
│
├── templates/
│ ├── excel_templates/ # Excel模板
│ ├── word_templates/ # Word模板
│ └── html_templates/ # HTML模板
│
├── config/
│ ├── settings.yaml # 配置文件
│ └── styles.yaml # 样式配置
│
├── examples/
│ ├── basic_usage.py # 基础使用示例
│ ├── advanced_features.py # 高级功能示例
│ └── batch_processing.py # 批量处理示例
│
└── tests/
├── test_generators.py # 生成器测试
└── test_data_sources.py # 数据源测试
环境验证
import sys
import pandas as pd
import openpyxl
import docx
from reportlab.lib import colors
import jinja2
def verify_environment():
"""验证开发环境是否正确配置"""
print("=== 环境验证 ===")
print(f"Python版本: {sys.version}")
print(f"Pandas版本: {pd.__version__}")
print(f"OpenPyXL版本: {openpyxl.__version__}")
print(f"Python-docx版本: {docx.__version__}")
print(f"Jinja2版本: {jinja2.__version__}")
# 测试基本功能
try:
# 测试pandas
df = pd.DataFrame({'A': [1, 2], 'B': [3, 4]})
print("✓ Pandas工作正常")
# 测试openpyxl
from openpyxl import Workbook
wb = Workbook()
print("✓ OpenPyXL工作正常")
# 测试python-docx
from docx import Document
doc = Document()
print("✓ Python-docx工作正常")
print("✓ 所有依赖库验证通过")
return True
except Exception as e:
print(f"✗ 环境验证失败: {e}")
return False
if __name__ == "__main__":
verify_environment()
数据源处理与标准化
在生成表格之前，我们需要处理各种不同的数据源，并将它们标准化为统一的格式。
数据源基类设计
from abc import ABC, abstractmethod
import pandas as pd
from typing import Dict, Any, Optional, List
import logging
class DataSource(ABC):
"""数据源基类"""
def __init__(self, config: Dict[str, Any]):
self.config = config
self.logger = logging.getLogger(self.__class__.__name__)
self._data = None
self._metadata = {}
@abstractmethod
def load_data(self) -> pd.DataFrame:
"""加载数据的抽象方法"""
pass
@abstractmethod
def validate_data(self, data: pd.DataFrame) -> bool:
"""验证数据的抽象方法"""
pass
def get_data(self, force_reload: bool = False) -> pd.DataFrame:
"""获取数据，支持缓存"""
if self._data is None or force_reload:
self._data = self.load_data()
if not self.validate_data(self._data):
raise ValueError("数据验证失败")
return self._data.copy()
def get_metadata(self) -> Dict[str, Any]:
"""获取数据元信息"""
return self._metadata.copy()
def preprocess_data(self, data: pd.DataFrame) -> pd.DataFrame:
"""数据预处理"""
# 处理缺失值
if self.config.get('fill_na', False):
fill_value = self.config.get('fill_na_value', '')
data = data.fillna(fill_value)
# 数据类型转换
if 'column_types' in self.config:
for col, dtype in self.config['column_types'].items():
if col in data.columns:
try:
data[col] = data[col].astype(dtype)
except Exception as e:
self.logger.warning(f"无法转换列 {col} 的数据类型: {e}")
# 列重命名
if 'column_mapping' in self.config:
data = data.rename(columns=self.config['column_mapping'])
# 数据过滤
if 'filters' in self.config:
for filter_config in self.config['filters']:
data = self._apply_filter(data, filter_config)
return data
def _apply_filter(self, data: pd.DataFrame, filter_config: Dict) -> pd.DataFrame:
"""应用数据过滤器"""
column = filter_config.get('column')
operator = filter_config.get('operator', '==')
value = filter_config.get('value')
if column not in data.columns:
self.logger.warning(f"过滤列 {column} 不存在")
return data
try:
if operator == '==':
return data[data[column] == value]
elif operator == '!=':
return data[data[column] != value]
elif operator == '>':
return data[data[column] > value]
elif operator == '<':
return data[data[column] < value]
elif operator == '>=':
return data[data[column] >= value]
elif operator == '<=':
return data[data[column] <= value]
elif operator == 'in':
return data[data[column].isin(value)]
elif operator == 'contains':
return data[data[column].str.contains(value, na=False)]
else:
self.logger.warning(f"不支持的过滤操作符: {operator}")
return data
except Exception as e:
self.logger.error(f"应用过滤器时出错: {e}")
return data
CSV数据源实现
import pandas as pd
import os
from typing import Dict, Any
class CSVDataSource(DataSource):
"""CSV数据源"""
def load_data(self) -> pd.DataFrame:
"""从CSV文件加载数据"""
file_path = self.config.get('file_path')
if not file_path or not os.path.exists(file_path):
raise FileNotFoundError(f"CSV文件不存在: {file_path}")
# CSV读取参数
csv_params = {
'encoding': self.config.get('encoding', 'utf-8'),
'sep': self.config.get('separator', ','),
'header': self.config.get('header', 0),
'skiprows': self.config.get('skip_rows', None),
'nrows': self.config.get('max_rows', None)
}
try:
data = pd.read_csv(file_path, **csv_params)
self.logger.info(f"成功加载CSV文件: {file_path}, 数据形状: {data.shape}")
# 更新元数据
self._metadata = {
'source_type': 'csv',
'file_path': file_path,
'file_size': os.path.getsize(file_path),
'rows': len(data),
'columns': len(data.columns),
'column_names': list(data.columns)
}
return self.preprocess_data(data)
except Exception as e:
self.logger.error(f"加载CSV文件失败: {e}")
raise
def validate_data(self, data: pd.DataFrame) -> bool:
"""验证CSV数据"""
if data.empty:
self.logger.error("CSV数据为空")
return False
# 检查必需列
required_columns = self.config.get('required_columns', [])
missing_columns = set(required_columns) - set(data.columns)
if missing_columns:
self.logger.error(f"缺少必需列: {missing_columns}")
return False
# 检查数据类型
if 'column_types' in self.config:
for col, expected_type in self.config['column_types'].items():
if col in data.columns:
try:
pd.api.types.is_dtype_equal(data[col].dtype, expected_type)
except Exception:
self.logger.warning(f"列 {col} 的数据类型可能不匹配")
return True
JSON数据源实现
import json
import pandas as pd
from typing import Dict, Any, List, Union
class JSONDataSource(DataSource):
"""JSON数据源"""
def load_data(self) -> pd.DataFrame:
"""从JSON文件或API加载数据"""
if 'file_path' in self.config:
return self._load_from_file()
elif 'url' in self.config:
return self._load_from_api()
else:
raise ValueError("必须指定file_path或url")
def _load_from_file(self) -> pd.DataFrame:
"""从JSON文件加载数据"""
file_path = self.config['file_path']
if not os.path.exists(file_path):
raise FileNotFoundError(f"JSON文件不存在: {file_path}")
try:
with open(file_path, 'r', encoding=self.config.get('encoding', 'utf-8')) as f:
json_data = json.load(f)
data = self._json_to_dataframe(json_data)
self._metadata = {
'source_type': 'json_file',
'file_path': file_path,
'file_size': os.path.getsize(file_path),
'rows': len(data),
'columns': len(data.columns)
}
return self.preprocess_data(data)
except Exception as e:
self.logger.error(f"加载JSON文件失败: {e}")
raise
def _load_from_api(self) -> pd.DataFrame:
"""从API加载JSON数据"""
import requests
url = self.config['url']
headers = self.config.get('headers', {})
params = self.config.get('params', {})
timeout = self.config.get('timeout', 30)
try:
response = requests.get(url, headers=headers, params=params, timeout=timeout)
response.raise_for_status()
json_data = response.json()
data = self._json_to_dataframe(json_data)
self._metadata = {
'source_type': 'json_api',
'url': url,
'status_code': response.status_code,
'rows': len(data),
'columns': len(data.columns)
}
return self.preprocess_data(data)
except Exception as e:
self.logger.error(f"从API加载JSON数据失败: {e}")
raise
def _json_to_dataframe(self, json_data: Union[Dict, List]) -> pd.DataFrame:
"""将JSON数据转换为DataFrame"""
data_path = self.config.get('data_path', None)
# 如果指定了数据路径，提取嵌套数据
if data_path:
current_data = json_data
for key in data_path.split('.'):
if isinstance(current_data, dict) and key in current_data:
current_data = current_data[key]
else:
raise ValueError(f"无法找到数据路径: {data_path}")
json_data = current_data
# 转换为DataFrame
if isinstance(json_data, list):
return pd.DataFrame(json_data)
elif isinstance(json_data, dict):
# 如果是字典，尝试转换为记录列表
if all(isinstance(v, (list, tuple)) for v in json_data.values()):
return pd.DataFrame(json_data)
else:
# 单条记录
return pd.DataFrame([json_data])
else:
raise ValueError("不支持的JSON数据格式")
def validate_data(self, data: pd.DataFrame) -> bool:
"""验证JSON数据"""
if data.empty:
self.logger.error("JSON数据为空")
return False
return True
数据库数据源实现
import pandas as pd
from sqlalchemy import create_engine, text
from typing import Dict, Any
class DatabaseDataSource(DataSource):
"""数据库数据源"""
def __init__(self, config: Dict[str, Any]):
super().__init__(config)
self.engine = None
self._create_engine()
def _create_engine(self):
"""创建数据库引擎"""
db_config = self.config.get('database', {})
# 构建连接字符串
db_type = db_config.get('type', 'mysql')
host = db_config.get('host', 'localhost')
port = db_config.get('port', 3306)
username = db_config.get('username')
password = db_config.get('password')
database = db_config.get('database')
if db_type == 'mysql':
connection_string = f"mysql+pymysql://{username}:{password}@{host}:{port}/{database}"
elif db_type == 'postgresql':
connection_string = f"postgresql+psycopg2://{username}:{password}@{host}:{port}/{database}"
elif db_type == 'sqlite':
connection_string = f"sqlite:///{database}"
else:
raise ValueError(f"不支持的数据库类型: {db_type}")
try:
self.engine = create_engine(
connection_string,
pool_pre_ping=True,
pool_recycle=3600
)
self.logger.info(f"成功连接到数据库: {db_type}")
except Exception as e:
self.logger.error(f"数据库连接失败: {e}")
raise
def load_data(self) -> pd.DataFrame:
"""从数据库加载数据"""
query = self.config.get('query')
table_name = self.config.get('table_name')
if query:
return self._load_from_query(query)
elif table_name:
return self._load_from_table(table_name)
else:
raise ValueError("必须指定query或table_name")
def _load_from_query(self, query: str) -> pd.DataFrame:
"""从SQL查询加载数据"""
try:
# 支持参数化查询
query_params = self.config.get('query_params', {})
data = pd.read_sql(
text(query),
self.engine,
params=query_params
)
self.logger.info(f"成功执行查询，返回 {len(data)} 行数据")
self._metadata = {
'source_type': 'database_query',
'query': query,
'rows': len(data),
'columns': len(data.columns)
}
return self.preprocess_data(data)
except Exception as e:
self.logger.error(f"执行数据库查询失败: {e}")
raise
def _load_from_table(self, table_name: str) -> pd.DataFrame:
"""从数据表加载数据"""
try:
# 构建查询
columns = self.config.get('columns', '*')
if isinstance(columns, list):
columns = ', '.join(columns)
where_clause = self.config.get('where_clause', '')
order_by = self.config.get('order_by', '')
limit = self.config.get('limit', '')
query = f"SELECT {columns} FROM {table_name}"
if where_clause:
query += f" WHERE {where_clause}"
if order_by:
query += f" ORDER BY {order_by}"
if limit:
query += f" LIMIT {limit}"
return self._load_from_query(query)
except Exception as e:
self.logger.error(f"从数据表加载数据失败: {e}")
raise
def validate_data(self, data: pd.DataFrame) -> bool:
"""验证数据库数据"""
if data.empty:
self.logger.warning("数据库查询返回空结果")
return True # 空结果可能是正常的
return True
def __del__(self):
"""清理数据库连接"""
if self.engine:
self.engine.dispose()
Excel表格自动生成
Excel是最常用的表格格式之一，我们将实现一个功能强大的Excel表格生成器。
Excel生成器核心实现
import pandas as pd
from openpyxl import Workbook, load_workbook
from openpyxl.styles import Font, PatternFill, Border, Side, Alignment
from openpyxl.utils.dataframe import dataframe_to_rows
from openpyxl.chart import BarChart, LineChart, PieChart, Reference
from openpyxl.worksheet.table import Table, TableStyleInfo
import os
from typing import Dict, Any, List, Optional, Union
import logging
class ExcelTableGenerator:
"""Excel表格生成器"""
def __init__(self, config: Dict[str, Any] = None):
self.config = config or {}
self.logger = logging.getLogger(self.__class__.__name__)
self.workbook = None
self.worksheet = None
# 默认样式配置
self.default_styles = {
'header': {
'font': Font(bold=True, color='FFFFFF'),
'fill': PatternFill(start_color='366092', end_color='366092', fill_type='solid'),
'alignment': Alignment(horizontal='center', vertical='center')
},
'data': {
'font': Font(color='000000'),
'alignment': Alignment(horizontal='left', vertical='center')
},
'number': {
'font': Font(color='000000'),
'alignment': Alignment(horizontal='right', vertical='center'),
'number_format': '#,##0.00'
},
'border': Border(
left=Side(style='thin'),
right=Side(style='thin'),
top=Side(style='thin'),
bottom=Side(style='thin')
)
}
def create_table(self, data: pd.DataFrame, output_path: str,
sheet_name: str = 'Sheet1', **kwargs) -> str:
"""
创建Excel表格
Args:
data: 数据DataFrame
output_path: 输出文件路径
sheet_name: 工作表名称
**kwargs: 其他配置参数
Returns:
生成的文件路径
"""
try:
# 创建工作簿
self.workbook = Workbook()
self.worksheet = self.workbook.active
self.worksheet.title = sheet_name
# 写入数据
self._write_data(data, **kwargs)
# 应用样式
self._apply_styles(data, **kwargs)
# 添加表格功能
if kwargs.get('enable_table', True):
self._create_excel_table(data, **kwargs)
# 添加图表
if kwargs.get('add_charts', False):
self._add_charts(data, **kwargs)
# 自动调整列宽
if kwargs.get('auto_fit_columns', True):
self._auto_fit_columns()
# 保存文件
os.makedirs(os.path.dirname(output_path), exist_ok=True)
self.workbook.save(output_path)
self.logger.info(f"Excel表格已生成: {output_path}")
return output_path
except Exception as e:
self.logger.error(f"生成Excel表格失败: {e}")
raise
def _write_data(self, data: pd.DataFrame, start_row: int = 1,
start_col: int = 1, include_index: bool = False, **kwargs):
"""写入数据到工作表"""
# 添加标题
title = kwargs.get('title')
if title:
self.worksheet.cell(row=start_row, column=start_col, value=title)
self.worksheet.merge_cells(
start_row=start_row, start_column=start_col,
end_row=start_row, end_column=start_col + len(data.columns) - 1
)
# 标题样式
title_cell = self.worksheet.cell(row=start_row, column=start_col)
title_cell.font = Font(bold=True, size=16)
title_cell.alignment = Alignment(horizontal='center', vertical='center')
start_row += 2
# 写入DataFrame数据
for r in dataframe_to_rows(data, index=include_index, header=True):
for c_idx, value in enumerate(r, start_col):
self.worksheet.cell(row=start_row, column=c_idx, value=value)
start_row += 1
# 记录数据范围
self.data_start_row = 2 if title else 1
self.data_end_row = start_row - 1
self.data_start_col = start_col
self.data_end_col = start_col + len(data.columns) - 1
def _apply_styles(self, data: pd.DataFrame, **kwargs):
"""应用样式"""
# 获取样式配置
styles = kwargs.get('styles', {})
header_style = {**self.default_styles['header'], **styles.get('header', {})}
data_style = {**self.default_styles['data'], **styles.get('data', {})}
# 应用表头样式
for col in range(self.data_start_col, self.data_end_col + 1):
cell = self.worksheet.cell(row=self.data_start_row, column=col)
self._apply_cell_style(cell, header_style)
# 应用数据样式
for row in range(self.data_start_row + 1, self.data_end_row + 1):
for col in range(self.data_start_col, self.data_end_col + 1):
cell = self.worksheet.cell(row=row, column=col)
# 根据数据类型选择样式
col_name = data.columns[col - self.data_start_col]
if pd.api.types.is_numeric_dtype(data[col_name]):
style = {**self.default_styles['number'], **styles.get('number', {})}
else:
style = data_style
self._apply_cell_style(cell, style)
# 应用边框
self._apply_borders()
def _apply_cell_style(self, cell, style: Dict):
"""应用单元格样式"""
if 'font' in style:
cell.font = style['font']
if 'fill' in style:
cell.fill = style['fill']
if 'alignment' in style:
cell.alignment = style['alignment']
if 'number_format' in style:
cell.number_format = style['number_format']
if 'border' in style:
cell.border = style['border']
def _apply_borders(self):
"""应用边框"""
border = self.default_styles['border']
for row in range(self.data_start_row, self.data_end_row + 1):
for col in range(self.data_start_col, self.data_end_col + 1):
cell = self.worksheet.cell(row=row, column=col)
cell.border = border
def _create_excel_table(self, data: pd.DataFrame, **kwargs):
"""创建Excel表格对象"""
table_name = kwargs.get('table_name', 'DataTable')
table_style = kwargs.get('table_style', 'TableStyleMedium9')
# 定义表格范围
table_range = f"{self.worksheet.cell(self.data_start_row, self.data_start_col).coordinate}:" \
table_range = f"{self.worksheet.cell(self.data_start_row, self.data_start_col).coordinate}:" \
f"{self.worksheet.cell(self.data_end_row, self.data_end_col).coordinate}"
# 创建表格
table = Table(displayName=table_name, ref=table_range)
# 设置表格样式
style = TableStyleInfo(
name=table_style,
showFirstColumn=False,
showLastColumn=False,
showRowStripes=True,
showColumnStripes=False
)
table.tableStyleInfo = style
# 添加表格到工作表
self.worksheet.add_table(table)
def _add_charts(self, data: pd.DataFrame, **kwargs):
"""添加图表"""
chart_configs = kwargs.get('chart_configs', [])
for i, chart_config in enumerate(chart_configs):
chart_type = chart_config.get('type', 'bar')
chart_title = chart_config.get('title', f'图表 {i+1}')
data_columns = chart_config.get('columns', [])
if not data_columns:
continue
# 创建图表
if chart_type == 'bar':
chart = BarChart()
elif chart_type == 'line':
chart = LineChart()
elif chart_type == 'pie':
chart = PieChart()
else:
continue
chart.title = chart_title
# 添加数据系列
for col_name in data_columns:
if col_name in data.columns:
col_idx = list(data.columns).index(col_name) + self.data_start_col
data_ref = Reference(
self.worksheet,
min_col=col_idx,
min_row=self.data_start_row + 1,
max_row=self.data_end_row
)
chart.add_data(data_ref, titles_from_data=False)
# 设置类别标签（通常是第一列）
if len(data.columns) > 0:
cats_ref = Reference(
self.worksheet,
min_col=self.data_start_col,
min_row=self.data_start_row + 1,
max_row=self.data_end_row
)
chart.set_categories(cats_ref)
# 添加图表到工作表
chart_position = chart_config.get('position', f'H{5 + i * 15}')
self.worksheet.add_chart(chart, chart_position)
def _auto_fit_columns(self):
"""自动调整列宽"""
for column in self.worksheet.columns:
max_length = 0
column_letter = column[0].column_letter
for cell in column:
try:
if len(str(cell.value)) > max_length:
max_length = len(str(cell.value))
except:
pass
adjusted_width = min(max_length + 2, 50) # 限制最大宽度
self.worksheet.column_dimensions[column_letter].width = adjusted_width
def create_multi_sheet_workbook(self, data_dict: Dict[str, pd.DataFrame],
output_path: str, **kwargs) -> str:
"""
创建多工作表Excel文件
Args:
data_dict: 工作表名称到DataFrame的映射
output_path: 输出文件路径
**kwargs: 其他配置参数
Returns:
生成的文件路径
"""
try:
self.workbook = Workbook()
# 删除默认工作表
default_sheet = self.workbook.active
self.workbook.remove(default_sheet)
# 为每个数据集创建工作表
for sheet_name, data in data_dict.items():
self.worksheet = self.workbook.create_sheet(title=sheet_name)
self._write_data(data, **kwargs)
self._apply_styles(data, **kwargs)
if kwargs.get('enable_table', True):
self._create_excel_table(data, table_name=f"Table_{sheet_name}", **kwargs)
if kwargs.get('auto_fit_columns', True):
self._auto_fit_columns()
# 保存文件
os.makedirs(os.path.dirname(output_path), exist_ok=True)
self.workbook.save(output_path)
self.logger.info(f"多工作表Excel文件已生成: {output_path}")
return output_path
except Exception as e:
self.logger.error(f"生成多工作表Excel文件失败: {e}")
raise
def add_summary_sheet(self, data_dict: Dict[str, pd.DataFrame], **kwargs):
"""添加汇总工作表"""
summary_data = []
for sheet_name, data in data_dict.items():
summary_row = {
'工作表名称': sheet_name,
'数据行数': len(data),
'数据列数': len(data.columns),
'数值列数': len(data.select_dtypes(include=['number']).columns),
'文本列数': len(data.select_dtypes(include=['object']).columns)
}
# 添加数值列的统计信息
numeric_data = data.select_dtypes(include=['number'])
if not numeric_data.empty:
summary_row['数值总和'] = numeric_data.sum().sum()
summary_row['数值平均值'] = numeric_data.mean().mean()
summary_data.append(summary_row)
summary_df = pd.DataFrame(summary_data)
# 创建汇总工作表
self.worksheet = self.workbook.create_sheet(title='数据汇总', index=0)
self._write_data(summary_df, title='数据汇总报告', **kwargs)
self._apply_styles(summary_df, **kwargs)
if kwargs.get('auto_fit_columns', True):
self._auto_fit_columns()
Excel高级功能实现
class AdvancedExcelGenerator(ExcelTableGenerator):
"""高级Excel生成器"""
def __init__(self, config: Dict[str, Any] = None):
super().__init__(config)
self.conditional_formats = []
self.data_validations = []
def add_conditional_formatting(self, data: pd.DataFrame, rules: List[Dict], **kwargs):
"""添加条件格式"""
from openpyxl.formatting.rule import ColorScaleRule, DataBarRule, IconSetRule
from openpyxl.styles import PatternFill
for rule in rules:
rule_type = rule.get('type')
column = rule.get('column')
if column not in data.columns:
continue
col_idx = list(data.columns).index(column) + self.data_start_col
range_string = f"{self.worksheet.cell(self.data_start_row + 1, col_idx).coordinate}:" \
f"{self.worksheet.cell(self.data_end_row, col_idx).coordinate}"
if rule_type == 'color_scale':
# 颜色刻度
color_rule = ColorScaleRule(
start_type='min', start_color=rule.get('start_color', 'FF0000'),
end_type='max', end_color=rule.get('end_color', '00FF00')
)
self.worksheet.conditional_formatting.add(range_string, color_rule)
elif rule_type == 'data_bar':
# 数据条
data_bar_rule = DataBarRule(
start_type='min', end_type='max',
color=rule.get('color', '0066CC')
)
self.worksheet.conditional_formatting.add(range_string, data_bar_rule)
elif rule_type == 'icon_set':
# 图标集
icon_rule = IconSetRule(
icon_style=rule.get('icon_style', '3TrafficLights1'),
type='percent',
values=[33, 67]
)
self.worksheet.conditional_formatting.add(range_string, icon_rule)
elif rule_type == 'highlight_cells':
# 突出显示单元格
from openpyxl.formatting.rule import CellIsRule
operator = rule.get('operator', 'greaterThan')
value = rule.get('value', 0)
fill_color = rule.get('fill_color', 'FFFF00')
highlight_rule = CellIsRule(
operator=operator,
formula=[value],
fill=PatternFill(start_color=fill_color, end_color=fill_color, fill_type='solid')
)
self.worksheet.conditional_formatting.add(range_string, highlight_rule)
def add_data_validation(self, data: pd.DataFrame, validations: List[Dict], **kwargs):
"""添加数据验证"""
from openpyxl.worksheet.datavalidation import DataValidation
for validation in validations:
column = validation.get('column')
if column not in data.columns:
continue
col_idx = list(data.columns).index(column) + self.data_start_col
range_string = f"{self.worksheet.cell(self.data_start_row + 1, col_idx).coordinate}:" \
f"{self.worksheet.cell(self.data_end_row, col_idx).coordinate}"
validation_type = validation.get('type', 'list')
if validation_type == 'list':
# 下拉列表验证
formula = f'"{",".join(validation.get("values", []))}"'
dv = DataValidation(type="list", formula1=formula, allow_blank=True)
elif validation_type == 'whole':
# 整数验证
dv = DataValidation(
type="whole",
operator=validation.get('operator', 'between'),
formula1=validation.get('min_value', 0),
formula2=validation.get('max_value', 100)
)
elif validation_type == 'decimal':
# 小数验证
dv = DataValidation(
type="decimal",
operator=validation.get('operator', 'between'),
formula1=validation.get('min_value', 0.0),
formula2=validation.get('max_value', 100.0)
)
else:
continue
# 设置错误消息
dv.error = validation.get('error_message', '输入的数据无效')
dv.errorTitle = validation.get('error_title', '数据验证错误')
# 设置提示消息
dv.prompt = validation.get('prompt_message', '请输入有效数据')
dv.promptTitle = validation.get('prompt_title', '数据输入提示')
self.worksheet.add_data_validation(dv)
dv.add(range_string)
def add_pivot_table_data(self, data: pd.DataFrame, output_path: str, **kwargs):
"""为数据透视表准备数据"""
# 创建数据源工作表
data_sheet = self.workbook.create_sheet(title='数据源')
# 写入数据
for r in dataframe_to_rows(data, index=False, header=True):
data_sheet.append(r)
# 应用基本格式
for cell in data_sheet[1]: # 表头
cell.font = Font(bold=True)
cell.fill = PatternFill(start_color='D9E1F2', end_color='D9E1F2', fill_type='solid')
# 创建数据透视表配置说明
instructions = [
"数据透视表创建说明：",
"1. 选择'数据源'工作表中的所有数据",
"2. 插入 -> 数据透视表",
"3. 根据需要配置行、列、值字段",
"",
"数据字段说明："
]
for col in data.columns:
dtype = str(data[col].dtype)
instructions.append(f"- {col}: {dtype}")
# 添加说明工作表
instruction_sheet = self.workbook.create_sheet(title='使用说明')
for i, instruction in enumerate(instructions, 1):
instruction_sheet.cell(row=i, column=1, value=instruction)
def create_dashboard(self, data: pd.DataFrame, output_path: str, **kwargs):
"""创建数据仪表板"""
# 创建仪表板工作表
dashboard_sheet = self.workbook.create_sheet(title='数据仪表板', index=0)
# 添加标题
title = kwargs.get('dashboard_title', '数据仪表板')
dashboard_sheet.cell(row=1, column=1, value=title)
dashboard_sheet.merge_cells('A1:H1')
title_cell = dashboard_sheet.cell(row=1, column=1)
title_cell.font = Font(bold=True, size=20, color='FFFFFF')
title_cell.fill = PatternFill(start_color='2F5597', end_color='2F5597', fill_type='solid')
title_cell.alignment = Alignment(horizontal='center', vertical='center')
# 添加关键指标
metrics = self._calculate_key_metrics(data)
self._add_metrics_section(dashboard_sheet, metrics, start_row=3)
# 添加图表
self._add_dashboard_charts(dashboard_sheet, data, start_row=10)
# 设置行高
dashboard_sheet.row_dimensions[1].height = 40
def _calculate_key_metrics(self, data: pd.DataFrame) -> Dict:
"""计算关键指标"""
metrics = {
'总记录数': len(data),
'数据列数': len(data.columns)
}
# 数值列统计
numeric_data = data.select_dtypes(include=['number'])
if not numeric_data.empty:
metrics['数值列数'] = len(numeric_data.columns)
metrics['数值总和'] = numeric_data.sum().sum()
metrics['平均值'] = numeric_data.mean().mean()
# 文本列统计
text_data = data.select_dtypes(include=['object'])
if not text_data.empty:
metrics['文本列数'] = len(text_data.columns)
return metrics
def _add_metrics_section(self, worksheet, metrics: Dict, start_row: int):
"""添加指标区域"""
col = 1
for metric_name, metric_value in metrics.items():
# 指标名称
name_cell = worksheet.cell(row=start_row, column=col)
name_cell.value = metric_name
name_cell.font = Font(bold=True, size=12)
name_cell.alignment = Alignment(horizontal='center')
# 指标值
value_cell = worksheet.cell(row=start_row + 1, column=col)
value_cell.value = metric_value
value_cell.font = Font(bold=True, size=16, color='2F5597')
value_cell.alignment = Alignment(horizontal='center')
# 样式
for r in range(start_row, start_row + 2):
cell = worksheet.cell(row=r, column=col)
cell.border = Border(
left=Side(style='thin'),
right=Side(style='thin'),
top=Side(style='thin'),
bottom=Side(style='thin')
)
col += 2 # 间隔一列
def _add_dashboard_charts(self, worksheet, data: pd.DataFrame, start_row: int):
"""添加仪表板图表"""
# 这里可以添加各种图表
# 由于篇幅限制，这里只是示例框架
pass
Word文档表格生成
Word文档中的表格生成同样重要，特别是在报告和文档自动化中。
Word表格生成器实现
from docx import Document
from docx.shared import Inches, Cm, Pt
from docx.enum.table import WD_TABLE_ALIGNMENT, WD_ALIGN_VERTICAL
from docx.enum.text import WD_ALIGN_PARAGRAPH
from docx.oxml.shared import OxmlElement, qn
import pandas as pd
from typing import Dict, Any, List, Optional
import logging
class WordTableGenerator:
"""Word表格生成器"""
def __init__(self, config: Dict[str, Any] = None):
self.config = config or {}
self.logger = logging.getLogger(self.__class__.__name__)
self.document = None
# 默认样式配置
self.default_styles = {
'table_style': 'Table Grid',
'header_style': {
'bold': True,
'font_size': Pt(12),
'alignment': WD_ALIGN_PARAGRAPH.CENTER
},
'data_style': {
'font_size': Pt(10),
'alignment': WD_ALIGN_PARAGRAPH.LEFT
},
'number_style': {
'font_size': Pt(10),
'alignment': WD_ALIGN_PARAGRAPH.RIGHT
}
}
def create_table(self, data: pd.DataFrame, output_path: str, **kwargs) -> str:
"""
创建Word表格文档
Args:
data: 数据DataFrame
output_path: 输出文件路径
**kwargs: 其他配置参数
Returns:
生成的文件路径
"""
try:
# 创建或加载文档
template_path = kwargs.get('template_path')
if template_path and os.path.exists(template_path):
self.document = Document(template_path)
else:
self.document = Document()
# 添加标题
title = kwargs.get('title')
if title:
self._add_title(title, **kwargs)
# 添加表格
table = self._create_word_table(data, **kwargs)
# 添加表格说明
description = kwargs.get('description')
if description:
self._add_description(description, **kwargs)
# 添加统计信息
if kwargs.get('add_statistics', False):
self._add_statistics(data, **kwargs)
# 保存文档
os.makedirs(os.path.dirname(output_path), exist_ok=True)
self.document.save(output_path)
self.logger.info(f"Word表格文档已生成: {output_path}")
return output_path
except Exception as e:
self.logger.error(f"生成Word表格文档失败: {e}")
raise
def _add_title(self, title: str, **kwargs):
"""添加文档标题"""
title_paragraph = self.document.add_heading(title, level=1)
title_paragraph.alignment = WD_ALIGN_PARAGRAPH.CENTER
# 添加副标题
subtitle = kwargs.get('subtitle')
if subtitle:
subtitle_paragraph = self.document.add_heading(subtitle, level=2)
subtitle_paragraph.alignment = WD_ALIGN_PARAGRAPH.CENTER
# 添加空行
self.document.add_paragraph()
def _create_word_table(self, data: pd.DataFrame, **kwargs):
"""创建Word表格"""
# 创建表格
table = self.document.add_table(rows=1, cols=len(data.columns))
table.style = kwargs.get('table_style', self.default_styles['table_style'])
table.alignment = WD_TABLE_ALIGNMENT.CENTER
# 设置表头
header_cells = table.rows[0].cells
for i, column_name in enumerate(data.columns):
header_cells[i].text = str(column_name)
self._apply_cell_style(header_cells[i], self.default_styles['header_style'])
# 添加数据行
for _, row in data.iterrows():
row_cells = table.add_row().cells
for i, value in enumerate(row):
row_cells[i].text = str(value) if pd.notna(value) else ''
# 根据数据类型应用样式
if pd.api.types.is_numeric_dtype(data.iloc[:, i]):
style = self.default_styles['number_style']
else:
style = self.default_styles['data_style']
self._apply_cell_style(row_cells[i], style)
# 设置列宽
self._set_column_widths(table, data, **kwargs)
return table
def _apply_cell_style(self, cell, style: Dict):
"""应用单元格样式"""
paragraph = cell.paragraphs[0]
run = paragraph.runs[0] if paragraph.runs else paragraph.add_run()
if 'bold' in style:
run.bold = style['bold']
if 'italic' in style:
run.italic = style['italic']
if 'font_size' in style:
run.font.size = style['font_size']
if 'font_color' in style:
run.font.color.rgb = style['font_color']
if 'alignment' in style:
paragraph.alignment = style['alignment']
# 设置单元格垂直对齐
cell.vertical_alignment = WD_ALIGN_VERTICAL.CENTER
def _set_column_widths(self, table, data: pd.DataFrame, **kwargs):
"""设置列宽"""
column_widths = kwargs.get('column_widths')
if column_widths:
# 使用指定的列宽
for i, width in enumerate(column_widths):
if i < len(table.columns):
for cell in table.columns[i].cells:
cell.width = Inches(width)
else:
# 自动计算列宽
total_width = Inches(6.5) # A4纸张可用宽度
col_count = len(data.columns)
# 根据内容长度分配宽度
max_lengths = []
for col in data.columns:
max_len = max(
len(str(col)), # 列名长度
data[col].astype(str).str.len().max() if not data.empty else 0
)
max_lengths.append(max_len)
total_chars = sum(max_lengths)
for i, max_len in enumerate(max_lengths):
if total_chars > 0:
width_ratio = max_len / total_chars
column_width = total_width * width_ratio
else:
column_width = total_width / col_count
for cell in table.columns[i].cells:
cell.width = column_width
def _add_description(self, description: str, **kwargs):
"""添加表格说明"""
self.document.add_paragraph()
desc_paragraph = self.document.add_paragraph(description)
desc_paragraph.alignment = WD_ALIGN_PARAGRAPH.JUSTIFY
def _add_statistics(self, data: pd.DataFrame, **kwargs):
"""添加统计信息"""
self.document.add_paragraph()
stats_heading = self.document.add_heading('数据统计', level=2)
# 基本统计
stats_text = f"数据行数: {len(data)}\n"
stats_text += f"数据列数: {len(data.columns)}\n"
# 数值列统计
numeric_data = data.select_dtypes(include=['number'])
if not numeric_data.empty:
stats_text += f"数值列数: {len(numeric_data.columns)}\n"
stats_text += f"数值总和: {numeric_data.sum().sum():.2f}\n"
stats_text += f"数值平均值: {numeric_data.mean().mean():.2f}\n"
stats_paragraph = self.document.add_paragraph(stats_text)
def create_multi_table_document(self, data_dict: Dict[str, pd.DataFrame],
output_path: str, **kwargs) -> str:
"""
创建包含多个表格的Word文档
Args:
data_dict: 表格名称到DataFrame的映射
output_path: 输出文件路径
**kwargs: 其他配置参数
Returns:
生成的文件路径
"""
try:
self.document = Document()
# 添加文档标题
doc_title = kwargs.get('document_title', '数据报告')
self._add_title(doc_title, **kwargs)
# 为每个数据集创建表格
for i, (table_name, data) in enumerate(data_dict.items()):
if i > 0:
self.document.add_page_break()
# 添加表格标题
table_heading = self.document.add_heading(table_name, level=2)
# 创建表格
self._create_word_table(data, **kwargs)
# 添加表格统计
if kwargs.get('add_table_statistics', True):
self._add_table_summary(data, table_name)
# 保存文档
os.makedirs(os.path.dirname(output_path), exist_ok=True)
self.document.save(output_path)
self.logger.info(f"多表格Word文档已生成: {output_path}")
return output_path
except Exception as e:
self.logger.error(f"生成多表格Word文档失败: {e}")
raise
def _add_table_summary(self, data: pd.DataFrame, table_name: str):
"""添加表格摘要"""
summary_text = f"{table_name} 包含 {len(data)} 行数据，{len(data.columns)} 个字段。"
# 添加数值统计
numeric_data = data.select_dtypes(include=['number'])
if not numeric_data.empty:
summary_text += f" 其中数值字段 {len(numeric_data.columns)} 个。"
summary_paragraph = self.document.add_paragraph(summary_text)
summary_paragraph.alignment = WD_ALIGN_PARAGRAPH.JUSTIFY
# 添加空行
self.document.add_paragraph()
HTML表格生成与Web展示
HTML表格适用于Web展示和交互式报告。
HTML表格生成器实现
import pandas as pd
from jinja2 import Template, Environment, FileSystemLoader
import os
from typing import Dict, Any, List, Optional
import json
import base64
from io import BytesIO
import matplotlib.pyplot as plt
import seaborn as sns
class HTMLTableGenerator:
"""HTML表格生成器"""
def __init__(self, config: Dict[str, Any] = None):
self.config = config or {}
self.logger = logging.getLogger(self.__class__.__name__)
# 默认HTML模板
self.default_template = """
{{ title }}
{% if enable_datatables %}
{% endif %}
{{ title }}
{% if description %}
{{ description }}
{% endif %}
{% if enable_filters %}
数据筛选
{% for column in columns %}
{% endfor %}
{% endif %}
{% for column in columns %}
{{ column }}
{% endfor %}
{% for row in data %}
{% for cell in row %}
{{ cell }}
{% endfor %}
{% endfor %}
{% if statistics %}
数据统计
{% for key, value in statistics.items() %}
{{ key }}: {{ value }}
{% endfor %}
{% endif %}
{% if charts %}
数据图表
{% for chart in charts %}
{{ chart.title }}
{% endfor %}
{% endif %}
{% if enable_datatables %}
{% endif %}
"""
def create_table(self, data: pd.DataFrame, output_path: str, **kwargs) -> str:
"""
创建HTML表格
Args:
data: 数据DataFrame
output_path: 输出文件路径
**kwargs: 其他配置参数
Returns:
生成的文件路径
"""
try:
# 准备模板数据
template_data = {
'title': kwargs.get('title', '数据表格'),
'description': kwargs.get('description'),
'columns': list(data.columns),
'data': data.values.tolist(),
'enable_datatables': kwargs.get('enable_datatables', True),
'enable_filters': kwargs.get('enable_filters', False)
}
# 添加统计信息
if kwargs.get('add_statistics', True):
template_data['statistics'] = self._calculate_statistics(data)
# 添加图表
if kwargs.get('add_charts', False):
template_data['charts'] = self._generate_charts(data, **kwargs)
# 渲染模板
template = Template(self.default_template)
html_content = template.render(**template_data)
# 保存文件
os.makedirs(os.path.dirname(output_path), exist_ok=True)
with open(output_path, 'w', encoding='utf-8') as f:
f.write(html_content)
self.logger.info(f"HTML表格已生成: {output_path}")
return output_path
except Exception as e:
self.logger.error(f"生成HTML表格失败: {e}")
raise
def _calculate_statistics(self, data: pd.DataFrame) -> Dict[str, Any]:
"""计算统计信息"""
stats = {
'总行数': len(data),
'总列数': len(data.columns)
}
# 数值列统计
numeric_data = data.select_dtypes(include=['number'])
if not numeric_data.empty:
stats['数值列数'] = len(numeric_data.columns)
stats['数值总和'] = f"{numeric_data.sum().sum():.2f}"
stats['数值平均值'] = f"{numeric_data.mean().mean():.2f}"
stats['最大值'] = f"{numeric_data.max().max():.2f}"
stats['最小值'] = f"{numeric_data.min().min():.2f}"
# 文本列统计
text_data = data.select_dtypes(include=['object'])
if not text_data.empty:
stats['文本列数'] = len(text_data.columns)
# 缺失值统计
missing_count = data.isnull().sum().sum()
if missing_count > 0:
stats['缺失值数量'] = missing_count
stats['缺失值比例'] = f"{(missing_count / (len(data) * len(data.columns)) * 100):.2f}%"
return stats
def _generate_charts(self, data: pd.DataFrame, **kwargs) -> List[Dict]:
"""生成图表"""
charts = []
chart_configs = kwargs.get('chart_configs', [])
# 设置中文字体
plt.rcParams['font.sans-serif'] = ['SimHei', 'DejaVu Sans']
plt.rcParams['axes.unicode_minus'] = False
for chart_config in chart_configs:
try:
chart_type = chart_config.get('type', 'bar')
title = chart_config.get('title', '图表')
columns = chart_config.get('columns', [])
if not columns or not all(col in data.columns for col in columns):
continue
# 创建图表
fig, ax = plt.subplots(figsize=(10, 6))
if chart_type == 'bar':
if len(columns) >= 2:
x_col, y_col = columns[0], columns[1]
data_subset = data[[x_col, y_col]].dropna()
ax.bar(data_subset[x_col], data_subset[y_col])
ax.set_xlabel(x_col)
ax.set_ylabel(y_col)
elif chart_type == 'line':
if len(columns) >= 2:
x_col, y_col = columns[0], columns[1]
data_subset = data[[x_col, y_col]].dropna()
ax.plot(data_subset[x_col], data_subset[y_col], marker='o')
ax.set_xlabel(x_col)
ax.set_ylabel(y_col)
elif chart_type == 'pie':
if len(columns) >= 1:
col = columns[0]
value_counts = data[col].value_counts()
ax.pie(value_counts.values, labels=value_counts.index, autopct='%1.1f%%')
elif chart_type == 'histogram':
if len(columns) >= 1:
col = columns[0]
if pd.api.types.is_numeric_dtype(data[col]):
ax.hist(data[col].dropna(), bins=20, alpha=0.7)
ax.set_xlabel(col)
ax.set_ylabel('频次')
ax.set_title(title)
plt.tight_layout()
# 转换为base64图片
buffer = BytesIO()
plt.savefig(buffer, format='png', dpi=150, bbox_inches='tight')
buffer.seek(0)
image_base64 = base64.b64encode(buffer.getvalue()).decode()
buffer.close()
plt.close()
charts.append({
'title': title,
'image': image_base64
})
except Exception as e:
self.logger.warning(f"生成图表 {chart_config.get('title', '未知')} 失败: {e}")
return charts
def create_interactive_table(self, data: pd.DataFrame, output_path: str, **kwargs) -> str:
"""创建交互式HTML表格"""
# 使用DataTables库创建交互式表格
kwargs['enable_datatables'] = True
kwargs['enable_filters'] = True
return self.create_table(data, output_path, **kwargs)
def create_dashboard(self, data_dict: Dict[str, pd.DataFrame],
output_path: str, **kwargs) -> str:
"""创建HTML仪表板"""
dashboard_template = """
{{ title }}
{{ title }}
{{ description }}
{% for metric in metrics %}
{{ metric.value }}
{{ metric.label }}
{% endfor %}
{% for table_name, table_data in tables.items() %}
{{ table_name }}
{% for column in table_data.columns %}
{{ column }}
{% endfor %}
{% for row in table_data.data %}
{% for cell in row %}
{{ cell }}
{% endfor %}
{% endfor %}
{% endfor %}
"""
try:
# 计算总体指标
total_rows = sum(len(df) for df in data_dict.values())
total_columns = sum(len(df.columns) for df in data_dict.values())
metrics = [
{'label': '数据表数量', 'value': len(data_dict)},
{'label': '总数据行数', 'value': f"{total_rows:,}"},
{'label': '总数据列数', 'value': total_columns},
{'label': '平均行数', 'value': f"{total_rows // len(data_dict) if data_dict else 0:,}"}
]
# 准备表格数据
tables = {}
for name, df in data_dict.items():
# 只显示前10行数据
display_df = df.head(10)
tables[name] = {
'columns': list(display_df.columns),
'data': display_df.values.tolist()
}
# 渲染模板
template = Template(dashboard_template)
html_content = template.render(
title=kwargs.get('title', '数据仪表板'),
description=kwargs.get('description', '数据概览和统计信息'),
metrics=metrics,
tables=tables
)
# 保存文件
os.makedirs(os.path.dirname(output_path), exist_ok=True)
with open(output_path, 'w', encoding='utf-8') as f:
f.write(html_content)
self.logger.info(f"HTML仪表板已生成: {output_path}")
return output_path
except Exception as e:
self.logger.error(f"生成HTML仪表板失败: {e}")
raise
批量处理与模板系统
为了提高效率，我们需要实现批量处理和模板系统。
批量处理框架
import concurrent.futures
import threading
from pathlib import Path
from typing import Dict, List, Any, Callable
import yaml
import json
from datetime import datetime
import shutil
class BatchTableProcessor:
"""批量表格处理器"""
def __init__(self, config_path: str = None):
self.config = self._load_config(config_path) if config_path else {}
self.logger = logging.getLogger(self.__class__.__name__)
self.results = []
self.errors = []
self.lock = threading.Lock()
# 初始化生成器
self.generators = {
'excel': ExcelTableGenerator(),
'word': WordTableGenerator(),
'html': HTMLTableGenerator(),
'pdf': None # 将在后面实现
}
def _load_config(self, config_path: str) -> Dict:
"""加载配置文件"""
try:
with open(config_path, 'r', encoding='utf-8') as f:
if config_path.endswith('.yaml') or config_path.endswith('.yml'):
return yaml.safe_load(f)
elif config_path.endswith('.json'):
return json.load(f)
else:
raise ValueError("不支持的配置文件格式")
except Exception as e:
self.logger.error(f"加载配置文件失败: {e}")
return {}
def process_batch(self, tasks: List[Dict], output_dir: str,
max_workers: int = 4) -> Dict:
"""
批量处理任务
Args:
tasks: 任务列表，每个任务包含数据源和输出配置
output_dir: 输出目录
max_workers: 最大并发工作线程数
Returns:
处理结果摘要
"""
os.makedirs(output_dir, exist_ok=True)
self.logger.info(f"开始批量处理 {len(tasks)} 个任务")
# 使用线程池并行处理
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
# 提交所有任务
future_to_task = {
executor.submit(self._process_single_task, task, output_dir): task
for task in tasks
}
# 处理结果
for future in concurrent.futures.as_completed(future_to_task):
task = future_to_task[future]
try:
result = future.result()
with self.lock:
self.results.append(result)
except Exception as e:
error_info = {
'task': task,
'error': str(e),
'timestamp': datetime.now().isoformat()
}
with self.lock:
self.errors.append(error_info)
self.logger.error(f"处理任务失败: {e}")
# 生成处理摘要
summary = self._generate_batch_summary(output_dir)
return summary
def _process_single_task(self, task: Dict, output_dir: str) -> Dict:
"""处理单个任务"""
try:
# 解析任务配置
data_source_config = task.get('data_source', {})
output_config = task.get('output', {})
# 加载数据
data = self._load_task_data(data_source_config)
# 生成表格
results = {}
for output_format, format_config in output_config.items():
if output_format in self.generators and self.generators[output_format]:
generator = self.generators[output_format]
# 构建输出文件路径
filename = format_config.get('filename', f'table_{datetime.now().strftime("%Y%m%d_%H%M%S")}')
if not filename.endswith(f'.{output_format}'):
filename += f'.{output_format}'
output_path = os.path.join(output_dir, filename)
# 生成表格
if output_format == 'html':
result_path = generator.create_table(data, output_path, **format_config)
else:
result_path = generator.create_table(data, output_path, **format_config)
results[output_format] = result_path
return {
'task_id': task.get('id', 'unknown'),
'status': 'success',
'data_rows': len(data),
'data_columns': len(data.columns),
'outputs': results,
'timestamp': datetime.now().isoformat()
}
except Exception as e:
self.logger.error(f"处理任务失败: {e}")
raise
def _load_task_data(self, data_source_config: Dict) -> pd.DataFrame:
"""加载任务数据"""
source_type = data_source_config.get('type')
if source_type == 'csv':
source = CSVDataSource(data_source_config)
elif source_type == 'json':
source = JSONDataSource(data_source_config)
elif source_type == 'database':
source = DatabaseDataSource(data_source_config)
else:
raise ValueError(f"不支持的数据源类型: {source_type}")
return source.get_data()
def _generate_batch_summary(self, output_dir: str) -> Dict:
"""生成批量处理摘要"""
summary = {
'processing_time': datetime.now().isoformat(),
'total_tasks': len(self.results) + len(self.errors),
'successful_tasks': len(self.results),
'failed_tasks': len(self.errors),
'success_rate': len(self.results) / (len(self.results) + len(self.errors)) * 100 if (len(self.results) + len(self.errors)) > 0 else 0,
'output_directory': output_dir,
'results': self.results,
'errors': self.errors
}
# 保存摘要文件
summary_path = os.path.join(output_dir, 'batch_summary.json')
with open(summary_path, 'w', encoding='utf-8') as f:
json.dump(summary, f, ensure_ascii=False, indent=2)
return summary
def create_batch_config_template(self, output_path: str):
"""创建批量处理配置模板"""
template_config = {
"batch_settings": {
"max_workers": 4,
"output_directory": "./output",
"log_level": "INFO"
},
"tasks": [
{
"id": "task_1",
"description": "示例任务1 - CSV数据生成Excel表格",
"data_source": {
"type": "csv",
"file_path": "data/sample.csv",
"encoding": "utf-8",
"required_columns": ["name", "value"]
},
"output": {
"excel": {
"filename": "sample_report.xlsx",
"title": "样本数据报告",
"add_charts": True,
"chart_configs": [
{
"type": "bar",
"title": "数值分布",
"columns": ["name", "value"]
}
]
},
"html": {
"filename": "sample_report.html",
"title": "样本数据报告",
"enable_datatables": True,
"add_statistics": True
}
}
},
{
"id": "task_2",
"description": "示例任务2 - 数据库查询生成Word文档",
"data_source": {
"type": "database",
"database": {
"type": "mysql",
"host": "localhost",
"port": 3306,
"username": "user",
"password": "password",
"database": "testdb"
},
"query": "SELECT * FROM sales_data WHERE date >= '2023-01-01'"
},
"output": {
"word": {
"filename": "sales_report.docx",
"title": "销售数据报告",
"add_statistics": True
}
}
}
]
}
os.makedirs(os.path.dirname(output_path), exist_ok=True)
with open(output_path, 'w', encoding='utf-8') as f:
yaml.dump(template_config, f, default_flow_style=False, allow_unicode=True)
self.logger.info(f"批量处理配置模板已生成: {output_path}")
模板系统实现
class TableTemplateManager:
"""表格模板管理器"""
def __init__(self, template_dir: str = "templates"):
self.template_dir = template_dir
self.logger = logging.getLogger(self.__class__.__name__)
os.makedirs(template_dir, exist_ok=True)
# 初始化Jinja2环境
self.jinja_env = Environment(
loader=FileSystemLoader(template_dir),
autoescape=True
)
def create_excel_template(self, template_name: str, config: Dict) -> str:
"""创建Excel模板"""
template_path = os.path.join(self.template_dir, 'excel_templates', f'{template_name}.xlsx')
os.makedirs(os.path.dirname(template_path), exist_ok=True)
# 创建模板工作簿
wb = Workbook()
ws = wb.active
ws.title = config.get('sheet_name', 'Data')
# 设置模板样式
header_style = config.get('header_style', {})
if header_style:
# 应用表头样式到第一行
for col in range(1, config.get('max_columns', 26) + 1):
cell = ws.cell(row=1, column=col)
if 'font' in header_style:
cell.font = Font(**header_style['font'])
if 'fill' in header_style:
cell.fill = PatternFill(**header_style['fill'])
# 保存模板
wb.save(template_path)
self.logger.info(f"Excel模板已创建: {template_path}")
return template_path
def create_html_template(self, template_name: str, template_content: str) -> str:
"""创建HTML模板"""
template_path = os.path.join(self.template_dir, 'html_templates', f'{template_name}.html')
os.makedirs(os.path.dirname(template_path), exist_ok=True)
with open(template_path, 'w', encoding='utf-8') as f:
f.write(template_content)
self.logger.info(f"HTML模板已创建: {template_path}")
return template_path
def render_template(self, template_name: str, data: Dict) -> str:
"""渲染模板"""
try:
template = self.jinja_env.get_template(template_name)
return template.render(**data)
except Exception as e:
self.logger.error(f"渲染模板失败: {e}")
raise
def list_templates(self) -> Dict[str, List[str]]:
"""列出所有模板"""
templates = {
'excel': [],
'html': [],
'word': []
}
for template_type in templates.keys():
template_dir = os.path.join(self.template_dir, f'{template_type}_templates')
if os.path.exists(template_dir):
for file in os.listdir(template_dir):
if file.endswith(('.xlsx', '.html', '.docx')):
templates[template_type].append(file)
return templates
实际应用案例
让我们通过几个实际案例来展示如何使用这个表格生成系统。
案例1：销售数据报告自动化
def generate_sales_report():
"""生成销售数据报告"""
# 1. 从数据库加载销售数据
# 1. 从数据库加载销售数据
db_config = {
'type': 'database',
'database': {
'type': 'mysql',
'host': 'localhost',
'port': 3306,
'username': 'sales_user',
'password': 'password',
'database': 'sales_db'
},
'query': '''
SELECT
DATE_FORMAT(sale_date, '%Y-%m') as month,
product_category,
SUM(amount) as total_sales,
COUNT(*) as transaction_count,
AVG(amount) as avg_transaction
FROM sales_transactions
WHERE sale_date >= DATE_SUB(NOW(), INTERVAL 12 MONTH)
GROUP BY DATE_FORMAT(sale_date, '%Y-%m'), product_category
ORDER BY month DESC, total_sales DESC
'''
}
try:
# 加载数据
data_source = DatabaseDataSource(db_config)
sales_data = data_source.get_data()
# 2. 生成Excel报告
excel_generator = ExcelTableGenerator()
excel_path = excel_generator.create_table(
sales_data,
'reports/sales_report.xlsx',
title='月度销售数据报告',
enable_table=True,
add_charts=True,
chart_configs=[
{
'type': 'bar',
'title': '各类别销售额',
'columns': ['product_category', 'total_sales'],
'position': 'H2'
},
{
'type': 'line',
'title': '月度销售趋势',
'columns': ['month', 'total_sales'],
'position': 'H20'
}
]
)
# 3. 生成Word报告
word_generator = WordTableGenerator()
word_path = word_generator.create_table(
sales_data,
'reports/sales_report.docx',
title='销售数据分析报告',
subtitle='过去12个月销售业绩总结',
description='本报告展示了过去12个月各产品类别的销售表现，包括销售额、交易数量和平均交易金额等关键指标。',
add_statistics=True
)
# 4. 生成HTML仪表板
html_generator = HTMLTableGenerator()
html_path = html_generator.create_table(
sales_data,
'reports/sales_dashboard.html',
title='销售数据仪表板',
description='实时销售数据监控和分析',
enable_datatables=True,
add_statistics=True,
add_charts=True,
chart_configs=[
{
'type': 'bar',
'title': '产品类别销售对比',
'columns': ['product_category', 'total_sales']
},
{
'type': 'pie',
'title': '销售额分布',
'columns': ['product_category']
}
]
)
print(f"销售报告已生成:")
print(f"- Excel报告: {excel_path}")
print(f"- Word报告: {word_path}")
print(f"- HTML仪表板: {html_path}")
except Exception as e:
print(f"生成销售报告失败: {e}")
# 使用示例
if __name__ == "__main__":
generate_sales_report()
案例2：学生成绩统计系统
def generate_student_grade_reports():
"""生成学生成绩统计报告"""
# 1. 从CSV文件加载成绩数据
csv_config = {
'type': 'csv',
'file_path': 'data/student_grades.csv',
'encoding': 'utf-8',
'required_columns': ['student_id', 'student_name', 'subject', 'score'],
'column_types': {
'student_id': 'str',
'score': 'float'
}
}
try:
# 加载原始数据
data_source = CSVDataSource(csv_config)
raw_data = data_source.get_data()
# 2. 数据处理和分析
# 计算每个学生的总分和平均分
student_summary = raw_data.groupby(['student_id', 'student_name']).agg({
'score': ['sum', 'mean', 'count']
}).round(2)
student_summary.columns = ['总分', '平均分', '科目数']
student_summary = student_summary.reset_index()
# 计算每个科目的统计信息
subject_summary = raw_data.groupby('subject').agg({
'score': ['mean', 'max', 'min', 'std', 'count']
}).round(2)
subject_summary.columns = ['平均分', '最高分', '最低分', '标准差', '学生数']
subject_summary = subject_summary.reset_index()
# 3. 生成多工作表Excel报告
excel_generator = AdvancedExcelGenerator()
data_dict = {
'学生成绩汇总': student_summary,
'科目统计': subject_summary,
'原始数据': raw_data
}
excel_path = excel_generator.create_multi_sheet_workbook(
data_dict,
'reports/grade_analysis.xlsx',
title='学生成绩分析报告',
auto_fit_columns=True,
enable_table=True
)
# 添加条件格式
excel_generator.add_conditional_formatting(
student_summary,
[
{
'type': 'color_scale',
'column': '平均分',
'start_color': 'FF0000',
'end_color': '00FF00'
},
{
'type': 'data_bar',
'column': '总分',
'color': '0066CC'
}
]
)
# 4. 生成个人成绩单
for _, student in student_summary.iterrows():
student_id = student['student_id']
student_name = student['student_name']
# 获取该学生的详细成绩
student_data = raw_data[raw_data['student_id'] == student_id][['subject', 'score']]
# 生成个人Word成绩单
word_generator = WordTableGenerator()
word_path = word_generator.create_table(
student_data,
f'reports/individual/{student_id}_{student_name}_成绩单.docx',
title=f'{student_name} 成绩单',
subtitle=f'学号: {student_id}',
description=f'总分: {student["总分"]}, 平均分: {student["平均分"]:.2f}',
add_statistics=False
)
# 5. 生成班级HTML报告
html_generator = HTMLTableGenerator()
html_dashboard = html_generator.create_dashboard(
{
'学生成绩排名': student_summary.sort_values('平均分', ascending=False).head(20),
'科目难度分析': subject_summary.sort_values('平均分')
},
'reports/class_dashboard.html',
title='班级成绩分析仪表板',
description='全班成绩概览和统计分析'
)
print("学生成绩报告已生成:")
print(f"- 综合Excel报告: {excel_path}")
print(f"- 个人成绩单: reports/individual/")
print(f"- 班级HTML仪表板: {html_dashboard}")
except Exception as e:
print(f"生成成绩报告失败: {e}")
案例3：财务数据批量处理
def batch_process_financial_data():
"""批量处理财务数据"""
# 1. 创建批量处理配置
batch_config = {
"batch_settings": {
"max_workers": 6,
"output_directory": "./financial_reports",
"log_level": "INFO"
},
"tasks": [
{
"id": "monthly_revenue",
"description": "月度收入报告",
"data_source": {
"type": "database",
"database": {
"type": "mysql",
"host": "finance-db.company.com",
"port": 3306,
"username": "finance_user",
"password": "secure_password",
"database": "finance_db"
},
"query": """
SELECT
DATE_FORMAT(transaction_date, '%Y-%m') as month,
department,
SUM(CASE WHEN type = 'income' THEN amount ELSE 0 END) as revenue,
SUM(CASE WHEN type = 'expense' THEN amount ELSE 0 END) as expenses,
SUM(CASE WHEN type = 'income' THEN amount ELSE -amount END) as net_income
FROM financial_transactions
WHERE transaction_date >= DATE_SUB(NOW(), INTERVAL 12 MONTH)
GROUP BY DATE_FORMAT(transaction_date, '%Y-%m'), department
ORDER BY month DESC, net_income DESC
"""
},
"output": {
"excel": {
"filename": "monthly_revenue_report.xlsx",
"title": "月度收入分析报告",
"add_charts": True,
"chart_configs": [
{
"type": "line",
"title": "月度收入趋势",
"columns": ["month", "revenue"]
},
{
"type": "bar",
"title": "部门收入对比",
"columns": ["department", "net_income"]
}
]
},
"html": {
"filename": "monthly_revenue_dashboard.html",
"title": "月度收入仪表板",
"enable_datatables": True,
"add_statistics": True
}
}
},
{
"id": "expense_analysis",
"description": "费用分析报告",
"data_source": {
"type": "csv",
"file_path": "data/expense_data.csv",
"encoding": "utf-8",
"filters": [
{
"column": "amount",
"operator": ">",
"value": 0
}
]
},
"output": {
"word": {
"filename": "expense_analysis_report.docx",
"title": "费用分析报告",
"subtitle": "详细费用统计和分析",
"add_statistics": True
},
"excel": {
"filename": "expense_analysis.xlsx",
"title": "费用明细表",
"enable_table": True,
"auto_fit_columns": True
}
}
}
]
}
# 2. 执行批量处理
processor = BatchTableProcessor()
summary = processor.process_batch(
batch_config['tasks'],
batch_config['batch_settings']['output_directory'],
max_workers=batch_config['batch_settings']['max_workers']
)
# 3. 输出处理结果
print("批量处理完成:")
print(f"- 总任务数: {summary['total_tasks']}")
print(f"- 成功任务: {summary['successful_tasks']}")
print(f"- 失败任务: {summary['failed_tasks']}")
print(f"- 成功率: {summary['success_rate']:.1f}%")
# 4. 生成汇总报告
if summary['successful_tasks'] > 0:
# 创建处理结果汇总
results_df = pd.DataFrame([
{
'任务ID': result['task_id'],
'数据行数': result['data_rows'],
'数据列数': result['data_columns'],
'输出格式': ', '.join(result['outputs'].keys()),
'处理时间': result['timestamp']
}
for result in summary['results']
])
# 生成汇总Excel报告
excel_generator = ExcelTableGenerator()
summary_path = excel_generator.create_table(
results_df,
os.path.join(summary['output_directory'], 'batch_processing_summary.xlsx'),
title='批量处理结果汇总',
enable_table=True,
auto_fit_columns=True
)
print(f"- 汇总报告: {summary_path}")
性能优化与大数据处理
当处理大量数据时，性能优化变得至关重要。
内存优化策略
class OptimizedTableGenerator:
"""优化的表格生成器"""
def __init__(self, chunk_size: int = 10000):
self.chunk_size = chunk_size
self.logger = logging.getLogger(self.__class__.__name__)
def process_large_dataset(self, data_source: DataSource,
output_path: str, **kwargs):
"""处理大数据集"""
# 1. 分块读取数据
total_rows = 0
processed_chunks = 0
# 创建输出文件
if output_path.endswith('.xlsx'):
workbook = Workbook()
worksheet = workbook.active
current_row = 1
elif output_path.endswith('.csv'):
output_file = open(output_path, 'w', newline='', encoding='utf-8')
csv_writer = None
try:
# 分块处理数据
for chunk in self._read_data_chunks(data_source):
if chunk.empty:
continue
# 处理数据块
processed_chunk = self._process_chunk(chunk, **kwargs)
# 写入输出文件
if output_path.endswith('.xlsx'):
current_row = self._write_chunk_to_excel(
processed_chunk, worksheet, current_row,
include_header=(processed_chunks == 0)
)
elif output_path.endswith('.csv'):
if csv_writer is None:
csv_writer = csv.writer(output_file)
csv_writer.writerow(processed_chunk.columns)
for _, row in processed_chunk.iterrows():
csv_writer.writerow(row.values)
total_rows += len(processed_chunk)
processed_chunks += 1
# 内存清理
del processed_chunk
import gc
gc.collect()
self.logger.info(f"已处理 {processed_chunks} 个数据块，共 {total_rows} 行")
finally:
# 保存并关闭文件
if output_path.endswith('.xlsx'):
workbook.save(output_path)
elif output_path.endswith('.csv'):
output_file.close()
self.logger.info(f"大数据集处理完成: {output_path}, 总行数: {total_rows}")
return output_path
def _read_data_chunks(self, data_source: DataSource):
"""分块读取数据"""
if hasattr(data_source, 'read_chunks'):
# 如果数据源支持分块读取
yield from data_source.read_chunks(self.chunk_size)
else:
# 否则一次性读取后分块
data = data_source.get_data()
for i in range(0, len(data), self.chunk_size):
yield data.iloc[i:i + self.chunk_size]
def _process_chunk(self, chunk: pd.DataFrame, **kwargs) -> pd.DataFrame:
"""处理数据块"""
# 应用数据处理逻辑
if kwargs.get('sort_columns'):
chunk = chunk.sort_values(kwargs['sort_columns'])
if kwargs.get('filter_conditions'):
for condition in kwargs['filter_conditions']:
chunk = chunk.query(condition)
return chunk
def _write_chunk_to_excel(self, chunk: pd.DataFrame, worksheet,
start_row: int, include_header: bool = False) -> int:
"""将数据块写入Excel"""
current_row = start_row
# 写入表头
if include_header:
for col_idx, column_name in enumerate(chunk.columns, 1):
worksheet.cell(row=current_row, column=col_idx, value=column_name)
current_row += 1
# 写入数据
for _, row in chunk.iterrows():
for col_idx, value in enumerate(row.values, 1):
worksheet.cell(row=current_row, column=col_idx, value=value)
current_row += 1
return current_row
并行处理优化
import multiprocessing as mp
from functools import partial
class ParallelTableProcessor:
"""并行表格处理器"""
def __init__(self, n_processes: int = None):
self.n_processes = n_processes or mp.cpu_count()
self.logger = logging.getLogger(self.__class__.__name__)
def parallel_process_files(self, file_list: List[str],
process_func: Callable,
output_dir: str, **kwargs):
"""并行处理多个文件"""
os.makedirs(output_dir, exist_ok=True)
# 创建处理函数的偏函数
process_partial = partial(
self._process_single_file,
process_func=process_func,
output_dir=output_dir,
**kwargs
)
# 使用进程池并行处理
with mp.Pool(processes=self.n_processes) as pool:
results = pool.map(process_partial, file_list)
# 过滤成功的结果
successful_results = [r for r in results if r['success']]
failed_results = [r for r in results if not r['success']]
self.logger.info(f"并行处理完成: 成功 {len(successful_results)}, 失败 {len(failed_results)}")
return {
'successful': successful_results,
'failed': failed_results,
'total': len(file_list)
}
def _process_single_file(self, file_path: str, process_func: Callable,
output_dir: str, **kwargs):
"""处理单个文件"""
try:
result = process_func(file_path, output_dir, **kwargs)
return {
'file_path': file_path,
'result': result,
'success': True,
'error': None
}
except Exception as e:
return {
'file_path': file_path,
'result': None,
'success': False,
'error': str(e)
}
最佳实践与注意事项
数据安全与隐私保护
class SecureTableGenerator:
"""安全的表格生成器"""
def __init__(self):
self.logger = logging.getLogger(self.__class__.__name__)
self.sensitive_columns = set()
def add_sensitive_column(self, column_name: str):
"""添加敏感数据列"""
self.sensitive_columns.add(column_name)
def anonymize_data(self, data: pd.DataFrame) -> pd.DataFrame:
"""数据脱敏处理"""
anonymized_data = data.copy()
for column in self.sensitive_columns:
if column in anonymized_data.columns:
if anonymized_data[column].dtype == 'object':
# 文本数据脱敏
anonymized_data[column] = anonymized_data[column].apply(
lambda x: self._mask_text(str(x)) if pd.notna(x) else x
)
else:
# 数值数据脱敏
anonymized_data[column] = anonymized_data[column].apply(
lambda x: self._mask_number(x) if pd.notna(x) else x
)
return anonymized_data
def _mask_text(self, text: str) -> str:
"""文本脱敏"""
if len(text) <= 2:
return '*' * len(text)
return text[0] + '*' * (len(text) - 2) + text[-1]
def _mask_number(self, number) -> str:
"""数值脱敏"""
str_num = str(number)
if len(str_num) <= 2:
return '*' * len(str_num)
return str_num[0] + '*' * (len(str_num) - 2) + str_num[-1]
def create_secure_table(self, data: pd.DataFrame, output_path: str,
anonymize: bool = True, **kwargs):
"""创建安全的表格"""
if anonymize:
data = self.anonymize_data(data)
# 添加水印或标识
if kwargs.get('add_watermark', True):
kwargs['title'] = f"{kwargs.get('title', '数据表格')} [机密]"
kwargs['description'] = f"{kwargs.get('description', '')} \n注意：本报告包含敏感信息，请妥善保管。"
# 根据文件类型选择生成器
if output_path.endswith('.xlsx'):
generator = ExcelTableGenerator()
elif output_path.endswith('.docx'):
generator = WordTableGenerator()
elif output_path.endswith('.html'):
generator = HTMLTableGenerator()
else:
raise ValueError("不支持的文件格式")
return generator.create_table(data, output_path, **kwargs)
错误处理与日志记录
import logging
from functools import wraps
import traceback
def error_handler(func):
"""错误处理装饰器"""
@wraps(func)
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except Exception as e:
logger = logging.getLogger(func.__module__)
logger.error(f"函数 {func.__name__} 执行失败: {e}")
logger.error(f"错误详情: {traceback.format_exc()}")
raise
return wrapper
class TableGeneratorLogger:
"""表格生成器日志管理"""
@staticmethod
def setup_logging(log_file: str = 'table_generator.log',
log_level: str = 'INFO'):
"""设置日志配置"""
logging.basicConfig(
level=getattr(logging, log_level.upper()),
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
handlers=[
logging.FileHandler(log_file, encoding='utf-8'),
logging.StreamHandler()
]
)
@staticmethod
def log_performance(func):
"""性能日志装饰器"""
@wraps(func)
def wrapper(*args, **kwargs):
import time
start_time = time.time()
logger = logging.getLogger(func.__module__)
logger.info(f"开始执行 {func.__name__}")
result = func(*args, **kwargs)
execution_time = time.time() - start_time
logger.info(f"{func.__name__} 执行完成，耗时: {execution_time:.2f}秒")
return result
return wrapper
总结与展望
通过本文的详细介绍，我们构建了一个完整的数据自动生成表格系统，具备以下核心能力：
主要功能特性
-
多数据源支持
- CSV文件、JSON数据、数据库查询
- API接口数据、Excel文件等
- 灵活的数据预处理和过滤机制
-
多格式输出
- Excel表格（支持图表、条件格式、数据验证）
- Word文档表格（支持样式定制、统计信息）
- HTML表格（支持交互功能、响应式设计）
- PDF报告（适合正式文档）
-
高级功能
- 批量处理和并行计算
- 模板系统和样式管理
- 数据可视化和图表生成
- 条件格式和数据验证
-
企业级特性
- 数据安全和隐私保护
- 错误处理和日志记录
- 性能优化和大数据处理
- 配置管理和部署支持
技术优势
- 模块化架构：各组件独立，易于维护和扩展
- 高性能处理：支持大数据集的分块处理和并行计算
- 灵活配置：通过配置文件实现个性化定制
- 安全可靠：完善的错误处理和数据安全机制
实际应用价值
- 提高效率：自动化报表生成，节省大量人工时间
- 保证质量：统一的格式标准，减少人为错误
- 降低成本：减少重复性工作，提高资源利用率
- 增强分析：集成数据可视化，提供更好的洞察
未来发展方向
-
AI集成
- 智能数据分析和异常检测
- 自动生成数据洞察和建议
- 自然语言查询和报告生成
-
云原生支持
- 微服务架构改造
- 容器化部署
- 云存储和计算集成
-
实时处理
- 流数据处理支持
- 实时报表更新
- 事件驱动的报告生成
-
更多集成
- BI工具集成
- 企业系统对接
- 移动端支持
最佳实践建议
-
设计阶段
- 明确需求和输出格式
- 设计合理的数据模型
- 考虑性能和扩展性
-
开发阶段
- 遵循模块化原则
- 实施完善的测试
- 注重代码质量和文档
-
部署阶段
- 配置合适的运行环境
- 设置监控和日志
- 建立备份和恢复机制
-
维护阶段
- 定期更新和优化
- 监控性能指标
- 收集用户反馈
这个数据自动生成表格系统为现代数据处理和报告生成提供了强大而灵活的解决方案。通过合理的架构设计和丰富的功能特性，它能够满足从个人使用到企业级应用的各种需求，为数据驱动的决策提供有力支持。
希望本文能够帮助您在数据表格自动化方面取得更好的成果，提高工作效率，并为进一步的技术创新奠定基础。随着技术的不断发展，这个系统还有很大的扩展空间，期待在实际应用中不断完善和优化。
Python实现数据自动生成表格：从数据源到可视化表格的完整解决方案
在现代数据处理和报告生成中，将原始数据转换为结构化、美观的表格是一个常见且重要的需求。无论是生成Excel报表、Word文档中的表格，还是HTML网页表格，自动化的表格生成能够大大提高工作效率，减少人工错误，并确保数据展示的一致性。本文将深入探讨如何使用Python实现数据自动生成表格的完整解决方案，涵盖多种数据源、多种输出格式，以及高级的表格样式和交互功能。
目录
- 数据自动生成表格概述
- 技术栈与环境准备
- 数据源处理与标准化
- Excel表格自动生成
- Word文档表格生成
- HTML表格生成与Web展示
- PDF表格生成
- 高级表格样式与格式化
- 动态表格与交互功能
- 批量处理与模板系统
- 性能优化与大数据处理
- 实际应用案例
- 最佳实践与注意事项
- 总结与展望
数据自动生成表格概述
什么是数据自动生成表格
数据自动生成表格是指通过编程方式，将各种格式的原始数据（如CSV、JSON、数据库查询结果等）自动转换为格式化的表格文档。这个过程包括数据读取、处理、格式化、样式应用和输出等多个步骤。
核心优势
- 效率提升：自动化处理大量数据，避免手工制表的繁琐工作
- 一致性保证：确保所有表格遵循统一的格式和样式标准
- 错误减少：消除人工操作中的计算错误和格式错误
- 可重复性：相同的数据处理逻辑可以重复应用于不同的数据集
- 灵活性：支持多种数据源和输出格式，适应不同的业务需求
应用场景
- 财务报表生成：自动生成月度、季度、年度财务报表
- 销售数据分析：生成销售业绩表格和趋势分析
- 库存管理报告：自动更新库存状态表格
- 学生成绩统计：批量生成成绩单和统计报表
- 项目进度跟踪：生成项目状态和里程碑表格
- 数据分析报告：将分析结果以表格形式展示
技术栈与环境准备
核心Python库
我们将使用以下Python库来实现完整的表格生成功能：
# 数据处理
pip install pandas numpy
# Excel处理
pip install openpyxl xlsxwriter
# Word文档处理
pip install python-docx
# PDF生成
pip install reportlab
# HTML生成
pip install jinja2 beautifulsoup4
# 数据库连接
pip install sqlalchemy pymysql psycopg2-binary
# 图表生成
pip install matplotlib seaborn plotly
# 进度条和日志
pip install tqdm loguru
# 配置文件处理
pip install pyyaml configparser
项目结构设计
auto_table_generator/
│
├── core/
│ ├── __init__.py
│ ├── data_processor.py # 数据处理核心
│ ├── table_generator.py # 表格生成器基类
│ └── style_manager.py # 样式管理器
│
├── generators/
│ ├── __init__.py
│ ├── excel_generator.py # Excel表格生成器
│ ├── word_generator.py # Word表格生成器
│ ├── html_generator.py # HTML表格生成器
│ └── pdf_generator.py # PDF表格生成器
│
├── data_sources/
│ ├── __init__.py
│ ├── csv_source.py # CSV数据源
│ ├── json_source.py # JSON数据源
│ ├── database_source.py # 数据库数据源
│ └── api_source.py # API数据源
│
├── templates/
│ ├── excel_templates/ # Excel模板
│ ├── word_templates/ # Word模板
│ └── html_templates/ # HTML模板
│
├── config/
│ ├── settings.yaml # 配置文件
│ └── styles.yaml # 样式配置
│
├── examples/
│ ├── basic_usage.py # 基础使用示例
│ ├── advanced_features.py # 高级功能示例
│ └── batch_processing.py # 批量处理示例
│
└── tests/
├── test_generators.py # 生成器测试
└── test_data_sources.py # 数据源测试
环境验证
import sys
import pandas as pd
import openpyxl
import docx
from reportlab.lib import colors
import jinja2
def verify_environment():
"""验证开发环境是否正确配置"""
print("=== 环境验证 ===")
print(f"Python版本: {sys.version}")
print(f"Pandas版本: {pd.__version__}")
print(f"OpenPyXL版本: {openpyxl.__version__}")
print(f"Python-docx版本: {docx.__version__}")
print(f"Jinja2版本: {jinja2.__version__}")
# 测试基本功能
try:
# 测试pandas
df = pd.DataFrame({'A': [1, 2], 'B': [3, 4]})
print("✓ Pandas工作正常")
# 测试openpyxl
from openpyxl import Workbook
wb = Workbook()
print("✓ OpenPyXL工作正常")
# 测试python-docx
from docx import Document
doc = Document()
print("✓ Python-docx工作正常")
print("✓ 所有依赖库验证通过")
return True
except Exception as e:
print(f"✗ 环境验证失败: {e}")
return False
if __name__ == "__main__":
verify_environment()
数据源处理与标准化
在生成表格之前，我们需要处理各种不同的数据源，并将它们标准化为统一的格式。
数据源基类设计
from abc import ABC, abstractmethod
import pandas as pd
from typing import Dict, Any, Optional, List
import logging
class DataSource(ABC):
"""数据源基类"""
def __init__(self, config: Dict[str, Any]):
self.config = config
self.logger = logging.getLogger(self.__class__.__name__)
self._data = None
self._metadata = {}
@abstractmethod
def load_data(self) -> pd.DataFrame:
"""加载数据的抽象方法"""
pass
@abstractmethod
def validate_data(self, data: pd.DataFrame) -> bool:
"""验证数据的抽象方法"""
pass
def get_data(self, force_reload: bool = False) -> pd.DataFrame:
"""获取数据，支持缓存"""
if self._data is None or force_reload:
self._data = self.load_data()
if not self.validate_data(self._data):
raise ValueError("数据验证失败")
return self._data.copy()
def get_metadata(self) -> Dict[str, Any]:
"""获取数据元信息"""
return self._metadata.copy()
def preprocess_data(self, data: pd.DataFrame) -> pd.DataFrame:
"""数据预处理"""
# 处理缺失值
if self.config.get('fill_na', False):
fill_value = self.config.get('fill_na_value', '')
data = data.fillna(fill_value)
# 数据类型转换
if 'column_types' in self.config:
for col, dtype in self.config['column_types'].items():
if col in data.columns:
try:
data[col] = data[col].astype(dtype)
except Exception as e:
self.logger.warning(f"无法转换列 {col} 的数据类型: {e}")
# 列重命名
if 'column_mapping' in self.config:
data = data.rename(columns=self.config['column_mapping'])
# 数据过滤
if 'filters' in self.config:
for filter_config in self.config['filters']:
data = self._apply_filter(data, filter_config)
return data
def _apply_filter(self, data: pd.DataFrame, filter_config: Dict) -> pd.DataFrame:
"""应用数据过滤器"""
column = filter_config.get('column')
operator = filter_config.get('operator', '==')
value = filter_config.get('value')
if column not in data.columns:
self.logger.warning(f"过滤列 {column} 不存在")
return data
try:
if operator == '==':
return data[data[column] == value]
elif operator == '!=':
return data[data[column] != value]
elif operator == '>':
return data[data[column] > value]
elif operator == '<':
return data[data[column] < value]
elif operator == '>=':
return data[data[column] >= value]
elif operator == '<=':
return data[data[column] <= value]
elif operator == 'in':
return data[data[column].isin(value)]
elif operator == 'contains':
return data[data[column].str.contains(value, na=False)]
else:
self.logger.warning(f"不支持的过滤操作符: {operator}")
return data
except Exception as e:
self.logger.error(f"应用过滤器时出错: {e}")
return data
CSV数据源实现
import pandas as pd
import os
from typing import Dict, Any
class CSVDataSource(DataSource):
"""CSV数据源"""
def load_data(self) -> pd.DataFrame:
"""从CSV文件加载数据"""
file_path = self.config.get('file_path')
if not file_path or not os.path.exists(file_path):
raise FileNotFoundError(f"CSV文件不存在: {file_path}")
# CSV读取参数
csv_params = {
'encoding': self.config.get('encoding', 'utf-8'),
'sep': self.config.get('separator', ','),
'header': self.config.get('header', 0),
'skiprows': self.config.get('skip_rows', None),
'nrows': self.config.get('max_rows', None)
}
try:
data = pd.read_csv(file_path, **csv_params)
self.logger.info(f"成功加载CSV文件: {file_path}, 数据形状: {data.shape}")
# 更新元数据
self._metadata = {
'source_type': 'csv',
'file_path': file_path,
'file_size': os.path.getsize(file_path),
'rows': len(data),
'columns': len(data.columns),
'column_names': list(data.columns)
}
return self.preprocess_data(data)
except Exception as e:
self.logger.error(f"加载CSV文件失败: {e}")
raise
def validate_data(self, data: pd.DataFrame) -> bool:
"""验证CSV数据"""
if data.empty:
self.logger.error("CSV数据为空")
return False
# 检查必需列
required_columns = self.config.get('required_columns', [])
missing_columns = set(required_columns) - set(data.columns)
if missing_columns:
self.logger.error(f"缺少必需列: {missing_columns}")
return False
# 检查数据类型
if 'column_types' in self.config:
for col, expected_type in self.config['column_types'].items():
if col in data.columns:
try:
pd.api.types.is_dtype_equal(data[col].dtype, expected_type)
except Exception:
self.logger.warning(f"列 {col} 的数据类型可能不匹配")
return True
JSON数据源实现
import json
import pandas as pd
from typing import Dict, Any, List, Union
class JSONDataSource(DataSource):
"""JSON数据源"""
def load_data(self) -> pd.DataFrame:
"""从JSON文件或API加载数据"""
if 'file_path' in self.config:
return self._load_from_file()
elif 'url' in self.config:
return self._load_from_api()
else:
raise ValueError("必须指定file_path或url")
def _load_from_file(self) -> pd.DataFrame:
"""从JSON文件加载数据"""
file_path = self.config['file_path']
if not os.path.exists(file_path):
raise FileNotFoundError(f"JSON文件不存在: {file_path}")
try:
with open(file_path, 'r', encoding=self.config.get('encoding', 'utf-8')) as f:
json_data = json.load(f)
data = self._json_to_dataframe(json_data)
self._metadata = {
'source_type': 'json_file',
'file_path': file_path,
'file_size': os.path.getsize(file_path),
'rows': len(data),
'columns': len(data.columns)
}
return self.preprocess_data(data)
except Exception as e:
self.logger.error(f"加载JSON文件失败: {e}")
raise
def _load_from_api(self) -> pd.DataFrame:
"""从API加载JSON数据"""
import requests
url = self.config['url']
headers = self.config.get('headers', {})
params = self.config.get('params', {})
timeout = self.config.get('timeout', 30)
try:
response = requests.get(url, headers=headers, params=params, timeout=timeout)
response.raise_for_status()
json_data = response.json()
data = self._json_to_dataframe(json_data)
self._metadata = {
'source_type': 'json_api',
'url': url,
'status_code': response.status_code,
'rows': len(data),
'columns': len(data.columns)
}
return self.preprocess_data(data)
except Exception as e:
self.logger.error(f"从API加载JSON数据失败: {e}")
raise
def _json_to_dataframe(self, json_data: Union[Dict, List]) -> pd.DataFrame:
"""将JSON数据转换为DataFrame"""
data_path = self.config.get('data_path', None)
# 如果指定了数据路径，提取嵌套数据
if data_path:
current_data = json_data
for key in data_path.split('.'):
if isinstance(current_data, dict) and key in current_data:
current_data = current_data[key]
else:
raise ValueError(f"无法找到数据路径: {data_path}")
json_data = current_data
# 转换为DataFrame
if isinstance(json_data, list):
return pd.DataFrame(json_data)
elif isinstance(json_data, dict):
# 如果是字典，尝试转换为记录列表
if all(isinstance(v, (list, tuple)) for v in json_data.values()):
return pd.DataFrame(json_data)
else:
# 单条记录
return pd.DataFrame([json_data])
else:
raise ValueError("不支持的JSON数据格式")
def validate_data(self, data: pd.DataFrame) -> bool:
"""验证JSON数据"""
if data.empty:
self.logger.error("JSON数据为空")
return False
return True
数据库数据源实现
import pandas as pd
from sqlalchemy import create_engine, text
from typing import Dict, Any
class DatabaseDataSource(DataSource):
"""数据库数据源"""
def __init__(self, config: Dict[str, Any]):
super().__init__(config)
self.engine = None
self._create_engine()
def _create_engine(self):
"""创建数据库引擎"""
db_config = self.config.get('database', {})
# 构建连接字符串
db_type = db_config.get('type', 'mysql')
host = db_config.get('host', 'localhost')
port = db_config.get('port', 3306)
username = db_config.get('username')
password = db_config.get('password')
database = db_config.get('database')
if db_type == 'mysql':
connection_string = f"mysql+pymysql://{username}:{password}@{host}:{port}/{database}"
elif db_type == 'postgresql':
connection_string = f"postgresql+psycopg2://{username}:{password}@{host}:{port}/{database}"
elif db_type == 'sqlite':
connection_string = f"sqlite:///{database}"
else:
raise ValueError(f"不支持的数据库类型: {db_type}")
try:
self.engine = create_engine(
connection_string,
pool_pre_ping=True,
pool_recycle=3600
)
self.logger.info(f"成功连接到数据库: {db_type}")
except Exception as e:
self.logger.error(f"数据库连接失败: {e}")
raise
def load_data(self) -> pd.DataFrame:
"""从数据库加载数据"""
query = self.config.get('query')
table_name = self.config.get('table_name')
if query:
return self._load_from_query(query)
elif table_name:
return self._load_from_table(table_name)
else:
raise ValueError("必须指定query或table_name")
def _load_from_query(self, query: str) -> pd.DataFrame:
"""从SQL查询加载数据"""
try:
# 支持参数化查询
query_params = self.config.get('query_params', {})
data = pd.read_sql(
text(query),
self.engine,
params=query_params
)
self.logger.info(f"成功执行查询，返回 {len(data)} 行数据")
self._metadata = {
'source_type': 'database_query',
'query': query,
'rows': len(data),
'columns': len(data.columns)
}
return self.preprocess_data(data)
except Exception as e:
self.logger.error(f"执行数据库查询失败: {e}")
raise
def _load_from_table(self, table_name: str) -> pd.DataFrame:
"""从数据表加载数据"""
try:
# 构建查询
columns = self.config.get('columns', '*')
if isinstance(columns, list):
columns = ', '.join(columns)
where_clause = self.config.get('where_clause', '')
order_by = self.config.get('order_by', '')
limit = self.config.get('limit', '')
query = f"SELECT {columns} FROM {table_name}"
if where_clause:
query += f" WHERE {where_clause}"
if order_by:
query += f" ORDER BY {order_by}"
if limit:
query += f" LIMIT {limit}"
return self._load_from_query(query)
except Exception as e:
self.logger.error(f"从数据表加载数据失败: {e}")
raise
def validate_data(self, data: pd.DataFrame) -> bool:
"""验证数据库数据"""
if data.empty:
self.logger.warning("数据库查询返回空结果")
return True # 空结果可能是正常的
return True
def __del__(self):
"""清理数据库连接"""
if self.engine:
self.engine.dispose()
Excel表格自动生成
Excel是最常用的表格格式之一，我们将实现一个功能强大的Excel表格生成器。
Excel生成器核心实现
import pandas as pd
from openpyxl import Workbook, load_workbook
from openpyxl.styles import Font, PatternFill, Border, Side, Alignment
from openpyxl.utils.dataframe import dataframe_to_rows
from openpyxl.chart import BarChart, LineChart, PieChart, Reference
from openpyxl.worksheet.table import Table, TableStyleInfo
import os
from typing import Dict, Any, List, Optional, Union
import logging
class ExcelTableGenerator:
"""Excel表格生成器"""
def __init__(self, config: Dict[str, Any] = None):
self.config = config or {}
self.logger = logging.getLogger(self.__class__.__name__)
self.workbook = None
self.worksheet = None
# 默认样式配置
self.default_styles = {
'header': {
'font': Font(bold=True, color='FFFFFF'),
'fill': PatternFill(start_color='366092', end_color='366092', fill_type='solid'),
'alignment': Alignment(horizontal='center', vertical='center')
},
'data': {
'font': Font(color='000000'),
'alignment': Alignment(horizontal='left', vertical='center')
},
'number': {
'font': Font(color='000000'),
'alignment': Alignment(horizontal='right', vertical='center'),
'number_format': '#,##0.00'
},
'border': Border(
left=Side(style='thin'),
right=Side(style='thin'),
top=Side(style='thin'),
bottom=Side(style='thin')
)
}
def create_table(self, data: pd.DataFrame, output_path: str,
sheet_name: str = 'Sheet1', **kwargs) -> str:
"""
创建Excel表格
Args:
data: 数据DataFrame
output_path: 输出文件路径
sheet_name: 工作表名称
**kwargs: 其他配置参数
Returns:
生成的文件路径
"""
try:
# 创建工作簿
self.workbook = Workbook()
self.worksheet = self.workbook.active
self.worksheet.title = sheet_name
# 写入数据
self._write_data(data, **kwargs)
# 应用样式
self._apply_styles(data, **kwargs)
# 添加表格功能
if kwargs.get('enable_table', True):
self._create_excel_table(data, **kwargs)
# 添加图表
if kwargs.get('add_charts', False):
self._add_charts(data, **kwargs)
# 自动调整列宽
if kwargs.get('auto_fit_columns', True):
self._auto_fit_columns()
# 保存文件
os.makedirs(os.path.dirname(output_path), exist_ok=True)
self.workbook.save(output_path)
self.logger.info(f"Excel表格已生成: {output_path}")
return output_path
except Exception as e:
self.logger.error(f"生成Excel表格失败: {e}")
raise
def _write_data(self, data: pd.DataFrame, start_row: int = 1,
start_col: int = 1, include_index: bool = False, **kwargs):
"""写入数据到工作表"""
# 添加标题
title = kwargs.get('title')
if title:
self.worksheet.cell(row=start_row, column=start_col, value=title)
self.worksheet.merge_cells(
start_row=start_row, start_column=start_col,
end_row=start_row, end_column=start_col + len(data.columns) - 1
)
# 标题样式
title_cell = self.worksheet.cell(row=start_row, column=start_col)
title_cell.font = Font(bold=True, size=16)
title_cell.alignment = Alignment(horizontal='center', vertical='center')
start_row += 2
# 写入DataFrame数据
for r in dataframe_to_rows(data, index=include_index, header=True):
for c_idx, value in enumerate(r, start_col):
self.worksheet.cell(row=start_row, column=c_idx, value=value)
start_row += 1
# 记录数据范围
self.data_start_row = 2 if title else 1
self.data_end_row = start_row - 1
self.data_start_col = start_col
self.data_end_col = start_col + len(data.columns) - 1
def _apply_styles(self, data: pd.DataFrame, **kwargs):
"""应用样式"""
# 获取样式配置
styles = kwargs.get('styles', {})
header_style = {**self.default_styles['header'], **styles.get('header', {})}
data_style = {**self.default_styles['data'], **styles.get('data', {})}
# 应用表头样式
for col in range(self.data_start_col, self.data_end_col + 1):
cell = self.worksheet.cell(row=self.data_start_row, column=col)
self._apply_cell_style(cell, header_style)
# 应用数据样式
for row in range(self.data_start_row + 1, self.data_end_row + 1):
for col in range(self.data_start_col, self.data_end_col + 1):
cell = self.worksheet.cell(row=row, column=col)
# 根据数据类型选择样式
col_name = data.columns[col - self.data_start_col]
if pd.api.types.is_numeric_dtype(data[col_name]):
style = {**self.default_styles['number'], **styles.get('number', {})}
else:
style = data_style
self._apply_cell_style(cell, style)
# 应用边框
self._apply_borders()
def _apply_cell_style(self, cell, style: Dict):
"""应用单元格样式"""
if 'font' in style:
cell.font = style['font']
if 'fill' in style:
cell.fill = style['fill']
if 'alignment' in style:
cell.alignment = style['alignment']
if 'number_format' in style:
cell.number_format = style['number_format']
if 'border' in style:
cell.border = style['border']
def _apply_borders(self):
"""应用边框"""
border = self.default_styles['border']
for row in range(self.data_start_row, self.data_end_row + 1):
for col in range(self.data_start_col, self.data_end_col + 1):
cell = self.worksheet.cell(row=row, column=col)
cell.border = border
def _create_excel_table(self, data: pd.DataFrame, **kwargs):
"""创建Excel表格对象"""
table_name = kwargs.get('table_name', 'DataTable')
table_style = kwargs.get('table_style', 'TableStyleMedium9')
# 定义表格范围
table_range = f"{self.worksheet.cell(self.data_start_row, self.data_start_col).coordinate}:" \
table_range = f"{self.worksheet.cell(self.data_start_row, self.data_start_col).coordinate}:" \
f"{self.worksheet.cell(self.data_end_row, self.data_end_col).coordinate}"
# 创建表格
table = Table(displayName=table_name, ref=table_range)
# 设置表格样式
style = TableStyleInfo(
name=table_style,
showFirstColumn=False,
showLastColumn=False,
showRowStripes=True,
showColumnStripes=False
)
table.tableStyleInfo = style
# 添加表格到工作表
self.worksheet.add_table(table)
def _add_charts(self, data: pd.DataFrame, **kwargs):
"""添加图表"""
chart_configs = kwargs.get('chart_configs', [])
for i, chart_config in enumerate(chart_configs):
chart_type = chart_config.get('type', 'bar')
chart_title = chart_config.get('title', f'图表 {i+1}')
data_columns = chart_config.get('columns', [])
if not data_columns:
continue
# 创建图表
if chart_type == 'bar':
chart = BarChart()
elif chart_type == 'line':
chart = LineChart()
elif chart_type == 'pie':
chart = PieChart()
else:
continue
chart.title = chart_title
# 添加数据系列
for col_name in data_columns:
if col_name in data.columns:
col_idx = list(data.columns).index(col_name) + self.data_start_col
data_ref = Reference(
self.worksheet,
min_col=col_idx,
min_row=self.data_start_row + 1,
max_row=self.data_end_row
)
chart.add_data(data_ref, titles_from_data=False)
# 设置类别标签（通常是第一列）
if len(data.columns) > 0:
cats_ref = Reference(
self.worksheet,
min_col=self.data_start_col,
min_row=self.data_start_row + 1,
max_row=self.data_end_row
)
chart.set_categories(cats_ref)
# 添加图表到工作表
chart_position = chart_config.get('position', f'H{5 + i * 15}')
self.worksheet.add_chart(chart, chart_position)
def _auto_fit_columns(self):
"""自动调整列宽"""
for column in self.worksheet.columns:
max_length = 0
column_letter = column[0].column_letter
for cell in column:
try:
if len(str(cell.value)) > max_length:
max_length = len(str(cell.value))
except:
pass
adjusted_width = min(max_length + 2, 50) # 限制最大宽度
self.worksheet.column_dimensions[column_letter].width = adjusted_width
def create_multi_sheet_workbook(self, data_dict: Dict[str, pd.DataFrame],
output_path: str, **kwargs) -> str:
"""
创建多工作表Excel文件
Args:
data_dict: 工作表名称到DataFrame的映射
output_path: 输出文件路径
**kwargs: 其他配置参数
Returns:
生成的文件路径
"""
try:
self.workbook = Workbook()
# 删除默认工作表
default_sheet = self.workbook.active
self.workbook.remove(default_sheet)
# 为每个数据集创建工作表
for sheet_name, data in data_dict.items():
self.worksheet = self.workbook.create_sheet(title=sheet_name)
self._write_data(data, **kwargs)
self._apply_styles(data, **kwargs)
if kwargs.get('enable_table', True):
self._create_excel_table(data, table_name=f"Table_{sheet_name}", **kwargs)
if kwargs.get('auto_fit_columns', True):
self._auto_fit_columns()
# 保存文件
os.makedirs(os.path.dirname(output_path), exist_ok=True)
self.workbook.save(output_path)
self.logger.info(f"多工作表Excel文件已生成: {output_path}")
return output_path
except Exception as e:
self.logger.error(f"生成多工作表Excel文件失败: {e}")
raise
def add_summary_sheet(self, data_dict: Dict[str, pd.DataFrame], **kwargs):
"""添加汇总工作表"""
summary_data = []
for sheet_name, data in data_dict.items():
summary_row = {
'工作表名称': sheet_name,
'数据行数': len(data),
'数据列数': len(data.columns),
'数值列数': len(data.select_dtypes(include=['number']).columns),
'文本列数': len(data.select_dtypes(include=['object']).columns)
}
# 添加数值列的统计信息
numeric_data = data.select_dtypes(include=['number'])
if not numeric_data.empty:
summary_row['数值总和'] = numeric_data.sum().sum()
summary_row['数值平均值'] = numeric_data.mean().mean()
summary_data.append(summary_row)
summary_df = pd.DataFrame(summary_data)
# 创建汇总工作表
self.worksheet = self.workbook.create_sheet(title='数据汇总', index=0)
self._write_data(summary_df, title='数据汇总报告', **kwargs)
self._apply_styles(summary_df, **kwargs)
if kwargs.get('auto_fit_columns', True):
self._auto_fit_columns()
Excel高级功能实现
class AdvancedExcelGenerator(ExcelTableGenerator):
"""高级Excel生成器"""
def __init__(self, config: Dict[str, Any] = None):
super().__init__(config)
self.conditional_formats = []
self.data_validations = []
def add_conditional_formatting(self, data: pd.DataFrame, rules: List[Dict], **kwargs):
"""添加条件格式"""
from openpyxl.formatting.rule import ColorScaleRule, DataBarRule, IconSetRule
from openpyxl.styles import PatternFill
for rule in rules:
rule_type = rule.get('type')
column = rule.get('column')
if column not in data.columns:
continue
col_idx = list(data.columns).index(column) + self.data_start_col
range_string = f"{self.worksheet.cell(self.data_start_row + 1, col_idx).coordinate}:" \
f"{self.worksheet.cell(self.data_end_row, col_idx).coordinate}"
if rule_type == 'color_scale':
# 颜色刻度
color_rule = ColorScaleRule(
start_type='min', start_color=rule.get('start_color', 'FF0000'),
end_type='max', end_color=rule.get('end_color', '00FF00')
)
self.worksheet.conditional_formatting.add(range_string, color_rule)
elif rule_type == 'data_bar':
# 数据条
data_bar_rule = DataBarRule(
start_type='min', end_type='max',
color=rule.get('color', '0066CC')
)
self.worksheet.conditional_formatting.add(range_string, data_bar_rule)
elif rule_type == 'icon_set':
# 图标集
icon_rule = IconSetRule(
icon_style=rule.get('icon_style', '3TrafficLights1'),
type='percent',
values=[33, 67]
)
self.worksheet.conditional_formatting.add(range_string, icon_rule)
elif rule_type == 'highlight_cells':
# 突出显示单元格
from openpyxl.formatting.rule import CellIsRule
operator = rule.get('operator', 'greaterThan')
value = rule.get('value', 0)
fill_color = rule.get('fill_color', 'FFFF00')
highlight_rule = CellIsRule(
operator=operator,
formula=[value],
fill=PatternFill(start_color=fill_color, end_color=fill_color, fill_type='solid')
)
self.worksheet.conditional_formatting.add(range_string, highlight_rule)
def add_data_validation(self, data: pd.DataFrame, validations: List[Dict], **kwargs):
"""添加数据验证"""
from openpyxl.worksheet.datavalidation import DataValidation
for validation in validations:
column = validation.get('column')
if column not in data.columns:
continue
col_idx = list(data.columns).index(column) + self.data_start_col
range_string = f"{self.worksheet.cell(self.data_start_row + 1, col_idx).coordinate}:" \
f"{self.worksheet.cell(self.data_end_row, col_idx).coordinate}"
validation_type = validation.get('type', 'list')
if validation_type == 'list':
# 下拉列表验证
formula = f'"{",".join(validation.get("values", []))}"'
dv = DataValidation(type="list", formula1=formula, allow_blank=True)
elif validation_type == 'whole':
# 整数验证
dv = DataValidation(
type="whole",
operator=validation.get('operator', 'between'),
formula1=validation.get('min_value', 0),
formula2=validation.get('max_value', 100)
)
elif validation_type == 'decimal':
# 小数验证
dv = DataValidation(
type="decimal",
operator=validation.get('operator', 'between'),
formula1=validation.get('min_value', 0.0),
formula2=validation.get('max_value', 100.0)
)
else:
continue
# 设置错误消息
dv.error = validation.get('error_message', '输入的数据无效')
dv.errorTitle = validation.get('error_title', '数据验证错误')
# 设置提示消息
dv.prompt = validation.get('prompt_message', '请输入有效数据')
dv.promptTitle = validation.get('prompt_title', '数据输入提示')
self.worksheet.add_data_validation(dv)
dv.add(range_string)
def add_pivot_table_data(self, data: pd.DataFrame, output_path: str, **kwargs):
"""为数据透视表准备数据"""
# 创建数据源工作表
data_sheet = self.workbook.create_sheet(title='数据源')
# 写入数据
for r in dataframe_to_rows(data, index=False, header=True):
data_sheet.append(r)
# 应用基本格式
for cell in data_sheet[1]: # 表头
cell.font = Font(bold=True)
cell.fill = PatternFill(start_color='D9E1F2', end_color='D9E1F2', fill_type='solid')
# 创建数据透视表配置说明
instructions = [
"数据透视表创建说明：",
"1. 选择'数据源'工作表中的所有数据",
"2. 插入 -> 数据透视表",
"3. 根据需要配置行、列、值字段",
"",
"数据字段说明："
]
for col in data.columns:
dtype = str(data[col].dtype)
instructions.append(f"- {col}: {dtype}")
# 添加说明工作表
instruction_sheet = self.workbook.create_sheet(title='使用说明')
for i, instruction in enumerate(instructions, 1):
instruction_sheet.cell(row=i, column=1, value=instruction)
def create_dashboard(self, data: pd.DataFrame, output_path: str, **kwargs):
"""创建数据仪表板"""
# 创建仪表板工作表
dashboard_sheet = self.workbook.create_sheet(title='数据仪表板', index=0)
# 添加标题
title = kwargs.get('dashboard_title', '数据仪表板')
dashboard_sheet.cell(row=1, column=1, value=title)
dashboard_sheet.merge_cells('A1:H1')
title_cell = dashboard_sheet.cell(row=1, column=1)
title_cell.font = Font(bold=True, size=20, color='FFFFFF')
title_cell.fill = PatternFill(start_color='2F5597', end_color='2F5597', fill_type='solid')
title_cell.alignment = Alignment(horizontal='center', vertical='center')
# 添加关键指标
metrics = self._calculate_key_metrics(data)
self._add_metrics_section(dashboard_sheet, metrics, start_row=3)
# 添加图表
self._add_dashboard_charts(dashboard_sheet, data, start_row=10)
# 设置行高
dashboard_sheet.row_dimensions[1].height = 40
def _calculate_key_metrics(self, data: pd.DataFrame) -> Dict:
"""计算关键指标"""
metrics = {
'总记录数': len(data),
'数据列数': len(data.columns)
}
# 数值列统计
numeric_data = data.select_dtypes(include=['number'])
if not numeric_data.empty:
metrics['数值列数'] = len(numeric_data.columns)
metrics['数值总和'] = numeric_data.sum().sum()
metrics['平均值'] = numeric_data.mean().mean()
# 文本列统计
text_data = data.select_dtypes(include=['object'])
if not text_data.empty:
metrics['文本列数'] = len(text_data.columns)
return metrics
def _add_metrics_section(self, worksheet, metrics: Dict, start_row: int):
"""添加指标区域"""
col = 1
for metric_name, metric_value in metrics.items():
# 指标名称
name_cell = worksheet.cell(row=start_row, column=col)
name_cell.value = metric_name
name_cell.font = Font(bold=True, size=12)
name_cell.alignment = Alignment(horizontal='center')
# 指标值
value_cell = worksheet.cell(row=start_row + 1, column=col)
value_cell.value = metric_value
value_cell.font = Font(bold=True, size=16, color='2F5597')
value_cell.alignment = Alignment(horizontal='center')
# 样式
for r in range(start_row, start_row + 2):
cell = worksheet.cell(row=r, column=col)
cell.border = Border(
left=Side(style='thin'),
right=Side(style='thin'),
top=Side(style='thin'),
bottom=Side(style='thin')
)
col += 2 # 间隔一列
def _add_dashboard_charts(self, worksheet, data: pd.DataFrame, start_row: int):
"""添加仪表板图表"""
# 这里可以添加各种图表
# 由于篇幅限制，这里只是示例框架
pass
Word文档表格生成
Word文档中的表格生成同样重要，特别是在报告和文档自动化中。
Word表格生成器实现
from docx import Document
from docx.shared import Inches, Cm, Pt
from docx.enum.table import WD_TABLE_ALIGNMENT, WD_ALIGN_VERTICAL
from docx.enum.text import WD_ALIGN_PARAGRAPH
from docx.oxml.shared import OxmlElement, qn
import pandas as pd
from typing import Dict, Any, List, Optional
import logging
class WordTableGenerator:
"""Word表格生成器"""
def __init__(self, config: Dict[str, Any] = None):
self.config = config or {}
self.logger = logging.getLogger(self.__class__.__name__)
self.document = None
# 默认样式配置
self.default_styles = {
'table_style': 'Table Grid',
'header_style': {
'bold': True,
'font_size': Pt(12),
'alignment': WD_ALIGN_PARAGRAPH.CENTER
},
'data_style': {
'font_size': Pt(10),
'alignment': WD_ALIGN_PARAGRAPH.LEFT
},
'number_style': {
'font_size': Pt(10),
'alignment': WD_ALIGN_PARAGRAPH.RIGHT
}
}
def create_table(self, data: pd.DataFrame, output_path: str, **kwargs) -> str:
"""
创建Word表格文档
Args:
data: 数据DataFrame
output_path: 输出文件路径
**kwargs: 其他配置参数
Returns:
生成的文件路径
"""
try:
# 创建或加载文档
template_path = kwargs.get('template_path')
if template_path and os.path.exists(template_path):
self.document = Document(template_path)
else:
self.document = Document()
# 添加标题
title = kwargs.get('title')
if title:
self._add_title(title, **kwargs)
# 添加表格
table = self._create_word_table(data, **kwargs)
# 添加表格说明
description = kwargs.get('description')
if description:
self._add_description(description, **kwargs)
# 添加统计信息
if kwargs.get('add_statistics', False):
self._add_statistics(data, **kwargs)
# 保存文档
os.makedirs(os.path.dirname(output_path), exist_ok=True)
self.document.save(output_path)
self.logger.info(f"Word表格文档已生成: {output_path}")
return output_path
except Exception as e:
self.logger.error(f"生成Word表格文档失败: {e}")
raise
def _add_title(self, title: str, **kwargs):
"""添加文档标题"""
title_paragraph = self.document.add_heading(title, level=1)
title_paragraph.alignment = WD_ALIGN_PARAGRAPH.CENTER
# 添加副标题
subtitle = kwargs.get('subtitle')
if subtitle:
subtitle_paragraph = self.document.add_heading(subtitle, level=2)
subtitle_paragraph.alignment = WD_ALIGN_PARAGRAPH.CENTER
# 添加空行
self.document.add_paragraph()
def _create_word_table(self, data: pd.DataFrame, **kwargs):
"""创建Word表格"""
# 创建表格
table = self.document.add_table(rows=1, cols=len(data.columns))
table.style = kwargs.get('table_style', self.default_styles['table_style'])
table.alignment = WD_TABLE_ALIGNMENT.CENTER
# 设置表头
header_cells = table.rows[0].cells
for i, column_name in enumerate(data.columns):
header_cells[i].text = str(column_name)
self._apply_cell_style(header_cells[i], self.default_styles['header_style'])
# 添加数据行
for _, row in data.iterrows():
row_cells = table.add_row().cells
for i, value in enumerate(row):
row_cells[i].text = str(value) if pd.notna(value) else ''
# 根据数据类型应用样式
if pd.api.types.is_numeric_dtype(data.iloc[:, i]):
style = self.default_styles['number_style']
else:
style = self.default_styles['data_style']
self._apply_cell_style(row_cells[i], style)
# 设置列宽
self._set_column_widths(table, data, **kwargs)
return table
def _apply_cell_style(self, cell, style: Dict):
"""应用单元格样式"""
paragraph = cell.paragraphs[0]
run = paragraph.runs[0] if paragraph.runs else paragraph.add_run()
if 'bold' in style:
run.bold = style['bold']
if 'italic' in style:
run.italic = style['italic']
if 'font_size' in style:
run.font.size = style['font_size']
if 'font_color' in style:
run.font.color.rgb = style['font_color']
if 'alignment' in style:
paragraph.alignment = style['alignment']
# 设置单元格垂直对齐
cell.vertical_alignment = WD_ALIGN_VERTICAL.CENTER
def _set_column_widths(self, table, data: pd.DataFrame, **kwargs):
"""设置列宽"""
column_widths = kwargs.get('column_widths')
if column_widths:
# 使用指定的列宽
for i, width in enumerate(column_widths):
if i < len(table.columns):
for cell in table.columns[i].cells:
cell.width = Inches(width)
else:
# 自动计算列宽
total_width = Inches(6.5) # A4纸张可用宽度
col_count = len(data.columns)
# 根据内容长度分配宽度
max_lengths = []
for col in data.columns:
max_len = max(
len(str(col)), # 列名长度
data[col].astype(str).str.len().max() if not data.empty else 0
)
max_lengths.append(max_len)
total_chars = sum(max_lengths)
for i, max_len in enumerate(max_lengths):
if total_chars > 0:
width_ratio = max_len / total_chars
column_width = total_width * width_ratio
else:
column_width = total_width / col_count
for cell in table.columns[i].cells:
cell.width = column_width
def _add_description(self, description: str, **kwargs):
"""添加表格说明"""
self.document.add_paragraph()
desc_paragraph = self.document.add_paragraph(description)
desc_paragraph.alignment = WD_ALIGN_PARAGRAPH.JUSTIFY
def _add_statistics(self, data: pd.DataFrame, **kwargs):
"""添加统计信息"""
self.document.add_paragraph()
stats_heading = self.document.add_heading('数据统计', level=2)
# 基本统计
stats_text = f"数据行数: {len(data)}\n"
stats_text += f"数据列数: {len(data.columns)}\n"
# 数值列统计
numeric_data = data.select_dtypes(include=['number'])
if not numeric_data.empty:
stats_text += f"数值列数: {len(numeric_data.columns)}\n"
stats_text += f"数值总和: {numeric_data.sum().sum():.2f}\n"
stats_text += f"数值平均值: {numeric_data.mean().mean():.2f}\n"
stats_paragraph = self.document.add_paragraph(stats_text)
def create_multi_table_document(self, data_dict: Dict[str, pd.DataFrame],
output_path: str, **kwargs) -> str:
"""
创建包含多个表格的Word文档
Args:
data_dict: 表格名称到DataFrame的映射
output_path: 输出文件路径
**kwargs: 其他配置参数
Returns:
生成的文件路径
"""
try:
self.document = Document()
# 添加文档标题
doc_title = kwargs.get('document_title', '数据报告')
self._add_title(doc_title, **kwargs)
# 为每个数据集创建表格
for i, (table_name, data) in enumerate(data_dict.items()):
if i > 0:
self.document.add_page_break()
# 添加表格标题
table_heading = self.document.add_heading(table_name, level=2)
# 创建表格
self._create_word_table(data, **kwargs)
# 添加表格统计
if kwargs.get('add_table_statistics', True):
self._add_table_summary(data, table_name)
# 保存文档
os.makedirs(os.path.dirname(output_path), exist_ok=True)
self.document.save(output_path)
self.logger.info(f"多表格Word文档已生成: {output_path}")
return output_path
except Exception as e:
self.logger.error(f"生成多表格Word文档失败: {e}")
raise
def _add_table_summary(self, data: pd.DataFrame, table_name: str):
"""添加表格摘要"""
summary_text = f"{table_name} 包含 {len(data)} 行数据，{len(data.columns)} 个字段。"
# 添加数值统计
numeric_data = data.select_dtypes(include=['number'])
if not numeric_data.empty:
summary_text += f" 其中数值字段 {len(numeric_data.columns)} 个。"
summary_paragraph = self.document.add_paragraph(summary_text)
summary_paragraph.alignment = WD_ALIGN_PARAGRAPH.JUSTIFY
# 添加空行
self.document.add_paragraph()
HTML表格生成与Web展示
HTML表格适用于Web展示和交互式报告。
HTML表格生成器实现
import pandas as pd
from jinja2 import Template, Environment, FileSystemLoader
import os
from typing import Dict, Any, List, Optional
import json
import base64
from io import BytesIO
import matplotlib.pyplot as plt
import seaborn as sns
class HTMLTableGenerator:
"""HTML表格生成器"""
def __init__(self, config: Dict[str, Any] = None):
self.config = config or {}
self.logger = logging.getLogger(self.__class__.__name__)
# 默认HTML模板
self.default_template = """
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>{{ title }}</title>
<style>
body {
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
margin: 20px;
background-color: #f5f5f5;
}
.container {
max-width: 1200px;
margin: 0 auto;
background-color: white;
padding: 20px;
border-radius: 8px;
box-shadow: 0 2px 10px rgba(0,0,0,0.1);
}
h1 {
color: #333;
text-align: center;
margin-bottom: 30px;
}
.table-container {
overflow-x: auto;
margin-bottom: 30px;
}
table {
width: 100%;
border-collapse: collapse;
margin-bottom: 20px;
}
th, td {
padding: 12px;
text-align: left;
border-bottom: 1px solid #ddd;
}
th {
background-color: #4CAF50;
color: white;
font-weight: bold;
text-align: center;
}
tr:nth-child(even) {
background-color: #f2f2f2;
}
tr:hover {
background-color: #f5f5f5;
}
.numeric {
text-align: right;
}
.statistics {
background-color: #e8f4f8;
padding: 15px;
border-radius: 5px;
margin-
# Python实现数据自动生成表格：从数据源到可视化表格的完整解决方案
在现代数据处理和报告生成中，将原始数据转换为结构化、美观的表格是一个常见且重要的需求。无论是生成Excel报表、Word文档中的表格，还是HTML网页表格，自动化的表格生成能够大大提高工作效率，减少人工错误，并确保数据展示的一致性。本文将深入探讨如何使用Python实现数据自动生成表格的完整解决方案，涵盖多种数据源、多种输出格式，以及高级的表格样式和交互功能。
## 目录
1. [数据自动生成表格概述](#数据自动生成表格概述)
2. [技术栈与环境准备](#技术栈与环境准备)
3. [数据源处理与标准化](#数据源处理与标准化)
4. [Excel表格自动生成](#excel表格自动生成)
5. [Word文档表格生成](#word文档表格生成)
6. [HTML表格生成与Web展示](#html表格生成与web展示)
7. [PDF表格生成](#pdf表格生成)
8. [高级表格样式与格式化](#高级表格样式与格式化)
9. [动态表格与交互功能](#动态表格与交互功能)
10. [批量处理与模板系统](#批量处理与模板系统)
11. [性能优化与大数据处理](#性能优化与大数据处理)
12. [实际应用案例](#实际应用案例)
13. [最佳实践与注意事项](#最佳实践与注意事项)
14. [总结与展望](#总结与展望)
## 数据自动生成表格概述
### 什么是数据自动生成表格
数据自动生成表格是指通过编程方式，将各种格式的原始数据（如CSV、JSON、数据库查询结果等）自动转换为格式化的表格文档。这个过程包括数据读取、处理、格式化、样式应用和输出等多个步骤。
### 核心优势
1. **效率提升**：自动化处理大量数据，避免手工制表的繁琐工作
2. **一致性保证**：确保所有表格遵循统一的格式和样式标准
3. **错误减少**：消除人工操作中的计算错误和格式错误
4. **可重复性**：相同的数据处理逻辑可以重复应用于不同的数据集
5. **灵活性**：支持多种数据源和输出格式，适应不同的业务需求
### 应用场景
- **财务报表生成**：自动生成月度、季度、年度财务报表
- **销售数据分析**：生成销售业绩表格和趋势分析
- **库存管理报告**：自动更新库存状态表格
- **学生成绩统计**：批量生成成绩单和统计报表
- **项目进度跟踪**：生成项目状态和里程碑表格
- **数据分析报告**：将分析结果以表格形式展示
## 技术栈与环境准备
### 核心Python库
我们将使用以下Python库来实现完整的表格生成功能：
```bash
# 数据处理
pip install pandas numpy
# Excel处理
pip install openpyxl xlsxwriter
# Word文档处理
pip install python-docx
# PDF生成
pip install reportlab
# HTML生成
pip install jinja2 beautifulsoup4
# 数据库连接
pip install sqlalchemy pymysql psycopg2-binary
# 图表生成
pip install matplotlib seaborn plotly
# 进度条和日志
pip install tqdm loguru
# 配置文件处理
pip install pyyaml configparser
项目结构设计
auto_table_generator/
│
├── core/
│ ├── __init__.py
│ ├── data_processor.py # 数据处理核心
│ ├── table_generator.py # 表格生成器基类
│ └── style_manager.py # 样式管理器
│
├── generators/
│ ├── __init__.py
│ ├── excel_generator.py # Excel表格生成器
│ ├── word_generator.py # Word表格生成器
│ ├── html_generator.py # HTML表格生成器
│ └── pdf_generator.py # PDF表格生成器
│
├── data_sources/
│ ├── __init__.py
│ ├── csv_source.py # CSV数据源
│ ├── json_source.py # JSON数据源
│ ├── database_source.py # 数据库数据源
│ └── api_source.py # API数据源
│
├── templates/
│ ├── excel_templates/ # Excel模板
│ ├── word_templates/ # Word模板
│ └── html_templates/ # HTML模板
│
├── config/
│ ├── settings.yaml # 配置文件
│ └── styles.yaml # 样式配置
│
├── examples/
│ ├── basic_usage.py # 基础使用示例
│ ├── advanced_features.py # 高级功能示例
│ └── batch_processing.py # 批量处理示例
│
└── tests/
├── test_generators.py # 生成器测试
└── test_data_sources.py # 数据源测试
环境验证
import sys
import pandas as pd
import openpyxl
import docx
from reportlab.lib import colors
import jinja2
def verify_environment():
"""验证开发环境是否正确配置"""
print("=== 环境验证 ===")
print(f"Python版本: {sys.version}")
print(f"Pandas版本: {pd.__version__}")
print(f"OpenPyXL版本: {openpyxl.__version__}")
print(f"Python-docx版本: {docx.__version__}")
print(f"Jinja2版本: {jinja2.__version__}")
# 测试基本功能
try:
# 测试pandas
df = pd.DataFrame({'A': [1, 2], 'B': [3, 4]})
print("✓ Pandas工作正常")
# 测试openpyxl
from openpyxl import Workbook
wb = Workbook()
print("✓ OpenPyXL工作正常")
# 测试python-docx
from docx import Document
doc = Document()
print("✓ Python-docx工作正常")
print("✓ 所有依赖库验证通过")
return True
except Exception as e:
print(f"✗ 环境验证失败: {e}")
return False
if __name__ == "__main__":
verify_environment()
数据源处理与标准化
在生成表格之前，我们需要处理各种不同的数据源，并将它们标准化为统一的格式。
数据源基类设计
from abc import ABC, abstractmethod
import pandas as pd
from typing import Dict, Any, Optional, List
import logging
class DataSource(ABC):
"""数据源基类"""
def __init__(self, config: Dict[str, Any]):
self.config = config
self.logger = logging.getLogger(self.__class__.__name__)
self._data = None
self._metadata = {}
@abstractmethod
def load_data(self) -> pd.DataFrame:
"""加载数据的抽象方法"""
pass
@abstractmethod
def validate_data(self, data: pd.DataFrame) -> bool:
"""验证数据的抽象方法"""
pass
def get_data(self, force_reload: bool = False) -> pd.DataFrame:
"""获取数据，支持缓存"""
if self._data is None or force_reload:
self._data = self.load_data()
if not self.validate_data(self._data):
raise ValueError("数据验证失败")
return self._data.copy()
def get_metadata(self) -> Dict[str, Any]:
"""获取数据元信息"""
return self._metadata.copy()
def preprocess_data(self, data: pd.DataFrame) -> pd.DataFrame:
"""数据预处理"""
# 处理缺失值
if self.config.get('fill_na', False):
fill_value = self.config.get('fill_na_value', '')
data = data.fillna(fill_value)
# 数据类型转换
if 'column_types' in self.config:
for col, dtype in self.config['column_types'].items():
if col in data.columns:
try:
data[col] = data[col].astype(dtype)
except Exception as e:
self.logger.warning(f"无法转换列 {col} 的数据类型: {e}")
# 列重命名
if 'column_mapping' in self.config:
data = data.rename(columns=self.config['column_mapping'])
# 数据过滤
if 'filters' in self.config:
for filter_config in self.config['filters']:
data = self._apply_filter(data, filter_config)
return data
def _apply_filter(self, data: pd.DataFrame, filter_config: Dict) -> pd.DataFrame:
"""应用数据过滤器"""
column = filter_config.get('column')
operator = filter_config.get('operator', '==')
value = filter_config.get('value')
if column not in data.columns:
self.logger.warning(f"过滤列 {column} 不存在")
return data
try:
if operator == '==':
return data[data[column] == value]
elif operator == '!=':
return data[data[column] != value]
elif operator == '>':
return data[data[column] > value]
elif operator == '<':
return data[data[column] < value]
elif operator == '>=':
return data[data[column] >= value]
elif operator == '<=':
return data[data[column] <= value]
elif operator == 'in':
return data[data[column].isin(value)]
elif operator == 'contains':
return data[data[column].str.contains(value, na=False)]
else:
self.logger.warning(f"不支持的过滤操作符: {operator}")
return data
except Exception as e:
self.logger.error(f"应用过滤器时出错: {e}")
return data
CSV数据源实现
import pandas as pd
import os
from typing import Dict, Any
class CSVDataSource(DataSource):
"""CSV数据源"""
def load_data(self) -> pd.DataFrame:
"""从CSV文件加载数据"""
file_path = self.config.get('file_path')
if not file_path or not os.path.exists(file_path):
raise FileNotFoundError(f"CSV文件不存在: {file_path}")
# CSV读取参数
csv_params = {
'encoding': self.config.get('encoding', 'utf-8'),
'sep': self.config.get('separator', ','),
'header': self.config.get('header', 0),
'skiprows': self.config.get('skip_rows', None),
'nrows': self.config.get('max_rows', None)
}
try:
data = pd.read_csv(file_path, **csv_params)
self.logger.info(f"成功加载CSV文件: {file_path}, 数据形状: {data.shape}")
# 更新元数据
self._metadata = {
'source_type': 'csv',
'file_path': file_path,
'file_size': os.path.getsize(file_path),
'rows': len(data),
'columns': len(data.columns),
'column_names': list(data.columns)
}
return self.preprocess_data(data)
except Exception as e:
self.logger.error(f"加载CSV文件失败: {e}")
raise
def validate_data(self, data: pd.DataFrame) -> bool:
"""验证CSV数据"""
if data.empty:
self.logger.error("CSV数据为空")
return False
# 检查必需列
required_columns = self.config.get('required_columns', [])
missing_columns = set(required_columns) - set(data.columns)
if missing_columns:
self.logger.error(f"缺少必需列: {missing_columns}")
return False
# 检查数据类型
if 'column_types' in self.config:
for col, expected_type in self.config['column_types'].items():
if col in data.columns:
try:
pd.api.types.is_dtype_equal(data[col].dtype, expected_type)
except Exception:
self.logger.warning(f"列 {col} 的数据类型可能不匹配")
return True
JSON数据源实现
import json
import pandas as pd
from typing import Dict, Any, List, Union
class JSONDataSource(DataSource):
"""JSON数据源"""
def load_data(self) -> pd.DataFrame:
"""从JSON文件或API加载数据"""
if 'file_path' in self.config:
return self._load_from_file()
elif 'url' in self.config:
return self._load_from_api()
else:
raise ValueError("必须指定file_path或url")
def _load_from_file(self) -> pd.DataFrame:
"""从JSON文件加载数据"""
file_path = self.config['file_path']
if not os.path.exists(file_path):
raise FileNotFoundError(f"JSON文件不存在: {file_path}")
try:
with open(file_path, 'r', encoding=self.config.get('encoding', 'utf-8')) as f:
json_data = json.load(f)
data = self._json_to_dataframe(json_data)
self._metadata = {
'source_type': 'json_file',
'file_path': file_path,
'file_size': os.path.getsize(file_path),
'rows': len(data),
'columns': len(data.columns)
}
return self.preprocess_data(data)
except Exception as e:
self.logger.error(f"加载JSON文件失败: {e}")
raise
def _load_from_api(self) -> pd.DataFrame:
"""从API加载JSON数据"""
import requests
url = self.config['url']
headers = self.config.get('headers', {})
params = self.config.get('params', {})
timeout = self.config.get('timeout', 30)
try:
response = requests.get(url, headers=headers, params=params, timeout=timeout)
response.raise_for_status()
json_data = response.json()
data = self._json_to_dataframe(json_data)
self._metadata = {
'source_type': 'json_api',
'url': url,
'status_code': response.status_code,
'rows': len(data),
'columns': len(data.columns)
}
return self.preprocess_data(data)
except Exception as e:
self.logger.error(f"从API加载JSON数据失败: {e}")
raise
def _json_to_dataframe(self, json_data: Union[Dict, List]) -> pd.DataFrame:
"""将JSON数据转换为DataFrame"""
data_path = self.config.get('data_path', None)
# 如果指定了数据路径，提取嵌套数据
if data_path:
current_data = json_data
for key in data_path.split('.'):
if isinstance(current_data, dict) and key in current_data:
current_data = current_data[key]
else:
raise ValueError(f"无法找到数据路径: {data_path}")
json_data = current_data
# 转换为DataFrame
if isinstance(json_data, list):
return pd.DataFrame(json_data)
elif isinstance(json_data, dict):
# 如果是字典，尝试转换为记录列表
if all(isinstance(v, (list, tuple)) for v in json_data.values()):
return pd.DataFrame(json_data)
else:
# 单条记录
return pd.DataFrame([json_data])
else:
raise ValueError("不支持的JSON数据格式")
def validate_data(self, data: pd.DataFrame) -> bool:
"""验证JSON数据"""
if data.empty:
self.logger.error("JSON数据为空")
return False
return True
数据库数据源实现
import pandas as pd
from sqlalchemy import create_engine, text
from typing import Dict, Any
class DatabaseDataSource(DataSource):
"""数据库数据源"""
def __init__(self, config: Dict[str, Any]):
super().__init__(config)
self.engine = None
self._create_engine()
def _create_engine(self):
"""创建数据库引擎"""
db_config = self.config.get('database', {})
# 构建连接字符串
db_type = db_config.get('type', 'mysql')
host = db_config.get('host', 'localhost')
port = db_config.get('port', 3306)
username = db_config.get('username')
password = db_config.get('password')
database = db_config.get('database')
if db_type == 'mysql':
connection_string = f"mysql+pymysql://{username}:{password}@{host}:{port}/{database}"
elif db_type == 'postgresql':
connection_string = f"postgresql+psycopg2://{username}:{password}@{host}:{port}/{database}"
elif db_type == 'sqlite':
connection_string = f"sqlite:///{database}"
else:
raise ValueError(f"不支持的数据库类型: {db_type}")
try:
self.engine = create_engine(
connection_string,
pool_pre_ping=True,
pool_recycle=3600
)
self.logger.info(f"成功连接到数据库: {db_type}")
except Exception as e:
self.logger.error(f"数据库连接失败: {e}")
raise
def load_data(self) -> pd.DataFrame:
"""从数据库加载数据"""
query = self.config.get('query')
table_name = self.config.get('table_name')
if query:
return self._load_from_query(query)
elif table_name:
return self._load_from_table(table_name)
else:
raise ValueError("必须指定query或table_name")
def _load_from_query(self, query: str) -> pd.DataFrame:
"""从SQL查询加载数据"""
try:
# 支持参数化查询
query_params = self.config.get('query_params', {})
data = pd.read_sql(
text(query),
self.engine,
params=query_params
)
self.logger.info(f"成功执行查询，返回 {len(data)} 行数据")
self._metadata = {
'source_type': 'database_query',
'query': query,
'rows': len(data),
'columns': len(data.columns)
}
return self.preprocess_data(data)
except Exception as e:
self.logger.error(f"执行数据库查询失败: {e}")
raise
def _load_from_table(self, table_name: str) -> pd.DataFrame:
"""从数据表加载数据"""
try:
# 构建查询
columns = self.config.get('columns', '*')
if isinstance(columns, list):
columns = ', '.join(columns)
where_clause = self.config.get('where_clause', '')
order_by = self.config.get('order_by', '')
limit = self.config.get('limit', '')
query = f"SELECT {columns} FROM {table_name}"
if where_clause:
query += f" WHERE {where_clause}"
if order_by:
query += f" ORDER BY {order_by}"
if limit:
query += f" LIMIT {limit}"
return self._load_from_query(query)
except Exception as e:
self.logger.error(f"从数据表加载数据失败: {e}")
raise
def validate_data(self, data: pd.DataFrame) -> bool:
"""验证数据库数据"""
if data.empty:
self.logger.warning("数据库查询返回空结果")
return True # 空结果可能是正常的
return True
def __del__(self):
"""清理数据库连接"""
if self.engine:
self.engine.dispose()
Excel表格自动生成
Excel是最常用的表格格式之一，我们将实现一个功能强大的Excel表格生成器。
Excel生成器核心实现
import pandas as pd
from openpyxl import Workbook, load_workbook
from openpyxl.styles import Font, PatternFill, Border, Side, Alignment
from openpyxl.utils.dataframe import dataframe_to_rows
from openpyxl.chart import BarChart, LineChart, PieChart, Reference
from openpyxl.worksheet.table import Table, TableStyleInfo
import os
from typing import Dict, Any, List, Optional, Union
import logging
class ExcelTableGenerator:
"""Excel表格生成器"""
def __init__(self, config: Dict[str, Any] = None):
self.config = config or {}
self.logger = logging.getLogger(self.__class__.__name__)
self.workbook = None
self.worksheet = None
# 默认样式配置
self.default_styles = {
'header': {
'font': Font(bold=True, color='FFFFFF'),
'fill': PatternFill(start_color='366092', end_color='366092', fill_type='solid'),
'alignment': Alignment(horizontal='center', vertical='center')
},
'data': {
'font': Font(color='000000'),
'alignment': Alignment(horizontal='left', vertical='center')
},
'number': {
'font': Font(color='000000'),
'alignment': Alignment(horizontal='right', vertical='center'),
'number_format': '#,##0.00'
},
'border': Border(
left=Side(style='thin'),
right=Side(style='thin'),
top=Side(style='thin'),
bottom=Side(style='thin')
)
}
def create_table(self, data: pd.DataFrame, output_path: str,
sheet_name: str = 'Sheet1', **kwargs) -> str:
"""
创建Excel表格
Args:
data: 数据DataFrame
output_path: 输出文件路径
sheet_name: 工作表名称
**kwargs: 其他配置参数
Returns:
生成的文件路径
"""
try:
# 创建工作簿
self.workbook = Workbook()
self.worksheet = self.workbook.active
self.worksheet.title = sheet_name
# 写入数据
self._write_data(data, **kwargs)
# 应用样式
self._apply_styles(data, **kwargs)
# 添加表格功能
if kwargs.get('enable_table', True):
self._create_excel_table(data, **kwargs)
# 添加图表
if kwargs.get('add_charts', False):
self._add_charts(data, **kwargs)
# 自动调整列宽
if kwargs.get('auto_fit_columns', True):
self._auto_fit_columns()
# 保存文件
os.makedirs(os.path.dirname(output_path), exist_ok=True)
self.workbook.save(output_path)
self.logger.info(f"Excel表格已生成: {output_path}")
return output_path
except Exception as e:
self.logger.error(f"生成Excel表格失败: {e}")
raise
def _write_data(self, data: pd.DataFrame, start_row: int = 1,
start_col: int = 1, include_index: bool = False, **kwargs):
"""写入数据到工作表"""
# 添加标题
title = kwargs.get('title')
if title:
self.worksheet.cell(row=start_row, column=start_col, value=title)
self.worksheet.merge_cells(
start_row=start_row, start_column=start_col,
end_row=start_row, end_column=start_col + len(data.columns) - 1
)
# 标题样式
title_cell = self.worksheet.cell(row=start_row, column=start_col)
title_cell.font = Font(bold=True, size=16)
title_cell.alignment = Alignment(horizontal='center', vertical='center')
start_row += 2
# 写入DataFrame数据
for r in dataframe_to_rows(data, index=include_index, header=True):
for c_idx, value in enumerate(r, start_col):
self.worksheet.cell(row=start_row, column=c_idx, value=value)
start_row += 1
# 记录数据范围
self.data_start_row = 2 if title else 1
self.data_end_row = start_row - 1
self.data_start_col = start_col
self.data_end_col = start_col + len(data.columns) - 1
def _apply_styles(self, data: pd.DataFrame, **kwargs):
"""应用样式"""
# 获取样式配置
styles = kwargs.get('styles', {})
header_style = {**self.default_styles['header'], **styles.get('header', {})}
data_style = {**self.default_styles['data'], **styles.get('data', {})}
# 应用表头样式
for col in range(self.data_start_col, self.data_end_col + 1):
cell = self.worksheet.cell(row=self.data_start_row, column=col)
self._apply_cell_style(cell, header_style)
# 应用数据样式
for row in range(self.data_start_row + 1, self.data_end_row + 1):
for col in range(self.data_start_col, self.data_end_col + 1):
cell = self.worksheet.cell(row=row, column=col)
# 根据数据类型选择样式
col_name = data.columns[col - self.data_start_col]
if pd.api.types.is_numeric_dtype(data[col_name]):
style = {**self.default_styles['number'], **styles.get('number', {})}
else:
style = data_style
self._apply_cell_style(cell, style)
# 应用边框
self._apply_borders()
def _apply_cell_style(self, cell, style: Dict):
"""应用单元格样式"""
if 'font' in style:
cell.font = style['font']
if 'fill' in style:
cell.fill = style['fill']
if 'alignment' in style:
cell.alignment = style['alignment']
if 'number_format' in style:
cell.number_format = style['number_format']
if 'border' in style:
cell.border = style['border']
def _apply_borders(self):
"""应用边框"""
border = self.default_styles['border']
for row in range(self.data_start_row, self.data_end_row + 1):
for col in range(self.data_start_col, self.data_end_col + 1):
cell = self.worksheet.cell(row=row, column=col)
cell.border = border
def _create_excel_table(self, data: pd.DataFrame, **kwargs):
"""创建Excel表格对象"""
table_name = kwargs.get('table_name', 'DataTable')
table_style = kwargs.get('table_style', 'TableStyleMedium9')
# 定义表格范围
table_range = f"{self.worksheet.cell(self.data_start_row, self.data_start_col).coordinate}:" \
f"{self.worksheet