|
|
"""
|
|
|
缓存服务 - SQLite 数据库操作
|
|
|
"""
|
|
|
import json
|
|
|
import logging
|
|
|
from datetime import datetime, timedelta
|
|
|
from typing import Dict, List, Optional
|
|
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
|
|
from app.models import MarketData, ScheduledTask, SymbolTimestamp
|
|
|
from app.config import CACHE_TTL_SECONDS
|
|
|
from app.storage_manager import get_storage_manager
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
# ===== 市场数据缓存 =====
|
|
|
|
|
|
def is_cache_valid(
|
|
|
db: Session,
|
|
|
symbol: str,
|
|
|
data_type: str,
|
|
|
period: str,
|
|
|
ttl_seconds: int = CACHE_TTL_SECONDS,
|
|
|
) -> bool:
|
|
|
"""检查指定品种+周期的缓存是否在有效期内"""
|
|
|
record = db.query(MarketData).filter_by(
|
|
|
symbol=symbol,
|
|
|
data_type=data_type,
|
|
|
period=period,
|
|
|
).first()
|
|
|
if not record:
|
|
|
return False
|
|
|
age = (datetime.now() - record.fetched_at).total_seconds()
|
|
|
return age < ttl_seconds
|
|
|
|
|
|
|
|
|
def check_cache_status(
|
|
|
db: Session,
|
|
|
symbol: str,
|
|
|
data_type: str,
|
|
|
periods: List[str],
|
|
|
ttl_seconds: int = CACHE_TTL_SECONDS,
|
|
|
) -> dict:
|
|
|
"""
|
|
|
检查一组周期的缓存状态。
|
|
|
Returns:
|
|
|
{
|
|
|
"all_valid": bool, # 所有周期都有有效缓存
|
|
|
"valid_periods": [...],
|
|
|
"missing_periods": [...],
|
|
|
}
|
|
|
"""
|
|
|
valid = []
|
|
|
missing = []
|
|
|
for p in periods:
|
|
|
if is_cache_valid(db, symbol, data_type, p, ttl_seconds):
|
|
|
valid.append(p)
|
|
|
else:
|
|
|
missing.append(p)
|
|
|
return {
|
|
|
"all_valid": len(missing) == 0,
|
|
|
"valid_periods": valid,
|
|
|
"missing_periods": missing,
|
|
|
}
|
|
|
|
|
|
|
|
|
def _save_to_sqlite(db: Session, symbol: str, data: Dict) -> MarketData:
|
|
|
"""
|
|
|
将采集结果写入 SQLite 缓存，并同步更新合约时间戳。
|
|
|
|
|
|
Args:
|
|
|
db: 数据库会话
|
|
|
symbol: 品种代码
|
|
|
data: 采集脚本返回的完整数据
|
|
|
|
|
|
Returns:
|
|
|
保存的 MarketData 记录
|
|
|
"""
|
|
|
now = datetime.now()
|
|
|
|
|
|
# 按 period 拆分存储（每个周期一条记录）
|
|
|
for period, candles in data.get("timeframes", {}).items():
|
|
|
record = db.query(MarketData).filter_by(
|
|
|
symbol=symbol,
|
|
|
data_type=data.get("type", "futures"),
|
|
|
period=period,
|
|
|
).first()
|
|
|
|
|
|
candles_json = json.dumps(candles, ensure_ascii=False)
|
|
|
|
|
|
if record:
|
|
|
record.candles_json = candles_json
|
|
|
record.current_price = data.get("current_price")
|
|
|
record.fetched_at = now
|
|
|
record.candle_count = len(candles)
|
|
|
else:
|
|
|
record = MarketData(
|
|
|
symbol=symbol,
|
|
|
data_type=data.get("type", "futures"),
|
|
|
period=period,
|
|
|
candles_json=candles_json,
|
|
|
current_price=data.get("current_price"),
|
|
|
fetched_at=now,
|
|
|
candle_count=len(candles),
|
|
|
)
|
|
|
db.add(record)
|
|
|
|
|
|
# 更新合约时间戳
|
|
|
_update_symbol_timestamp_sqlite(db, symbol, data.get("type", "futures"), now)
|
|
|
|
|
|
db.commit()
|
|
|
logger.info(f"缓存已更新: {symbol}, {len(data.get('timeframes', {}))} 个周期")
|
|
|
|
|
|
# 返回最新的一条作为代表
|
|
|
return db.query(MarketData).filter_by(
|
|
|
symbol=symbol,
|
|
|
data_type=data.get("type", "futures"),
|
|
|
).order_by(MarketData.fetched_at.desc()).first()
|
|
|
|
|
|
|
|
|
def _update_symbol_timestamp_sqlite(db: Session, symbol: str, data_type: str, refresh_time: datetime) -> None:
|
|
|
"""更新或创建合约时间戳记录（SQLite 兜底）。"""
|
|
|
timestamp_record = db.query(SymbolTimestamp).filter_by(
|
|
|
symbol=symbol,
|
|
|
data_type=data_type
|
|
|
).first()
|
|
|
|
|
|
if timestamp_record:
|
|
|
timestamp_record.last_refresh_at = refresh_time
|
|
|
timestamp_record.refresh_count += 1
|
|
|
else:
|
|
|
timestamp_record = SymbolTimestamp(
|
|
|
symbol=symbol,
|
|
|
data_type=data_type,
|
|
|
last_refresh_at=refresh_time,
|
|
|
refresh_count=1
|
|
|
)
|
|
|
db.add(timestamp_record)
|
|
|
|
|
|
|
|
|
def _get_symbol_timestamp_sqlite(db: Session, symbol: str, data_type: str = "futures") -> Optional[datetime]:
|
|
|
"""获取合约最后刷新时间（SQLite 兜底）。"""
|
|
|
record = db.query(SymbolTimestamp).filter_by(
|
|
|
symbol=symbol,
|
|
|
data_type=data_type
|
|
|
).first()
|
|
|
return record.last_refresh_at if record else None
|
|
|
|
|
|
|
|
|
def _get_from_sqlite(
|
|
|
db: Session,
|
|
|
symbol: str,
|
|
|
data_type: str = "futures",
|
|
|
periods: Optional[List[str]] = None,
|
|
|
end_time: Optional[datetime] = None,
|
|
|
max_candles: int = 100,
|
|
|
) -> Optional[Dict]:
|
|
|
"""
|
|
|
从 SQLite 缓存中获取完整的多周期数据。
|
|
|
|
|
|
Args:
|
|
|
db: 数据库会话
|
|
|
symbol: 品种代码
|
|
|
data_type: 数据类型
|
|
|
periods: 周期列表
|
|
|
end_time: 结束时间(可选),默认为当前时间
|
|
|
max_candles: 每个周期最大K线数量,默认100
|
|
|
|
|
|
Returns:
|
|
|
与采集脚本相同格式的数据，或 None
|
|
|
"""
|
|
|
query = db.query(MarketData).filter_by(symbol=symbol, data_type=data_type)
|
|
|
if periods:
|
|
|
query = query.filter(MarketData.period.in_(periods))
|
|
|
|
|
|
records = query.all()
|
|
|
if not records:
|
|
|
return None
|
|
|
|
|
|
# 检查缓存是否过期
|
|
|
now = datetime.now()
|
|
|
newest = max(r.fetched_at for r in records)
|
|
|
is_fresh = (now - newest).total_seconds() < CACHE_TTL_SECONDS
|
|
|
|
|
|
# 如果未指定结束时间,默认为当前时间
|
|
|
filter_end_time = end_time if end_time else now
|
|
|
|
|
|
# 确保filter_end_time是naive datetime(无时区)
|
|
|
if filter_end_time.tzinfo is not None:
|
|
|
filter_end_time = filter_end_time.replace(tzinfo=None)
|
|
|
|
|
|
timeframes = {}
|
|
|
current_price = None
|
|
|
for r in records:
|
|
|
candles = json.loads(r.candles_json)
|
|
|
|
|
|
# 过滤结束时间之前的K线数据
|
|
|
filtered_candles = []
|
|
|
for candle in candles:
|
|
|
candle_time = candle.get('datetime') or candle.get('time')
|
|
|
if candle_time:
|
|
|
# 解析K线时间
|
|
|
if isinstance(candle_time, str):
|
|
|
try:
|
|
|
candle_dt = datetime.fromisoformat(candle_time.replace('Z', '+00:00'))
|
|
|
# 转换为naive datetime进行比较
|
|
|
if candle_dt.tzinfo is not None:
|
|
|
candle_dt = candle_dt.replace(tzinfo=None)
|
|
|
except:
|
|
|
filtered_candles.append(candle)
|
|
|
continue
|
|
|
else:
|
|
|
candle_dt = candle_time
|
|
|
# 如果是aware datetime,转换为naive
|
|
|
if candle_dt.tzinfo is not None:
|
|
|
candle_dt = candle_dt.replace(tzinfo=None)
|
|
|
|
|
|
# 只保留结束时间之前的数据
|
|
|
if candle_dt <= filter_end_time:
|
|
|
filtered_candles.append(candle)
|
|
|
else:
|
|
|
filtered_candles.append(candle)
|
|
|
|
|
|
# 限制K线数量,超过max_candles则取最新的max_candles条
|
|
|
if len(filtered_candles) > max_candles:
|
|
|
filtered_candles = filtered_candles[-max_candles:]
|
|
|
|
|
|
timeframes[r.period] = filtered_candles
|
|
|
|
|
|
if current_price is None:
|
|
|
current_price = r.current_price
|
|
|
|
|
|
return {
|
|
|
"symbol": symbol,
|
|
|
"type": data_type,
|
|
|
"current_price": current_price,
|
|
|
"timestamp": newest.isoformat(),
|
|
|
"timeframes": timeframes,
|
|
|
"is_fresh": is_fresh,
|
|
|
"fetched_at": newest.isoformat(),
|
|
|
}
|
|
|
|
|
|
|
|
|
def save_market_data(db: Session, symbol: str, data: Dict) -> Optional[MarketData]:
|
|
|
"""
|
|
|
保存采集结果到缓存，优先写入 StorageManager（Redis/MySQL），失败时降级到 SQLite。
|
|
|
|
|
|
Args:
|
|
|
symbol: 品种代码
|
|
|
data: 采集脚本返回的完整数据
|
|
|
|
|
|
Returns:
|
|
|
SQLite 写入时返回 MarketData 记录；StorageManager 成功时返回 None
|
|
|
"""
|
|
|
storage = get_storage_manager()
|
|
|
data_type = data.get("type", "futures")
|
|
|
now = datetime.now()
|
|
|
|
|
|
if storage.is_available():
|
|
|
try:
|
|
|
if storage.save_market_data(symbol, data):
|
|
|
storage.save_symbol_timestamp(symbol, data_type, now)
|
|
|
return None
|
|
|
else:
|
|
|
raise Exception("save_market_data returned False")
|
|
|
except Exception as e:
|
|
|
logger.warning(f"StorageManager 写入失败，降级到 SQLite: {e}")
|
|
|
|
|
|
return _save_to_sqlite(db, symbol, data)
|
|
|
|
|
|
|
|
|
def update_symbol_timestamp(db: Session, symbol: str, data_type: str, refresh_time: datetime) -> None:
|
|
|
"""更新或创建合约时间戳记录，优先使用 StorageManager。"""
|
|
|
storage = get_storage_manager()
|
|
|
|
|
|
if storage.is_available():
|
|
|
try:
|
|
|
storage.save_symbol_timestamp(symbol, data_type, refresh_time)
|
|
|
return
|
|
|
except Exception as e:
|
|
|
logger.warning(f"StorageManager 时间戳写入失败，降级到 SQLite: {e}")
|
|
|
|
|
|
_update_symbol_timestamp_sqlite(db, symbol, data_type, refresh_time)
|
|
|
db.commit()
|
|
|
|
|
|
|
|
|
def get_symbol_timestamp(db: Session, symbol: str, data_type: str = "futures") -> Optional[datetime]:
|
|
|
"""获取合约最后刷新时间，优先使用 StorageManager。"""
|
|
|
storage = get_storage_manager()
|
|
|
|
|
|
if storage.is_available():
|
|
|
try:
|
|
|
result = storage.get_symbol_timestamp(symbol, data_type)
|
|
|
if result:
|
|
|
return datetime.fromisoformat(result["last_refresh_at"])
|
|
|
except Exception as e:
|
|
|
logger.warning(f"StorageManager 时间戳读取失败，降级到 SQLite: {e}")
|
|
|
|
|
|
return _get_symbol_timestamp_sqlite(db, symbol, data_type)
|
|
|
|
|
|
|
|
|
def needs_refresh(db: Session, symbol: str, data_type: str = "futures", threshold_seconds: int = 300) -> bool:
|
|
|
"""
|
|
|
检查合约是否需要刷新（数据是否超过阈值时间）
|
|
|
|
|
|
Args:
|
|
|
db: 数据库会话
|
|
|
symbol: 品种代码
|
|
|
data_type: 数据类型
|
|
|
threshold_seconds: 阈值时间（秒），默认300秒（5分钟）
|
|
|
|
|
|
Returns:
|
|
|
True 表示需要刷新，False 表示数据仍然新鲜
|
|
|
"""
|
|
|
last_refresh = get_symbol_timestamp(db, symbol, data_type)
|
|
|
if last_refresh is None:
|
|
|
return True # 从未刷新过，需要刷新
|
|
|
|
|
|
age = (datetime.now() - last_refresh).total_seconds()
|
|
|
return age > threshold_seconds
|
|
|
|
|
|
|
|
|
def get_latest_cached(
|
|
|
db: Session,
|
|
|
symbol: str,
|
|
|
data_type: str = "futures",
|
|
|
period: Optional[str] = None,
|
|
|
) -> List[MarketData]:
|
|
|
"""获取最新缓存数据"""
|
|
|
query = db.query(MarketData).filter_by(symbol=symbol, data_type=data_type)
|
|
|
if period:
|
|
|
query = query.filter_by(period=period)
|
|
|
return query.order_by(MarketData.fetched_at.desc()).all()
|
|
|
|
|
|
|
|
|
def _merge_storage_market_data(
|
|
|
symbol: str,
|
|
|
data_type: str,
|
|
|
periods: Optional[List[str]],
|
|
|
storage_results: Dict[str, Optional[Dict]],
|
|
|
) -> Optional[Dict]:
|
|
|
"""将 StorageManager 按周期返回的数据合并为 get_cached_data 标准格式。"""
|
|
|
timeframes = {}
|
|
|
current_price = None
|
|
|
newest = None
|
|
|
|
|
|
for period, result in storage_results.items():
|
|
|
if not result:
|
|
|
continue
|
|
|
candles = result.get("candles", [])
|
|
|
timeframes[period] = candles
|
|
|
if current_price is None:
|
|
|
current_price = result.get("current_price")
|
|
|
timestamp_raw = result.get("timestamp")
|
|
|
if timestamp_raw:
|
|
|
try:
|
|
|
timestamp_dt = datetime.fromisoformat(timestamp_raw)
|
|
|
if newest is None or timestamp_dt > newest:
|
|
|
newest = timestamp_dt
|
|
|
except Exception:
|
|
|
pass
|
|
|
|
|
|
if not timeframes:
|
|
|
return None
|
|
|
|
|
|
if newest is None:
|
|
|
newest = datetime.now()
|
|
|
|
|
|
now = datetime.now()
|
|
|
is_fresh = (now - newest).total_seconds() < CACHE_TTL_SECONDS
|
|
|
|
|
|
return {
|
|
|
"symbol": symbol,
|
|
|
"type": data_type,
|
|
|
"current_price": current_price,
|
|
|
"timestamp": newest.isoformat(),
|
|
|
"timeframes": timeframes,
|
|
|
"is_fresh": is_fresh,
|
|
|
"fetched_at": newest.isoformat(),
|
|
|
}
|
|
|
|
|
|
|
|
|
def get_cached_data(
|
|
|
db: Session,
|
|
|
symbol: str,
|
|
|
data_type: str = "futures",
|
|
|
periods: Optional[List[str]] = None,
|
|
|
end_time: Optional[datetime] = None,
|
|
|
max_candles: int = 100,
|
|
|
) -> Optional[Dict]:
|
|
|
"""
|
|
|
从缓存中获取完整的多周期数据，优先使用 StorageManager，失败时降级到 SQLite。
|
|
|
|
|
|
Args:
|
|
|
db: 数据库会话
|
|
|
symbol: 品种代码
|
|
|
data_type: 数据类型
|
|
|
periods: 周期列表
|
|
|
end_time: 结束时间(可选),默认为当前时间
|
|
|
max_candles: 每个周期最大K线数量,默认100
|
|
|
|
|
|
Returns:
|
|
|
与采集脚本相同格式的数据，或 None
|
|
|
"""
|
|
|
storage = get_storage_manager()
|
|
|
|
|
|
if storage.is_available():
|
|
|
try:
|
|
|
storage_results = {}
|
|
|
query_periods = periods if periods else []
|
|
|
for period in query_periods:
|
|
|
storage_results[period] = storage.get_market_data(symbol, data_type, period)
|
|
|
|
|
|
merged = _merge_storage_market_data(symbol, data_type, periods, storage_results)
|
|
|
if merged:
|
|
|
return merged
|
|
|
except Exception as e:
|
|
|
logger.warning(f"StorageManager 读取失败，降级到 SQLite: {e}")
|
|
|
|
|
|
return _get_from_sqlite(db, symbol, data_type, periods, end_time, max_candles)
|
|
|
|
|
|
|
|
|
# ===== 定时任务管理 =====
|
|
|
|
|
|
def create_task(
|
|
|
db: Session,
|
|
|
symbol: str,
|
|
|
data_type: str,
|
|
|
periods: str,
|
|
|
interval_seconds: int,
|
|
|
task_type: str = "interval",
|
|
|
run_time: Optional[str] = None,
|
|
|
) -> ScheduledTask:
|
|
|
"""创建定时任务配置"""
|
|
|
existing = db.query(ScheduledTask).filter_by(
|
|
|
symbol=symbol, data_type=data_type
|
|
|
).first()
|
|
|
if existing:
|
|
|
existing.periods = periods
|
|
|
existing.interval_seconds = interval_seconds
|
|
|
existing.task_type = task_type
|
|
|
existing.run_time = run_time
|
|
|
existing.enabled = True
|
|
|
existing.updated_at = datetime.now()
|
|
|
db.commit()
|
|
|
db.refresh(existing)
|
|
|
return existing
|
|
|
|
|
|
task = ScheduledTask(
|
|
|
symbol=symbol,
|
|
|
data_type=data_type,
|
|
|
periods=periods,
|
|
|
interval_seconds=interval_seconds,
|
|
|
task_type=task_type,
|
|
|
run_time=run_time,
|
|
|
enabled=True,
|
|
|
)
|
|
|
db.add(task)
|
|
|
db.commit()
|
|
|
db.refresh(task)
|
|
|
return task
|
|
|
|
|
|
|
|
|
def list_tasks(db: Session) -> List[ScheduledTask]:
|
|
|
"""列出所有任务"""
|
|
|
return db.query(ScheduledTask).order_by(ScheduledTask.created_at.desc()).all()
|
|
|
|
|
|
|
|
|
def get_task(db: Session, task_id: int) -> Optional[ScheduledTask]:
|
|
|
"""获取单个任务"""
|
|
|
return db.query(ScheduledTask).filter_by(id=task_id).first()
|
|
|
|
|
|
|
|
|
def disable_task(db: Session, task_id: int) -> Optional[ScheduledTask]:
|
|
|
"""禁用任务"""
|
|
|
task = db.query(ScheduledTask).filter_by(id=task_id).first()
|
|
|
if task:
|
|
|
task.enabled = False
|
|
|
task.updated_at = datetime.now()
|
|
|
db.commit()
|
|
|
db.refresh(task)
|
|
|
return task
|
|
|
|
|
|
|
|
|
def enable_task(db: Session, task_id: int) -> Optional[ScheduledTask]:
|
|
|
"""启用任务"""
|
|
|
task = db.query(ScheduledTask).filter_by(id=task_id).first()
|
|
|
if task:
|
|
|
task.enabled = True
|
|
|
task.updated_at = datetime.now()
|
|
|
db.commit()
|
|
|
db.refresh(task)
|
|
|
return task
|
|
|
|
|
|
|
|
|
def delete_task(db: Session, task_id: int) -> bool:
|
|
|
"""删除任务"""
|
|
|
task = db.query(ScheduledTask).filter_by(id=task_id).first()
|
|
|
if task:
|
|
|
db.delete(task)
|
|
|
db.commit()
|
|
|
return True
|
|
|
return False
|
|
|
|
|
|
|
|
|
def update_task_status(
|
|
|
db: Session, task_id: int, status: str
|
|
|
) -> None:
|
|
|
"""更新任务执行状态"""
|
|
|
task = db.query(ScheduledTask).filter_by(id=task_id).first()
|
|
|
if task:
|
|
|
task.last_run = datetime.now()
|
|
|
task.last_status = status
|
|
|
db.commit()
|
