import akshare as ak import pandas as pd import random import time import logging import requests from typing import List, Optional, Dict from datetime import datetime from app.services.datasource.base import DataSourceBase logger = logging.getLogger(__name__) class SmartRequester: """ 反爬综合管理器：集成 Headers 伪装、拟人化延时、重试机制 (IP 代理部分暂按文档要求空缺，后续扩展) """ def __init__(self, max_retries: int = 3): self.max_retries = max_retries self.session = requests.Session() # User-Agent 池 self.user_agents = [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.42", ] def get_random_headers(self, referer: str = "https://finance.sina.com.cn/") -> Dict[str, str]: """生成随机请求头，模拟真实浏览器""" return { "User-Agent": random.choice(self.user_agents), "Referer": referer, "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", "Accept-Encoding": "gzip, deflate, br", "Connection": "keep-alive", "Upgrade-Insecure-Requests": "1", } def request(self, url, referer="https://finance.sina.com.cn/", method="GET", **kwargs) -> Optional[requests.Response]: """执行智能请求""" last_error = None for attempt in range(self.max_retries): try: # 1. 拟人化延时 if attempt == 0: time.sleep(random.uniform(0.5, 1.5)) else: delay = (2 ** attempt) + random.uniform(1, 3) logging.warning(f"第 {attempt+1} 次重试，等待 {delay:.1f} 秒...") time.sleep(delay) # 2. 轮换 Headers headers = kwargs.pop("headers", {}) random_headers = self.get_random_headers(referer=referer) headers.update(random_headers) kwargs["headers"] = headers # 3. 发送请求 logging.debug(f"请求: {method} {url}") response = self.session.request(method, url, timeout=10, **kwargs) # 4. 检查状态 if response.status_code == 200: logging.info("✅ 请求成功") return response elif response.status_code == 403: logging.warning("⚠️ 收到 403 Forbidden，将尝试重试") raise requests.exceptions.HTTPError("403 Forbidden") else: response.raise_for_status() except Exception as e: last_error = e logging.warning(f"❌ 请求失败: {str(e)}") logging.error(f"🚫 所有 {self.max_retries} 次尝试均失败") return None class AkshareSource(DataSourceBase): """AKShare 数据源适配器""" def __init__(self, config: dict): super().__init__(config) self.name = "akshare" self.requester = SmartRequester(max_retries=config.get("max_retries", 3)) def initialize(self) -> bool: """初始化检查""" try: ak.__version__ self._initialized = True return True except Exception as e: self._initialized = False logging.error(f"AkshareSource 初始化失败: {e}") return False def get_contract_list(self, exchange: Optional[str] = None) -> List[dict]: """获取期货合约列表""" if not self._initialized: self.initialize() results = [] try: exchanges_to_fetch = ["CZCE", "DCE", "SHFE", "INE", "CFFEX", "GFEX"] if exchange: exchanges_to_fetch = [exchange] for ex in exchanges_to_fetch: try: func_name = f"futures_contract_info_{ex.lower()}" if hasattr(ak, func_name): df = getattr(ak, func_name)() else: continue if df is not None and not df.empty: # 统一列名映射 col_map = {} if '合约代码' in df.columns: col_map['合约代码'] = 'symbol' if '产品代码' in df.columns: col_map['产品代码'] = 'product' if '品种' in df.columns: col_map['品种'] = 'product' if '交易单位' in df.columns: col_map['交易单位'] = 'multiplier' if '最小变动价位' in df.columns: col_map['最小变动价位'] = 'price_tick' if '上市日' in df.columns: col_map['上市日'] = 'list_date' if '到期日' in df.columns: col_map['到期日'] = 'expire_date' df = df.rename(columns=col_map) for _, row in df.iterrows(): symbol = row.get("symbol") if not symbol: continue # 提取品种代码 (通常 symbol 最后两位是年月，前面是品种，如 rb2401 -> rb) product = row.get("product", "") if not product and len(symbol) > 2: # 简单提取字母部分 import re match = re.match(r"([a-zA-Z]+)", symbol) if match: product = match.group(1).lower() multiplier = row.get("multiplier") if multiplier: try: multiplier = int(float(str(multiplier).replace(',', ''))) except: multiplier = 10 else: multiplier = 10 # 默认值 price_tick = row.get("price_tick") if price_tick: try: price_tick = float(str(price_tick).replace(',', '')) except: price_tick = None results.append({ "symbol": symbol, "exchange": ex, "name": symbol, # AKShare 通常不返回中文名称，用代码代替 "product": product, "multiplier": multiplier, "price_tick": price_tick, "expire_date": self._parse_date(row.get("expire_date")), "is_active": True, }) except Exception as e: logging.warning(f"获取 {ex} 合约列表失败: {e}") continue except Exception as e: logging.error(f"获取合约列表异常: {e}") return results def _parse_date(self, date_str) -> Optional[datetime]: """解析日期""" if not date_str: return None try: if isinstance(date_str, str): return datetime.strptime(date_str, "%Y-%m-%d") elif isinstance(date_str, pd.Timestamp): return date_str.to_pydatetime() return None except: return None def get_kline_daily(self, symbol: str, start_date: str, end_date: str) -> pd.DataFrame: """获取日 K 线数据""" logger.info(f"[AKShare-日K线] 开始获取 symbol={symbol}, start_date={start_date}, end_date={end_date}") if not self._initialized: logger.info(f"[AKShare-日K线] 初始化 AKShare") self.initialize() try: # AKShare 期货日 K 线接口：futures_zh_daily_sina logger.info(f"[AKShare-日K线] 调用 ak.futures_zh_daily_sina(symbol='{symbol}')") df = ak.futures_zh_daily_sina(symbol=symbol) if df is None or df.empty: logger.warning(f"[AKShare-日K线] AKShare 返回空数据，symbol={symbol}") return pd.DataFrame() logger.info(f"[AKShare-日K线] AKShare 返回 {len(df)} 条原始数据") logger.debug(f"[AKShare-日K线] 原始数据列: {df.columns.tolist()}") logger.debug(f"[AKShare-日K线] 原始数据样例:\n{df.head()}") # 过滤日期范围 df['date'] = pd.to_datetime(df['date']) logger.debug(f"[AKShare-日K线] 日期范围: {df['date'].min()} ~ {df['date'].max()}") mask = (df['date'] >= start_date) & (df['date'] <= end_date) df_filtered = df.loc[mask].copy() logger.info(f"[AKShare-日K线] 日期过滤后剩余 {len(df_filtered)} 条记录") # 统一列名 df_filtered = df_filtered.rename(columns={ "date": "trade_date", "open": "open", "high": "high", "low": "low", "close": "close", "volume": "volume", "hold": "open_interest", "settle": "settle", }) # AKShare 日线通常没有 turnover 和 pre_settle，置空 df_filtered["turnover"] = None df_filtered["pre_settle"] = None df_filtered["trade_date"] = pd.to_datetime(df_filtered["trade_date"]) logger.info(f"[AKShare-日K线] 最终返回 {len(df_filtered)} 条记录") return df_filtered[["trade_date", "open", "high", "low", "close", "volume", "turnover", "open_interest", "settle", "pre_settle"]] except Exception as e: logger.error(f"[AKShare-日K线] 获取 {symbol} 日 K 线失败: {e}", exc_info=True) return pd.DataFrame() def get_kline_weekly(self, symbol: str, start_date: str, end_date: str) -> pd.DataFrame: """获取周 K 线数据 (通过日 K 聚合)""" daily_df = self.get_kline_daily(symbol, start_date, end_date) if daily_df.empty: return pd.DataFrame() daily_df = daily_df.set_index("trade_date") weekly = daily_df.resample("W-FRI").agg({ "open": "first", "high": "max", "low": "min", "close": "last", "volume": "sum", "turnover": "sum", "open_interest": "last", }).dropna() weekly = weekly.reset_index() weekly = weekly.rename(columns={"trade_date": "trade_date"}) return weekly def get_kline_intraday(self, symbol: str, period: str, start_date: str, end_date: str) -> pd.DataFrame: """获取分钟级 K 线数据""" if not self._initialized: self.initialize() try: # AKShare 期货分钟 K 线接口：futures_zh_minute_sina period_map = {"5m": "5", "15m": "15", "30m": "30", "60m": "60"} freq = period_map.get(period, "5") df = ak.futures_zh_minute_sina(symbol=symbol, period=freq) if df is None or df.empty: return pd.DataFrame() # 过滤日期 df['datetime'] = pd.to_datetime(df['datetime']) mask = (df['datetime'] >= start_date) & (df['datetime'] <= end_date) df = df.loc[mask].copy() # 统一列名 df = df.rename(columns={ "datetime": "trade_time", "open": "open", "high": "high", "low": "low", "close": "close", "volume": "volume", "hold": "open_interest", }) df["turnover"] = None return df[["trade_time", "open", "high", "low", "close", "volume", "turnover", "open_interest"]] except Exception as e: logging.error(f"获取 {symbol} 分钟 K 线失败: {e}") return pd.DataFrame()