claude强化功能

backend/sentiment_monitor.py

@@ -0,0 +1,556 @@
+"""社区情绪监控 — 爬取分析东方财富/雪球热帖，量化散户情绪。
+
+功能：
+1. 爬取社区热帖
+2. 情绪分析（乐观/悲观）
+3. 热议股票排行
+4. 关键词提取和词云
+5. 情绪与股价相关性分析
+"""
+import datetime as dt
+import json
+import re
+from typing import List, Dict, Any, Optional
+from collections import Counter, defaultdict
+import requests
+from bs4 import BeautifulSoup
+import jieba
+import jieba.analyse
+from sqlalchemy import select, func, and_, desc
+
+from db import get_session
+from models import SocialPost, SentimentIndex, DailyQuote, StockMetric
+
+# 情绪关键词库
+BULLISH_KEYWORDS = [
+    '看多', '看好', '买入', '加仓', '抄底', '突破', '上涨', '暴涨', '牛市',
+    '利好', '反弹', '强势', '拉升', '涨停', '走强', '看涨', '做多'
+]
+
+BEARISH_KEYWORDS = [
+    '看空', '看跌', '卖出', '减仓', '止损', '下跌', '暴跌', '熊市',
+    '利空', '回调', '弱势', '下杀', '跌停', '走弱', '做空', '被套'
+]
+
+# 停用词
+STOP_WORDS = set([
+    '的', '了', '是', '在', '我', '有', '和', '就', '不', '人', '都', '一',
+    '一个', '上', '也', '很', '到', '说', '要', '去', '你', '会', '着', '没有',
+    '看', '好', '自己', '这', '那', '以', '为', '而', '能', '他', '对', '于'
+])
+
+
+def crawl_eastmoney_hot(limit: int = 50) -> List[Dict[str, Any]]:
+    """爬取东方财富热帖（简化版，实际需要处理反爬）
+    
+    注意：由于反爬限制，这里返回模拟数据
+    实际生产环境需要：
+    1. 使用代理IP
+    2. 模拟浏览器headers
+    3. 控制请求频率
+    4. 处理验证码
+    """
+    # 模拟数据（实际应该爬取真实数据）
+    mock_posts = [
+        {
+            'source': 'eastmoney',
+            'post_id': f'em_{i}',
+            'title': f'模拟帖子{i}：今天大盘要反弹了',
+            'content': '技术分析显示底部信号明显，建议逢低买入',
+            'author': f'用户{i}',
+            'comment_count': 100 + i * 10,
+            'view_count': 1000 + i * 100,
+        }
+        for i in range(limit)
+    ]
+    
+    return mock_posts
+
+
+def crawl_xueqiu_hot(limit: int = 50) -> List[Dict[str, Any]]:
+    """爬取雪球热帖（简化版）"""
+    # 雪球API（需要cookie和token）
+    # 实际使用需要登录后获取token
+    
+    mock_posts = [
+        {
+            'source': 'xueqiu',
+            'post_id': f'xq_{i}',
+            'title': f'雪球热议{i}：半导体板块分析',
+            'content': '从产业链角度看，半导体景气度回升',
+            'author': f'雪球用户{i}',
+            'comment_count': 50 + i * 5,
+            'view_count': 500 + i * 50,
+        }
+        for i in range(limit)
+    ]
+    
+    return mock_posts
+
+
+def analyze_sentiment(text: str) -> str:
+    """分析文本情绪
+    
+    Args:
+        text: 待分析文本
+    
+    Returns:
+        情绪标签：bullish/bearish/neutral
+    """
+    text_lower = text.lower()
+    
+    bullish_score = sum(1 for kw in BULLISH_KEYWORDS if kw in text_lower)
+    bearish_score = sum(1 for kw in BEARISH_KEYWORDS if kw in text_lower)
+    
+    if bullish_score > bearish_score and bullish_score >= 2:
+        return 'bullish'
+    elif bearish_score > bullish_score and bearish_score >= 2:
+        return 'bearish'
+    else:
+        return 'neutral'
+
+
+def extract_keywords(text: str, top_n: int = 10) -> List[str]:
+    """提取关键词
+    
+    Args:
+        text: 文本内容
+        top_n: 返回前N个关键词
+    
+    Returns:
+        关键词列表
+    """
+    # 使用jieba提取关键词
+    keywords = jieba.analyse.extract_tags(text, topK=top_n, withWeight=False)
+    
+    # 过滤停用词
+    keywords = [kw for kw in keywords if kw not in STOP_WORDS and len(kw) > 1]
+    
+    return keywords[:top_n]
+
+
+def extract_stock_codes(text: str) -> List[str]:
+    """从文本中提取股票代码
+    
+    Args:
+        text: 文本内容
+    
+    Returns:
+        股票代码列表
+    """
+    # 匹配6位数字的股票代码
+    pattern = r'\b[036]\d{5}\b'
+    codes = re.findall(pattern, text)
+    return list(set(codes))
+
+
+def collect_posts(limit_per_source: int = 50) -> Dict[str, Any]:
+    """采集社区帖子
+    
+    Args:
+        limit_per_source: 每个来源采集数量
+    
+    Returns:
+        采集结果
+    """
+    all_posts = []
+    
+    # 采集东方财富
+    try:
+        em_posts = crawl_eastmoney_hot(limit_per_source)
+        all_posts.extend(em_posts)
+    except Exception as e:
+        print(f"[eastmoney] crawl error: {e}")
+    
+    # 采集雪球
+    try:
+        xq_posts = crawl_xueqiu_hot(limit_per_source)
+        all_posts.extend(xq_posts)
+    except Exception as e:
+        print(f"[xueqiu] crawl error: {e}")
+    
+    # 分析并存储
+    saved_count = 0
+    with get_session() as s:
+        for post in all_posts:
+            # 检查是否已存在
+            exists = s.execute(
+                select(SocialPost).where(SocialPost.post_id == post['post_id'])
+            ).scalar_one_or_none()
+            
+            if exists:
+                continue
+            
+            # 情绪分析
+            text = post['title'] + ' ' + post.get('content', '')
+            sentiment = analyze_sentiment(text)
+            
+            # 提取关键词
+            keywords = extract_keywords(text, top_n=5)
+            
+            # 提取股票代码
+            codes = extract_stock_codes(text)
+            code = codes[0] if codes else ''
+            
+            # 存储
+            record = SocialPost(
+                source=post['source'],
+                post_id=post['post_id'],
+                code=code,
+                title=post['title'],
+                content=post.get('content', ''),
+                author=post.get('author', ''),
+                comment_count=post.get('comment_count', 0),
+                view_count=post.get('view_count', 0),
+                sentiment=sentiment,
+                keywords=','.join(keywords)
+            )
+            s.add(record)
+            saved_count += 1
+        
+        s.commit()
+    
+    return {
+        'ok': True,
+        'collected': len(all_posts),
+        'saved': saved_count
+    }
+
+
+def calculate_sentiment_index(date: Optional[dt.date] = None) -> Dict[str, Any]:
+    """计算情绪指数
+    
+    Args:
+        date: 统计日期，None表示今天
+    
+    Returns:
+        情绪指数数据
+    """
+    if date is None:
+        date = dt.date.today()
+    
+    start = dt.datetime.combine(date, dt.time.min)
+    end = dt.datetime.combine(date, dt.time.max)
+    
+    with get_session() as s:
+        # 统计各情绪数量
+        posts = s.execute(
+            select(SocialPost)
+            .where(
+                and_(
+                    SocialPost.created_at >= start,
+                    SocialPost.created_at <= end
+                )
+            )
+        ).scalars().all()
+        
+        if not posts:
+            return {'ok': False, 'msg': '暂无数据'}
+        
+        bullish_count = sum(1 for p in posts if p.sentiment == 'bullish')
+        bearish_count = sum(1 for p in posts if p.sentiment == 'bearish')
+        neutral_count = sum(1 for p in posts if p.sentiment == 'neutral')
+        total = len(posts)
+        
+        bullish_ratio = bullish_count / total * 100 if total > 0 else 0
+        
+        # 提取热门关键词
+        all_keywords = []
+        for p in posts:
+            if p.keywords:
+                all_keywords.extend(p.keywords.split(','))
+        
+        keyword_counter = Counter(all_keywords)
+        top_keywords = [
+            {'word': kw, 'count': cnt}
+            for kw, cnt in keyword_counter.most_common(20)
+        ]
+        
+        # 存储情绪指数
+        index_record = s.execute(
+            select(SentimentIndex).where(SentimentIndex.date == date)
+        ).scalar_one_or_none()
+        
+        if index_record:
+            index_record.bullish_count = bullish_count
+            index_record.bearish_count = bearish_count
+            index_record.neutral_count = neutral_count
+            index_record.bullish_ratio = bullish_ratio
+            index_record.total_posts = total
+            index_record.top_keywords = json.dumps(top_keywords, ensure_ascii=False)
+            index_record.updated_at = dt.datetime.now()
+        else:
+            index_record = SentimentIndex(
+                date=date,
+                bullish_count=bullish_count,
+                bearish_count=bearish_count,
+                neutral_count=neutral_count,
+                bullish_ratio=bullish_ratio,
+                total_posts=total,
+                top_keywords=json.dumps(top_keywords, ensure_ascii=False)
+            )
+            s.add(index_record)
+        
+        s.commit()
+    
+    return {
+        'ok': True,
+        'date': date.isoformat(),
+        'bullish_count': bullish_count,
+        'bearish_count': bearish_count,
+        'neutral_count': neutral_count,
+        'bullish_ratio': round(bullish_ratio, 2),
+        'total_posts': total,
+        'top_keywords': top_keywords
+    }
+
+
+def get_hot_stocks(days: int = 1, limit: int = 20) -> Dict[str, Any]:
+    """获取热议股票排行
+    
+    Args:
+        days: 统计天数
+        limit: 返回数量
+    
+    Returns:
+        热议股票列表
+    """
+    since = dt.datetime.now() - dt.timedelta(days=days)
+    
+    with get_session() as s:
+        # 按股票代码分组统计
+        stmt = (
+            select(
+                SocialPost.code,
+                func.count().label('post_count'),
+                func.sum(SocialPost.comment_count).label('total_comments'),
+                func.sum(SocialPost.view_count).label('total_views')
+            )
+            .where(
+                and_(
+                    SocialPost.code != '',
+                    SocialPost.created_at >= since
+                )
+            )
+            .group_by(SocialPost.code)
+            .order_by(desc('post_count'))
+            .limit(limit)
+        )
+        
+        rows = s.execute(stmt).all()
+        
+        if not rows:
+            return {'ok': False, 'msg': '暂无数据'}
+        
+        # 获取股票名称和最新价格
+        codes = [r.code for r in rows]
+        metrics = {}
+        for m in s.execute(
+            select(StockMetric)
+            .where(StockMetric.code.in_(codes))
+        ).scalars():
+            metrics[m.code] = {
+                'name': m.name,
+                'close': m.close,
+                'pct': m.pct
+            }
+        
+        results = []
+        for r in rows:
+            info = metrics.get(r.code, {'name': r.code, 'close': 0, 'pct': 0})
+            results.append({
+                'code': r.code,
+                'name': info['name'],
+                'post_count': r.post_count,
+                'total_comments': r.total_comments or 0,
+                'total_views': r.total_views or 0,
+                'heat_score': r.post_count * 10 + (r.total_comments or 0),
+                'close': info['close'],
+                'pct': info['pct']
+            })
+        
+        # 按热度评分排序
+        results.sort(key=lambda x: x['heat_score'], reverse=True)
+    
+    return {
+        'ok': True,
+        'days': days,
+        'stocks': results
+    }
+
+
+def get_sentiment_history(days: int = 30) -> Dict[str, Any]:
+    """获取情绪指数历史
+    
+    Args:
+        days: 统计天数
+    
+    Returns:
+        历史数据
+    """
+    since = dt.date.today() - dt.timedelta(days=days)
+    
+    with get_session() as s:
+        rows = s.execute(
+            select(SentimentIndex)
+            .where(SentimentIndex.date >= since)
+            .order_by(SentimentIndex.date)
+        ).scalars().all()
+    
+    if not rows:
+        return {'ok': False, 'msg': '暂无历史数据'}
+    
+    return {
+        'ok': True,
+        'dates': [r.date.isoformat() for r in rows],
+        'bullish_ratio': [round(r.bullish_ratio, 2) for r in rows],
+        'total_posts': [r.total_posts for r in rows]
+    }
+
+
+def analyze_sentiment_correlation(code: str, days: int = 60) -> Dict[str, Any]:
+    """分析情绪与股价相关性
+    
+    Args:
+        code: 股票代码
+        days: 分析天数
+    
+    Returns:
+        相关性分析结果
+    """
+    since = dt.date.today() - dt.timedelta(days=days)
+    
+    with get_session() as s:
+        # 获取该股票的讨论量和情绪
+        posts = s.execute(
+            select(SocialPost)
+            .where(
+                and_(
+                    SocialPost.code == code,
+                    func.date(SocialPost.created_at) >= since
+                )
+            )
+        ).scalars().all()
+        
+        if not posts:
+            return {'ok': False, 'msg': '该股票暂无社区数据'}
+        
+        # 按日期聚合
+        daily_sentiment = defaultdict(lambda: {'bullish': 0, 'bearish': 0, 'neutral': 0, 'total': 0})
+        for p in posts:
+            date = p.created_at.date()
+            daily_sentiment[date][p.sentiment] += 1
+            daily_sentiment[date]['total'] += 1
+        
+        # 获取股价数据
+        prices = {}
+        for q in s.execute(
+            select(DailyQuote)
+            .where(
+                and_(
+                    DailyQuote.code == code,
+                    DailyQuote.date >= since
+                )
+            )
+            .order_by(DailyQuote.date)
+        ).scalars():
+            prices[q.date] = {
+                'close': float(q.close),
+                'pct': ((float(q.close) - float(q.open)) / float(q.open) * 100) if q.open > 0 else 0
+            }
+    
+    if not prices:
+        return {'ok': False, 'msg': '缺少股价数据'}
+    
+    # 计算相关性（简化版）
+    dates = sorted(set(daily_sentiment.keys()) & set(prices.keys()))
+    
+    if len(dates) < 10:
+        return {'ok': False, 'msg': '数据点不足'}
+    
+    sentiment_scores = []
+    price_changes = []
+    
+    for date in dates:
+        s_data = daily_sentiment[date]
+        bullish_ratio = s_data['bullish'] / s_data['total'] * 100 if s_data['total'] > 0 else 50
+        sentiment_scores.append(bullish_ratio)
+        
+        price_changes.append(prices[date]['pct'])
+    
+    # 计算相关系数（简化版）
+    import numpy as np
+    if len(sentiment_scores) > 1:
+        correlation = np.corrcoef(sentiment_scores, price_changes)[0, 1]
+    else:
+        correlation = 0
+    
+    return {
+        'ok': True,
+        'code': code,
+        'days': days,
+        'data_points': len(dates),
+        'correlation': round(float(correlation), 3),
+        'interpretation': _interpret_correlation(correlation),
+        'dates': [d.isoformat() for d in dates],
+        'sentiment_scores': [round(s, 2) for s in sentiment_scores],
+        'price_changes': [round(p, 2) for p in price_changes]
+    }
+
+
+def _interpret_correlation(corr: float) -> str:
+    """解释相关系数"""
+    if corr > 0.7:
+        return '强正相关：情绪高涨时股价往往上涨'
+    elif corr > 0.3:
+        return '中度正相关：情绪与股价有一定同步性'
+    elif corr > -0.3:
+        return '弱相关：情绪与股价关系不明显'
+    elif corr > -0.7:
+        return '中度负相关：情绪高涨时股价反而下跌（反向指标）'
+    else:
+        return '强负相关：典型反向指标，情绪越乐观越要警惕'
+
+
+def get_keyword_cloud(days: int = 7, top_n: int = 50) -> Dict[str, Any]:
+    """获取关键词云数据
+    
+    Args:
+        days: 统计天数
+        top_n: 返回前N个关键词
+    
+    Returns:
+        词云数据
+    """
+    since = dt.datetime.now() - dt.timedelta(days=days)
+    
+    with get_session() as s:
+        posts = s.execute(
+            select(SocialPost)
+            .where(SocialPost.created_at >= since)
+        ).scalars().all()
+        
+        if not posts:
+            return {'ok': False, 'msg': '暂无数据'}
+        
+        # 收集所有关键词
+        all_keywords = []
+        for p in posts:
+            if p.keywords:
+                all_keywords.extend(p.keywords.split(','))
+        
+        # 统计词频
+        keyword_counter = Counter(all_keywords)
+        
+        # 格式化为词云数据
+        wordcloud_data = [
+            {'name': kw, 'value': cnt}
+            for kw, cnt in keyword_counter.most_common(top_n)
+        ]
+    
+    return {
+        'ok': True,
+        'days': days,
+        'keywords': wordcloud_data
+    }
+