← vibesdk

#!/usr/bin/env python3
"""
Conversation Messages Analyzer

This script analyzes the conversationMessages property to understand
why it's so large and provide specific recommendations for cleanup.

Usage: python conversation_analyzer.py
"""

import json
import os
from typing import List, Dict, Any
from dataclasses import dataclass
from collections import Counter, defaultdict

@dataclass
class ConversationAnalysis:
    total_messages: int
    total_size: int
    avg_message_size: int
    message_types: Dict[str, int]
    largest_messages: List[Dict[str, Any]]
    size_by_type: Dict[str, int]
    recommendations: List[str]

class ConversationAnalyzer:
    def __init__(self):
        self.size_thresholds = {
            'small': 1000,      # 1KB
            'medium': 5000,     # 5KB  
            'large': 20000,     # 20KB
            'huge': 100000      # 100KB
        }
    
    def analyze_conversation_messages(self, messages: List[Dict[str, Any]]) -> ConversationAnalysis:
        """Analyze conversation messages for size and content"""
        print(f"🔍 Analyzing {len(messages)} conversation messages...")
        
        total_size = 0
        message_types = Counter()
        size_by_type = defaultdict(int)
        largest_messages = []
        
        for i, msg in enumerate(messages):
            # Calculate message size
            msg_size = len(json.dumps(msg, default=str))
            total_size += msg_size
            
            # Categorize by type/role
            msg_type = msg.get('role', msg.get('type', 'unknown'))
            message_types[msg_type] += 1
            size_by_type[msg_type] += msg_size
            
            # Track largest messages
            msg_info = {
                'index': i,
                'size': msg_size,
                'type': msg_type,
                'content_preview': str(msg.get('content', ''))[:100],
                'timestamp': msg.get('timestamp', 'unknown')
            }
            largest_messages.append(msg_info)
        
        # Sort by size for analysis
        largest_messages.sort(key=lambda x: x['size'], reverse=True)
        
        avg_size = total_size // len(messages) if messages else 0
        
        # Generate recommendations
        recommendations = self._generate_recommendations(
            messages, total_size, avg_size, message_types, largest_messages
        )
        
        return ConversationAnalysis(
            total_messages=len(messages),
            total_size=total_size,
            avg_message_size=avg_size,
            message_types=dict(message_types),
            largest_messages=largest_messages[:20],  # Top 20
            size_by_type=dict(size_by_type),
            recommendations=recommendations
        )
    
    def _generate_recommendations(self, messages, total_size, avg_size, message_types, largest_messages):
        recommendations = []
        
        # Size-based recommendations
        if total_size > 1000000:  # 1MB
            recommendations.append("🚨 CRITICAL: Conversation messages exceed 1MB - implement immediate cleanup")
            recommendations.append("   Implement a maximum message history limit (e.g., 50-100 messages)")
        
        if avg_size > 10000:  # 10KB average
            recommendations.append("⚠️  Average message size is very large - check for data bloat in messages")
        
        # Check for huge individual messages
        huge_messages = [m for m in largest_messages if m['size'] > self.size_thresholds['huge']]
        if huge_messages:
            recommendations.append(f"🔍 Found {len(huge_messages)} messages larger than 100KB each")
            recommendations.append("   Consider truncating or summarizing very long messages")
        
        # Type-based recommendations
        if 'assistant' in message_types and message_types['assistant'] > 100:
            recommendations.append("📝 High number of assistant messages - consider keeping only recent ones")
        
        if 'user' in message_types and message_types['user'] > 50:
            recommendations.append("👤 High number of user messages - implement conversation pruning")
        
        # Specific code recommendations
        total_messages = len(messages)
        if total_messages > 100:
            keep_recent = min(50, total_messages // 2)
            recommendations.append(f"💡 SUGGESTED FIX: Keep only the {keep_recent} most recent messages")
            recommendations.append("   Implementation: conversationMessages = conversationMessages.slice(-{keep_recent})")
        
        return recommendations
    
    def generate_report(self, analysis: ConversationAnalysis) -> str:
        """Generate detailed conversation analysis report"""
        report = ["Conversation Messages Analysis Report", "=" * 50, ""]
        
        # Summary
        report.append("📊 CONVERSATION SUMMARY")
        report.append("-" * 30)
        report.append(f"Total messages: {analysis.total_messages:,}")
        report.append(f"Total size: {analysis.total_size:,} characters ({analysis.total_size/1024:.1f} KB)")
        report.append(f"Average message size: {analysis.avg_message_size:,} characters")
        report.append("")
        
        # Message types breakdown
        report.append("📈 MESSAGE TYPES")
        report.append("-" * 20)
        for msg_type, count in sorted(analysis.message_types.items(), key=lambda x: x[1], reverse=True):
            percentage = (count / analysis.total_messages * 100)
            avg_size_type = analysis.size_by_type[msg_type] // count if count > 0 else 0
            report.append(f"{msg_type:>15}: {count:>4} messages ({percentage:4.1f}%) - avg {avg_size_type:,} chars each")
        report.append("")
        
        # Largest messages
        report.append("🎯 LARGEST MESSAGES (Top 20)")
        report.append("-" * 40)
        report.append(f"{'Index':>5} {'Size':>8} {'Type':>12} {'Preview'}")
        report.append("-" * 70)
        
        for msg in analysis.largest_messages[:20]:
            preview = msg['content_preview'].replace('\n', ' ')[:50]
            report.append(f"{msg['index']:>5} {msg['size']:>8} {msg['type']:>12} {preview}")
        report.append("")
        
        # Size distribution
        report.append("📊 SIZE DISTRIBUTION")
        report.append("-" * 25)
        size_buckets = {
            'tiny (<1KB)': 0,
            'small (1-5KB)': 0, 
            'medium (5-20KB)': 0,
            'large (20-100KB)': 0,
            'huge (>100KB)': 0
        }
        
        for msg in analysis.largest_messages:
            size = msg['size']
            if size < 1000:
                size_buckets['tiny (<1KB)'] += 1
            elif size < 5000:
                size_buckets['small (1-5KB)'] += 1
            elif size < 20000:
                size_buckets['medium (5-20KB)'] += 1
            elif size < 100000:
                size_buckets['large (20-100KB)'] += 1
            else:
                size_buckets['huge (>100KB)'] += 1
        
        for bucket, count in size_buckets.items():
            percentage = (count / analysis.total_messages * 100) if analysis.total_messages > 0 else 0
            report.append(f"{bucket:>20}: {count:>4} messages ({percentage:4.1f}%)")
        report.append("")
        
        # Recommendations
        report.append("💡 RECOMMENDATIONS")
        report.append("-" * 20)
        for rec in analysis.recommendations:
            report.append(rec)
        report.append("")
        
        # Code suggestions
        report.append("🔧 CODE IMPLEMENTATION SUGGESTIONS")
        report.append("-" * 40)
        if analysis.total_messages > 50:
            keep_messages = min(50, analysis.total_messages // 2)
            report.append("// Add this to your setState calls to limit conversation history:")
            report.append(f"const MAX_CONVERSATION_MESSAGES = {keep_messages};")
            report.append("if (conversationMessages.length > MAX_CONVERSATION_MESSAGES) {")
            report.append("    conversationMessages = conversationMessages.slice(-MAX_CONVERSATION_MESSAGES);")
            report.append("}")
            report.append("")
            
            # Calculate potential savings
            messages_to_remove = analysis.total_messages - keep_messages
            estimated_savings = (messages_to_remove / analysis.total_messages) * analysis.total_size
            report.append(f"// Estimated size reduction: ~{estimated_savings:,.0f} characters ({estimated_savings/1024:.1f}KB)")
        
        return "\n".join(report)

def main():
    # Check if we have debug files from the main analyzer
    conversation_file = "debug_output/conversationMessages_new.json"
    
    if not os.path.exists(conversation_file):
        print("❌ Conversation messages debug file not found!")
        print("   Please run the main state analyzer first: python state_analyzer.py errorfile.json")
        print("   This will generate the required debug files in debug_output/")
        return
    
    print("🚀 Starting conversation messages analysis...")
    print(f"📁 Reading conversation data from: {conversation_file}")
    
    try:
        with open(conversation_file, 'r') as f:
            messages = json.load(f)
        
        print(f"📄 Loaded {len(messages)} conversation messages")
        
        analyzer = ConversationAnalyzer()
        analysis = analyzer.analyze_conversation_messages(messages)
        
        # Generate report
        report = analyzer.generate_report(analysis)
        
        # Save report
        report_file = "conversation_analysis_report.txt"
        with open(report_file, 'w') as f:
            f.write(report)
        
        print(f"📊 Conversation analysis saved to: {report_file}")
        
        # Print key findings
        print("\n" + "="*60)
        print("KEY FINDINGS")
        print("="*60)
        print(f"Total messages: {analysis.total_messages:,}")
        print(f"Total size: {analysis.total_size:,} chars ({analysis.total_size/1024:.1f}KB)")
        print(f"Average per message: {analysis.avg_message_size:,} chars")
        
        if analysis.largest_messages:
            largest = analysis.largest_messages[0]
            print(f"Largest single message: {largest['size']:,} chars ({largest['type']})")
        
        print(f"\n💡 Top recommendation:")
        if analysis.recommendations:
            print(f"   {analysis.recommendations[0]}")
        
        print(f"\n📋 Full analysis available in: {report_file}")
        
    except Exception as e:
        print(f"❌ Error during conversation analysis: {e}")
        import traceback
        traceback.print_exc()

if __name__ == "__main__":
    main()