test_loader.py

"""
Test Case Loader for User Memory Evaluation Framework
"""

import sys
import json
import subprocess
from pathlib import Path
from typing import Dict, List, Optional, Any
from config import EVAL_FRAMEWORK_PATH

class TestCaseLoader:
    """Load test cases from user-memory-evaluation framework"""
    
    def __init__(self):
        self.eval_framework_path = EVAL_FRAMEWORK_PATH
        if not self.eval_framework_path.exists():
            raise ValueError(f"Evaluation framework not found at {self.eval_framework_path}")
    
    def get_all_test_cases(self) -> List[Dict[str, Any]]:
        """Get all available test cases"""
        script = """
import sys
import json
from pathlib import Path
import io

# Suppress rich console output
import rich.console
rich.console.Console = lambda *args, **kwargs: type('FakeConsole', (), {
    'print': lambda self, *a, **k: None,
    '__getattr__': lambda self, name: lambda *a, **k: None
})()

# Redirect output
old_stdout = sys.stdout
sys.stdout = io.StringIO()

try:
    from framework import UserMemoryEvaluationFramework
    
    framework = UserMemoryEvaluationFramework()
    test_cases = []
    
    for tc in framework.list_test_cases():
        test_cases.append({
            'test_id': tc.test_id,
            'category': tc.category,
            'title': tc.title,
            'description': tc.description,
            'num_conversations': len(tc.conversation_histories),
            'user_question': tc.user_question
        })
    
    # Restore stdout for JSON output
    sys.stdout = old_stdout
    print(json.dumps(test_cases))
    
except Exception as e:
    sys.stdout = old_stdout
    print(json.dumps([]))
"""
        
        result = subprocess.run(
            [sys.executable, "-c", script],
            cwd=self.eval_framework_path,
            capture_output=True,
            text=True
        )
        
        if result.returncode != 0:
            print(f"Error getting test cases: {result.stderr}")
            return []
        
        try:
            return json.loads(result.stdout)
        except json.JSONDecodeError:
            print(f"Error parsing test cases JSON")
            return []
    
    def get_layer3_test_cases(self) -> List[Dict[str, Any]]:
        """Get only Layer 3 test cases (most complex)"""
        all_cases = self.get_all_test_cases()
        return [tc for tc in all_cases if tc['category'] == 'layer3']
    
    def get_test_case_conversations(self, test_id: str) -> List[Dict[str, Any]]:
        """Get detailed conversation histories for a specific test case"""
        script = f"""
import sys
import json
from pathlib import Path
import io

# Redirect stdout to suppress any print statements from framework
old_stdout = sys.stdout
sys.stdout = io.StringIO()

try:
    from framework import UserMemoryEvaluationFramework
    
    framework = UserMemoryEvaluationFramework()
    tc = framework.get_test_case("{test_id}")
    
    # Restore stdout for our JSON output
    sys.stdout = old_stdout
    
    if not tc:
        print(json.dumps([]))
    else:
        conversations = []
        for conv in tc.conversation_histories:
            conv_data = {{
                'conversation_id': conv.conversation_id,
                'timestamp': conv.timestamp,
                'messages': []
            }}
            
            for msg in conv.messages:
                msg_data = {{
                    'role': msg.role.value,
                    'content': msg.content
                }}
                # Add metadata if it exists
                if hasattr(msg, 'metadata'):
                    msg_data['metadata'] = msg.metadata
                conv_data['messages'].append(msg_data)
            
            conversations.append(conv_data)
        
        print(json.dumps(conversations))
        
except Exception as e:
    import traceback
    sys.stdout = old_stdout
    sys.stderr.write(traceback.format_exc())
    print(json.dumps([]))
"""
        
        result = subprocess.run(
            [sys.executable, "-c", script],
            cwd=self.eval_framework_path,
            capture_output=True,
            text=True
        )
        
        if result.returncode != 0:
            print(f"Error getting conversation histories: {result.stderr}")
            return []
        
        try:
            return json.loads(result.stdout)
        except json.JSONDecodeError as e:
            print(f"Error parsing conversation histories JSON: {e}")
            if result.stdout:
                print(f"stdout (first 500 chars): {result.stdout[:500]}")
            if result.stderr:
                print(f"stderr (first 500 chars): {result.stderr[:500]}")
            return []
    
    def format_conversation_text(self, conversation: Dict[str, Any]) -> str:
        """Format a conversation into readable text"""
        lines = []
        lines.append(f"Conversation ID: {conversation['conversation_id']}")
        lines.append(f"Timestamp: {conversation['timestamp']}")
        lines.append("-" * 50)
        
        for msg in conversation['messages']:
            role = msg['role'].upper()
            content = msg['content']
            lines.append(f"{role}: {content}")
            lines.append("")  # Empty line between messages
        
        return "\n".join(lines)