cache key error when user id changes -fixed task 1 31_10_2025 v2

migrate_database.py

@@ -0,0 +1,408 @@
+#!/usr/bin/env python
+"""
+Database Migration Script - Fresh Start
+Creates a new database with enhanced schema to prevent context loops.
+Optionally migrates data from existing database.
+"""
+
+import sqlite3
+import os
+import sys
+import json
+import shutil
+from datetime import datetime
+import hashlib
+
+class DatabaseMigration:
+    def __init__(self, old_db_path='sessions.db', new_db_path='sessions_v2.db'):
+        self.old_db_path = old_db_path
+        self.new_db_path = new_db_path
+        self.schema_file = 'database_schema.sql'
+        
+    def backup_existing_database(self):
+        """Create a backup of the existing database"""
+        if os.path.exists(self.old_db_path):
+            backup_path = f"{self.old_db_path}.backup_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+            shutil.copy2(self.old_db_path, backup_path)
+            print(f"✅ Created backup: {backup_path}")
+            return backup_path
+        else:
+            print(f"ℹ️ No existing database found at {self.old_db_path}")
+            return None
+    
+    def create_fresh_database(self):
+        """Create a new database with the updated schema"""
+        print(f"\n🔨 Creating fresh database: {self.new_db_path}")
+        
+        # Remove existing new database if it exists
+        if os.path.exists(self.new_db_path):
+            os.remove(self.new_db_path)
+            print(f"  Removed existing {self.new_db_path}")
+        
+        # Read schema from file or use embedded schema
+        if os.path.exists(self.schema_file):
+            print(f"  Loading schema from {self.schema_file}")
+            with open(self.schema_file, 'r') as f:
+                schema_sql = f.read()
+        else:
+            print("  Using embedded schema")
+            schema_sql = self.get_embedded_schema()
+        
+        # Create new database
+        conn = sqlite3.connect(self.new_db_path)
+        cursor = conn.cursor()
+        
+        # Execute schema
+        try:
+            cursor.executescript(schema_sql)
+            conn.commit()
+            print("✅ Database created successfully")
+            
+            # Verify tables were created
+            cursor.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name")
+            tables = cursor.fetchall()
+            print(f"  Created {len(tables)} tables: {', '.join([t[0] for t in tables])}")
+            
+            # Verify indexes were created
+            cursor.execute("SELECT name FROM sqlite_master WHERE type='index' AND name NOT LIKE 'sqlite_%' ORDER BY name")
+            indexes = cursor.fetchall()
+            print(f"  Created {len(indexes)} indexes")
+            
+        except Exception as e:
+            print(f"❌ Error creating database: {e}")
+            conn.close()
+            return False
+        
+        conn.close()
+        return True
+    
+    def migrate_data(self):
+        """Migrate data from old database to new one"""
+        if not os.path.exists(self.old_db_path):
+            print("ℹ️ No old database to migrate from")
+            return True
+        
+        print(f"\n📦 Migrating data from {self.old_db_path} to {self.new_db_path}")
+        
+        old_conn = sqlite3.connect(self.old_db_path)
+        old_conn.row_factory = sqlite3.Row
+        new_conn = sqlite3.connect(self.new_db_path)
+        
+        try:
+            # Check what tables exist in old database
+            old_cursor = old_conn.cursor()
+            old_cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
+            old_tables = [row[0] for row in old_cursor.fetchall()]
+            print(f"  Old database tables: {', '.join(old_tables)}")
+            
+            migrated_counts = {}
+            
+            # Migrate sessions
+            if 'sessions' in old_tables:
+                old_cursor.execute("SELECT * FROM sessions")
+                sessions = old_cursor.fetchall()
+                
+                new_cursor = new_conn.cursor()
+                for session in sessions:
+                    # Extract user_id if it exists, otherwise use default
+                    user_id = session['user_id'] if 'user_id' in session.keys() else 'Test_Any'
+                    
+                    new_cursor.execute("""
+                        INSERT OR IGNORE INTO sessions 
+                        (session_id, user_id, created_at, last_activity, context_data, user_metadata)
+                        VALUES (?, ?, ?, ?, ?, ?)
+                    """, (
+                        session['session_id'],
+                        user_id,
+                        session['created_at'] if 'created_at' in session.keys() else datetime.now().isoformat(),
+                        session['last_activity'] if 'last_activity' in session.keys() else datetime.now().isoformat(),
+                        session['context_data'] if 'context_data' in session.keys() else '{}',
+                        session['user_metadata'] if 'user_metadata' in session.keys() else '{}'
+                    ))
+                
+                migrated_counts['sessions'] = len(sessions)
+                print(f"  ✅ Migrated {len(sessions)} sessions")
+            
+            # Migrate interactions with deduplication
+            if 'interactions' in old_tables:
+                old_cursor.execute("SELECT * FROM interactions")
+                interactions = old_cursor.fetchall()
+                
+                new_cursor = new_conn.cursor()
+                for interaction in interactions:
+                    # Generate interaction hash for deduplication
+                    hash_content = f"{interaction['session_id']}:{interaction['user_input']}:{interaction['created_at']}"
+                    interaction_hash = hashlib.sha256(hash_content.encode()).hexdigest()
+                    
+                    # Check for required fields
+                    interaction_id = interaction['interaction_id'] if 'interaction_id' in interaction.keys() else interaction['id'] if 'id' in interaction.keys() else str(hash(hash_content))
+                    
+                    new_cursor.execute("""
+                        INSERT OR IGNORE INTO interactions 
+                        (interaction_id, interaction_hash, session_id, user_input, final_response, 
+                         processing_time, created_at)
+                        VALUES (?, ?, ?, ?, ?, ?, ?)
+                    """, (
+                        str(interaction_id),
+                        interaction_hash,
+                        interaction['session_id'],
+                        interaction['user_input'],
+                        interaction['final_response'] if 'final_response' in interaction.keys() else None,
+                        interaction['processing_time'] if 'processing_time' in interaction.keys() else None,
+                        interaction['created_at'] if 'created_at' in interaction.keys() else datetime.now().isoformat()
+                    ))
+                
+                migrated_counts['interactions'] = len(interactions)
+                print(f"  ✅ Migrated {len(interactions)} interactions")
+            
+            # Migrate user_contexts if they exist
+            if 'user_contexts' in old_tables:
+                old_cursor.execute("SELECT * FROM user_contexts")
+                user_contexts = old_cursor.fetchall()
+                
+                new_cursor = new_conn.cursor()
+                for context in user_contexts:
+                    new_cursor.execute("""
+                        INSERT OR REPLACE INTO user_contexts 
+                        (user_id, persona_summary, updated_at)
+                        VALUES (?, ?, ?)
+                    """, (
+                        context['user_id'],
+                        context['persona_summary'] if 'persona_summary' in context.keys() else None,
+                        context['updated_at'] if 'updated_at' in context.keys() else datetime.now().isoformat()
+                    ))
+                
+                migrated_counts['user_contexts'] = len(user_contexts)
+                print(f"  ✅ Migrated {len(user_contexts)} user contexts")
+            
+            # Migrate interaction_contexts if they exist
+            if 'interaction_contexts' in old_tables:
+                old_cursor.execute("SELECT * FROM interaction_contexts")
+                int_contexts = old_cursor.fetchall()
+                
+                new_cursor = new_conn.cursor()
+                for context in int_contexts:
+                    new_cursor.execute("""
+                        INSERT OR IGNORE INTO interaction_contexts 
+                        (interaction_id, session_id, user_input, system_response, 
+                         interaction_summary, created_at)
+                        VALUES (?, ?, ?, ?, ?, ?)
+                    """, (
+                        context['interaction_id'],
+                        context['session_id'],
+                        context['user_input'] if 'user_input' in context.keys() else None,
+                        context['system_response'] if 'system_response' in context.keys() else None,
+                        context['interaction_summary'] if 'interaction_summary' in context.keys() else None,
+                        context['created_at'] if 'created_at' in context.keys() else datetime.now().isoformat()
+                    ))
+                
+                migrated_counts['interaction_contexts'] = len(int_contexts)
+                print(f"  ✅ Migrated {len(int_contexts)} interaction contexts")
+            
+            new_conn.commit()
+            
+            # Print summary
+            print("\n📊 Migration Summary:")
+            for table, count in migrated_counts.items():
+                print(f"    {table}: {count} records")
+            
+        except Exception as e:
+            print(f"❌ Error during migration: {e}")
+            new_conn.rollback()
+            return False
+        finally:
+            old_conn.close()
+            new_conn.close()
+        
+        return True
+    
+    def verify_new_database(self):
+        """Verify the new database is working correctly"""
+        print(f"\n🔍 Verifying new database: {self.new_db_path}")
+        
+        conn = sqlite3.connect(self.new_db_path)
+        cursor = conn.cursor()
+        
+        try:
+            # Check tables
+            cursor.execute("SELECT COUNT(*) FROM sqlite_master WHERE type='table'")
+            table_count = cursor.fetchone()[0]
+            print(f"  Tables: {table_count}")
+            
+            # Check indexes
+            cursor.execute("SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name NOT LIKE 'sqlite_%'")
+            index_count = cursor.fetchone()[0]
+            print(f"  Indexes: {index_count}")
+            
+            # Check triggers
+            cursor.execute("SELECT COUNT(*) FROM sqlite_master WHERE type='trigger'")
+            trigger_count = cursor.fetchone()[0]
+            print(f"  Triggers: {trigger_count}")
+            
+            # Check views
+            cursor.execute("SELECT COUNT(*) FROM sqlite_master WHERE type='view'")
+            view_count = cursor.fetchone()[0]
+            print(f"  Views: {view_count}")
+            
+            # Check data
+            cursor.execute("SELECT COUNT(*) FROM sessions")
+            session_count = cursor.fetchone()[0]
+            print(f"  Sessions: {session_count}")
+            
+            cursor.execute("SELECT COUNT(*) FROM interactions")
+            interaction_count = cursor.fetchone()[0]
+            print(f"  Interactions: {interaction_count}")
+            
+            cursor.execute("SELECT COUNT(*) FROM user_contexts")
+            user_count = cursor.fetchone()[0]
+            print(f"  Users: {user_count}")
+            
+            # Test a view
+            cursor.execute("SELECT COUNT(*) FROM active_sessions")
+            active_count = cursor.fetchone()[0]
+            print(f"  Active sessions (view): {active_count}")
+            
+            print("\n✅ Database verification complete!")
+            
+        except Exception as e:
+            print(f"❌ Verification error: {e}")
+            return False
+        finally:
+            conn.close()
+        
+        return True
+    
+    def replace_old_database(self):
+        """Replace the old database with the new one"""
+        print(f"\n🔄 Replacing old database...")
+        
+        # Move old database to .old
+        if os.path.exists(self.old_db_path):
+            old_backup = f"{self.old_db_path}.old"
+            if os.path.exists(old_backup):
+                os.remove(old_backup)
+            os.rename(self.old_db_path, old_backup)
+            print(f"  Moved {self.old_db_path} to {old_backup}")
+        
+        # Move new database to production name
+        os.rename(self.new_db_path, self.old_db_path)
+        print(f"  Moved {self.new_db_path} to {self.old_db_path}")
+        
+        print("✅ Database replacement complete!")
+        return True
+    
+    def get_embedded_schema(self):
+        """Return the embedded schema if file is not available"""
+        return """
+-- Embedded minimal schema (use database_schema.sql for full version)
+PRAGMA journal_mode=WAL;
+PRAGMA foreign_keys=ON;
+
+CREATE TABLE sessions (
+    session_id TEXT PRIMARY KEY,
+    user_id TEXT NOT NULL DEFAULT 'Test_Any',
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    last_activity TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    context_data TEXT DEFAULT '{}',
+    user_metadata TEXT DEFAULT '{}'
+);
+
+CREATE TABLE interactions (
+    interaction_id TEXT PRIMARY KEY,
+    interaction_hash TEXT UNIQUE,
+    session_id TEXT NOT NULL,
+    user_input TEXT NOT NULL,
+    final_response TEXT,
+    processing_time REAL,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY(session_id) REFERENCES sessions(session_id)
+);
+
+CREATE TABLE user_contexts (
+    user_id TEXT PRIMARY KEY,
+    persona_summary TEXT,
+    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE TABLE interaction_contexts (
+    interaction_id TEXT PRIMARY KEY,
+    session_id TEXT NOT NULL,
+    user_input TEXT,
+    system_response TEXT,
+    interaction_summary TEXT,
+    needs_refresh INTEGER DEFAULT 0,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY(session_id) REFERENCES sessions(session_id)
+);
+
+CREATE TABLE user_change_log (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    session_id TEXT NOT NULL,
+    old_user_id TEXT,
+    new_user_id TEXT NOT NULL,
+    timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY(session_id) REFERENCES sessions(session_id)
+);
+
+-- Basic indexes
+CREATE INDEX idx_sessions_user_id ON sessions(user_id);
+CREATE INDEX idx_interactions_session_id ON interactions(session_id);
+CREATE INDEX idx_interactions_hash ON interactions(interaction_hash);
+"""
+
+def main():
+    """Main migration process"""
+    print("=" * 60)
+    print("DATABASE MIGRATION TOOL")
+    print("This will create a fresh database with loop prevention fixes")
+    print("=" * 60)
+    
+    # Parse arguments
+    migrate_data = True
+    replace_db = False
+    
+    if len(sys.argv) > 1:
+        if '--no-migration' in sys.argv:
+            migrate_data = False
+            print("ℹ️ Data migration disabled")
+        if '--replace' in sys.argv:
+            replace_db = True
+            print("ℹ️ Will replace existing database")
+    
+    migration = DatabaseMigration()
+    
+    # Step 1: Backup
+    backup_path = migration.backup_existing_database()
+    
+    # Step 2: Create fresh database
+    if not migration.create_fresh_database():
+        print("❌ Failed to create new database")
+        return 1
+    
+    # Step 3: Migrate data (optional)
+    if migrate_data and backup_path:
+        if not migration.migrate_data():
+            print("⚠️ Data migration had issues, but continuing...")
+    
+    # Step 4: Verify
+    if not migration.verify_new_database():
+        print("❌ Database verification failed")
+        return 1
+    
+    # Step 5: Replace (optional)
+    if replace_db:
+        if not migration.replace_old_database():
+            print("❌ Failed to replace database")
+            return 1
+    else:
+        print(f"\n✅ New database created at: {migration.new_db_path}")
+        print(f"To use it, either:")
+        print(f"  1. Run with --replace flag to replace the old database")
+        print(f"  2. Manually rename {migration.new_db_path} to sessions.db")
+        print(f"  3. Update your application to use {migration.new_db_path}")
+    
+    print("\n🎉 Migration complete!")
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())

update_database_schema.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python
+"""
+Database Schema Update Script
+Updates the sessions.db database with necessary schema changes for the context loop fix.
+"""
+
+import sqlite3
+import os
+import sys
+from datetime import datetime
+
+def update_database_schema(db_path='sessions.db'):
+    """
+    Update database schema with loop prevention improvements
+    
+    Args:
+        db_path: Path to the SQLite database file (default: sessions.db)
+    """
+    
+    # Check if database file exists
+    if not os.path.exists(db_path):
+        print(f"❌ Error: Database file '{db_path}' not found!")
+        print(f"Current directory: {os.getcwd()}")
+        print(f"Files in current directory: {os.listdir('.')}")
+        return False
+    
+    # Verify it's a valid SQLite database
+    try:
+        # Test connection
+        test_conn = sqlite3.connect(db_path)
+        test_cursor = test_conn.cursor()
+        test_cursor.execute("SELECT name FROM sqlite_master WHERE type='table' LIMIT 1")
+        test_conn.close()
+    except sqlite3.DatabaseError as e:
+        print(f"❌ Error: '{db_path}' is not a valid SQLite database!")
+        print(f"Error details: {e}")
+        return False
+    
+    print(f"✅ Found database: {db_path}")
+    print(f"📁 File size: {os.path.getsize(db_path):,} bytes")
+    print("=" * 50)
+    
+    applied_count = 0
+    skipped_count = 0
+    failed_count = 0
+    
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+        
+        # First, check existing schema
+        print("Checking existing schema...")
+        cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
+        existing_tables = [row[0] for row in cursor.fetchall()]
+        print(f"Existing tables: {', '.join(existing_tables)}")
+        print("-" * 50)
+        
+        # Enable WAL mode for better concurrency
+        try:
+            cursor.execute('PRAGMA journal_mode=WAL')
+            print("✓ Enabled WAL mode for better concurrency")
+            applied_count += 1
+        except sqlite3.OperationalError as e:
+            print(f"✗ Failed to enable WAL mode: {e}")
+            failed_count += 1
+        
+        # Schema updates to apply
+        schema_updates = [
+            # Add interaction_hash for deduplication
+            {
+                'name': 'interaction_hash column',
+                'check_sql': "PRAGMA table_info(interactions)",
+                'check_field': 'interaction_hash',
+                'update_sql': "ALTER TABLE interactions ADD COLUMN interaction_hash TEXT UNIQUE",
+                'table_required': 'interactions'
+            },
+            
+            # Add needs_refresh flag for cache invalidation
+            {
+                'name': 'needs_refresh column',
+                'check_sql': "PRAGMA table_info(interaction_contexts)",
+                'check_field': 'needs_refresh',
+                'update_sql': "ALTER TABLE interaction_contexts ADD COLUMN needs_refresh INTEGER DEFAULT 0",
+                'table_required': 'interaction_contexts'
+            },
+            
+            # Create user change log table
+            {
+                'name': 'user_change_log table',
+                'check_sql': "SELECT name FROM sqlite_master WHERE type='table' AND name='user_change_log'",
+                'check_exists': False,
+                'update_sql': """CREATE TABLE IF NOT EXISTS user_change_log (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    session_id TEXT,
+                    old_user_id TEXT,
+                    new_user_id TEXT,
+                    timestamp TIMESTAMP,
+                    FOREIGN KEY(session_id) REFERENCES sessions(session_id)
+                )"""
+            }
+        ]
+        
+        # Apply schema updates
+        for update in schema_updates:
+            try:
+                # Check if table exists (if required)
+                if 'table_required' in update:
+                    if update['table_required'] not in existing_tables:
+                        print(f"⚠ Skipping {update['name']} - table '{update['table_required']}' doesn't exist")
+                        skipped_count += 1
+                        continue
+                
+                # Check if update is needed
+                needs_update = True
+                if 'check_field' in update:
+                    cursor.execute(update['check_sql'])
+                    columns = [row[1] for row in cursor.fetchall()]
+                    if update['check_field'] in columns:
+                        print(f"⚠ {update['name']} already exists")
+                        skipped_count += 1
+                        needs_update = False
+                elif 'check_exists' in update and not update['check_exists']:
+                    cursor.execute(update['check_sql'])
+                    if cursor.fetchone():
+                        print(f"⚠ {update['name']} already exists")
+                        skipped_count += 1
+                        needs_update = False
+                
+                # Apply update if needed
+                if needs_update:
+                    cursor.execute(update['update_sql'])
+                    print(f"✓ Applied: {update['name']}")
+                    applied_count += 1
+                    
+            except sqlite3.OperationalError as e:
+                error_msg = str(e).lower()
+                if "duplicate column" in error_msg or "already exists" in error_msg:
+                    print(f"⚠ {update['name']} already exists")
+                    skipped_count += 1
+                else:
+                    print(f"✗ Failed: {update['name']} - {e}")
+                    failed_count += 1
+        
+        print("-" * 50)
+        
+        # Create indexes for better performance
+        indexes = [
+            ("idx_sessions_user_id", "CREATE INDEX IF NOT EXISTS idx_sessions_user_id ON sessions(user_id)", "sessions"),
+            ("idx_sessions_last_activity", "CREATE INDEX IF NOT EXISTS idx_sessions_last_activity ON sessions(last_activity)", "sessions"),
+            ("idx_interaction_hash", "CREATE INDEX IF NOT EXISTS idx_interaction_hash ON interactions(interaction_hash)", "interactions"),
+            ("idx_interaction_contexts_session_id", "CREATE INDEX IF NOT EXISTS idx_interaction_contexts_session_id ON interaction_contexts(session_id)", "interaction_contexts"),
+            ("idx_interaction_contexts_needs_refresh", "CREATE INDEX IF NOT EXISTS idx_interaction_contexts_needs_refresh ON interaction_contexts(needs_refresh)", "interaction_contexts"),
+            ("idx_user_change_log_session_id", "CREATE INDEX IF NOT EXISTS idx_user_change_log_session_id ON user_change_log(session_id)", "user_change_log")
+        ]
+        
+        print("Creating performance indexes...")
+        for index_name, index_sql, required_table in indexes:
+            try:
+                # Check if required table exists
+                if required_table not in existing_tables and required_table != "user_change_log":
+                    print(f"⚠ Skipping {index_name} - table '{required_table}' doesn't exist")
+                    skipped_count += 1
+                    continue
+                    
+                cursor.execute(index_sql)
+                print(f"✓ Created index: {index_name}")
+                applied_count += 1
+            except sqlite3.OperationalError as e:
+                if "already exists" in str(e).lower():
+                    print(f"⚠ Index {index_name} already exists")
+                    skipped_count += 1
+                else:
+                    print(f"✗ Failed: {index_name} - {e}")
+                    failed_count += 1
+        
+        # Run ANALYZE to optimize query planner
+        try:
+            cursor.execute("ANALYZE")
+            print("✓ Database statistics updated (ANALYZE)")
+            applied_count += 1
+        except Exception as e:
+            print(f"✗ Failed to run ANALYZE: {e}")
+            failed_count += 1
+        
+        # Commit all changes
+        conn.commit()
+        
+        # Get final schema info
+        print("-" * 50)
+        print("Final schema check:")
+        cursor.execute("SELECT name, type FROM sqlite_master WHERE type IN ('table', 'index') ORDER BY type, name")
+        for name, obj_type in cursor.fetchall():
+            print(f"  {obj_type.capitalize()}: {name}")
+        
+        conn.close()
+        
+    except Exception as e:
+        print(f"❌ Unexpected error: {e}")
+        if conn:
+            conn.rollback()
+            conn.close()
+        return False
+    
+    print("=" * 50)
+    print("Database schema update complete!")
+    print(f"  Applied: {applied_count}")
+    print(f"  Skipped: {skipped_count} (already exists)")
+    print(f"  Failed: {failed_count}")
+    print("=" * 50)
+    
+    return failed_count == 0
+
+def main():
+    """Main entry point"""
+    # Check command line arguments
+    if len(sys.argv) > 1:
+        db_path = sys.argv[1]
+        # Check if user accidentally passed the SQL schema file
+        if db_path.endswith('.sql'):
+            print("❌ Error: You provided a .sql file, but this script needs the SQLite database file!")
+            print("Usage: python update_database_schema.py [database_file]")
+            print("Example: python update_database_schema.py sessions.db")
+            print("\nLooking for SQLite database files in current directory...")
+            db_files = [f for f in os.listdir('.') if f.endswith('.db')]
+            if db_files:
+                print(f"Found: {', '.join(db_files)}")
+                if 'sessions.db' in db_files:
+                    print("\nWould you like to use 'sessions.db'? Run:")
+                    print("python update_database_schema.py sessions.db")
+            else:
+                print("No .db files found in current directory.")
+            return 1
+    else:
+        # Default to sessions.db
+        db_path = 'sessions.db'
+    
+    print(f"Database Schema Update Script")
+    print(f"Target database: {db_path}")
+    print("=" * 50)
+    
+    success = update_database_schema(db_path)
+    return 0 if success else 1
+
+if __name__ == "__main__":
+    sys.exit(main())