feat: Add topic similarity service using Google Generative AI embeddings, enabling improved topic matching and similarity analysis. Update topic extraction logic to utilize this service and enhance overall functionality.

.gitignore

@@ -10,4 +10,5 @@ __pycache__
 *.pyzw
 *.pyzwz
 *.pyzwzw
-*.pyzwzwz
+*.pyzwzwz
+data/topic_embeddings_cache.json

config.py

@@ -28,6 +28,9 @@ GENERATE_MODEL_ID = HUGGINGFACE_GENERATE_MODEL_ID
 # ============ GROQ MODELS ============
 GROQ_API_KEY = os.getenv("GROQ_API_KEY", "")
 
+# ============ GOOGLE MODELS ============
+GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY", "")
+
 # **Speech-to-Text**
 GROQ_STT_MODEL = "whisper-large-v3-turbo"
 
@@ -84,5 +87,6 @@ logger.info(f"  HF Label Model  : {HUGGINGFACE_LABEL_MODEL_ID}")
 logger.info(f"  GROQ STT Model  : {GROQ_STT_MODEL}")
 logger.info(f"  GROQ TTS Model  : {GROQ_TTS_MODEL}")
 logger.info(f"  GROQ Chat Model : {GROQ_CHAT_MODEL}")
+logger.info(f"  Google API Key  : {'✓ Configured' if GOOGLE_API_KEY else '✗ Not configured'}")
 logger.info(f"  Supabase URL    : {'✓ Configured' if SUPABASE_URL else '✗ Not configured'}")
 logger.info("="*60)

data/topics.json

@@ -0,0 +1,45 @@
+{
+    "topics": [
+        "Assisted suicide should be a criminal offence",
+        "We should abolish intellectual property rights",
+        "Homeschooling should be banned",
+        "The vow of celibacy should be abandoned",
+        "We should legalize prostitution",
+        "We should ban private military companies",
+        "We should abolish capital punishment",
+        "Foster care brings more harm than good",
+        "Routine child vaccinations should be mandatory",
+        "We should abolish the three-strikes laws",
+        "We should subsidize student loans",
+        "We should end the use of economic sanctions",
+        "We should end mandatory retirement",
+        "We should close Guantanamo Bay detention camp",
+        "We should subsidize space exploration",
+        "We should abandon the use of school uniform",
+        "The use of public defenders should be mandatory",
+        "We should adopt an austerity regime",
+        "Social media platforms should be regulated by the government",
+        "We should ban human cloning",
+        "We should adopt atheism",
+        "We should introduce compulsory voting",
+        "We should adopt libertarianism",
+        "We should abolish the right to keep and bear arms",
+        "We should legalize sex selection",
+        "We should abandon marriage",
+        "Entrapment should be legalized",
+        "We should end affirmative action",
+        "We should prohibit women in combat",
+        "We should adopt a zero-tolerance policy in schools",
+        "We should subsidize vocational education",
+        "We should ban the use of child actors",
+        "We should legalize cannabis",
+        "We should ban cosmetic surgery",
+        "We should end racial profiling",
+        "We should prohibit flag burning",
+        "The USA is a good country to live in",
+        "We should ban algorithmic trading",
+        "We should fight for the abolition of nuclear weapons",
+        "We should fight urbanization",
+        "We should subsidize journalism"
+    ]
+}

requirements.txt

@@ -8,6 +8,7 @@ pydantic>=2.5.0
 requests>=2.31.0
 groq>=0.9.0
 supabase>=2.0.0
+google-genai>=0.2.0
 
 # LangChain
 langchain>=0.1.0
@@ -24,6 +25,7 @@ torch>=2.0.1
 
 # Autres dépendances
 numpy>=1.26.4
+scikit-learn>=1.3.0
 
 mcp>=1.0.0
 # Note: fastapi-mcp peut ne pas exister officiellement, 

services/analysis_service.py

@@ -5,7 +5,7 @@ from typing import List, Dict, Optional
 from datetime import datetime
 
 from services.database_service import database_service
-from services.topic_service import topic_service
+from services.topic_similarity_service import topic_similarity_service
 from services.stance_model_manager import stance_model_manager
 
 logger = logging.getLogger(__name__)
@@ -42,9 +42,9 @@ class AnalysisService:
             
             logger.info(f"Starting analysis for {len(arguments)} arguments for user {user_id}")
             
-            # Step 1: Extract topics for all arguments
-            logger.info("Step 1: Extracting topics...")
-            topics = topic_service.batch_extract_topics(arguments)
+            # Step 1: Find most similar topics for all arguments
+            logger.info("Step 1: Finding most similar topics...")
+            topics = topic_similarity_service.batch_find_similar_topics(arguments)
             
             if len(topics) != len(arguments):
                 raise RuntimeError(f"Topic extraction returned {len(topics)} topics but expected {len(arguments)}")

services/topic_service.py

@@ -1,73 +1,32 @@
 """Service for topic extraction from text using LangChain Groq"""
 
 import logging
-import json
 from typing import Optional, List
 from langchain_core.messages import HumanMessage, SystemMessage
 from langchain_groq import ChatGroq
+from pydantic import BaseModel, Field
 from langsmith import traceable
 
 from config import GROQ_API_KEY
 
 logger = logging.getLogger(__name__)
 
-# Predefined topics list
-PREDEFINED_TOPICS = [
-    "Assisted suicide should be a criminal offence",
-    "We should abolish intellectual property rights",
-    "Homeschooling should be banned",
-    "The vow of celibacy should be abandoned",
-    "We should legalize prostitution",
-    "We should ban private military companies",
-    "We should abolish capital punishment",
-    "Foster care brings more harm than good",
-    "Routine child vaccinations should be mandatory",
-    "We should abolish the three-strikes laws",
-    "We should subsidize student loans",
-    "We should end the use of economic sanctions",
-    "We should end mandatory retirement",
-    "We should close Guantanamo Bay detention camp",
-    "We should subsidize space exploration",
-    "We should abandon the use of school uniform",
-    "The use of public defenders should be mandatory",
-    "We should adopt an austerity regime",
-    "Social media platforms should be regulated by the government",
-    "We should ban human cloning",
-    "We should adopt atheism",
-    "We should introduce compulsory voting",
-    "We should adopt libertarianism",
-    "We should abolish the right to keep and bear arms",
-    "We should legalize sex selection",
-    "We should abandon marriage",
-    "Entrapment should be legalized",
-    "We should end affirmative action",
-    "We should prohibit women in combat",
-    "We should adopt a zero-tolerance policy in schools",
-    "We should subsidize vocational education",
-    "We should ban the use of child actors",
-    "We should legalize cannabis",
-    "We should ban cosmetic surgery",
-    "We should end racial profiling",
-    "We should prohibit flag burning",
-    "The USA is a good country to live in",
-    "We should ban algorithmic trading",
-    "We should fight for the abolition of nuclear weapons",
-    "We should fight urbanization",
-    "We should subsidize journalism",
-]
+
+class TopicOutput(BaseModel):
+    """Pydantic schema for topic extraction output"""
+    topic: str = Field(..., description="A specific, detailed topic description")
 
 
 class TopicService:
-    """Service for extracting topics from text arguments by matching to predefined topics"""
+    """Service for extracting topics from text arguments"""
     
     def __init__(self):
         self.llm = None
-        self.model_name = "openai/gpt-oss-safeguard-20b"  # Default model
+        self.model_name = "openai/gpt-oss-safeguard-120b"  # another model meta-llama/llama-4-scout-17b-16e-instruct
         self.initialized = False
-        self.predefined_topics = PREDEFINED_TOPICS
         
     def initialize(self, model_name: Optional[str] = None):
-        """Initialize the Groq LLM"""
+        """Initialize the Groq LLM with structured output"""
         if self.initialized:
             logger.info("Topic service already initialized")
             return
@@ -81,13 +40,15 @@ class TopicService:
         try:
             logger.info(f"Initializing topic extraction service with model: {self.model_name}")
             
-            self.llm = ChatGroq(
+            llm = ChatGroq(
                 model=self.model_name,
                 api_key=GROQ_API_KEY,
                 temperature=0.0,
                 max_tokens=512,
             )
             
+            # Bind structured output directly to the model
+            self.llm = llm.with_structured_output(TopicOutput)
             self.initialized = True
             
             logger.info("✓ Topic extraction service initialized successfully")
@@ -96,46 +57,16 @@ class TopicService:
             logger.error(f"Error initializing topic service: {str(e)}")
             raise RuntimeError(f"Failed to initialize topic service: {str(e)}")
     
-    def _get_system_message(self) -> str:
-        """Generate system message with predefined topics list"""
-        topics_list = "\n".join([f"{i+1}. {topic}" for i, topic in enumerate(self.predefined_topics)])
-        
-        return f"""You are a topic classification model. Your task is to select the MOST SIMILAR topic from the predefined list below that best matches the user's input text.
-
-IMPORTANT: You MUST return EXACTLY one of the predefined topics below. Do not create new topics or modify the wording.
-
-Return your response as a JSON object with a single "topic" field containing the exact topic text from the list.
-
-Predefined Topics:
-{topics_list}
-
-Instructions:
-1. Analyze the user's input text carefully
-2. Identify the main theme, subject, or argument being discussed
-3. Find the topic from the predefined list that is MOST SIMILAR to the input text
-4. Return a JSON object with the EXACT topic text as it appears in the list above
-
-Examples:
-- Input: "I think we need to make assisted suicide illegal and punishable by law."
-  Output: {{"topic": "Assisted suicide should be a criminal offence"}}
-
-- Input: "Student debt is crushing young people. The government should help pay for college."
-  Output: {{"topic": "We should subsidize student loans"}}
-
-- Input: "Marijuana should be legal for adults to use recreationally."
-  Output: {{"topic": "We should legalize cannabis"}}
-"""
-    
     @traceable(name="extract_topic")
     def extract_topic(self, text: str) -> str:
         """
-        Extract a topic from the given text/argument by matching to predefined topics
+        Extract a topic from the given text/argument
         
         Args:
             text: The input text/argument to extract topic from
             
         Returns:
-            The extracted topic string (must be one of the predefined topics)
+            The extracted topic string
         """
         if not self.initialized:
             self.initialize()
@@ -147,7 +78,16 @@ Examples:
         if len(text) == 0:
             raise ValueError("Text cannot be empty")
         
-        system_message = self._get_system_message()
+        system_message = """You are an information extraction model.
+Extract a topic from the user text. The topic should be a single sentence that captures the main idea of the text in simple english.
+
+Examples:
+- Text: "Governments should subsidize electric cars to encourage adoption."
+  Output: topic="government subsidies for electric vehicle adoption"
+
+- Text: "Raising the minimum wage will hurt small businesses and cost jobs."
+  Output: topic="raising the minimum wage and its economic impact on small businesses"
+"""
         
         try:
             result = self.llm.invoke(
@@ -157,68 +97,7 @@ Examples:
                 ]
             )
             
-            # Extract content from the response
-            response_content = result.content.strip()
-            
-            # Try to parse as JSON first
-            try:
-                parsed_response = json.loads(response_content)
-                selected_topic = parsed_response.get("topic", "").strip()
-            except json.JSONDecodeError:
-                # If not JSON, try to extract topic from plain text
-                # Look for the topic in the response text
-                selected_topic = response_content.strip()
-                # Remove quotes if present
-                if selected_topic.startswith('"') and selected_topic.endswith('"'):
-                    selected_topic = selected_topic[1:-1]
-                elif selected_topic.startswith("'") and selected_topic.endswith("'"):
-                    selected_topic = selected_topic[1:-1]
-            
-            if not selected_topic:
-                raise ValueError("No topic found in LLM response")
-            
-            # Validate that the returned topic is in the predefined list
-            if selected_topic not in self.predefined_topics:
-                logger.warning(
-                    f"LLM returned topic not in predefined list: '{selected_topic}'. "
-                    f"Attempting to find closest match..."
-                )
-                # Try to find the closest match (case-insensitive)
-                selected_topic_lower = selected_topic.lower()
-                for predefined_topic in self.predefined_topics:
-                    if predefined_topic.lower() == selected_topic_lower:
-                        selected_topic = predefined_topic
-                        logger.info(f"Found case-insensitive match: '{selected_topic}'")
-                        break
-                else:
-                    # If still no match, try fuzzy matching by checking if the topic contains key words
-                    # This is a fallback for when the LLM returns something close but not exact
-                    best_match = None
-                    best_match_score = 0
-                    selected_words = set(selected_topic_lower.split())
-                    
-                    for predefined_topic in self.predefined_topics:
-                        predefined_words = set(predefined_topic.lower().split())
-                        # Calculate word overlap
-                        overlap = len(selected_words & predefined_words)
-                        if overlap > best_match_score and overlap >= 2:  # At least 2 words must match
-                            best_match_score = overlap
-                            best_match = predefined_topic
-                    
-                    if best_match:
-                        logger.info(f"Found fuzzy match: '{selected_topic}' -> '{best_match}'")
-                        selected_topic = best_match
-                    else:
-                        # If still no match, log error and raise
-                        logger.error(
-                            f"Could not match returned topic '{selected_topic}' to any predefined topic. "
-                            f"Available topics: {self.predefined_topics[:3]}..."
-                        )
-                        raise ValueError(
-                            f"Returned topic '{selected_topic}' is not in the predefined topics list"
-                        )
-            
-            return selected_topic
+            return result.topic
             
         except Exception as e:
             logger.error(f"Error extracting topic: {str(e)}")

services/topic_similarity_service.py

@@ -0,0 +1,250 @@
+"""Service for finding similar topics using Google Generative AI embeddings"""
+
+import logging
+import json
+import hashlib
+from pathlib import Path
+from typing import Optional, List, Dict
+from datetime import datetime
+import numpy as np
+from sklearn.metrics.pairwise import cosine_similarity
+from google import genai
+from google.genai import types
+
+from config import GOOGLE_API_KEY, PROJECT_ROOT
+
+logger = logging.getLogger(__name__)
+
+# Paths for topics and cache files
+TOPICS_FILE = PROJECT_ROOT / "data" / "topics.json"
+EMBEDDINGS_CACHE_FILE = PROJECT_ROOT / "data" / "topic_embeddings_cache.json"
+
+
+class TopicSimilarityService:
+    """Service for finding the most similar topic from a predefined list using embeddings"""
+    
+    def __init__(self):
+        self.client = None
+        self.topics = []
+        self.topic_embeddings = None
+        self.initialized = False
+        self.model_name = "models/text-embedding-004"
+        
+    def initialize(self):
+        """Initialize the Google Generative AI client and load topic embeddings"""
+        if self.initialized:
+            logger.info("Topic similarity service already initialized")
+            return
+            
+        if not GOOGLE_API_KEY:
+            raise ValueError("GOOGLE_API_KEY not found in environment variables")
+        
+        try:
+            logger.info("Initializing topic similarity service with Google Generative AI")
+            
+            # Create Google Generative AI client
+            self.client = genai.Client(api_key=GOOGLE_API_KEY)
+            
+            # Load topics
+            self.topics = self._load_topics()
+            logger.info(f"Loaded {len(self.topics)} topics from {TOPICS_FILE}")
+            
+            # Load or generate topic embeddings
+            self.topic_embeddings = self._get_topic_embeddings()
+            logger.info(f"Loaded {len(self.topic_embeddings)} topic embeddings")
+            
+            self.initialized = True
+            logger.info("✓ Topic similarity service initialized successfully")
+            
+        except Exception as e:
+            logger.error(f"Error initializing topic similarity service: {str(e)}")
+            raise RuntimeError(f"Failed to initialize topic similarity service: {str(e)}")
+    
+    def _load_topics(self) -> List[str]:
+        """Load topics from topics.json file"""
+        if not TOPICS_FILE.exists():
+            raise FileNotFoundError(f"Topics file not found: {TOPICS_FILE}")
+        
+        try:
+            with open(TOPICS_FILE, 'r', encoding='utf-8') as f:
+                data = json.load(f)
+            return data.get("topics", [])
+        except (json.JSONDecodeError, KeyError) as e:
+            raise ValueError(f"Error loading topics from {TOPICS_FILE}: {str(e)}")
+    
+    def _get_topics_hash(self, topics: List[str]) -> str:
+        """Generate a hash of the topics list to verify cache validity"""
+        topics_str = json.dumps(topics, sort_keys=True)
+        return hashlib.md5(topics_str.encode('utf-8')).hexdigest()
+    
+    def _load_cached_embeddings(self) -> Optional[np.ndarray]:
+        """Load cached topic embeddings if they exist and are valid"""
+        if not EMBEDDINGS_CACHE_FILE.exists():
+            return None
+        
+        try:
+            with open(EMBEDDINGS_CACHE_FILE, 'r', encoding='utf-8') as f:
+                cache_data = json.load(f)
+            
+            # Verify cache is valid by checking topics hash
+            current_hash = self._get_topics_hash(self.topics)
+            
+            if cache_data.get("topics_hash") == current_hash:
+                # Convert list embeddings back to numpy arrays
+                embeddings = [np.array(emb) for emb in cache_data.get("embeddings", [])]
+                logger.info(f"Loaded {len(embeddings)} topic embeddings from cache")
+                return np.array(embeddings)
+            else:
+                # Topics have changed, cache is invalid
+                logger.info("Topics have changed, cache is invalid")
+                return None
+        except (json.JSONDecodeError, KeyError, ValueError) as e:
+            logger.warning(f"Could not load cached embeddings: {e}")
+            return None
+    
+    def _save_cached_embeddings(self, embeddings: np.ndarray):
+        """Save topic embeddings to cache file"""
+        topics_hash = self._get_topics_hash(self.topics)
+        
+        # Convert numpy arrays to lists for JSON serialization
+        embeddings_list = [emb.tolist() for emb in embeddings]
+        
+        cache_data = {
+            "topics_hash": topics_hash,
+            "embeddings": embeddings_list,
+            "model": self.model_name,
+            "cached_at": datetime.now().isoformat()
+        }
+        
+        try:
+            # Ensure directory exists
+            EMBEDDINGS_CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
+            
+            with open(EMBEDDINGS_CACHE_FILE, 'w', encoding='utf-8') as f:
+                json.dump(cache_data, f, indent=2)
+            logger.info(f"Cached {len(embeddings)} topic embeddings to {EMBEDDINGS_CACHE_FILE}")
+        except Exception as e:
+            logger.warning(f"Could not save cached embeddings: {e}")
+    
+    def _get_topic_embeddings(self) -> np.ndarray:
+        """
+        Get topic embeddings, loading from cache if available, otherwise generating and caching them
+        
+        Returns:
+            numpy.ndarray: Array of topic embeddings
+        """
+        # Try to load from cache first
+        cached_embeddings = self._load_cached_embeddings()
+        if cached_embeddings is not None:
+            return cached_embeddings
+        
+        # Cache miss or invalid - generate embeddings
+        logger.info(f"Generating embeddings for {len(self.topics)} topics (this may take a moment)...")
+        
+        try:
+            embedding_response = self.client.models.embed_content(
+                model=self.model_name,
+                contents=self.topics,
+                config=types.EmbedContentConfig(task_type="SEMANTIC_SIMILARITY")
+            )
+            
+            if not hasattr(embedding_response, "embeddings") or embedding_response.embeddings is None:
+                raise RuntimeError("Embedding API did not return embeddings")
+            
+            embeddings = [np.array(e.values) for e in embedding_response.embeddings]
+            embeddings_array = np.array(embeddings)
+            
+            # Save to cache for future use
+            self._save_cached_embeddings(embeddings_array)
+            
+            return embeddings_array
+            
+        except Exception as e:
+            logger.error(f"Error generating topic embeddings: {str(e)}")
+            raise RuntimeError(f"Failed to generate topic embeddings: {str(e)}")
+    
+    def find_most_similar_topic(self, input_text: str) -> Dict[str, any]:
+        """
+        Compare a single input text to all topics and return the highest cosine similarity
+        
+        Args:
+            input_text: The text to compare against topics
+            
+        Returns:
+            dict: Contains 'topic', 'similarity', and 'index' of the most similar topic
+        """
+        if not self.initialized:
+            self.initialize()
+        
+        if not input_text or not isinstance(input_text, str):
+            raise ValueError("Input text must be a non-empty string")
+        
+        input_text = input_text.strip()
+        if len(input_text) == 0:
+            raise ValueError("Input text cannot be empty")
+        
+        if not self.topics:
+            raise ValueError("No topics found in topics.json")
+        
+        try:
+            # Embed the input text
+            embedding_response = self.client.models.embed_content(
+                model=self.model_name,
+                contents=[input_text],
+                config=types.EmbedContentConfig(task_type="SEMANTIC_SIMILARITY")
+            )
+            
+            if not hasattr(embedding_response, "embeddings") or embedding_response.embeddings is None:
+                raise RuntimeError("Embedding API did not return embeddings")
+            
+            # Extract input embedding
+            input_embedding = np.array(embedding_response.embeddings[0].values).reshape(1, -1)
+            
+            # Calculate cosine similarity between input and each topic
+            similarities = cosine_similarity(input_embedding, self.topic_embeddings)[0]
+            
+            # Find the highest similarity
+            max_index = np.argmax(similarities)
+            max_similarity = similarities[max_index]
+            most_similar_topic = self.topics[max_index]
+            
+            return {
+                "topic": most_similar_topic,
+                "similarity": float(max_similarity),
+                "index": int(max_index)
+            }
+            
+        except Exception as e:
+            logger.error(f"Error finding similar topic: {str(e)}")
+            raise RuntimeError(f"Failed to find similar topic: {str(e)}")
+    
+    def batch_find_similar_topics(self, input_texts: List[str]) -> List[str]:
+        """
+        Find the most similar topic for each input text
+        
+        Args:
+            input_texts: List of input texts to compare against topics
+            
+        Returns:
+            List of most similar topics (one per input text)
+        """
+        if not self.initialized:
+            self.initialize()
+        
+        if not input_texts or not isinstance(input_texts, list):
+            raise ValueError("Input texts must be a non-empty list")
+        
+        results = []
+        for text in input_texts:
+            try:
+                result = self.find_most_similar_topic(text)
+                results.append(result["topic"])
+            except Exception as e:
+                logger.error(f"Error finding similar topic for text '{text[:50]}...': {str(e)}")
+                results.append(None)  # Or raise, depending on desired behavior
+        
+        return results
+
+
+# Initialize singleton instance
+topic_similarity_service = TopicSimilarityService()