Fetch data from pincone

app.py

@@ -31,11 +31,8 @@ class CustomRAG:
 
 
 def answer_question(query):
-    docs = load_raw_dataset()
     llm = get_llm("google/flan-t5-base")
-    vector_database = create_vector_database(
-        docs, "sentence-transformers/all-MiniLM-L6-v2"
-    )
+    vector_database = create_vector_database("sentence-transformers/all-MiniLM-L6-v2")
     prompt_template = get_prompt_template()
     rag = CustomRAG(
         vector_database,

indexer.py

@@ -1,12 +1,20 @@
+from pinecone import Pinecone
 from datasets import load_dataset
 import pandas as pd
 from langchain.schema import Document
 from langchain.embeddings import HuggingFaceEmbeddings
-from langchain.vectorstores import FAISS
-from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline
+from transformers import (
+    AutoTokenizer,
+    pipeline,
+    AutoModelForSeq2SeqLM,
+)
 from langchain.llms import HuggingFacePipeline
 from langchain.prompts import PromptTemplate
+import os
 
+api_key = os.environ["PINECONE_API_KEY"]
+
+from langchain_pinecone import PineconeVectorStore
 
 def load_raw_dataset():
     dataset = load_dataset("lavita/ChatDoctor-HealthCareMagic-100k")
@@ -15,8 +23,6 @@ def load_raw_dataset():
 
     df["combined"] = df["input"] + " " + df["output"]
 
-    df = df.sample(n=min(5000, len(df)), random_state=42)
-
     docs = [
         Document(
             page_content=row["combined"],
@@ -28,9 +34,14 @@ def load_raw_dataset():
     return docs
 
 
-def create_vector_database(docs, model_name):
+def create_vector_database(model_name):
+    PINECONE_INDEX_NAME = "medical-rag-index"
+    pc = Pinecone(api_key=os.environ["PINECONE_API_KEY"])
+
     embedding_model = HuggingFaceEmbeddings(model_name=model_name)
-    vectorstore = FAISS.from_documents(docs, embedding_model)
+
+    index = pc.Index(PINECONE_INDEX_NAME)
+    vectorstore = PineconeVectorStore(index=index, embedding=embedding_model)
     return vectorstore
 
 

requirements.txt

@@ -7,4 +7,6 @@ faiss-cpu
 huggingface-hub
 praw
 langchain-community
-accelerate
+accelerate
+langchain-pinecone
+pinecone