Create app.py

app.py

@@ -0,0 +1,196 @@
+from tenacity import retry, stop_after_attempt, wait_random_exponential
+from tqdm import tqdm
+import time
+import sys
+
+# import openai
+import time
+# import pandas as pd
+import random
+import csv
+import os
+import pickle
+import json
+import nltk
+nltk.download('punkt')
+nltk.download('stopwords')
+from nltk.tokenize import sent_tokenize
+from nltk.corpus import stopwords
+import string
+from typing import List
+import difflib
+
+
+# import tiktoken
+
+import re
+from nltk.tokenize import sent_tokenize
+from collections import defaultdict
+
+
+import nltk
+from nltk.tokenize import sent_tokenize
+from nltk.tokenize import word_tokenize
+import numpy as np
+from retrieve import get_retrieved_results, get_slide
+
+# Ensure you have downloaded the 'punkt' tokenizer models
+nltk.download('punkt')
+
+
+
+
+
+import streamlit as st
+
+# Get the parent directory
+# parent_dir = os.path.abspath(os.path.join(os.getcwd(), os.pardir))
+# Add the parent directory to the system path
+# sys.path.append(parent_dir)
+
+from utils import AzureModels, write_to_file, read_from_file
+# from utils_open import OpenModels
+
+# Function to calculate similarity
+def calculate_similarity(sentence1: str, sentence2: str) -> float:
+    return difflib.SequenceMatcher(None, sentence1, sentence2).ratio()
+
+# Function to highlight sentences based on similarity
+def highlight_sentences(predicted: str, ground_truth: str) -> str:
+    ground_truth_sentences = nltk.sent_tokenize(ground_truth)
+    predicted_sentences = nltk.sent_tokenize(predicted)
+
+    highlighted_text = ""
+    
+    for pred_sentence in predicted_sentences:
+        max_similarity = 0
+        for gt_sentence in ground_truth_sentences:
+            similarity = calculate_similarity(pred_sentence, gt_sentence)
+            if similarity > max_similarity:
+                max_similarity = similarity
+        # Determine shade of green
+        shade = max_similarity  # No need to convert to int, max_similarity is already in [0, 1]
+        highlighted_text += f'<span style="background-color: rgba(0, 255, 0, {shade})">{pred_sentence}</span> '
+
+    return highlighted_text
+
+st.title('Multi-Document Narrative Generation')
+
+options = ["Select", "Adobe Firefly", "Adobe Acrobat"]
+selection = st.selectbox('Select an example', options)
+
+if selection=="Select":
+    pass
+elif selection=="Adobe Firefly":
+    with open('wiki_1.json', 'r') as fr:
+        list_1 = json.load(fr)
+    
+    with open('wiki_2.json', 'r') as fr:
+        list_2 = json.load(fr)
+    document_name = "Adobe Firefly"
+    section_names = ["Introduction"]*7+["History"]*2
+    ref_doc_indices = np.arange(1,8).tolist() + np.arange(1,3).tolist()
+else:
+    with open('wiki_2.json', 'r') as fr:
+        list_1 = json.load(fr)
+    
+    with open('wiki_1.json', 'r') as fr:
+        list_2 = json.load(fr)
+    document_name = "Adobe Acrobat"
+    section_names = ["Introduction"]*3+["History"]*3+["Document Cloud"]*2
+    ref_doc_indices = np.arange(1,4).tolist() + np.arange(1,4).tolist() + np.arange(1,3).tolist()
+    
+inp_doc_list = []
+inp_keys_list = []
+retrieved_doc_list = []
+
+
+if selection!='Select':
+    # for item, ret_item in zip(list_1, retrieved_out):
+    for item in list_1:
+        for key in item['ref_abstract']:
+            inp_doc_list.append(item['ref_abstract'][key])
+            inp_keys_list.append(key)
+            # retrieved_doc_list.append(ret_item['ref_abstract'][key]['abstract'])    
+    # Initialize session state
+    if 'retrieve_clicked' not in st.session_state:
+        st.session_state.retrieve_clicked = False
+    
+    retrieve_prompt_template = "{} : Document {} for the '{}' Section of the Article titled '{}'"
+    
+    ui_doc_list = []
+    ui_retrieved_doc_list = []
+    
+    # 5 input text boxes for 5 input documents
+    st.header('Input Documents')
+    # doc1 = st.text_area('Document 1', value="1. What up bruh??")
+    for i in range(len(section_names)):
+        ui_doc_list.append(st.text_area(retrieve_prompt_template.format(inp_keys_list[i], ref_doc_indices[i], section_names[i], document_name), value=inp_doc_list[i]))
+    
+    if st.button('Retrieve'):
+        if 'organize_clicked' not in st.session_state:
+            st.session_state.organize_clicked = False        
+        retrieved_out = get_retrieved_results("gpt4o", 0, "fixed", list_2, list_1)
+        write_to_file("retrieved_docs.json", retrieved_out)
+        retrieved_out_train = get_retrieved_results("gpt4o", 0, "fixed", list_1, list_2)
+        write_to_file("retrieved_docs_train.json", retrieved_out_train)
+    
+        for ret_item in retrieved_out:
+            for key in ret_item['ref_abstract']:
+                # inp_doc_list.append(item['ref_abstract'][key])    
+                retrieved_doc_list.append(ret_item['ref_abstract'][key]['abstract'])
+        
+        # Step 2: Lowercase the documents
+        st.session_state.retrieve_clicked = True
+        st.header('Retrieved Documents')
+        
+        for i in range(len(section_names)):
+            ui_retrieved_doc_list.append(st.text_area(retrieve_prompt_template.format(inp_keys_list[i], ref_doc_indices[i], section_names[i], document_name), value=retrieved_doc_list[i]))
+    if st.session_state.retrieve_clicked:
+        if st.button('Organize'):
+            if 'summarize_clicked' not in st.session_state:
+                st.session_state.summarize_clicked = False                    
+            st.session_state.organize_clicked = True
+            st.header("Organization of the documents in the narrative")
+            topics_list = ["Introduction", "History", "Document Cloud"]
+            
+            organize_list = []
+            ui_organize_list = []
+            
+            test_list = read_from_file("retrieved_docs.json")
+            train_list = read_from_file("retrieved_docs_train.json")
+            organize_out = get_retrieved_results("gpt4o", 1, "fixed", train_list, test_list, True)
+            for i in range(len(organize_out)):
+                organize_list.append(organize_out[i])
+                ui_organize_list.append(st.text_area("Section: " + topics_list[i], value=organize_out[i]))
+            write_to_file("organized_docs.json", organize_out)
+
+        if st.session_state.organize_clicked:
+            if st.button("Summarize"):
+                # if 'narrative_clicked' not in st.session_state:
+                #     st.session_state.narrative_clicked = False                                    
+                st.session_state.summarize_clicked = True
+                st.header("Intent-based multi-document summary")
+                topics_list = ["Introduction", "History", "Document Cloud"]
+                generate_list = []
+                ui_generate_list = []
+                slides_list = []
+                test_list = read_from_file("retrieved_docs.json")
+                train_list = read_from_file("retrieved_docs_train.json")                
+                organize_out = read_from_file("organized_docs.json")
+                gen_summary_dict = get_retrieved_results("gpt4o", 1, "fixed", train_list, test_list, False, organize_out)
+                for i in range(len(gen_summary_dict)):
+                    highlighted_summary = highlight_sentences(gen_summary_dict[i], test_list[i]['abstract'])
+                    slides_list.append(get_slide(topics_list[i], gen_summary_dict[i]))
+                    # generate_list.append(.format(topics_list[i], gen_summary_dict[i]))
+                    st.markdown(f"## {topics_list[i]}")
+                    # st.markdown(f"*{gen_summary_dict[i]}*")
+                    st.markdown(highlighted_summary, unsafe_allow_html=True)
+                st.header("Generated Narrative")
+                for i in range(len(slides_list)):
+                    st.markdown("---")
+                    st.markdown(slides_list[i])
+                    st.markdown("---")
+            # if st.session_state.summarize_clicked:
+            #     if st.button("Narrative"):                
+            #         st.session_state.narrative_clicked = True