Spaces:

bardsai
/

performance-llm-board

Sleeping

App Files Files Community

piotr-szleg-bards-ai commited on Feb 15, 2024

Commit

95117cd

1 Parent(s): 74c94d1

2024-02-15 13:26:46 Publish script update

Browse files

Files changed (4) hide show

app.py +26 -34
data/general_plots.csv +55 -39
data/time_of_day_plots.csv +0 -0
pipeline/models.py +70 -56

app.py CHANGED Viewed

@@ -1,12 +1,11 @@
 import re
 import gradio as gr
 import pandas as pd
 import plotly
-import io
 import plotly.express as px
-import json
 from pandas.api.types import is_numeric_dtype
 from pipeline.config import LLMBoardConfig, QueriesConfig
@@ -120,15 +119,12 @@ def dataframes():
     def for_dataframe(df):
         if not searched_model_names:
             return df
-        return dataframe_style(pd.concat(
-            df[df.model.str.lower().str.contains(n)]
-            for n in searched_model_names
-        ))
     return (
         for_dataframe(summary_df_processed),
         for_dataframe(time_of_day_comparison_df),
-        for_dataframe(model_costs_df)
     )
@@ -161,7 +157,8 @@ last_textbox = 0
 plots = []
 single_model_plots = []
-def filter_plots(searched_query:str):
     searched_model_names = searched_query.split("|")
     searched_model_names = [n.lower().strip() for n in searched_model_names]
     searched_model_names = [n for n in searched_model_names if n]
@@ -169,11 +166,8 @@ def filter_plots(searched_query:str):
     def filter_dataframe(df):
         if not searched_model_names:
             return df
-        return pd.concat(
-            df[df.model.str.lower().str.contains(n)]
-            for n in searched_model_names
-        )
     results = []
     for plot_display, plot, row in plots:
         visible = True
@@ -181,31 +175,27 @@ def filter_plots(searched_query:str):
             buffer = io.StringIO(row["df"])
             df = pd.read_csv(buffer)
             df = filter_dataframe(df)
-            plot = px.bar(
-                df,
-                **json.loads(row["arguments"])
-            )
             plot.update_layout(autosize=True)
         elif "for model" in row["header"] and searched_model_names:
             plot_model = row["header"].split("for model")[1].lower()
             if not any(n in plot_model for n in searched_model_names):
                 visible = False
         results.append(gr.Plot(plot, visible=visible))
     return results
 def display_plot(plot_df_row):
     row = dict(plot_df_row)
     plot = plotly.io.from_json(row["plot_json"])
     plot.update_layout(autosize=True)
-    plots.append((
-        gr.Plot(plot, label=row["header"], scale=1),
-        plot,
-        row))
     if "description" in row and pd.notna(row["description"]):
         gr.Markdown(str(row["description"]))
 with gr.Blocks() as demo:
     gr.HTML("<h1>Performance LLM Board</h1>")
@@ -213,7 +203,7 @@ with gr.Blocks() as demo:
         filter_textbox.render()
         filter_button.render()
     gr.Markdown(
-        "&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;\* You can use `|` operator to display multiple models at once, for example \"gpt|mistral|zephyr\""
     )
     with gr.Tab("About this project"):
@@ -238,8 +228,6 @@ Execution time refers to averaged time needed to execute one query.
 Hours and times of day in the table and in the plot are based on Central European Time.
 Measurements were made during a normal work week.
-To compare providers we checked execution times for the same model `Mistral (7B) Instruct v0.2` on both Hugging Face and Together AI.
 """
         )
         # display rest of the plots
@@ -272,14 +260,18 @@ for models hosted this way we calculated "Cost Per Token" column using data coll
 Note that pause and resume time cost was not included in the "Cost Per Token" column calculation.
             """
         )
-        general_plots[general_plots.plot_name=="execution_costs"].apply(display_plot, axis=1)
-    with gr.Tab("Model dimensions"):
-        general_plots[general_plots.plot_name!="execution_costs"].apply(display_plot, axis=1)
-        gr.Markdown("""
-Model dimensions based on release blogs and documentation of their respective developers.
-Note that models, which dimensions were not disclosed publicly, were omitted from the plots.
-""")
     filter_button.click(
         fn=filter_dataframes,
         inputs=filter_textbox,

+import io
+import json
 import re
 import gradio as gr
 import pandas as pd
 import plotly
 import plotly.express as px
 from pandas.api.types import is_numeric_dtype
 from pipeline.config import LLMBoardConfig, QueriesConfig
     def for_dataframe(df):
         if not searched_model_names:
             return df
+        return dataframe_style(pd.concat(df[df.model.str.lower().str.contains(n)] for n in searched_model_names))
     return (
         for_dataframe(summary_df_processed),
         for_dataframe(time_of_day_comparison_df),
+        for_dataframe(model_costs_df),
     )
 plots = []
 single_model_plots = []
+def filter_plots(searched_query: str):
     searched_model_names = searched_query.split("|")
     searched_model_names = [n.lower().strip() for n in searched_model_names]
     searched_model_names = [n for n in searched_model_names if n]
     def filter_dataframe(df):
         if not searched_model_names:
             return df
+        return pd.concat(df[df.model.str.lower().str.contains(n)] for n in searched_model_names)
     results = []
     for plot_display, plot, row in plots:
         visible = True
             buffer = io.StringIO(row["df"])
             df = pd.read_csv(buffer)
             df = filter_dataframe(df)
+            plot = px.bar(df, **json.loads(row["arguments"]))
             plot.update_layout(autosize=True)
         elif "for model" in row["header"] and searched_model_names:
             plot_model = row["header"].split("for model")[1].lower()
             if not any(n in plot_model for n in searched_model_names):
                 visible = False
         results.append(gr.Plot(plot, visible=visible))
     return results
 def display_plot(plot_df_row):
     row = dict(plot_df_row)
     plot = plotly.io.from_json(row["plot_json"])
     plot.update_layout(autosize=True)
+    plots.append((gr.Plot(plot, label=row["header"], scale=1), plot, row))
     if "description" in row and pd.notna(row["description"]):
         gr.Markdown(str(row["description"]))
 with gr.Blocks() as demo:
     gr.HTML("<h1>Performance LLM Board</h1>")
         filter_textbox.render()
         filter_button.render()
     gr.Markdown(
+        '&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;\* You can use `|` operator to display multiple models at once, for example "gpt|mistral|zephyr"'
     )
     with gr.Tab("About this project"):
 Hours and times of day in the table and in the plot are based on Central European Time.
 Measurements were made during a normal work week.
 """
         )
         # display rest of the plots
 Note that pause and resume time cost was not included in the "Cost Per Token" column calculation.
             """
         )
+        general_plots[general_plots.plot_name == "execution_costs"].apply(display_plot, axis=1)
+    with gr.Tab("Context length and parameters count"):
+        general_plots[general_plots.plot_name != "execution_costs"].apply(display_plot, axis=1)
+        gr.Markdown(
+            """
+LLM models context length and parameters count are based on release blogs and documentation of their respective developers.
+A lot of models had to be omitted due to their developers not disclosing their parameters count.
+Mainly OpenAI's GPT models and Google's Palm 2.
+"""
+        )
     filter_button.click(
         fn=filter_dataframes,
         inputs=filter_textbox,

data/general_plots.csv CHANGED Viewed

@@ -10,7 +10,7 @@ execution_costs,./html/plots/execution_costs.html,"Figure({
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.0249]),
               'xaxis': 'x',
               'y': array(['gpt-4'], dtype=object),
               'yaxis': 'y'},
@@ -24,7 +24,7 @@ execution_costs,./html/plots/execution_costs.html,"Figure({
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.0229]),
               'xaxis': 'x',
               'y': array(['gpt-4-turbo'], dtype=object),
               'yaxis': 'y'},
@@ -38,7 +38,7 @@ execution_costs,./html/plots/execution_costs.html,"Figure({
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.00083]),
               'xaxis': 'x',
               'y': array(['gpt-3.5-turbo'], dtype=object),
               'yaxis': 'y'},
@@ -52,7 +52,7 @@ execution_costs,./html/plots/execution_costs.html,"Figure({
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.000711]),
               'xaxis': 'x',
               'y': array(['llama-2-70b-chat'], dtype=object),
               'yaxis': 'y'},
@@ -66,7 +66,7 @@ execution_costs,./html/plots/execution_costs.html,"Figure({
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.000711]),
               'xaxis': 'x',
               'y': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
               'yaxis': 'y'},
@@ -80,7 +80,7 @@ execution_costs,./html/plots/execution_costs.html,"Figure({
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.0001975]),
               'xaxis': 'x',
               'y': array(['gemini-pro'], dtype=object),
               'yaxis': 'y'},
@@ -94,7 +94,7 @@ execution_costs,./html/plots/execution_costs.html,"Figure({
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.0001975]),
               'xaxis': 'x',
               'y': array(['chat-bison (PaLM 2)'], dtype=object),
               'yaxis': 'y'},
@@ -108,9 +108,23 @@ execution_costs,./html/plots/execution_costs.html,"Figure({
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.0001975]),
               'xaxis': 'x',
               'y': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
               'yaxis': 'y'}],
     'layout': {'barmode': 'relative',
                'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
@@ -118,7 +132,8 @@ execution_costs,./html/plots/execution_costs.html,"Figure({
                'title': {'text': 'Costs of execution of 20 test queries per model'},
                'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Execution cost ($)'}},
                'yaxis': {'anchor': 'x',
-                         'categoryarray': [chat-bison-32k (PaLM 2 32K), chat-bison
                                            (PaLM 2), gemini-pro,
                                            Mixtral-8x7B-Instruct-v0.1,
                                            llama-2-70b-chat, gpt-3.5-turbo,
@@ -126,15 +141,16 @@ execution_costs,./html/plots/execution_costs.html,"Figure({
                          'categoryorder': 'array',
                          'domain': [0.0, 1.0],
                          'title': {'text': 'Model'}}}
-})",Costs of execution of 20 test queries per model,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0249],""xaxis"":""x"",""y"":[""gpt-4""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0229],""xaxis"":""x"",""y"":[""gpt-4-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.00083],""xaxis"":""x"",""y"":[""gpt-3.5-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.000711],""xaxis"":""x"",""y"":[""llama-2-70b-chat""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.000711],""xaxis"":""x"",""y"":[""Mixtral-8x7B-Instruct-v0.1""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0001975],""xaxis"":""x"",""y"":[""gemini-pro""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0001975],""xaxis"":""x"",""y"":[""chat-bison (PaLM 2)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0001975],""xaxis"":""x"",""y"":[""chat-bison-32k (PaLM 2 32K)""],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Execution cost ($)""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""chat-bison-32k (PaLM 2 32K)"",""chat-bison (PaLM 2)"",""gemini-pro"",""Mixtral-8x7B-Instruct-v0.1"",""llama-2-70b-chat"",""gpt-3.5-turbo"",""gpt-4-turbo"",""gpt-4""]},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Costs of execution of 20 test queries per model""},""barmode"":""relative""}}","{""y"": ""model"", ""x"": ""model_query_costs"", ""color"": ""model"", ""orientation"": ""h"", ""title"": ""Costs of execution of 20 test queries per model"", ""labels"": {""model"": ""Model"", ""model_query_costs"": ""Execution cost ($)""}}",",model_query_costs,model
-2,0.0249,gpt-4
-1,0.0229,gpt-4-turbo
-0,0.00083,gpt-3.5-turbo
-3,0.000711,llama-2-70b-chat
-4,0.000711,Mixtral-8x7B-Instruct-v0.1
-8,0.0001975,gemini-pro
-9,0.0001975,chat-bison (PaLM 2)
-10,0.0001975,chat-bison-32k (PaLM 2 32K)
 "
 model_sizes,./html/plots/model_sizes.html,"Figure({
     'data': [{'alignmentgroup': 'True',
@@ -337,9 +353,9 @@ model_sizes,./html/plots/model_sizes.html,"Figure({
 9,7.0,RedPajama-INCITE Chat (7B)
 4,1.1,TinyLlama/TinyLlama-1.1B-Chat-v1.0
 "
-model_input_sizes,./html/plots/model_input_sizes.html,"Figure({
     'data': [{'alignmentgroup': 'True',
-              'hovertemplate': 'Model=%{y}<br>Model input size (tokens)=%{x}<extra></extra>',
               'legendgroup': 'gpt-4-turbo',
               'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
               'name': 'gpt-4-turbo',
@@ -353,7 +369,7 @@ model_input_sizes,./html/plots/model_input_sizes.html,"Figure({
               'y': array(['gpt-4-turbo'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
-              'hovertemplate': 'Model=%{y}<br>Model input size (tokens)=%{x}<extra></extra>',
               'legendgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
               'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
               'name': 'Mistral (7B) Instruct v0.2 (Together AI)',
@@ -367,7 +383,7 @@ model_input_sizes,./html/plots/model_input_sizes.html,"Figure({
               'y': array(['Mistral (7B) Instruct v0.2 (Together AI)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
-              'hovertemplate': 'Model=%{y}<br>Model input size (tokens)=%{x}<extra></extra>',
               'legendgroup': 'Snorkel Mistral PairRM DPO (7B)',
               'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
               'name': 'Snorkel Mistral PairRM DPO (7B)',
@@ -381,7 +397,7 @@ model_input_sizes,./html/plots/model_input_sizes.html,"Figure({
               'y': array(['Snorkel Mistral PairRM DPO (7B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
-              'hovertemplate': 'Model=%{y}<br>Model input size (tokens)=%{x}<extra></extra>',
               'legendgroup': 'Qwen 1.5 Chat (7B)',
               'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
               'name': 'Qwen 1.5 Chat (7B)',
@@ -395,7 +411,7 @@ model_input_sizes,./html/plots/model_input_sizes.html,"Figure({
               'y': array(['Qwen 1.5 Chat (7B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
-              'hovertemplate': 'Model=%{y}<br>Model input size (tokens)=%{x}<extra></extra>',
               'legendgroup': 'gpt-4',
               'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
               'name': 'gpt-4',
@@ -409,7 +425,7 @@ model_input_sizes,./html/plots/model_input_sizes.html,"Figure({
               'y': array(['gpt-4'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
-              'hovertemplate': 'Model=%{y}<br>Model input size (tokens)=%{x}<extra></extra>',
               'legendgroup': 'chat-bison (PaLM 2)',
               'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
               'name': 'chat-bison (PaLM 2)',
@@ -423,7 +439,7 @@ model_input_sizes,./html/plots/model_input_sizes.html,"Figure({
               'y': array(['chat-bison (PaLM 2)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
-              'hovertemplate': 'Model=%{y}<br>Model input size (tokens)=%{x}<extra></extra>',
               'legendgroup': 'OpenHermes-2.5-Mistral (7B)',
               'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
               'name': 'OpenHermes-2.5-Mistral (7B)',
@@ -437,7 +453,7 @@ model_input_sizes,./html/plots/model_input_sizes.html,"Figure({
               'y': array(['OpenHermes-2.5-Mistral (7B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
-              'hovertemplate': 'Model=%{y}<br>Model input size (tokens)=%{x}<extra></extra>',
               'legendgroup': 'gpt-3.5-turbo',
               'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
               'name': 'gpt-3.5-turbo',
@@ -451,7 +467,7 @@ model_input_sizes,./html/plots/model_input_sizes.html,"Figure({
               'y': array(['gpt-3.5-turbo'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
-              'hovertemplate': 'Model=%{y}<br>Model input size (tokens)=%{x}<extra></extra>',
               'legendgroup': 'WizardLM v1.2 (13B)',
               'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
               'name': 'WizardLM v1.2 (13B)',
@@ -465,7 +481,7 @@ model_input_sizes,./html/plots/model_input_sizes.html,"Figure({
               'y': array(['WizardLM v1.2 (13B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
-              'hovertemplate': 'Model=%{y}<br>Model input size (tokens)=%{x}<extra></extra>',
               'legendgroup': 'LLaMA-2 Chat (7B)',
               'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
               'name': 'LLaMA-2 Chat (7B)',
@@ -479,7 +495,7 @@ model_input_sizes,./html/plots/model_input_sizes.html,"Figure({
               'y': array(['LLaMA-2 Chat (7B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
-              'hovertemplate': 'Model=%{y}<br>Model input size (tokens)=%{x}<extra></extra>',
               'legendgroup': 'Vicuna v1.5 (7B)',
               'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
               'name': 'Vicuna v1.5 (7B)',
@@ -493,7 +509,7 @@ model_input_sizes,./html/plots/model_input_sizes.html,"Figure({
               'y': array(['Vicuna v1.5 (7B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
-              'hovertemplate': 'Model=%{y}<br>Model input size (tokens)=%{x}<extra></extra>',
               'legendgroup': '01-ai Yi Chat (34B)',
               'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
               'name': '01-ai Yi Chat (34B)',
@@ -507,7 +523,7 @@ model_input_sizes,./html/plots/model_input_sizes.html,"Figure({
               'y': array(['01-ai Yi Chat (34B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
-              'hovertemplate': 'Model=%{y}<br>Model input size (tokens)=%{x}<extra></extra>',
               'legendgroup': 'Upstage SOLAR Instruct v1 (11B)',
               'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
               'name': 'Upstage SOLAR Instruct v1 (11B)',
@@ -521,7 +537,7 @@ model_input_sizes,./html/plots/model_input_sizes.html,"Figure({
               'y': array(['Upstage SOLAR Instruct v1 (11B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
-              'hovertemplate': 'Model=%{y}<br>Model input size (tokens)=%{x}<extra></extra>',
               'legendgroup': 'Chronos Hermes (13B)',
               'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
               'name': 'Chronos Hermes (13B)',
@@ -535,7 +551,7 @@ model_input_sizes,./html/plots/model_input_sizes.html,"Figure({
               'y': array(['Chronos Hermes (13B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
-              'hovertemplate': 'Model=%{y}<br>Model input size (tokens)=%{x}<extra></extra>',
               'legendgroup': 'Falcon Instruct (7B)',
               'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
               'name': 'Falcon Instruct (7B)',
@@ -549,7 +565,7 @@ model_input_sizes,./html/plots/model_input_sizes.html,"Figure({
               'y': array(['Falcon Instruct (7B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
-              'hovertemplate': 'Model=%{y}<br>Model input size (tokens)=%{x}<extra></extra>',
               'legendgroup': 'RedPajama-INCITE Chat (7B)',
               'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
               'name': 'RedPajama-INCITE Chat (7B)',
@@ -563,7 +579,7 @@ model_input_sizes,./html/plots/model_input_sizes.html,"Figure({
               'y': array(['RedPajama-INCITE Chat (7B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
-              'hovertemplate': 'Model=%{y}<br>Model input size (tokens)=%{x}<extra></extra>',
               'legendgroup': 'llama-2-70b-chat',
               'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
               'name': 'llama-2-70b-chat',
@@ -579,8 +595,8 @@ model_input_sizes,./html/plots/model_input_sizes.html,"Figure({
     'layout': {'barmode': 'relative',
                'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
                'template': '...',
-               'title': {'text': 'Model input sizes in tokens'},
-               'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model input size (tokens)'}},
                'yaxis': {'anchor': 'x',
                          'categoryarray': [llama-2-70b-chat, RedPajama-INCITE Chat
                                            (7B), Falcon Instruct (7B), Chronos
@@ -596,7 +612,7 @@ model_input_sizes,./html/plots/model_input_sizes.html,"Figure({
                          'categoryorder': 'array',
                          'domain': [0.0, 1.0],
                          'title': {'text': 'Model'}}}
-})",Model input sizes in tokens,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[128000],""xaxis"":""x"",""y"":[""gpt-4-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mistral (7B) Instruct v0.2 (Together AI)"",""offsetgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[32768],""xaxis"":""x"",""y"":[""Mistral (7B) Instruct v0.2 (Together AI)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Snorkel Mistral PairRM DPO (7B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Snorkel Mistral PairRM DPO (7B)"",""offsetgroup"":""Snorkel Mistral PairRM DPO (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[32768],""xaxis"":""x"",""y"":[""Snorkel Mistral PairRM DPO (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Qwen 1.5 Chat (7B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Qwen 1.5 Chat (7B)"",""offsetgroup"":""Qwen 1.5 Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[32768],""xaxis"":""x"",""y"":[""Qwen 1.5 Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[32000],""xaxis"":""x"",""y"":[""gpt-4""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[8196],""xaxis"":""x"",""y"":[""chat-bison (PaLM 2)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""OpenHermes-2.5-Mistral (7B)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""OpenHermes-2.5-Mistral (7B)"",""offsetgroup"":""OpenHermes-2.5-Mistral (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[8192],""xaxis"":""x"",""y"":[""OpenHermes-2.5-Mistral (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""gpt-3.5-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""WizardLM v1.2 (13B)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""WizardLM v1.2 (13B)"",""offsetgroup"":""WizardLM v1.2 (13B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""WizardLM v1.2 (13B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""LLaMA-2 Chat (7B)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""LLaMA-2 Chat (7B)"",""offsetgroup"":""LLaMA-2 Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""LLaMA-2 Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Vicuna v1.5 (7B)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Vicuna v1.5 (7B)"",""offsetgroup"":""Vicuna v1.5 (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""Vicuna v1.5 (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""01-ai Yi Chat (34B)"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""01-ai Yi Chat (34B)"",""offsetgroup"":""01-ai Yi Chat (34B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""01-ai Yi Chat (34B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Upstage SOLAR Instruct v1 (11B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Upstage SOLAR Instruct v1 (11B)"",""offsetgroup"":""Upstage SOLAR Instruct v1 (11B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""Upstage SOLAR Instruct v1 (11B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Chronos Hermes (13B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Chronos Hermes (13B)"",""offsetgroup"":""Chronos Hermes (13B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[2048],""xaxis"":""x"",""y"":[""Chronos Hermes (13B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Falcon Instruct (7B)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Falcon Instruct (7B)"",""offsetgroup"":""Falcon Instruct (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[2048],""xaxis"":""x"",""y"":[""Falcon Instruct (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""RedPajama-INCITE Chat (7B)"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""RedPajama-INCITE Chat (7B)"",""offsetgroup"":""RedPajama-INCITE Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[2048],""xaxis"":""x"",""y"":[""RedPajama-INCITE Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[2048],""xaxis"":""x"",""y"":[""llama-2-70b-chat""],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model input size (tokens)""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""llama-2-70b-chat"",""RedPajama-INCITE Chat (7B)"",""Falcon Instruct (7B)"",""Chronos Hermes (13B)"",""Upstage SOLAR Instruct v1 (11B)"",""01-ai Yi Chat (34B)"",""Vicuna v1.5 (7B)"",""LLaMA-2 Chat (7B)"",""WizardLM v1.2 (13B)"",""gpt-3.5-turbo"",""OpenHermes-2.5-Mistral (7B)"",""chat-bison (PaLM 2)"",""gpt-4"",""Qwen 1.5 Chat (7B)"",""Snorkel Mistral PairRM DPO (7B)"",""Mistral (7B) Instruct v0.2 (Together AI)"",""gpt-4-turbo""]},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model input sizes in tokens""},""barmode"":""relative""}}","{""x"": ""model_input_sizes"", ""y"": ""model"", ""color"": ""model"", ""orientation"": ""h"", ""title"": ""Model input sizes in tokens"", ""labels"": {""model"": ""Model"", ""model_input_sizes"": ""Model input size (tokens)""}}",",model_input_sizes,model
 1,128000,gpt-4-turbo
 9,32768,Mistral (7B) Instruct v0.2 (Together AI)
 11,32768,Snorkel Mistral PairRM DPO (7B)

               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
+              'x': array([0.1446]),
               'xaxis': 'x',
               'y': array(['gpt-4'], dtype=object),
               'yaxis': 'y'},
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
+              'x': array([0.056]),
               'xaxis': 'x',
               'y': array(['gpt-4-turbo'], dtype=object),
               'yaxis': 'y'},
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
+              'x': array([0.00442]),
               'xaxis': 'x',
               'y': array(['gpt-3.5-turbo'], dtype=object),
               'yaxis': 'y'},
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
+              'x': array([0.002808]),
               'xaxis': 'x',
               'y': array(['llama-2-70b-chat'], dtype=object),
               'yaxis': 'y'},
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
+              'x': array([0.00207]),
               'xaxis': 'x',
               'y': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
               'yaxis': 'y'},
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
+              'x': array([0.001195]),
               'xaxis': 'x',
               'y': array(['gemini-pro'], dtype=object),
               'yaxis': 'y'},
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
+              'x': array([0.001075]),
               'xaxis': 'x',
               'y': array(['chat-bison (PaLM 2)'], dtype=object),
               'yaxis': 'y'},
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
+              'x': array([0.001025]),
               'xaxis': 'x',
               'y': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
+              'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
+              'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([0.00062769]),
+              'xaxis': 'x',
+              'y': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
               'yaxis': 'y'}],
     'layout': {'barmode': 'relative',
                'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
                'title': {'text': 'Costs of execution of 20 test queries per model'},
                'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Execution cost ($)'}},
                'yaxis': {'anchor': 'x',
+                         'categoryarray': [TinyLlama/TinyLlama-1.1B-Chat-v1.0,
+                                           chat-bison-32k (PaLM 2 32K), chat-bison
                                            (PaLM 2), gemini-pro,
                                            Mixtral-8x7B-Instruct-v0.1,
                                            llama-2-70b-chat, gpt-3.5-turbo,
                          'categoryorder': 'array',
                          'domain': [0.0, 1.0],
                          'title': {'text': 'Model'}}}
+})",Costs of execution of 20 test queries per model,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.1446],""xaxis"":""x"",""y"":[""gpt-4""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.056],""xaxis"":""x"",""y"":[""gpt-4-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.00442],""xaxis"":""x"",""y"":[""gpt-3.5-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.002808],""xaxis"":""x"",""y"":[""llama-2-70b-chat""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0020700000000000002],""xaxis"":""x"",""y"":[""Mixtral-8x7B-Instruct-v0.1""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0011949999999999999],""xaxis"":""x"",""y"":[""gemini-pro""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.001075],""xaxis"":""x"",""y"":[""chat-bison (PaLM 2)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0010249999999999999],""xaxis"":""x"",""y"":[""chat-bison-32k (PaLM 2 32K)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0006276866594950359],""xaxis"":""x"",""y"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Execution cost ($)""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""chat-bison-32k (PaLM 2 32K)"",""chat-bison (PaLM 2)"",""gemini-pro"",""Mixtral-8x7B-Instruct-v0.1"",""llama-2-70b-chat"",""gpt-3.5-turbo"",""gpt-4-turbo"",""gpt-4""]},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Costs of execution of 20 test queries per model""},""barmode"":""relative""}}","{""y"": ""model"", ""x"": ""model_query_costs"", ""color"": ""model"", ""orientation"": ""h"", ""title"": ""Costs of execution of 20 test queries per model"", ""labels"": {""model"": ""Model"", ""model_query_costs"": ""Execution cost ($)""}}",",model_query_costs,model
+2,0.1446,gpt-4
+1,0.056,gpt-4-turbo
+0,0.00442,gpt-3.5-turbo
+3,0.002808,llama-2-70b-chat
+4,0.0020700000000000002,Mixtral-8x7B-Instruct-v0.1
+8,0.0011949999999999999,gemini-pro
+9,0.001075,chat-bison (PaLM 2)
+10,0.0010249999999999999,chat-bison-32k (PaLM 2 32K)
+7,0.0006276866594950359,TinyLlama/TinyLlama-1.1B-Chat-v1.0
 "
 model_sizes,./html/plots/model_sizes.html,"Figure({
     'data': [{'alignmentgroup': 'True',
 9,7.0,RedPajama-INCITE Chat (7B)
 4,1.1,TinyLlama/TinyLlama-1.1B-Chat-v1.0
 "
+model_context_lengths,./html/plots/model_context_lengths.html,"Figure({
     'data': [{'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Model context length (tokens)=%{x}<extra></extra>',
               'legendgroup': 'gpt-4-turbo',
               'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
               'name': 'gpt-4-turbo',
               'y': array(['gpt-4-turbo'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Model context length (tokens)=%{x}<extra></extra>',
               'legendgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
               'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
               'name': 'Mistral (7B) Instruct v0.2 (Together AI)',
               'y': array(['Mistral (7B) Instruct v0.2 (Together AI)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Model context length (tokens)=%{x}<extra></extra>',
               'legendgroup': 'Snorkel Mistral PairRM DPO (7B)',
               'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
               'name': 'Snorkel Mistral PairRM DPO (7B)',
               'y': array(['Snorkel Mistral PairRM DPO (7B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Model context length (tokens)=%{x}<extra></extra>',
               'legendgroup': 'Qwen 1.5 Chat (7B)',
               'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
               'name': 'Qwen 1.5 Chat (7B)',
               'y': array(['Qwen 1.5 Chat (7B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Model context length (tokens)=%{x}<extra></extra>',
               'legendgroup': 'gpt-4',
               'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
               'name': 'gpt-4',
               'y': array(['gpt-4'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Model context length (tokens)=%{x}<extra></extra>',
               'legendgroup': 'chat-bison (PaLM 2)',
               'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
               'name': 'chat-bison (PaLM 2)',
               'y': array(['chat-bison (PaLM 2)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Model context length (tokens)=%{x}<extra></extra>',
               'legendgroup': 'OpenHermes-2.5-Mistral (7B)',
               'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
               'name': 'OpenHermes-2.5-Mistral (7B)',
               'y': array(['OpenHermes-2.5-Mistral (7B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Model context length (tokens)=%{x}<extra></extra>',
               'legendgroup': 'gpt-3.5-turbo',
               'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
               'name': 'gpt-3.5-turbo',
               'y': array(['gpt-3.5-turbo'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Model context length (tokens)=%{x}<extra></extra>',
               'legendgroup': 'WizardLM v1.2 (13B)',
               'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
               'name': 'WizardLM v1.2 (13B)',
               'y': array(['WizardLM v1.2 (13B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Model context length (tokens)=%{x}<extra></extra>',
               'legendgroup': 'LLaMA-2 Chat (7B)',
               'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
               'name': 'LLaMA-2 Chat (7B)',
               'y': array(['LLaMA-2 Chat (7B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Model context length (tokens)=%{x}<extra></extra>',
               'legendgroup': 'Vicuna v1.5 (7B)',
               'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
               'name': 'Vicuna v1.5 (7B)',
               'y': array(['Vicuna v1.5 (7B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Model context length (tokens)=%{x}<extra></extra>',
               'legendgroup': '01-ai Yi Chat (34B)',
               'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
               'name': '01-ai Yi Chat (34B)',
               'y': array(['01-ai Yi Chat (34B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Model context length (tokens)=%{x}<extra></extra>',
               'legendgroup': 'Upstage SOLAR Instruct v1 (11B)',
               'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
               'name': 'Upstage SOLAR Instruct v1 (11B)',
               'y': array(['Upstage SOLAR Instruct v1 (11B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Model context length (tokens)=%{x}<extra></extra>',
               'legendgroup': 'Chronos Hermes (13B)',
               'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
               'name': 'Chronos Hermes (13B)',
               'y': array(['Chronos Hermes (13B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Model context length (tokens)=%{x}<extra></extra>',
               'legendgroup': 'Falcon Instruct (7B)',
               'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
               'name': 'Falcon Instruct (7B)',
               'y': array(['Falcon Instruct (7B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Model context length (tokens)=%{x}<extra></extra>',
               'legendgroup': 'RedPajama-INCITE Chat (7B)',
               'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
               'name': 'RedPajama-INCITE Chat (7B)',
               'y': array(['RedPajama-INCITE Chat (7B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Model context length (tokens)=%{x}<extra></extra>',
               'legendgroup': 'llama-2-70b-chat',
               'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
               'name': 'llama-2-70b-chat',
     'layout': {'barmode': 'relative',
                'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
                'template': '...',
+               'title': {'text': 'Model context lengths in tokens'},
+               'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model context length (tokens)'}},
                'yaxis': {'anchor': 'x',
                          'categoryarray': [llama-2-70b-chat, RedPajama-INCITE Chat
                                            (7B), Falcon Instruct (7B), Chronos
                          'categoryorder': 'array',
                          'domain': [0.0, 1.0],
                          'title': {'text': 'Model'}}}
+})",Model context lengths in tokens,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel context length (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[128000],""xaxis"":""x"",""y"":[""gpt-4-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel context length (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mistral (7B) Instruct v0.2 (Together AI)"",""offsetgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[32768],""xaxis"":""x"",""y"":[""Mistral (7B) Instruct v0.2 (Together AI)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel context length (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Snorkel Mistral PairRM DPO (7B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Snorkel Mistral PairRM DPO (7B)"",""offsetgroup"":""Snorkel Mistral PairRM DPO (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[32768],""xaxis"":""x"",""y"":[""Snorkel Mistral PairRM DPO (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel context length (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Qwen 1.5 Chat (7B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Qwen 1.5 Chat (7B)"",""offsetgroup"":""Qwen 1.5 Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[32768],""xaxis"":""x"",""y"":[""Qwen 1.5 Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel context length (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[32000],""xaxis"":""x"",""y"":[""gpt-4""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel context length (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[8196],""xaxis"":""x"",""y"":[""chat-bison (PaLM 2)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel context length (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""OpenHermes-2.5-Mistral (7B)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""OpenHermes-2.5-Mistral (7B)"",""offsetgroup"":""OpenHermes-2.5-Mistral (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[8192],""xaxis"":""x"",""y"":[""OpenHermes-2.5-Mistral (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel context length (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""gpt-3.5-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel context length (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""WizardLM v1.2 (13B)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""WizardLM v1.2 (13B)"",""offsetgroup"":""WizardLM v1.2 (13B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""WizardLM v1.2 (13B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel context length (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""LLaMA-2 Chat (7B)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""LLaMA-2 Chat (7B)"",""offsetgroup"":""LLaMA-2 Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""LLaMA-2 Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel context length (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Vicuna v1.5 (7B)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Vicuna v1.5 (7B)"",""offsetgroup"":""Vicuna v1.5 (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""Vicuna v1.5 (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel context length (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""01-ai Yi Chat (34B)"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""01-ai Yi Chat (34B)"",""offsetgroup"":""01-ai Yi Chat (34B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""01-ai Yi Chat (34B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel context length (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Upstage SOLAR Instruct v1 (11B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Upstage SOLAR Instruct v1 (11B)"",""offsetgroup"":""Upstage SOLAR Instruct v1 (11B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""Upstage SOLAR Instruct v1 (11B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel context length (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Chronos Hermes (13B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Chronos Hermes (13B)"",""offsetgroup"":""Chronos Hermes (13B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[2048],""xaxis"":""x"",""y"":[""Chronos Hermes (13B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel context length (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Falcon Instruct (7B)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Falcon Instruct (7B)"",""offsetgroup"":""Falcon Instruct (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[2048],""xaxis"":""x"",""y"":[""Falcon Instruct (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel context length (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""RedPajama-INCITE Chat (7B)"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""RedPajama-INCITE Chat (7B)"",""offsetgroup"":""RedPajama-INCITE Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[2048],""xaxis"":""x"",""y"":[""RedPajama-INCITE Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel context length (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[2048],""xaxis"":""x"",""y"":[""llama-2-70b-chat""],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model context length (tokens)""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""llama-2-70b-chat"",""RedPajama-INCITE Chat (7B)"",""Falcon Instruct (7B)"",""Chronos Hermes (13B)"",""Upstage SOLAR Instruct v1 (11B)"",""01-ai Yi Chat (34B)"",""Vicuna v1.5 (7B)"",""LLaMA-2 Chat (7B)"",""WizardLM v1.2 (13B)"",""gpt-3.5-turbo"",""OpenHermes-2.5-Mistral (7B)"",""chat-bison (PaLM 2)"",""gpt-4"",""Qwen 1.5 Chat (7B)"",""Snorkel Mistral PairRM DPO (7B)"",""Mistral (7B) Instruct v0.2 (Together AI)"",""gpt-4-turbo""]},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model context lengths in tokens""},""barmode"":""relative""}}","{""x"": ""model_context_lengths"", ""y"": ""model"", ""color"": ""model"", ""orientation"": ""h"", ""title"": ""Model context lengths in tokens"", ""labels"": {""model"": ""Model"", ""model_context_lengths"": ""Model context length (tokens)""}}",",model_context_lengths,model
 1,128000,gpt-4-turbo
 9,32768,Mistral (7B) Instruct v0.2 (Together AI)
 11,32768,Snorkel Mistral PairRM DPO (7B)

data/time_of_day_plots.csv CHANGED Viewed

The diff for this file is too large to render. See raw diff

pipeline/models.py CHANGED Viewed

@@ -15,7 +15,7 @@ class Model(object):
     cost_per_million_tokens: int = None
     cost_per_million_input_tokens: int = None
     cost_per_million_output_tokens: int = None
-    input_size: int = None
     selected: bool = False
     def __post_init__(self):
@@ -48,7 +48,7 @@ MODELS = [
         cost_per_million_input_tokens=1,
         cost_per_million_output_tokens=2,
         # https://learn.microsoft.com/en-us/answers/questions/1356487/what-is-the-exact-maximum-input-tokens-of-azure-gp
-        input_size=4096,
     ),
     Model(
         "gpt-4-turbo",
@@ -59,7 +59,7 @@ MODELS = [
         cost_per_million_input_tokens=10,
         cost_per_million_output_tokens=30,
         # https://writesonic.com/blog/gpt-4-turbo-vs-gpt-4
-        input_size=128_000,
     ),
     Model(
         "gpt-4",
@@ -69,7 +69,7 @@ MODELS = [
         supports_functions=True,
         cost_per_million_input_tokens=30,
         cost_per_million_output_tokens=60,
-        input_size=32_000,
     ),
     # source: https://www.together.ai/pricing
     Model(
@@ -80,7 +80,7 @@ MODELS = [
         cost_per_million_tokens=0.9,
         size_billion_parameters=70,
         # https://github.com/facebookresearch/llama/issues/148
-        input_size=2048,
     ),
     Model(
         "Mixtral-8x7B-Instruct-v0.1",
@@ -106,6 +106,7 @@ MODELS = [
         "Hugging Face Inference Endpoint",
         hourly_cost=1.30,
         size_billion_parameters=7,
     ),
     Model(
         "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
@@ -134,7 +135,7 @@ MODELS = [
         cost_per_million_input_tokens=0.25,
         cost_per_million_output_tokens=0.5,
         # https://ai.google.dev/models/palm
-        input_size=8196,
     ),
     Model(
         "chat-bison-32k (PaLM 2 32K)",
@@ -151,7 +152,7 @@ MODELS = [
         "together_ai/zero-one-ai/Yi-34B-Chat",
         None,
         "Together AI",
-        input_size=4096,
         # selected=True,
     ),
     Model(
@@ -159,7 +160,7 @@ MODELS = [
         "together_ai/Austism/chronos-hermes-13b",
         None,
         "Together AI",
-        input_size=2048,
         # selected=True,
     ),
     Model(
@@ -167,50 +168,58 @@ MODELS = [
         "together_ai/deepseek-ai/deepseek-coder-33b-instruct",
         None,
         "Together AI",
-        input_size=16384,
     ),
     Model(
         "Platypus2 Instruct (70B)",
         "together_ai/garage-bAInd/Platypus2-70B-instruct",
         None,
         "Together AI",
-        input_size=4096,
     ),
     Model(
         "MythoMax-L2 (13B)",
         "together_ai/Gryphe/MythoMax-L2-13b",
         None,
         "Together AI",
-        input_size=4096,
     ),
     Model(
         "Vicuna v1.5 (13B)",
         "together_ai/lmsys/vicuna-13b-v1.5",
         None,
         "Together AI",
-        input_size=4096,
     ),
-    Model("Vicuna v1.5 (7B)", "together_ai/lmsys/vicuna-7b-v1.5", None, "Together AI", input_size=4096, size_billion_parameters=7, selected=True),
     Model(
         "Code Llama Instruct (13B)",
         "together_ai/codellama/CodeLlama-13b-Instruct-hf",
         None,
         "Together AI",
-        input_size=16384,
     ),
     Model(
         "Code Llama Instruct (34B)",
         "together_ai/codellama/CodeLlama-34b-Instruct-hf",
         None,
         "Together AI",
-        input_size=16384,
     ),
     Model(
         "Code Llama Instruct (70B)",
         "together_ai/codellama/CodeLlama-70b-Instruct-hf",
         None,
         "Together AI",
-        input_size=4096,
     ),
     Model(
         "Code Llama Instruct (7B)",
@@ -218,7 +227,7 @@ MODELS = [
         None,
         "Together AI",
         size_billion_parameters=7,
-        input_size=16384,
     ),
     Model(
         "LLaMA-2 Chat (13B)",
@@ -226,7 +235,7 @@ MODELS = [
         None,
         "Together AI",
         size_billion_parameters=13,
-        input_size=4096,
     ),
     Model(
         "LLaMA-2 Chat (70B)",
@@ -234,7 +243,7 @@ MODELS = [
         None,
         "Together AI",
         size_billion_parameters=70,
-        input_size=4096,
     ),
     Model(
         "LLaMA-2 Chat (7B)",
@@ -242,7 +251,7 @@ MODELS = [
         None,
         "Together AI",
         size_billion_parameters=7,
-        input_size=4096,
         # selected=True,
     ),
     Model(
@@ -251,7 +260,7 @@ MODELS = [
         None,
         "Together AI",
         size_billion_parameters=7,
-        input_size=4096,
     ),
     Model(
         "Mistral (7B) Instruct v0.2 (Together AI)",
@@ -259,7 +268,7 @@ MODELS = [
         None,
         "Together AI",
         size_billion_parameters=7,
-        input_size=32768,
         selected=True,
     ),
     Model(
@@ -267,8 +276,8 @@ MODELS = [
         "together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1",
         None,
         "Together AI",
-        size_billion_parameters=8*7,
-        input_size=32768,
     ),
     Model(
         "Nous Capybara v1.9 (7B)",
@@ -276,23 +285,23 @@ MODELS = [
         None,
         "Together AI",
         size_billion_parameters=7,
-        input_size=8192,
     ),
     Model(
         "Nous Hermes 2 - Mixtral 8x7B-DPO (46.7B)",
         "together_ai/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
         None,
         "Together AI",
-        size_billion_parameters=8*7,
-        input_size=32768,
     ),
     Model(
         "Nous Hermes 2 - Mixtral 8x7B-SFT (46.7B)",
         "together_ai/NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT",
         None,
         "Together AI",
-        size_billion_parameters=8*7,
-        input_size=32768,
     ),
     Model(
         "Nous Hermes LLaMA-2 (7B)",
@@ -300,7 +309,7 @@ MODELS = [
         None,
         "Together AI",
         size_billion_parameters=7,
-        input_size=4096,
     ),
     Model(
         "Nous Hermes Llama-2 (13B)",
@@ -308,7 +317,7 @@ MODELS = [
         None,
         "Together AI",
         size_billion_parameters=13,
-        input_size=4096,
     ),
     Model(
         "Nous Hermes-2 Yi (34B)",
@@ -316,7 +325,7 @@ MODELS = [
         None,
         "Together AI",
         size_billion_parameters=34,
-        input_size=4096,
     ),
     Model(
         "OpenChat 3.5 (7B)",
@@ -324,7 +333,7 @@ MODELS = [
         None,
         "Together AI",
         size_billion_parameters=7,
-        input_size=8192,
     ),
     Model(
         "OpenOrca Mistral (7B) 8K",
@@ -332,7 +341,7 @@ MODELS = [
         None,
         "Together AI",
         size_billion_parameters=7,
-        input_size=8192,
     ),
     Model(
         "Qwen-Chat (7B)",
@@ -340,7 +349,7 @@ MODELS = [
         None,
         "Together AI",
         size_billion_parameters=7,
-        input_size=8192,
     ),
     Model(
         "Qwen 1.5 Chat (0.5B)",
@@ -348,14 +357,14 @@ MODELS = [
         None,
         "Together AI",
         size_billion_parameters=0.5,
-        input_size=32768,
     ),
     Model(
         "Qwen 1.5 Chat (1.8B)",
         "together_ai/Qwen/Qwen1.5-1.8B-Chat",
         None,
         "Together AI",
-        input_size=32768,
         size_billion_parameters=1.8,
     ),
     Model(
@@ -364,10 +373,15 @@ MODELS = [
         None,
         "Together AI",
         size_billion_parameters=4,
-        input_size=32768,
     ),
     Model(
-        "Qwen 1.5 Chat (7B)", "together_ai/Qwen/Qwen1.5-7B-Chat", None, "Together AI", input_size=32768, size_billion_parameters=7
         # selected=True
     ),
     Model(
@@ -376,21 +390,21 @@ MODELS = [
         None,
         "Together AI",
         size_billion_parameters=14,
-        input_size=32768,
     ),
     Model(
         "Qwen 1.5 Chat (72B)",
         "together_ai/Qwen/Qwen1.5-72B-Chat",
         None,
         "Together AI",
-        input_size=4096,
     ),
     Model(
         "Snorkel Mistral PairRM DPO (7B)",
         "together_ai/snorkelai/Snorkel-Mistral-PairRM-DPO",
         None,
         "Together AI",
-        input_size=32768,
         # selected=True,
     ),
     Model(
@@ -398,21 +412,21 @@ MODELS = [
         "together_ai/togethercomputer/alpaca-7b",
         None,
         "Together AI",
-        input_size=2048,
     ),
     Model(
         "OpenHermes-2-Mistral (7B)",
         "teknium/OpenHermes-2-Mistral-7B",
         None,
         "Together AI",
-        input_size=8192,
     ),
     Model(
         "OpenHermes-2.5-Mistral (7B)",
         "together_ai/teknium/OpenHermes-2p5-Mistral-7B",
         None,
         "Together AI",
-        input_size=8192,
         # selected=True,
     ),
     Model(
@@ -420,14 +434,14 @@ MODELS = [
         "together_ai/togethercomputer/falcon-40b-instruct",
         None,
         "Together AI",
-        input_size=2048,
     ),
     Model(
         "Falcon Instruct (7B)",
         "together_ai/togethercomputer/falcon-7b-instruct",
         None,
         "Together AI",
-        input_size=2048,
         # selected=True,
     ),
     Model(
@@ -435,7 +449,7 @@ MODELS = [
         "together_ai/togethercomputer/Llama-2-7B-32K-Instruct",
         None,
         "Together AI",
-        input_size=32768,
     ),
     Model(
         "RedPajama-INCITE Chat (3B)",
@@ -443,14 +457,14 @@ MODELS = [
         None,
         "Together AI",
         size_billion_parameters=3,
-        input_size=2048,
     ),
     Model(
         "RedPajama-INCITE Chat (7B)",
         "together_ai/togethercomputer/RedPajama-INCITE-7B-Chat",
         None,
         "Together AI",
-        input_size=2048,
         size_billion_parameters=7,
         # selected=True,
     ),
@@ -459,7 +473,7 @@ MODELS = [
         "together_ai/togethercomputer/StripedHyena-Nous-7B",
         None,
         "Together AI",
-        input_size=32768,
         size_billion_parameters=7,
     ),
     Model(
@@ -467,7 +481,7 @@ MODELS = [
         "together_ai/Undi95/ReMM-SLERP-L2-13B",
         None,
         "Together AI",
-        input_size=4096,
         size_billion_parameters=13,
     ),
     Model(
@@ -475,15 +489,15 @@ MODELS = [
         "together_ai/Undi95/Toppy-M-7B",
         None,
         "Together AI",
-        input_size=4096,
-        size_billion_parameters=7
     ),
     Model(
         "WizardLM v1.2 (13B)",
         "together_ai/WizardLM/WizardLM-13B-V1.2",
         None,
         "Together AI",
-        input_size=4096,
         size_billion_parameters=13,
         # selected=True,
     ),
@@ -492,7 +506,7 @@ MODELS = [
         "together_ai/upstage/SOLAR-10.7B-Instruct-v1.0",
         None,
         "Together AI",
-        input_size=4096,
         size_billion_parameters=11,
         # selected=True,
     ),

     cost_per_million_tokens: int = None
     cost_per_million_input_tokens: int = None
     cost_per_million_output_tokens: int = None
+    context_length: int = None
     selected: bool = False
     def __post_init__(self):
         cost_per_million_input_tokens=1,
         cost_per_million_output_tokens=2,
         # https://learn.microsoft.com/en-us/answers/questions/1356487/what-is-the-exact-maximum-input-tokens-of-azure-gp
+        context_length=4096,
     ),
     Model(
         "gpt-4-turbo",
         cost_per_million_input_tokens=10,
         cost_per_million_output_tokens=30,
         # https://writesonic.com/blog/gpt-4-turbo-vs-gpt-4
+        context_length=128_000,
     ),
     Model(
         "gpt-4",
         supports_functions=True,
         cost_per_million_input_tokens=30,
         cost_per_million_output_tokens=60,
+        context_length=32_000,
     ),
     # source: https://www.together.ai/pricing
     Model(
         cost_per_million_tokens=0.9,
         size_billion_parameters=70,
         # https://github.com/facebookresearch/llama/issues/148
+        context_length=2048,
     ),
     Model(
         "Mixtral-8x7B-Instruct-v0.1",
         "Hugging Face Inference Endpoint",
         hourly_cost=1.30,
         size_billion_parameters=7,
+        selected=True,
     ),
     Model(
         "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
         cost_per_million_input_tokens=0.25,
         cost_per_million_output_tokens=0.5,
         # https://ai.google.dev/models/palm
+        context_length=8196,
     ),
     Model(
         "chat-bison-32k (PaLM 2 32K)",
         "together_ai/zero-one-ai/Yi-34B-Chat",
         None,
         "Together AI",
+        context_length=4096,
         # selected=True,
     ),
     Model(
         "together_ai/Austism/chronos-hermes-13b",
         None,
         "Together AI",
+        context_length=2048,
         # selected=True,
     ),
     Model(
         "together_ai/deepseek-ai/deepseek-coder-33b-instruct",
         None,
         "Together AI",
+        context_length=16384,
     ),
     Model(
         "Platypus2 Instruct (70B)",
         "together_ai/garage-bAInd/Platypus2-70B-instruct",
         None,
         "Together AI",
+        context_length=4096,
     ),
     Model(
         "MythoMax-L2 (13B)",
         "together_ai/Gryphe/MythoMax-L2-13b",
         None,
         "Together AI",
+        context_length=4096,
     ),
     Model(
         "Vicuna v1.5 (13B)",
         "together_ai/lmsys/vicuna-13b-v1.5",
         None,
         "Together AI",
+        context_length=4096,
+    ),
+    Model(
+        "Vicuna v1.5 (7B)",
+        "together_ai/lmsys/vicuna-7b-v1.5",
+        None,
+        "Together AI",
+        context_length=4096,
+        size_billion_parameters=7,
+        # selected=True
     ),
     Model(
         "Code Llama Instruct (13B)",
         "together_ai/codellama/CodeLlama-13b-Instruct-hf",
         None,
         "Together AI",
+        context_length=16384,
     ),
     Model(
         "Code Llama Instruct (34B)",
         "together_ai/codellama/CodeLlama-34b-Instruct-hf",
         None,
         "Together AI",
+        context_length=16384,
     ),
     Model(
         "Code Llama Instruct (70B)",
         "together_ai/codellama/CodeLlama-70b-Instruct-hf",
         None,
         "Together AI",
+        context_length=4096,
     ),
     Model(
         "Code Llama Instruct (7B)",
         None,
         "Together AI",
         size_billion_parameters=7,
+        context_length=16384,
     ),
     Model(
         "LLaMA-2 Chat (13B)",
         None,
         "Together AI",
         size_billion_parameters=13,
+        context_length=4096,
     ),
     Model(
         "LLaMA-2 Chat (70B)",
         None,
         "Together AI",
         size_billion_parameters=70,
+        context_length=4096,
     ),
     Model(
         "LLaMA-2 Chat (7B)",
         None,
         "Together AI",
         size_billion_parameters=7,
+        context_length=4096,
         # selected=True,
     ),
     Model(
         None,
         "Together AI",
         size_billion_parameters=7,
+        context_length=4096,
     ),
     Model(
         "Mistral (7B) Instruct v0.2 (Together AI)",
         None,
         "Together AI",
         size_billion_parameters=7,
+        context_length=32768,
         selected=True,
     ),
     Model(
         "together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1",
         None,
         "Together AI",
+        size_billion_parameters=8 * 7,
+        context_length=32768,
     ),
     Model(
         "Nous Capybara v1.9 (7B)",
         None,
         "Together AI",
         size_billion_parameters=7,
+        context_length=8192,
     ),
     Model(
         "Nous Hermes 2 - Mixtral 8x7B-DPO (46.7B)",
         "together_ai/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
         None,
         "Together AI",
+        size_billion_parameters=8 * 7,
+        context_length=32768,
     ),
     Model(
         "Nous Hermes 2 - Mixtral 8x7B-SFT (46.7B)",
         "together_ai/NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT",
         None,
         "Together AI",
+        size_billion_parameters=8 * 7,
+        context_length=32768,
     ),
     Model(
         "Nous Hermes LLaMA-2 (7B)",
         None,
         "Together AI",
         size_billion_parameters=7,
+        context_length=4096,
     ),
     Model(
         "Nous Hermes Llama-2 (13B)",
         None,
         "Together AI",
         size_billion_parameters=13,
+        context_length=4096,
     ),
     Model(
         "Nous Hermes-2 Yi (34B)",
         None,
         "Together AI",
         size_billion_parameters=34,
+        context_length=4096,
     ),
     Model(
         "OpenChat 3.5 (7B)",
         None,
         "Together AI",
         size_billion_parameters=7,
+        context_length=8192,
     ),
     Model(
         "OpenOrca Mistral (7B) 8K",
         None,
         "Together AI",
         size_billion_parameters=7,
+        context_length=8192,
     ),
     Model(
         "Qwen-Chat (7B)",
         None,
         "Together AI",
         size_billion_parameters=7,
+        context_length=8192,
     ),
     Model(
         "Qwen 1.5 Chat (0.5B)",
         None,
         "Together AI",
         size_billion_parameters=0.5,
+        context_length=32768,
     ),
     Model(
         "Qwen 1.5 Chat (1.8B)",
         "together_ai/Qwen/Qwen1.5-1.8B-Chat",
         None,
         "Together AI",
+        context_length=32768,
         size_billion_parameters=1.8,
     ),
     Model(
         None,
         "Together AI",
         size_billion_parameters=4,
+        context_length=32768,
     ),
     Model(
+        "Qwen 1.5 Chat (7B)",
+        "together_ai/Qwen/Qwen1.5-7B-Chat",
+        None,
+        "Together AI",
+        context_length=32768,
+        size_billion_parameters=7
         # selected=True
     ),
     Model(
         None,
         "Together AI",
         size_billion_parameters=14,
+        context_length=32768,
     ),
     Model(
         "Qwen 1.5 Chat (72B)",
         "together_ai/Qwen/Qwen1.5-72B-Chat",
         None,
         "Together AI",
+        context_length=4096,
     ),
     Model(
         "Snorkel Mistral PairRM DPO (7B)",
         "together_ai/snorkelai/Snorkel-Mistral-PairRM-DPO",
         None,
         "Together AI",
+        context_length=32768,
         # selected=True,
     ),
     Model(
         "together_ai/togethercomputer/alpaca-7b",
         None,
         "Together AI",
+        context_length=2048,
     ),
     Model(
         "OpenHermes-2-Mistral (7B)",
         "teknium/OpenHermes-2-Mistral-7B",
         None,
         "Together AI",
+        context_length=8192,
     ),
     Model(
         "OpenHermes-2.5-Mistral (7B)",
         "together_ai/teknium/OpenHermes-2p5-Mistral-7B",
         None,
         "Together AI",
+        context_length=8192,
         # selected=True,
     ),
     Model(
         "together_ai/togethercomputer/falcon-40b-instruct",
         None,
         "Together AI",
+        context_length=2048,
     ),
     Model(
         "Falcon Instruct (7B)",
         "together_ai/togethercomputer/falcon-7b-instruct",
         None,
         "Together AI",
+        context_length=2048,
         # selected=True,
     ),
     Model(
         "together_ai/togethercomputer/Llama-2-7B-32K-Instruct",
         None,
         "Together AI",
+        context_length=32768,
     ),
     Model(
         "RedPajama-INCITE Chat (3B)",
         None,
         "Together AI",
         size_billion_parameters=3,
+        context_length=2048,
     ),
     Model(
         "RedPajama-INCITE Chat (7B)",
         "together_ai/togethercomputer/RedPajama-INCITE-7B-Chat",
         None,
         "Together AI",
+        context_length=2048,
         size_billion_parameters=7,
         # selected=True,
     ),
         "together_ai/togethercomputer/StripedHyena-Nous-7B",
         None,
         "Together AI",
+        context_length=32768,
         size_billion_parameters=7,
     ),
     Model(
         "together_ai/Undi95/ReMM-SLERP-L2-13B",
         None,
         "Together AI",
+        context_length=4096,
         size_billion_parameters=13,
     ),
     Model(
         "together_ai/Undi95/Toppy-M-7B",
         None,
         "Together AI",
+        context_length=4096,
+        size_billion_parameters=7,
     ),
     Model(
         "WizardLM v1.2 (13B)",
         "together_ai/WizardLM/WizardLM-13B-V1.2",
         None,
         "Together AI",
+        context_length=4096,
         size_billion_parameters=13,
         # selected=True,
     ),
         "together_ai/upstage/SOLAR-10.7B-Instruct-v1.0",
         None,
         "Together AI",
+        context_length=4096,
         size_billion_parameters=11,
         # selected=True,
     ),