app.R

library(shiny)
library(ellmer)
library(pdftools)
library(magick)
library(base64enc)
library(jsonlite)

# App logic lives in R/ and is sourced here in a fixed order
# (R/_disable_autoload.R stops Shiny from sourcing the directory a second time)
for (r_file in c("config.R", "fields.R", "sources.R", "extraction.R", "snippets.R", "help.R")) {
  source(file.path("R", r_file), local = TRUE)
}

# Uploads larger than this are rejected by Shiny
options(shiny.maxRequestSize = MAX_UPLOAD_MB * 1024^2)

# Model used for every call and named in every piece of UI text
EXTRACTION_MODEL = get_extraction_model()

APP_VERSION = paste0("Version ", APP_VERSION_NUMBER, " | ", APP_VERSION_DATE)

PAPER_TITLE = "What Quality Engineers Need to Know About Generative AI: Structured Data Extraction"

# Field templates (one JSON file each in config/templates)
FIELD_TEMPLATES = load_templates(file.path("config", "templates"))

# Fields shown at start-up and loaded with the demo data
demo_fields = function() {
  FIELD_TEMPLATES[["recall_notice"]]$fields[seq_len(NUM_EXAMPLE_FIELDS)]
}

# One input tab button (the panels are shown and hidden by www/app.js)
input_tab = function(id, label, icon_name, selected = FALSE) {
  shiny::tags$button(
    id = paste0("tab-", id),
    type = "button",
    class = if (selected) "input-tab active" else "input-tab",
    role = "tab",
    `aria-selected` = if (selected) "true" else "false",
    `aria-controls` = paste0("panel-", id),
    tabindex = if (selected) "0" else "-1",
    shiny::icon(icon_name),
    paste0(" ", label)
  )
}

# Container of one input tab's content
input_panel = function(id, ..., selected = FALSE) {
  shiny::div(
    id = paste0("panel-", id),
    class = "input-panel",
    role = "tabpanel",
    `aria-labelledby` = paste0("tab-", id),
    style = if (selected) NULL else "display: none;",
    ...
  )
}

# Heading followed by its "?" help button
heading_with_help = function(heading, topic) {
  shiny::div(class = "heading-with-help", heading, help_button(topic))
}

# Define UI for the app
ui = shiny::fluidPage(
  lang = "en",
  # Add Miami theme CSS and favicon
  shiny::tags$head(
    shiny::tags$link(rel = "icon", type = "image/svg+xml", href = "favicon.svg"),
    shiny::tags$link(rel = "stylesheet", type = "text/css", href = "miami-theme.css"),
    shiny::tags$title("AI-Powered Text Extraction Tool")
  ),

  # Custom header with Miami branding
  shiny::div(
    class = "app-header",
    shiny::div(
      class = "header-content",
      shiny::div(
        class = "header-left",
        shiny::tags$h1("AI-Powered Text Extraction Tool"),
        shiny::p(class = "subtitle", "Extract Structured Data from Text, Documents, and Images")
      ),
      shiny::div(
        class = "header-right",
        shiny::p(APP_VERSION, " | ", changes_link()),
        shiny::p(
          shiny::tags$strong("Authors: "),
          "Fadel M. Megahed, Ying-Ju Chen, Yamin Dahwich, Arthur Carvalho, L. Allison Jones-Farmer, Ibrahim Yousif, and Inez M. Zwetsloot"
        ),
        shiny::p(
          shiny::tags$strong("Contact: "),
          shiny::tags$a(
            href = "mailto:fmegahed@miamioh.edu",
            style = "color: #EFDB72;",
            "fmegahed@miamioh.edu"
          )
        )
      )
    )
  ),

  # University logos section
  shiny::div(
    class = "logo-container",
    shiny::tags$img(
      src = "miami-logo.png",
      alt = "Miami University Logo",
      style = "height: 55px;"
    ),
    shiny::div(class = "logo-divider"),
    shiny::tags$img(
      src = "university-of-dayton-vector-logo.png",
      alt = "University of Dayton Logo",
      style = "height: 50px;"
    ),
    shiny::div(class = "logo-divider"),
    shiny::tags$img(
      src = "university-of-cambridge-logo.png",
      alt = "University of Cambridge Logo",
      style = "height: 38px;"
    ),
    shiny::div(class = "logo-divider"),
    shiny::tags$img(
      src = "uva-compacte-logo.png",
      alt = "University of Amsterdam Logo",
      style = "height: 50px;"
    ),
    shiny::div(
      style = "margin-left: auto; font-size: 0.85em; color: #666;",
      shiny::p(
        style = "margin: 0;",
        "A collaboration between",
        shiny::tags$strong("Miami University,"),
        "the",
        shiny::tags$strong("University of Dayton,"),
        "the",
        shiny::tags$strong("University of Cambridge,"),
        "and the",
        shiny::tags$strong("University of Amsterdam")
      )
    )
  ),

  # How to use section
  shiny::div(
    class = "how-to-use",
    shiny::div(
      style = "display: flex; justify-content: space-between; align-items: center; margin-bottom: 10px;",
      shiny::tags$h4(
        style = "margin: 0;",
        shiny::icon("circle-info"),
        " How to Use This App"
      ),
      shiny::tags$button(
        id = "open-video-modal",
        type = "button",
        class = "btn btn-video-tutorial",
        `aria-haspopup` = "dialog",
        shiny::icon("play-circle"),
        " Watch Video Tutorial"
      )
    ),
    shiny::tags$h5(
      style = "color: #C41230; margin-top: 0;",
      shiny::icon("bolt"),
      " Quick Demo"
    ),
    shiny::p(
      "Click the ",
      shiny::tags$strong("'Load Demo Data'"),
      " button to load two sample NHTSA recall notices with pre-configured extraction fields. ",
      "Then click ",
      shiny::tags$strong("'Extract Data'"),
      " to see the model extract structured information from the text. ",
      "You can also add more fields, change field labels and types, and modify descriptions. ",
      shiny::tags$em(
        "(Note: The field description is the instruction the model receives. The label names the column in the results.)"
      )
    ),
    shiny::tags$h5(
      style = "color: #C41230; margin-top: 15px;",
      shiny::icon("file-import"),
      " Input Methods"
    ),
    shiny::p(
      "Choose from six input methods to provide your source content. Only the tab that is open when you click 'Extract Data' is used."
    ),
    shiny::tags$ul(
      style = "margin-bottom: 10px;",
      shiny::tags$li(
        shiny::tags$strong("Demo Data:"),
        " Try the app with pre-loaded NHTSA vehicle recall examples."
      ),
      shiny::tags$li(
        shiny::tags$strong("Paste Text:"),
        " Directly paste text content. Separate multiple items with double line breaks."
      ),
      shiny::tags$li(
        shiny::tags$strong("Text File:"),
        " Upload .txt, .csv (single column), or .md files."
      ),
      shiny::tags$li(
        shiny::tags$strong("Readable PDF:"),
        " Upload a machine-readable PDF. Text is extracted automatically."
      ),
      shiny::tags$li(
        shiny::tags$strong("Scanned PDF:"),
        " Upload scanned/image-based PDFs. Pages are converted to images for vision processing."
      ),
      shiny::tags$li(
        shiny::tags$strong("Upload Images:"),
        " Upload images (PNG, JPEG, WebP, GIF) for vision-based extraction."
      )
    ),
    shiny::div(
      style = "font-size: 0.9em; color: #666;",
      shiny::icon("info-circle"),
      " ",
      shiny::tags$em("Notes:"),
      shiny::tags$ul(
        style = "margin: 0.25em 0 0 1.25em; padding: 0; line-height: 1.3;",
        shiny::tags$li(
          style = "margin: 0; padding: 0;",
          sprintf(
            "Readable PDF and Scanned PDF are limited to the first %d pages, and Upload Images to %d images, in this free app.",
            MAX_PAGES_OR_IMAGES, MAX_PAGES_OR_IMAGES
          )
        ),
        shiny::tags$li(
          style = "margin: 0; padding: 0;",
          sprintf(
            "Any uploaded file must be at most %d MB, to keep the app responsive and avoid upload timeouts.",
            MAX_UPLOAD_MB
          )
        ),
        shiny::tags$li(
          style = "margin: 0; padding: 0;",
          "The '?' buttons next to each part of the app explain what it does and what is sent to the model."
        )
      )
    ),
    shiny::tags$h5(
      style = "color: #C41230; margin-top: 15px;",
      shiny::icon("edit"),
      " Custom Fields"
    ),
    shiny::p(
      "Customize your extraction by adjusting:"
    ),
    shiny::tags$ul(
      style = "margin-bottom: 10px;",
      shiny::tags$li(
        "The ", shiny::tags$strong("number of fields"), " to extract (up to ", MAX_FIELDS, ")"
      ),
      shiny::tags$li("Your own ", shiny::tags$strong("field labels"), " (the column names of the results)"),
      shiny::tags$li("Detailed ", shiny::tags$strong("field descriptions"), " to guide the model"),
      shiny::tags$li(
        "The ", shiny::tags$strong("field type"), " (text, number, yes/no, or list of text)"
      )
    ),
    shiny::p(
      "You can start from a ",
      shiny::tags$strong("template"),
      " for a common quality document (recall notice, 8D / corrective action report, customer complaint record, inspection or nonconformance note), and save your field definitions to a file to reuse them later."
    ),
    shiny::p(
      shiny::icon("lightbulb"),
      " ",
      shiny::tags$em(
        "Specific field descriptions give the model less room for interpretation. Always check the extracted values against the source documents."
      )
    )
  ),

  # Video Modal
  shiny::div(
    id = "video-modal",
    class = "video-modal",
    role = "dialog",
    `aria-modal` = "true",
    `aria-labelledby` = "video-modal-title",
    shiny::div(
      class = "video-modal-content",
      shiny::tags$button(
        id = "close-video-modal",
        type = "button",
        class = "video-modal-close",
        `aria-label` = "Close video tutorial",
        shiny::HTML("×")
      ),
      shiny::tags$h4(
        id = "video-modal-title",
        style = "color: #C41230; margin-top: 0; margin-bottom: 15px;",
        shiny::icon("play-circle"),
        " Video Tutorial"
      ),
      shiny::div(
        class = "video-modal-wrapper",
        # The video address is set by www/app.js when the dialog opens
        shiny::tags$iframe(
          id = "tutorial-video",
          title = "Video tutorial",
          src = "",
          allow = "accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture",
          allowfullscreen = NA
        )
      )
    )
  ),

  # JavaScript for the input tabs, the video dialog and the copy buttons
  shiny::tags$script(src = "app.js"),

  shiny::sidebarLayout(
    shiny::sidebarPanel(
      width = 4,
      class = "sidebar-panel",

      heading_with_help(
        shiny::tags$h4(
          id = "input-heading",
          style = "color: #C41230; margin-top: 0; margin-bottom: 0;",
          shiny::icon("file-alt"),
          " Input Configuration"
        ),
        "source"
      ),

      # Input method tabs (3 rows of 2 tabs)
      shiny::div(
        class = "input-tabs-container",
        role = "tablist",
        `aria-labelledby` = "input-heading",
        shiny::div(
          class = "input-tabs",
          role = "presentation",
          input_tab("demo", "Demo Data", "flask", selected = TRUE),
          input_tab("paste", "Paste Text", "paste")
        ),
        shiny::div(
          class = "input-tabs",
          role = "presentation",
          input_tab("textfile", "Text File", "file-lines"),
          input_tab("pdf", "Readable PDF", "file-pdf")
        ),
        shiny::div(
          class = "input-tabs",
          role = "presentation",
          input_tab("scanned", "Scanned PDF", "file-image"),
          input_tab("image", "Upload Images", "images")
        )
      ),

      # Demo data panel
      input_panel(
        "demo",
        selected = TRUE,
        shiny::p(
          "Try the app with pre-loaded NHTSA vehicle recall data. Click the button below to load sample text and pre-configured extraction fields."
        ),
        shiny::actionButton(
          "load_example",
          shiny::tags$span(shiny::icon("play"), " Load Demo Data"),
          class = "btn-primary",
          style = "margin-top: 10px;"
        ),
        shiny::uiOutput("demo_preview_ui")
      ),

      # Paste text panel
      input_panel(
        "paste",
        shiny::textAreaInput(
          "input_text",
          shiny::tags$span(class = "sr-only", "Text to extract from"),
          rows = 8,
          placeholder = "Paste your text content here.\n\nTo process multiple items at once, separate each block of text with a double line break (press Enter twice).\n\nIMPORTANT: Before extracting, scroll down to configure your extraction fields:\n1. Choose a template or set the number of fields you need\n2. Update each field label (e.g., 'product_name', 'price')\n3. Write clear descriptions for each field\n\nThen click 'Extract Data' to see results."
        )
      ),

      # Text file upload panel
      input_panel(
        "textfile",
        shiny::fileInput(
          "text_file",
          shiny::tags$span(class = "sr-only", "Text file"),
          accept = c(".txt", ".csv", ".md"),
          placeholder = "No file selected",
          buttonLabel = shiny::tags$span(shiny::icon("upload"), " Browse...")
        ),
        shiny::helpText(
          shiny::icon("info-circle"),
          " Upload a text file (.txt, .csv, .md). Content will be extracted automatically."
        ),
        shiny::div(
          class = "file-format-note",
          shiny::tags$strong("File format guidelines:"),
          shiny::tags$ul(
            style = "margin: 5px 0 0 0; padding-left: 20px; font-size: 0.85em;",
            shiny::tags$li(".txt: Plain text, separate items with double line breaks"),
            shiny::tags$li(".csv: Single column of text entries (one per row, first row is a header)"),
            shiny::tags$li(".md: Markdown file treated as plain text, split at double line breaks")
          )
        ),
        shiny::uiOutput("textfile_preview_ui")
      ),

      # PDF upload panel (machine-readable)
      input_panel(
        "pdf",
        shiny::fileInput(
          "pdf_file",
          shiny::tags$span(class = "sr-only", "Readable PDF file"),
          accept = ".pdf",
          placeholder = "No file selected",
          buttonLabel = shiny::tags$span(shiny::icon("upload"), " Browse...")
        ),
        shiny::helpText(
          shiny::icon("info-circle"),
          " Upload a machine-readable PDF. Text will be extracted automatically and sent as one item."
        ),
        shiny::div(
          class = "limit-note",
          shiny::icon("exclamation-circle"),
          shiny::tags$em(limit_note("pages"))
        ),
        shiny::uiOutput("pdf_preview_ui")
      ),

      # Image upload panel
      input_panel(
        "image",
        shiny::fileInput(
          "image_file",
          shiny::tags$span(class = "sr-only", "Image files"),
          accept = c(".png", ".jpg", ".jpeg", ".webp", ".gif"),
          multiple = TRUE,
          placeholder = "No file selected",
          buttonLabel = shiny::tags$span(shiny::icon("upload"), " Browse...")
        ),
        shiny::helpText(
          shiny::icon("info-circle"),
          " Upload image(s) containing text to extract. Supports PNG, JPEG, WebP, and GIF."
        ),
        shiny::div(
          class = "limit-note",
          shiny::icon("exclamation-circle"),
          shiny::tags$em(limit_note("images"))
        ),
        shiny::uiOutput("image_preview_ui")
      ),

      # Scanned PDF panel
      input_panel(
        "scanned",
        shiny::fileInput(
          "scanned_pdf_file",
          shiny::tags$span(class = "sr-only", "Scanned PDF file"),
          accept = ".pdf",
          placeholder = "No file selected",
          buttonLabel = shiny::tags$span(shiny::icon("upload"), " Browse...")
        ),
        shiny::helpText(
          shiny::icon("info-circle"),
          " Upload a scanned/image-based PDF. Pages will be converted to images for vision processing."
        ),
        shiny::div(
          class = "limit-note",
          shiny::icon("exclamation-circle"),
          shiny::tags$em(limit_note("pages"))
        ),
        shiny::uiOutput("scanned_pdf_preview_ui")
      ),

      shiny::hr(class = "field-separator"),

      heading_with_help(
        shiny::tags$h4(
          style = "color: #C41230; margin-top: 0; margin-bottom: 0;",
          shiny::icon("tags"),
          " Field Definitions"
        ),
        "fields"
      ),

      shiny::div(
        class = "template-row",
        shiny::selectInput(
          "template_id",
          "Start from a template:",
          choices = template_choices(FIELD_TEMPLATES),
          selectize = FALSE
        ),
        shiny::actionButton("apply_template", "Use template", class = "btn-info")
      ),
      shiny::uiOutput("template_description_ui"),

      shiny::numericInput(
        "num_fields",
        shiny::tags$span(
          shiny::icon("list-ol"),
          " Number of Fields to Extract:"
        ),
        value = NUM_EXAMPLE_FIELDS,
        min = 1,
        max = MAX_FIELDS
      ),

      shiny::helpText(
        shiny::icon("info-circle"),
        " Define each field with a clear label (e.g., 'manufacturer') and description (e.g., 'Name of the company that is recalling the vehicles. Leave empty if no company is named.')."
      ),

      shiny::uiOutput("fields_ui"),

      shiny::div(
        class = "fields-file-row",
        shiny::downloadButton("download_fields", "Save fields (JSON)", class = "btn-info"),
        shiny::fileInput(
          "fields_upload",
          "Load fields (JSON):",
          accept = c(".json", "application/json"),
          placeholder = "No file selected",
          buttonLabel = shiny::tags$span(shiny::icon("upload"), " Browse...")
        )
      ),

      shiny::hr(class = "field-separator"),

      shiny::div(
        class = "heading-with-help",
        shiny::actionButton(
          "extract_btn",
          shiny::tags$span(shiny::icon("magic"), " Extract Data"),
          class = "btn-primary"
        ),
        help_button("extract")
      )
    ),

    shiny::mainPanel(
      width = 8,
      class = "main-panel",

      shiny::div(
        class = "info-card",
        shiny::div(
          class = "heading-with-help section-heading",
          shiny::tags$h3(
            style = "color: #C41230; font-weight: 600; margin: 0;",
            shiny::icon("table"),
            " Structured Data Extracted Using AI"
          ),
          help_button("results")
        ),
        shiny::uiOutput("results_status_ui"),
        shiny::div(class = "results-table-wrapper", shiny::tableOutput("extracted_table")),
        shiny::uiOutput("download_btn_ui")
      ),

      shiny::div(
        class = "info-card",
        shiny::tags$h3(
          class = "section-heading",
          shiny::icon("code"),
          " Reproduce in Code"
        ),
        shiny::p(
          "The code below runs the same extraction, with your current field definitions and the model ",
          shiny::tags$code(EXTRACTION_MODEL),
          ", on your own computer and your own data. It needs your own ",
          MODEL_PROVIDER,
          " API key."
        ),
        shiny::uiOutput("snippet_status_ui"),
        shiny::tags$details(
          class = "code-details",
          shiny::tags$summary("R code (ellmer package)"),
          shiny::tags$button(
            type = "button",
            class = "btn btn-info copy-btn",
            `data-target` = "snippet_r",
            "Copy R code"
          ),
          shiny::verbatimTextOutput("snippet_r")
        ),
        shiny::tags$details(
          class = "code-details",
          shiny::tags$summary("Python code (chatlas package)"),
          shiny::tags$button(
            type = "button",
            class = "btn btn-info copy-btn",
            `data-target` = "snippet_python",
            "Copy Python code"
          ),
          shiny::verbatimTextOutput("snippet_python")
        )
      ),

      shiny::div(
        class = "tips-section",
        shiny::tags$h4(shiny::icon("lightbulb"), " Tips for Better Results"),
        shiny::tags$ul(
          shiny::tags$li("Use ", shiny::tags$strong("specific field descriptions"), " and say what to return when the information is absent"),
          shiny::tags$li("Start with ", shiny::tags$strong("more fields"), " and remove unnecessary ones later"),
          shiny::tags$li("If results are inaccurate, try ", shiny::tags$strong("rephrasing"), " your field descriptions"),
          shiny::tags$li("For ", shiny::tags$strong("multiple text blocks"), ", separate each with a double line break"),
          shiny::tags$li("Each text block should contain ", shiny::tags$strong("complete information"), " for all fields you want to extract"),
          shiny::tags$li(shiny::tags$strong("Check the results"), " against the source documents before you use them")
        )
      ),

      shiny::div(
        class = "note-section",
        shiny::tags$h4(shiny::icon("cog"), " Technical Note"),
        shiny::p(
          "This app sends its requests to the ",
          MODEL_PROVIDER,
          " model ",
          shiny::tags$code(EXTRACTION_MODEL),
          " through ",
          shiny::tags$code("chat_openai()"),
          " from the ",
          shiny::tags$a(
            href = "https://ellmer.tidyverse.org/",
            target = "_blank",
            "ellmer"
          ),
          " package. The app does not provide an option to change the model."
        ),
        shiny::p(
          "For ",
          shiny::tags$strong("complete privacy"),
          ", consider using local open-weight models via ",
          shiny::tags$a(
            href = "https://ellmer.tidyverse.org/reference/chat_ollama.html",
            target = "_blank",
            shiny::tags$code("chat_ollama()")
          ),
          " from ellmer, which connects to ",
          shiny::tags$a(
            href = "https://ollama.com/",
            target = "_blank",
            "Ollama"
          ),
          " for running models locally on your machine."
        ),
        shiny::p(
          "ellmer also supports models from other providers. For example, Claude models require ",
          shiny::tags$a(
            href = "https://ellmer.tidyverse.org/reference/chat_anthropic.html",
            target = "_blank",
            shiny::tags$code("chat_anthropic()")
          ),
          " and Gemini models require ",
          shiny::tags$a(
            href = "https://ellmer.tidyverse.org/reference/chat_google_gemini.html",
            target = "_blank",
            shiny::tags$code("chat_google_gemini()")
          ),
          ", each with their respective API keys configured in the R environment. ",
          "The code under 'Reproduce in Code' can be adapted by replacing the ",
          shiny::tags$code("chat_openai()"),
          " call."
        )
      )
    )
  ),

  # Footer
  shiny::div(
    class = "app-footer",
    shiny::p(
      "Built with ",
      shiny::tags$a(href = "https://shiny.posit.co/", target = "_blank", "Shiny"),
      " and ",
      shiny::tags$a(href = "https://ellmer.tidyverse.org/", target = "_blank", "ellmer"),
      " | ",
      "Powered by ",
      MODEL_PROVIDER
    ),
    shiny::p(
      shiny::tags$em(
        paste0(
          "Companion app to the paper '",
          PAPER_TITLE,
          "', in preparation by the app's authors for submission to "
        ),
        shiny::tags$a(
          href = "https://www.tandfonline.com/journals/lqen20",
          target = "_blank",
          "Quality Engineering."
        )
      )
    )
  )
)

# Define server logic required to generate dynamic UI and extract data
server = function(input, output, session) {

  # ---- State ---------------------------------------------------------------

  # What each input tab currently holds. Each tab has its own store, and the
  # extraction reads only the store of the active tab.
  demo_text = shiny::reactiveVal("")
  textfile_items = shiny::reactiveVal(character(0))
  pdf_text = shiny::reactiveVal("")
  image_files = shiny::reactiveVal(NULL)
  scanned_pdf_paths = shiny::reactiveVal(NULL)

  # Result of the last extraction: list(table, summary, n_failed, source)
  extraction = shiny::reactiveVal(NULL)

  # Field definitions. field_state holds the definitions the inputs were built
  # from; field_gen is the generation number used in the input ids. The inputs
  # are rebuilt only by set_fields(), never by a tab change.
  field_state = shiny::reactiveVal(demo_fields())
  field_gen = shiny::reactiveVal(0)

  # Creates the chat for each item. Tests replace it through the option.
  chat_factory = getOption("ste.chat_factory", default = make_chat_factory(EXTRACTION_MODEL))

  # ---- Helpers -------------------------------------------------------------

  notify = function(message, type = "message", duration = 3) {
    icon_name = switch(
      type,
      error = "exclamation-triangle",
      warning = "exclamation-circle",
      "check-circle"
    )
    shiny::showNotification(
      shiny::tags$span(shiny::icon(icon_name), " ", message),
      type = type,
      duration = duration
    )
  }

  # Active input tab (the client reports it through input$input_method)
  active_method = shiny::reactive({
    method = input$input_method
    if (is.null(method) || !(method %in% INPUT_METHODS)) "demo" else method
  })

  # Field definitions as currently shown, including what the user has typed
  current_fields = shiny::reactive({
    merge_field_inputs(field_state(), function(id) input[[id]], field_gen())
  })

  # Replace the field definitions and rebuild their inputs
  set_fields = function(fields) {
    field_state(fields)
    field_gen(shiny::isolate(field_gen()) + 1)
    shiny::updateNumericInput(session, "num_fields", value = length(fields))
  }

  # ---- Input sources -------------------------------------------------------

  # Load example data
  shiny::observeEvent(input$load_example, {
    demo_text(DEMO_TEXT)
    set_fields(demo_fields())
    notify("Demo data loaded. Click 'Extract Data' to see results.")
  })

  # Render demo preview
  output$demo_preview_ui = shiny::renderUI({
    text = demo_text()
    if (is.null(text) || nchar(text) == 0) return(NULL)

    shiny::div(
      class = "pdf-preview",
      style = "margin-top: 15px;",
      shiny::tags$strong("Loaded Demo Text:"),
      shiny::tags$pre(preview_text(text))
    )
  })

  # Process uploaded PDF
  shiny::observeEvent(input$pdf_file, {
    shiny::req(input$pdf_file)

    tryCatch({
      extracted = read_pdf_text(input$pdf_file$datapath, MAX_PAGES_OR_IMAGES)

      if (extracted$total_pages > extracted$used_pages) {
        notify(
          sprintf(
            "The PDF has %d pages. Only the first %d will be processed (free app limit).",
            extracted$total_pages, extracted$used_pages
          ),
          type = "warning",
          duration = 5
        )
      }

      pdf_text(extracted$text)

      if (nchar(extracted$text) == 0) {
        notify(
          "No text was found in this PDF. If it is a scan, use the Scanned PDF tab.",
          type = "warning",
          duration = 8
        )
      } else {
        notify(sprintf("PDF processed. Text of %d page(s) read.", extracted$used_pages))
      }
    }, error = function(e) {
      pdf_text("")
      notify(paste("Error reading PDF:", conditionMessage(e)), type = "error", duration = NULL)
    })
  })

  # Render PDF text preview
  output$pdf_preview_ui = shiny::renderUI({
    text = pdf_text()
    if (is.null(text) || nchar(text) == 0) return(NULL)

    shiny::div(
      class = "pdf-preview",
      shiny::tags$strong("Extracted Text Preview:"),
      shiny::tags$pre(preview_text(text))
    )
  })

  # Process uploaded text file
  shiny::observeEvent(input$text_file, {
    shiny::req(input$text_file)

    tryCatch({
      items = read_text_file_items(input$text_file$datapath, input$text_file$name)
      textfile_items(items)

      if (length(items) == 0) {
        notify("No text was found in this file.", type = "warning", duration = 8)
      } else {
        notify(sprintf("File processed. %d item(s) found.", length(items)))
      }
    }, error = function(e) {
      textfile_items(character(0))
      notify(paste("Error reading file:", conditionMessage(e)), type = "error", duration = NULL)
    })
  })

  # Render text file preview
  output$textfile_preview_ui = shiny::renderUI({
    items = textfile_items()
    if (length(items) == 0) return(NULL)

    shiny::div(
      class = "pdf-preview",
      shiny::tags$strong(sprintf("Extracted Text Preview (%d item(s)):", length(items))),
      shiny::tags$pre(preview_text(items))
    )
  })

  # Process uploaded images
  shiny::observeEvent(input$image_file, {
    shiny::req(input$image_file)

    tryCatch({
      files = input$image_file
      files$mime = vapply(
        seq_len(nrow(files)),
        function(i) detect_mime(files$datapath[i], files$name[i]),
        character(1)
      )

      not_images = files$name[is.na(files$mime)]
      files = files[!is.na(files$mime), , drop = FALSE]
      if (length(not_images) > 0) {
        notify(
          paste("Not a supported image, skipped:", paste(not_images, collapse = ", ")),
          type = "warning",
          duration = 8
        )
      }

      if (nrow(files) > MAX_PAGES_OR_IMAGES) {
        notify(
          sprintf(
            "Only the first %d images will be processed (free app limit).",
            MAX_PAGES_OR_IMAGES
          ),
          type = "warning",
          duration = 5
        )
        files = files[seq_len(MAX_PAGES_OR_IMAGES), , drop = FALSE]
      }

      image_files(if (nrow(files) > 0) files else NULL)
      if (nrow(files) > 0) notify(sprintf("%d image(s) uploaded.", nrow(files)))
    }, error = function(e) {
      image_files(NULL)
      notify(paste("Error uploading images:", conditionMessage(e)), type = "error", duration = NULL)
    })
  })

  # Render image preview
  output$image_preview_ui = shiny::renderUI({
    files = image_files()
    if (is.null(files) || nrow(files) == 0) return(NULL)

    shiny::tagList(
      shiny::div(
        class = "image-preview-grid",
        lapply(seq_len(nrow(files)), function(i) {
          shiny::div(
            class = "image-preview-item",
            shiny::tags$img(
              src = base64enc::dataURI(file = files$datapath[i], mime = files$mime[i]),
              alt = paste0("Image ", i, ": ", files$name[i])
            )
          )
        })
      ),
      shiny::div(
        class = "image-preview-count",
        paste(nrow(files), "image(s) ready for extraction")
      )
    )
  })

  # Process scanned PDF (convert pages to images)
  shiny::observeEvent(input$scanned_pdf_file, {
    shiny::req(input$scanned_pdf_file)

    tryCatch({
      shiny::showNotification(
        "Converting PDF pages to images...",
        type = "message",
        duration = NULL,
        id = "convert_notif"
      )

      rendered = render_pdf_pages(input$scanned_pdf_file$datapath, MAX_PAGES_OR_IMAGES)
      scanned_pdf_paths(rendered$paths)

      shiny::removeNotification(id = "convert_notif")
      if (rendered$total_pages > length(rendered$paths)) {
        notify(
          sprintf(
            "The PDF has %d pages. Only the first %d will be processed (free app limit).",
            rendered$total_pages, length(rendered$paths)
          ),
          type = "warning",
          duration = 5
        )
      }
      notify(sprintf("%d page(s) converted.", length(rendered$paths)))
    }, error = function(e) {
      scanned_pdf_paths(NULL)
      shiny::removeNotification(id = "convert_notif")
      notify(paste("Error processing scanned PDF:", conditionMessage(e)), type = "error", duration = NULL)
    })
  })

  # Render scanned PDF preview
  output$scanned_pdf_preview_ui = shiny::renderUI({
    paths = scanned_pdf_paths()
    if (is.null(paths) || length(paths) == 0) return(NULL)

    shiny::tagList(
      shiny::div(
        class = "image-preview-grid",
        lapply(seq_along(paths), function(i) {
          shiny::div(
            class = "image-preview-item",
            shiny::tags$img(
              src = base64enc::dataURI(file = paths[i], mime = detect_mime(paths[i])),
              alt = paste("Page", i)
            )
          )
        })
      ),
      shiny::div(
        class = "image-preview-count",
        paste(length(paths), "page(s) ready for extraction")
      )
    )
  })

  # The previews sit in panels that www/app.js shows and hides
  for (preview_id in c("demo_preview_ui", "pdf_preview_ui", "textfile_preview_ui",
                       "image_preview_ui", "scanned_pdf_preview_ui")) {
    shiny::outputOptions(output, preview_id, suspendWhenHidden = FALSE)
  }

  # What each tab holds right now, in the form resolve_source_items() expects
  current_sources = function() {
    files = image_files()
    list(
      demo = demo_text(),
      paste = input$input_text,
      textfile = textfile_items(),
      pdf = pdf_text(),
      scanned = scanned_pdf_paths(),
      image = if (is.null(files)) NULL else files$datapath
    )
  }

  # ---- Field definitions ---------------------------------------------------

  # Change the number of fields, keeping everything the user has entered
  shiny::observeEvent(input$num_fields, {
    n = clamp_num_fields(input$num_fields, MAX_FIELDS)
    if (is.na(n)) return()

    if (!isTRUE(n == input$num_fields)) {
      shiny::updateNumericInput(session, "num_fields", value = n)
      notify(
        sprintf("The number of fields must be between 1 and %d.", MAX_FIELDS),
        type = "warning",
        duration = 5
      )
    }

    fields = current_fields()
    if (n == length(fields)) return()
    field_state(resize_fields(fields, n))
    field_gen(field_gen() + 1)
  }, ignoreInit = TRUE)

  # Fill the fields from a template
  shiny::observeEvent(input$apply_template, {
    template = FIELD_TEMPLATES[[input$template_id]]
    if (is.null(template)) return()
    set_fields(template$fields)
    notify(sprintf("Template '%s' loaded. You can edit the fields.", template$name))
  })

  output$template_description_ui = shiny::renderUI({
    template = FIELD_TEMPLATES[[input$template_id]]
    if (is.null(template)) return(NULL)
    shiny::helpText(sprintf("%s %d fields.", template$description, length(template$fields)))
  })

  # Restore the fields from an uploaded JSON file
  shiny::observeEvent(input$fields_upload, {
    shiny::req(input$fields_upload)
    loaded = read_fields_json(input$fields_upload$datapath, MAX_FIELDS)
    if (!loaded$ok) {
      notify(
        paste("The field definitions were not loaded.", loaded$error),
        type = "error",
        duration = NULL
      )
      return()
    }
    set_fields(loaded$fields)
    notify(sprintf("%d field(s) loaded from the file.", length(loaded$fields)))
  })

  # Download the current field definitions as JSON
  output$download_fields = shiny::downloadHandler(
    filename = function() {
      paste0("field_definitions_", format(Sys.time(), "%Y%m%d_%H%M%S"), ".json")
    },
    content = function(file) {
      writeLines(fields_to_json(current_fields()), file, useBytes = TRUE)
    }
  )

  # Generate the inputs for each field's label, description and type.
  # Depends only on field_gen, so typing and tab changes never rebuild them.
  output$fields_ui = shiny::renderUI({
    gen = field_gen()
    fields = shiny::isolate(field_state())

    do.call(shiny::tagList, lapply(seq_along(fields), function(i) {
      ids = field_input_ids(gen, i)
      field = fields[[i]]

      shiny::div(
        class = "field-card",
        shiny::tags$span(
          style = "color: #C41230; font-weight: 600; font-size: 0.9em;",
          paste("Field", i)
        ),
        shiny::textInput(ids$label, "Label:", value = field$label),
        shiny::textAreaInput(
          ids$description,
          "Description:",
          value = field$description,
          rows = 5,
          placeholder = "Describe what to extract for this field, and what to return when it is absent"
        ),
        shiny::selectInput(
          ids$type,
          "Type:",
          choices = stats::setNames(names(FIELD_TYPES), FIELD_TYPES),
          selected = if (field$type %in% names(FIELD_TYPES)) field$type else "text",
          selectize = FALSE
        )
      )
    }))
  })

  # ---- Help ----------------------------------------------------------------

  lapply(HELP_TOPICS, function(topic) {
    shiny::observeEvent(input[[paste0("help_", topic)]], {
      shiny::showModal(help_modal(topic))
    })
  })

  # ---- Extraction ----------------------------------------------------------

  # When the extract button is clicked, perform extraction
  shiny::observeEvent(input$extract_btn, {
    # Items come from the active tab only
    resolved = resolve_source_items(active_method(), current_sources(), MAX_PAGES_OR_IMAGES)
    if (!resolved$ok) {
      notify(resolved$error, type = "error", duration = 5)
      return()
    }

    checked = validate_fields(current_fields(), MAX_FIELDS)
    if (!checked$ok) {
      notify(
        shiny::tagList(
          "Please correct the field definitions:",
          shiny::tags$ul(lapply(checked$errors, shiny::tags$li))
        ),
        type = "error",
        duration = NULL
      )
      return()
    }
    if (length(checked$notes) > 0) {
      notify(
        shiny::tags$ul(lapply(checked$notes, shiny::tags$li)),
        type = "warning",
        duration = 10
      )
    }

    if (is.null(getOption("ste.chat_factory")) && !api_key_available()) {
      notify(
        "The extraction service is not available right now. Please try again later, or contact the authors if the problem continues.",
        type = "error",
        duration = NULL
      )
      return()
    }

    items = resolved$items
    source_label = INPUT_METHOD_LABELS[[active_method()]]
    custom_type_object = build_type_object(checked$fields)

    results = shiny::withProgress(
      message = "Extracting",
      value = 0,
      {
        extracted = extract_items(
          items,
          custom_type_object,
          chat_factory,
          on_progress = function(i, n, item) {
            shiny::setProgress(
              value = (i - 1) / n,
              detail = sprintf("item %d of %d (source_id: %s)", i, n, item$source_id)
            )
          }
        )
        shiny::setProgress(value = 1, detail = "done")
        extracted
      }
    )

    results_table = assemble_results(results, checked$fields)
    n_failed = sum(results_table$status != "ok")
    extraction(list(
      table = results_table,
      summary = summarize_results(results_table),
      n_failed = n_failed,
      source = source_label
    ))

    if (n_failed == 0) {
      notify("Extraction complete.")
    } else {
      notify(summarize_results(results_table), type = "warning", duration = 8)
    }
  })

  # Summary line above the table, or the placeholder before the first run
  output$results_status_ui = shiny::renderUI({
    result = extraction()
    if (is.null(result)) {
      return(shiny::div(
        class = "results-placeholder",
        shiny::icon("arrow-left"),
        " Configure your extraction fields and click 'Extract Data' to see results here"
      ))
    }

    shiny::div(
      class = if (result$n_failed > 0) "results-summary has-failures" else "results-summary",
      role = "status",
      shiny::tags$strong(result$summary),
      sprintf(" Source: %s.", result$source),
      if (result$n_failed > 0) {
        " The status_message column gives the reason for each failure."
      },
      " Check the values against the source documents before you use them."
    )
  })

  output$extracted_table = shiny::renderTable(
    {
      result = extraction()
      if (is.null(result)) return(NULL)
      display_table(result$table)
    },
    striped = TRUE,
    hover = TRUE,
    bordered = TRUE
  )

  # Render download button only when results are available
  output$download_btn_ui = shiny::renderUI({
    result = extraction()
    if (is.null(result) || nrow(result$table) == 0) return(NULL)

    shiny::div(
      style = "margin-top: 15px;",
      shiny::downloadButton(
        "download_csv",
        "Download as CSV",
        class = "btn-info"
      )
    )
  })

  # Download handler for CSV export
  output$download_csv = shiny::downloadHandler(
    filename = function() {
      paste0("extracted_data_", format(Sys.time(), "%Y%m%d_%H%M%S"), ".csv")
    },
    content = function(file) {
      result = extraction()
      if (!is.null(result)) {
        utils::write.csv(result$table, file, row.names = FALSE, na = "")
      }
    }
  )

  # ---- Reproduce in code ---------------------------------------------------

  # Fields used in the snippets (sanitized, as they would be sent)
  snippet_fields = shiny::reactive({
    validate_fields(current_fields(), MAX_FIELDS)
  })

  output$snippet_status_ui = shiny::renderUI({
    checked = snippet_fields()
    if (checked$ok) return(NULL)
    shiny::div(
      class = "limit-note",
      "The code is shown once the field definitions are valid: ",
      paste(checked$errors, collapse = " ")
    )
  })

  output$snippet_r = shiny::renderText({
    checked = snippet_fields()
    if (!checked$ok) return("")
    r_snippet(checked$fields, EXTRACTION_MODEL)
  })

  output$snippet_python = shiny::renderText({
    checked = snippet_fields()
    if (!checked$ok) return("")
    python_snippet(checked$fields, EXTRACTION_MODEL)
  })

  # The code blocks sit inside collapsed <details> elements
  shiny::outputOptions(output, "snippet_r", suspendWhenHidden = FALSE)
  shiny::outputOptions(output, "snippet_python", suspendWhenHidden = FALSE)
}

# Run the Shiny app
shiny::shinyApp(ui = ui, server = server)