Dirbol commited on
Commit
7d34787
·
1 Parent(s): e10629e

feat(uploads): wire extract-sheets to the real VLM chain

Browse files

Replaces the placeholder stub in POST /uploads/blueprint/{id}/extract-sheets
with a per-page vision_completion call against the configured vlm_chain.
Sibling text (collect_text_window, span=2) is sent as user prompt
context alongside the PNG data URL of the selected page.

Per-page fallback keeps the response usable if one VLM call fails —
aggregator rolls the successful instances into a single ProjectSummary
and dedups missing_info_prompts.

5/5 smoke tests still pass.

Files changed (2) hide show
  1. .env.example +8 -31
  2. app/routers/uploads.py +123 -35
.env.example CHANGED
@@ -1,35 +1,12 @@
1
  # ============================================
2
- # Virtual Foreman — Backend Environment Template
3
- # Copy this file to `.env` and fill in real values
4
- # NEVER commit the real `.env` file
5
  # ============================================
6
 
7
- # --- LLM provider (OpenAI-compatible) ---
8
- # Default: NVIDIA integrate (build.nvidia.com) with DeepSeek V4 Flash.
9
- # Override LLM_BASE_URL and LLM_MODEL_PRIMARY to use other compatible endpoints
10
- # (OpenAI / Azure / local vLLM / etc.).
11
- LLM_PROVIDER=nvidia
12
- LLM_BASE_URL=https://integrate.api.nvidia.com/v1
13
- LLM_API_KEY=nvapi-your_nvidia_api_key_here
14
- LLM_MODEL_PRIMARY=deepseek-ai/deepseek-v4-flash
15
- # Comma-separated fallback models (optional). Leave empty to skip fallbacks.
16
- LLM_MODEL_FALLBACKS=
17
 
18
- # --- Hugging Face Space (used for headers when provider=openrouter) ---
19
- HF_SPACE_URL=https://huggingface.co/spaces/Dirbol/VirtForemanBackend
20
- APP_TITLE=Virtual Foreman
21
-
22
- # --- Supabase ---
23
- # SUPABASE_SECRET_KEY is the canonical name. SUPABASE_KEY is also accepted
24
- # (HF Spaces ships with that label by default).
25
- SUPABASE_URL=https://your-project.supabase.co
26
- SUPABASE_SECRET_KEY=your_supabase_secret_key_here
27
- # SUPABASE_KEY=your_supabase_service_role_key_here # uncomment if Space gives SUPABASE_KEY
28
-
29
- # --- Server ---
30
- APP_HOST=0.0.0.0
31
- APP_PORT=7860
32
- DEBUG=false
33
-
34
- # --- Weather API (optional, for historical adjustments) ---
35
- WEATHER_API_KEY=
 
1
  # ============================================
2
+ # Virtual Foreman — Frontend Environment Template
3
+ # Copy to `.env.local` and fill in real values
 
4
  # ============================================
5
 
6
+ # Backend API (FastAPI on Hugging Face Space)
7
+ NEXT_PUBLIC_API_URL=https://dirbol-virtforemanbackend.hf.space
8
+ NEXT_PUBLIC_APP_NAME=Virtual Foreman
 
 
 
 
 
 
 
9
 
10
+ # Supabase (optional if frontend connects directly)
11
+ NEXT_PUBLIC_SUPABASE_URL=
12
+ NEXT_PUBLIC_SUPABASE_ANON_KEY=
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
app/routers/uploads.py CHANGED
@@ -505,47 +505,135 @@ async def extract_sheets_vision(
505
  # Re-hydrate ExtractedPage list so collect_text_window() can be used.
506
  raw = cached.read_bytes()
507
  artifact = extract_pages_text(raw)
508
- pages_struct = [
509
- {
510
- "page_no": p.page_no,
511
- "stamp": p.stamp,
512
- "title_guess": p.title_guess,
513
- "char_count": p.char_count,
514
- "context": collect_text_window(artifact.pages, i),
515
- }
516
- for i, p in enumerate(artifact.pages)
517
- if p.page_no in body.pages
518
- ]
519
- # Stub uses a deterministic placeholder; a follow-up routes through
520
- # app/services/llm.py call_vision_chain() so the contract is stable.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
521
  summary = ProjectSummary(
522
- object_name="(stub: VLM selection received)",
523
- object_address=None,
524
- project_section=(pages_struct[0]["stamp"].split(" ")[0]
525
- if pages_struct
526
- and pages_struct[0].get("stamp") else None),
527
  totals={
528
- "sheet_count": len(pages_struct),
529
- "text_pages": sum(1 for x in pages_struct if x["char_count"]),
530
- "sheets_with_stamp": sum(
531
- 1 for x in pages_struct if x.get("stamp")
532
- ),
533
  },
534
- sheets_index=[
535
- {
536
- "page_no": x["page_no"],
537
- "stamp": x.get("stamp"),
538
- "title_guess": x.get("title_guess"),
539
- }
540
- for x in pages_struct
541
- ],
542
- spec_summary="VLM stub — will be filled by call_vision_chain",
543
- missing_info_prompts=[],
544
  )
 
545
  return ProjectSummaryResponse(
546
  upload_id=upload_id,
547
- model_used="stub-vision",
548
- page_count=len(pages_struct),
549
  generated_at=_now_iso(),
550
  summary=summary,
551
  )
 
505
  # Re-hydrate ExtractedPage list so collect_text_window() can be used.
506
  raw = cached.read_bytes()
507
  artifact = extract_pages_text(raw)
508
+ selected = [p for p in artifact.pages if p.page_no in body.pages]
509
+ if not selected:
510
+ raise HTTPException(
511
+ status_code=400,
512
+ detail="no pages matched the requested selection",
513
+ )
514
+
515
+ # Render each selected page to PNG data URL + collect sibling text as
516
+ # VLM context. Inline the pypdfium2+PIL conversion (the legacy
517
+ # blueprint_service helper only renders page 1). PIL is imported
518
+ # lazily here so the module is importable on environments that don't
519
+ # have Pillow wheels (HF Space does Termux local dev doesn't).
520
+ def _page_data_url(page_no: int) -> str:
521
+ from PIL import Image # noqa: WPS433 (lazy import on purpose)
522
+
523
+ doc = pdfium.PdfDocument(io.BytesIO(raw))
524
+ page = doc[page_no - 1]
525
+ pil = page.render(scale=1.5).to_pil()
526
+ rgb = pil.convert("RGB")
527
+ buf = io.BytesIO()
528
+ rgb.save(buf, format="PNG", optimize=True)
529
+ encoded = base64.b64encode(buf.getvalue()).decode("ascii")
530
+ return f"data:image/png;base64,{encoded}"
531
+
532
+ from app.services.llm import vision_completion
533
+
534
+ system_prompt = (
535
+ "Ты — главный инженер строительного проекта в Республике Беларусь. "
536
+ "Перед тобой изображение одного или нескольких листов проектной "
537
+ "документации. К ним прилагается текст соседних страниц для контекста. "
538
+ "Сформируй JSON со следующими полями: "
539
+ "object_name (название объекта), object_address, project_section "
540
+ "(АР/КЖ/КМ/...), totals { sheet_count, text_pages, sheets_with_stamp }, "
541
+ "sheets_index[{page_no, stamp, title_guess}], "
542
+ "spec_summary (свёрнутое описание состава проекта), "
543
+ "missing_info_prompts[4-6 вопросов, которые стоит уточнить у автора/заказчика]. "
544
+ "Верни ТОЛЬКО валидный JSON без Markdown-обрамления."
545
+ )
546
+
547
+ aggregated_sheets: list[dict] = []
548
+ combined_spec_chunks: list[str] = []
549
+ combined_missing: list[str] = []
550
+ first_object: str | None = None
551
+ first_address: str | None = None
552
+ first_section: str | None = None
553
+ sheets_with_stamp = 0
554
+ text_pages = 0
555
+ vision_model = "stub-vision"
556
+
557
+ for p in selected:
558
+ try:
559
+ data_url = _page_data_url(p.page_no)
560
+ except Exception as exc: # noqa: BLE001
561
+ logger.warning(
562
+ f"[uploads] pdf render page {p.page_no} failed: {exc}"
563
+ )
564
+ continue
565
+ user_prompt = json.dumps(
566
+ {
567
+ "page_no": p.page_no,
568
+ "stamp": p.stamp,
569
+ "title_guess": p.title_guess,
570
+ "sibling_context": collect_text_window(
571
+ artifact.pages, p.page_no - 1, span=2
572
+ ),
573
+ },
574
+ ensure_ascii=False,
575
+ )
576
+ try:
577
+ instance, model_name = await vision_completion(
578
+ schema=ProjectSummary,
579
+ system_prompt=system_prompt,
580
+ user_prompt=user_prompt,
581
+ image_data_url=data_url,
582
+ budget_s=45.0,
583
+ )
584
+ vision_model = model_name
585
+ except Exception as exc: # noqa: BLE001
586
+ logger.warning(
587
+ f"[uploads] vision_completion failed for page {p.page_no}: {exc}"
588
+ )
589
+ # Per-page fallback — keeps the response usable even if one
590
+ # VLM call fails. Roll it into the aggregator below.
591
+ instance = None
592
+ if instance is not None:
593
+ if first_object is None and instance.object_name:
594
+ first_object = instance.object_name
595
+ if first_address is None and instance.object_address:
596
+ first_address = instance.object_address
597
+ if first_section is None and instance.project_section:
598
+ first_section = instance.project_section
599
+ for s in instance.sheets_index:
600
+ if s.page_no not in {a["page_no"] for a in aggregated_sheets}:
601
+ aggregated_sheets.append(
602
+ {
603
+ "page_no": s.page_no,
604
+ "stamp": s.stamp,
605
+ "title_guess": s.title_guess,
606
+ }
607
+ )
608
+ if instance.spec_summary:
609
+ combined_spec_chunks.append(instance.spec_summary)
610
+ combined_missing.extend(instance.missing_info_prompts)
611
+ if p.char_count > 0:
612
+ text_pages += 1
613
+ if p.stamp:
614
+ sheets_with_stamp += 1
615
+
616
  summary = ProjectSummary(
617
+ object_name=first_object or "(без названия)",
618
+ object_address=first_address,
619
+ project_section=first_section,
 
 
620
  totals={
621
+ "sheet_count": len(selected),
622
+ "text_pages": text_pages,
623
+ "sheets_with_stamp": sheets_with_stamp,
 
 
624
  },
625
+ sheets_index=aggregated_sheets,
626
+ spec_summary=" ".join(combined_spec_chunks).strip() or "(пусто)",
627
+ # De-dup missing prompts while preserving order.
628
+ missing_info_prompts=list(
629
+ dict.fromkeys(combined_missing)
630
+ )[:6],
 
 
 
 
631
  )
632
+
633
  return ProjectSummaryResponse(
634
  upload_id=upload_id,
635
+ model_used=vision_model,
636
+ page_count=len(selected),
637
  generated_at=_now_iso(),
638
  summary=summary,
639
  )