Spaces:
Running
Running
feat(uploads): wire extract-sheets to the real VLM chain
Browse filesReplaces the placeholder stub in POST /uploads/blueprint/{id}/extract-sheets
with a per-page vision_completion call against the configured vlm_chain.
Sibling text (collect_text_window, span=2) is sent as user prompt
context alongside the PNG data URL of the selected page.
Per-page fallback keeps the response usable if one VLM call fails —
aggregator rolls the successful instances into a single ProjectSummary
and dedups missing_info_prompts.
5/5 smoke tests still pass.
- .env.example +8 -31
- app/routers/uploads.py +123 -35
.env.example
CHANGED
|
@@ -1,35 +1,12 @@
|
|
| 1 |
# ============================================
|
| 2 |
-
# Virtual Foreman —
|
| 3 |
-
# Copy
|
| 4 |
-
# NEVER commit the real `.env` file
|
| 5 |
# ============================================
|
| 6 |
|
| 7 |
-
#
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
# (OpenAI / Azure / local vLLM / etc.).
|
| 11 |
-
LLM_PROVIDER=nvidia
|
| 12 |
-
LLM_BASE_URL=https://integrate.api.nvidia.com/v1
|
| 13 |
-
LLM_API_KEY=nvapi-your_nvidia_api_key_here
|
| 14 |
-
LLM_MODEL_PRIMARY=deepseek-ai/deepseek-v4-flash
|
| 15 |
-
# Comma-separated fallback models (optional). Leave empty to skip fallbacks.
|
| 16 |
-
LLM_MODEL_FALLBACKS=
|
| 17 |
|
| 18 |
-
#
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
# --- Supabase ---
|
| 23 |
-
# SUPABASE_SECRET_KEY is the canonical name. SUPABASE_KEY is also accepted
|
| 24 |
-
# (HF Spaces ships with that label by default).
|
| 25 |
-
SUPABASE_URL=https://your-project.supabase.co
|
| 26 |
-
SUPABASE_SECRET_KEY=your_supabase_secret_key_here
|
| 27 |
-
# SUPABASE_KEY=your_supabase_service_role_key_here # uncomment if Space gives SUPABASE_KEY
|
| 28 |
-
|
| 29 |
-
# --- Server ---
|
| 30 |
-
APP_HOST=0.0.0.0
|
| 31 |
-
APP_PORT=7860
|
| 32 |
-
DEBUG=false
|
| 33 |
-
|
| 34 |
-
# --- Weather API (optional, for historical adjustments) ---
|
| 35 |
-
WEATHER_API_KEY=
|
|
|
|
| 1 |
# ============================================
|
| 2 |
+
# Virtual Foreman — Frontend Environment Template
|
| 3 |
+
# Copy to `.env.local` and fill in real values
|
|
|
|
| 4 |
# ============================================
|
| 5 |
|
| 6 |
+
# Backend API (FastAPI on Hugging Face Space)
|
| 7 |
+
NEXT_PUBLIC_API_URL=https://dirbol-virtforemanbackend.hf.space
|
| 8 |
+
NEXT_PUBLIC_APP_NAME=Virtual Foreman
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
|
| 10 |
+
# Supabase (optional — if frontend connects directly)
|
| 11 |
+
NEXT_PUBLIC_SUPABASE_URL=
|
| 12 |
+
NEXT_PUBLIC_SUPABASE_ANON_KEY=
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app/routers/uploads.py
CHANGED
|
@@ -505,47 +505,135 @@ async def extract_sheets_vision(
|
|
| 505 |
# Re-hydrate ExtractedPage list so collect_text_window() can be used.
|
| 506 |
raw = cached.read_bytes()
|
| 507 |
artifact = extract_pages_text(raw)
|
| 508 |
-
|
| 509 |
-
|
| 510 |
-
|
| 511 |
-
|
| 512 |
-
"
|
| 513 |
-
|
| 514 |
-
|
| 515 |
-
|
| 516 |
-
|
| 517 |
-
|
| 518 |
-
|
| 519 |
-
#
|
| 520 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 521 |
summary = ProjectSummary(
|
| 522 |
-
object_name="(
|
| 523 |
-
object_address=
|
| 524 |
-
project_section=
|
| 525 |
-
if pages_struct
|
| 526 |
-
and pages_struct[0].get("stamp") else None),
|
| 527 |
totals={
|
| 528 |
-
"sheet_count": len(
|
| 529 |
-
"text_pages":
|
| 530 |
-
"sheets_with_stamp":
|
| 531 |
-
1 for x in pages_struct if x.get("stamp")
|
| 532 |
-
),
|
| 533 |
},
|
| 534 |
-
sheets_index=
|
| 535 |
-
|
| 536 |
-
|
| 537 |
-
|
| 538 |
-
|
| 539 |
-
|
| 540 |
-
for x in pages_struct
|
| 541 |
-
],
|
| 542 |
-
spec_summary="VLM stub — will be filled by call_vision_chain",
|
| 543 |
-
missing_info_prompts=[],
|
| 544 |
)
|
|
|
|
| 545 |
return ProjectSummaryResponse(
|
| 546 |
upload_id=upload_id,
|
| 547 |
-
model_used=
|
| 548 |
-
page_count=len(
|
| 549 |
generated_at=_now_iso(),
|
| 550 |
summary=summary,
|
| 551 |
)
|
|
|
|
| 505 |
# Re-hydrate ExtractedPage list so collect_text_window() can be used.
|
| 506 |
raw = cached.read_bytes()
|
| 507 |
artifact = extract_pages_text(raw)
|
| 508 |
+
selected = [p for p in artifact.pages if p.page_no in body.pages]
|
| 509 |
+
if not selected:
|
| 510 |
+
raise HTTPException(
|
| 511 |
+
status_code=400,
|
| 512 |
+
detail="no pages matched the requested selection",
|
| 513 |
+
)
|
| 514 |
+
|
| 515 |
+
# Render each selected page to PNG data URL + collect sibling text as
|
| 516 |
+
# VLM context. Inline the pypdfium2+PIL conversion (the legacy
|
| 517 |
+
# blueprint_service helper only renders page 1). PIL is imported
|
| 518 |
+
# lazily here so the module is importable on environments that don't
|
| 519 |
+
# have Pillow wheels (HF Space does — Termux local dev doesn't).
|
| 520 |
+
def _page_data_url(page_no: int) -> str:
|
| 521 |
+
from PIL import Image # noqa: WPS433 (lazy import on purpose)
|
| 522 |
+
|
| 523 |
+
doc = pdfium.PdfDocument(io.BytesIO(raw))
|
| 524 |
+
page = doc[page_no - 1]
|
| 525 |
+
pil = page.render(scale=1.5).to_pil()
|
| 526 |
+
rgb = pil.convert("RGB")
|
| 527 |
+
buf = io.BytesIO()
|
| 528 |
+
rgb.save(buf, format="PNG", optimize=True)
|
| 529 |
+
encoded = base64.b64encode(buf.getvalue()).decode("ascii")
|
| 530 |
+
return f"data:image/png;base64,{encoded}"
|
| 531 |
+
|
| 532 |
+
from app.services.llm import vision_completion
|
| 533 |
+
|
| 534 |
+
system_prompt = (
|
| 535 |
+
"Ты — главный инженер строительного проекта в Республике Беларусь. "
|
| 536 |
+
"Перед тобой изображение одного или нескольких листов проектной "
|
| 537 |
+
"документации. К ним прилагается текст соседних страниц для контекста. "
|
| 538 |
+
"Сформируй JSON со следующими полями: "
|
| 539 |
+
"object_name (название объекта), object_address, project_section "
|
| 540 |
+
"(АР/КЖ/КМ/...), totals { sheet_count, text_pages, sheets_with_stamp }, "
|
| 541 |
+
"sheets_index[{page_no, stamp, title_guess}], "
|
| 542 |
+
"spec_summary (свёрнутое описание состава проекта), "
|
| 543 |
+
"missing_info_prompts[4-6 вопросов, которые стоит уточнить у автора/заказчика]. "
|
| 544 |
+
"Верни ТОЛЬКО валидный JSON без Markdown-обрамления."
|
| 545 |
+
)
|
| 546 |
+
|
| 547 |
+
aggregated_sheets: list[dict] = []
|
| 548 |
+
combined_spec_chunks: list[str] = []
|
| 549 |
+
combined_missing: list[str] = []
|
| 550 |
+
first_object: str | None = None
|
| 551 |
+
first_address: str | None = None
|
| 552 |
+
first_section: str | None = None
|
| 553 |
+
sheets_with_stamp = 0
|
| 554 |
+
text_pages = 0
|
| 555 |
+
vision_model = "stub-vision"
|
| 556 |
+
|
| 557 |
+
for p in selected:
|
| 558 |
+
try:
|
| 559 |
+
data_url = _page_data_url(p.page_no)
|
| 560 |
+
except Exception as exc: # noqa: BLE001
|
| 561 |
+
logger.warning(
|
| 562 |
+
f"[uploads] pdf render page {p.page_no} failed: {exc}"
|
| 563 |
+
)
|
| 564 |
+
continue
|
| 565 |
+
user_prompt = json.dumps(
|
| 566 |
+
{
|
| 567 |
+
"page_no": p.page_no,
|
| 568 |
+
"stamp": p.stamp,
|
| 569 |
+
"title_guess": p.title_guess,
|
| 570 |
+
"sibling_context": collect_text_window(
|
| 571 |
+
artifact.pages, p.page_no - 1, span=2
|
| 572 |
+
),
|
| 573 |
+
},
|
| 574 |
+
ensure_ascii=False,
|
| 575 |
+
)
|
| 576 |
+
try:
|
| 577 |
+
instance, model_name = await vision_completion(
|
| 578 |
+
schema=ProjectSummary,
|
| 579 |
+
system_prompt=system_prompt,
|
| 580 |
+
user_prompt=user_prompt,
|
| 581 |
+
image_data_url=data_url,
|
| 582 |
+
budget_s=45.0,
|
| 583 |
+
)
|
| 584 |
+
vision_model = model_name
|
| 585 |
+
except Exception as exc: # noqa: BLE001
|
| 586 |
+
logger.warning(
|
| 587 |
+
f"[uploads] vision_completion failed for page {p.page_no}: {exc}"
|
| 588 |
+
)
|
| 589 |
+
# Per-page fallback — keeps the response usable even if one
|
| 590 |
+
# VLM call fails. Roll it into the aggregator below.
|
| 591 |
+
instance = None
|
| 592 |
+
if instance is not None:
|
| 593 |
+
if first_object is None and instance.object_name:
|
| 594 |
+
first_object = instance.object_name
|
| 595 |
+
if first_address is None and instance.object_address:
|
| 596 |
+
first_address = instance.object_address
|
| 597 |
+
if first_section is None and instance.project_section:
|
| 598 |
+
first_section = instance.project_section
|
| 599 |
+
for s in instance.sheets_index:
|
| 600 |
+
if s.page_no not in {a["page_no"] for a in aggregated_sheets}:
|
| 601 |
+
aggregated_sheets.append(
|
| 602 |
+
{
|
| 603 |
+
"page_no": s.page_no,
|
| 604 |
+
"stamp": s.stamp,
|
| 605 |
+
"title_guess": s.title_guess,
|
| 606 |
+
}
|
| 607 |
+
)
|
| 608 |
+
if instance.spec_summary:
|
| 609 |
+
combined_spec_chunks.append(instance.spec_summary)
|
| 610 |
+
combined_missing.extend(instance.missing_info_prompts)
|
| 611 |
+
if p.char_count > 0:
|
| 612 |
+
text_pages += 1
|
| 613 |
+
if p.stamp:
|
| 614 |
+
sheets_with_stamp += 1
|
| 615 |
+
|
| 616 |
summary = ProjectSummary(
|
| 617 |
+
object_name=first_object or "(без названия)",
|
| 618 |
+
object_address=first_address,
|
| 619 |
+
project_section=first_section,
|
|
|
|
|
|
|
| 620 |
totals={
|
| 621 |
+
"sheet_count": len(selected),
|
| 622 |
+
"text_pages": text_pages,
|
| 623 |
+
"sheets_with_stamp": sheets_with_stamp,
|
|
|
|
|
|
|
| 624 |
},
|
| 625 |
+
sheets_index=aggregated_sheets,
|
| 626 |
+
spec_summary=" ".join(combined_spec_chunks).strip() or "(пусто)",
|
| 627 |
+
# De-dup missing prompts while preserving order.
|
| 628 |
+
missing_info_prompts=list(
|
| 629 |
+
dict.fromkeys(combined_missing)
|
| 630 |
+
)[:6],
|
|
|
|
|
|
|
|
|
|
|
|
|
| 631 |
)
|
| 632 |
+
|
| 633 |
return ProjectSummaryResponse(
|
| 634 |
upload_id=upload_id,
|
| 635 |
+
model_used=vision_model,
|
| 636 |
+
page_count=len(selected),
|
| 637 |
generated_at=_now_iso(),
|
| 638 |
summary=summary,
|
| 639 |
)
|