KGQA-Navigation-Challenge / graph-worker.js
muttsurini's picture
Adding MetaQA + cache support
8b39f58 verified
Raw History Blame Contribute Delete
9.27 kB
/* A dedicated browser Web Worker builds a directed per-node adjacency index.
It never reads benchmark answers or reference paths. */
'use strict';
const graphs = new Map();
const choose = (key) => graphs.get(key);
function emit(message) { self.postMessage(message); }
async function loadGraph(key, url, id) {
if (graphs.has(key)) { emit({id, type:'ready', key, edges:graphs.get(key).edges, vertices:graphs.get(key).index.size, cached:true}); return; }
const response = await fetch(url);
if (!response.ok) throw Error(`Graph download HTTP ${response.status}`);
const expected = Number(response.headers.get('content-length') || 0);
const index = new Map();
let bytes = 0, edges = 0, remaining = '';
function consume(line) {
if (!line) return;
const fields = line.replace(/\r$/, '').split('\t');
if (fields.length !== 3) return;
const [h, r, t] = fields;
if (!h || !r || !t) return;
let rels = index.get(h);
if (!rels) { rels = new Map(); index.set(h, rels); }
let neighbors = rels.get(r);
if (!neighbors) { neighbors = []; rels.set(r, neighbors); }
neighbors.push(t);
edges++;
}
if (response.body && response.body.getReader) {
const reader = response.body.getReader();
const decoder = new TextDecoder();
let lastUpdate=0;
while (true) {
const {done,value} = await reader.read();
if (done) break;
bytes += value.byteLength;
remaining += decoder.decode(value,{stream:true});
let end = 0, start = 0;
while ((end = remaining.indexOf('\n', start)) >= 0) {
consume(remaining.slice(start, end)); start = end + 1;
}
remaining = remaining.slice(start);
const now = Date.now();
if (now - lastUpdate > 250) {
emit({id,type:'progress',key,edges,bytes,expected}); lastUpdate=now;
}
}
remaining += decoder.decode();
} else {
const body = await response.text(); bytes = body.length;
for (const line of body.split('\n')) consume(line);
remaining = '';
}
consume(remaining);
if (!edges) throw Error('No triples found. Expected tab-separated head, relation, tail.');
graphs.set(key, {index, edges});
emit({id,type:'ready',key,edges,vertices:index.size,cached:false});
}
// MetaQA is distributed as an encoded ZIP archive rather than a Hugging Face
// Dataset Viewer config. The same archive supplies questions, graph and labels.
// Unlike MQuAKE-ST, MetaQA navigation is undirected: reverse actions are
// represented by underscore-prefixed relation IDs, following MINERVA's format.
let metaqaArchivePromise=null;
let metaqaRowsPromise=null;
let metaqaLabelsPromise=null;
let metaqaNodeNames=new Map();
const METAQA_PREFIX='metaqa_dataset/';
function progress(id,message){emit({id,type:'progress',message});}
async function getMetaqaArchive(url,id){
if(!metaqaArchivePromise){
metaqaArchivePromise=(async()=>{
progress(id,'Downloading the encoded MetaQA archive from GCS…');
if(!self.JSZip)importScripts('./vendor/jszip.min.js?v=4');
const response=await fetch(url);
if(!response.ok){let reason='';try{reason=(await response.text()).slice(0,240);}catch(_){}throw Error('MetaQA archive HTTP '+response.status+(reason?' · '+reason:''));}
const bytes=await response.arrayBuffer();
progress(id,'Extracting the encoded MetaQA archive…');
return await self.JSZip.loadAsync(bytes);
})().catch(err=>{metaqaArchivePromise=null;throw err;});
}
return metaqaArchivePromise;
}
async function archiveText(archive,path){
const entry=archive.file(METAQA_PREFIX+path)||archive.file(path);
if(!entry)throw Error('MetaQA archive is missing '+path);
return await entry.async('string');
}
function csvRecords(raw){
const lines=[];let row=[],field='',quoted=false;
for(let i=0;i<raw.length;i++){
const c=raw[i];
if(c==='"'){
if(quoted&&raw[i+1]==='"'){field+='"';i++;}else quoted=!quoted;
}else if(c===','&&!quoted){row.push(field);field='';}
else if((c==='\n'||c==='\r')&&!quoted){
if(c==='\r'&&raw[i+1]==='\n')i++;
row.push(field);if(row.some(x=>x!==''))lines.push(row);
row=[];field='';
}else field+=c;
}
if(field||row.length){row.push(field);lines.push(row);}
return lines;
}
async function getMetaqaRows(url,id){
if(!metaqaRowsPromise){
metaqaRowsPromise=(async()=>{
const zip=await getMetaqaArchive(url,id);
progress(id,'Reading MetaQA questions…');
const csv=await archiveText(zip,'qa/metaqa_nhop.csv');
const lines=csvRecords(csv);
const headers=(lines.shift()||[]).map(s=>s.trim().replace(/^\ufeff/,''));
for(const col of ['Question','Source-Entity','Answer-Entity','SplitLabel']){
if(!headers.includes(col))throw Error('MetaQA QA file is missing '+col);
}
const q=headers.indexOf('Question'),s=headers.indexOf('Source-Entity'),a=headers.indexOf('Answer-Entity'),p=headers.indexOf('SplitLabel'),h=headers.indexOf('Hops');
const rows=lines.filter(row=>row[q]&&row[s]&&row[a]).map(row=>({
Question:row[q], 'Source-Entity':row[s], 'Answer-Entity':row[a],
SplitLabel:row[p], Hops:h<0?'':row[h]
}));
if(!rows.length)throw Error('MetaQA QA file has no usable questions');
return rows;
})().catch(err=>{metaqaRowsPromise=null;throw err;});
}
return metaqaRowsPromise;
}
function normalizedSplit(s){
const label=String(s||'').trim().toLowerCase();
return ['dev','development','valid','validation','val'].includes(label)?'dev':label==='training'?'train':label==='testing'?'test':label;
}
async function sampleMetaqa(url,split,id){
const rows=await getMetaqaRows(url,id);
const eligible=split==='all'?rows:rows.filter(row=>normalizedSplit(row.SplitLabel)===split);
if(!eligible.length)throw Error('MetaQA has no questions in the '+split+' partition');
const row=eligible[Math.floor(Math.random()*eligible.length)];
emit({id,type:'sample-metaqa',row,count:eligible.length});
}
async function getMetaqaLabels(url,id){
if(!metaqaLabelsPromise){
metaqaLabelsPromise=(async()=>{
const zip=await getMetaqaArchive(url,id);
progress(id,'Reading MetaQA entity and relation labels…');
const labels=[];
for(const [path,key] of [['metadata/node_data.csv','EID'],['metadata/relation_data.csv','RID']]){
const lines=csvRecords(await archiveText(zip,path));
const head=(lines.shift()||[]).map(s=>s.trim().replace(/^\ufeff/,''));
const idCol=head.indexOf(key),nameCol=head.indexOf('Title');
if(idCol<0||nameCol<0)throw Error('Missing '+key+' / Title metadata columns');
labels.push(lines.filter(r=>r[idCol]&&r[nameCol]).map(r=>[r[idCol].trim(),r[nameCol].trim()]));
}
return {nodes:labels[0],relations:labels[1]};
})().catch(err=>{metaqaLabelsPromise=null;throw err;});
}
return metaqaLabelsPromise;
}
async function loadMetaqaGraph(key,url,id){
if(graphs.has(key)){
const labels=await getMetaqaLabels(url,id);
emit({id,type:'ready',key,edges:graphs.get(key).edges,vertices:graphs.get(key).index.size,cached:true,...labels});
return;
}
const zip=await getMetaqaArchive(url,id);
progress(id,'Indexing MetaQA graph and reverse actions…');
const raw=await archiveText(zip,'kg/triplets.txt');
const index=new Map();let edges=0;
function add(from,relation,to){
let rels=index.get(from);
if(!rels){rels=new Map();index.set(from,rels);}
let neighbors=rels.get(relation);
if(!neighbors){neighbors=[];rels.set(relation,neighbors);}
neighbors.push(to);edges++;
}
for(const line of raw.split('\n')){
if(!line.trim())continue;
const [head,rel,tail]=line.replace(/\r$/,'').split('\t');
if(!head||!rel||!tail)continue;
add(head,rel,tail);
add(tail,'_'+rel,head);
}
if(!edges)throw Error('MetaQA graph contained no valid head–relation–tail triples');
const labels=await getMetaqaLabels(url,id);
metaqaNodeNames=new Map(labels.nodes);
graphs.set(key,{index,edges});
emit({id,type:'ready',key,edges,vertices:index.size,cached:false,...labels});
}
self.onmessage = async (event) => {
const {id,type,key,node,relation,offset=0,limit=50,query='',url} = event.data;
try {
if (type === 'sample-metaqa') return await sampleMetaqa(url,event.data.split,id);
if (type === 'load-metaqa') return await loadMetaqaGraph(key,url,id);
if (type === 'load') return await loadGraph(key,url,id);
const graph=choose(key);
if (!graph) throw Error('Graph not loaded');
const rels=graph.index.get(String(node)) || new Map();
if (type === 'relations') {
emit({id,type:'relations',node,items:[...rels].map(([relation,values]) => ({relation,count:values.length})).sort((a,b)=>a.relation.localeCompare(b.relation)),total:[...rels.values()].reduce((n,v)=>n+v.length,0)});
} else if (type === 'neighbors') {
let vals=rels.get(String(relation)) || [];
const q=String(query).trim().toLowerCase();
if(q) vals=vals.filter(n=>n.toLowerCase().includes(q)||(key==='metaqa'&&(metaqaNodeNames.get(n)||'').toLowerCase().includes(q)));
// Sort a single relation's targets only, never the global graph.
const total=vals.length;
emit({id,type:'neighbors',node,relation,total,offset,items:vals.slice(offset,offset+limit)});
}
} catch(error) {
if (type === 'load' || type === 'load-metaqa') graphs.delete(key);
emit({id,type:'error',message:String(error?.message||error)});
}
};