File size: 9,274 Bytes
f9edbec
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8b39f58
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f9edbec
 
 
8b39f58
 
f9edbec
 
 
 
 
 
 
 
 
8b39f58
f9edbec
 
 
 
 
8b39f58
f9edbec
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
/* A dedicated browser Web Worker builds a directed per-node adjacency index.
   It never reads benchmark answers or reference paths. */
'use strict';
const graphs = new Map();
const choose = (key) => graphs.get(key);
function emit(message) { self.postMessage(message); }
async function loadGraph(key, url, id) {
  if (graphs.has(key)) { emit({id, type:'ready', key, edges:graphs.get(key).edges, vertices:graphs.get(key).index.size, cached:true}); return; }
  const response = await fetch(url);
  if (!response.ok) throw Error(`Graph download HTTP ${response.status}`);
  const expected = Number(response.headers.get('content-length') || 0);
  const index = new Map();
  let bytes = 0, edges = 0, remaining = '';
  function consume(line) {
    if (!line) return;
    const fields = line.replace(/\r$/, '').split('\t');
    if (fields.length !== 3) return;
    const [h, r, t] = fields;
    if (!h || !r || !t) return;
    let rels = index.get(h);
    if (!rels) { rels = new Map(); index.set(h, rels); }
    let neighbors = rels.get(r);
    if (!neighbors) { neighbors = []; rels.set(r, neighbors); }
    neighbors.push(t);
    edges++;
  }
  if (response.body && response.body.getReader) {
    const reader = response.body.getReader();
    const decoder = new TextDecoder();
    let lastUpdate=0;
    while (true) {
      const {done,value} = await reader.read();
      if (done) break;
      bytes += value.byteLength;
      remaining += decoder.decode(value,{stream:true});
      let end = 0, start = 0;
      while ((end = remaining.indexOf('\n', start)) >= 0) {
        consume(remaining.slice(start, end)); start = end + 1;
      }
      remaining = remaining.slice(start);
      const now = Date.now();
      if (now - lastUpdate > 250) {
        emit({id,type:'progress',key,edges,bytes,expected}); lastUpdate=now;
      }
    }
    remaining += decoder.decode();
  } else {
    const body = await response.text(); bytes = body.length;
    for (const line of body.split('\n')) consume(line);
    remaining = '';
  }
  consume(remaining);
  if (!edges) throw Error('No triples found. Expected tab-separated head, relation, tail.');
  graphs.set(key, {index, edges});
  emit({id,type:'ready',key,edges,vertices:index.size,cached:false});
}


// MetaQA is distributed as an encoded ZIP archive rather than a Hugging Face
// Dataset Viewer config. The same archive supplies questions, graph and labels.
// Unlike MQuAKE-ST, MetaQA navigation is undirected: reverse actions are
// represented by underscore-prefixed relation IDs, following MINERVA's format.
let metaqaArchivePromise=null;
let metaqaRowsPromise=null;
let metaqaLabelsPromise=null;
let metaqaNodeNames=new Map();
const METAQA_PREFIX='metaqa_dataset/';
function progress(id,message){emit({id,type:'progress',message});}
async function getMetaqaArchive(url,id){
 if(!metaqaArchivePromise){
  metaqaArchivePromise=(async()=>{
   progress(id,'Downloading the encoded MetaQA archive from GCS…');
   if(!self.JSZip)importScripts('./vendor/jszip.min.js?v=4');
   const response=await fetch(url);
   if(!response.ok){let reason='';try{reason=(await response.text()).slice(0,240);}catch(_){}throw Error('MetaQA archive HTTP '+response.status+(reason?' · '+reason:''));}
   const bytes=await response.arrayBuffer();
   progress(id,'Extracting the encoded MetaQA archive…');
   return await self.JSZip.loadAsync(bytes);
  })().catch(err=>{metaqaArchivePromise=null;throw err;});
 }
 return metaqaArchivePromise;
}
async function archiveText(archive,path){
 const entry=archive.file(METAQA_PREFIX+path)||archive.file(path);
 if(!entry)throw Error('MetaQA archive is missing '+path);
 return await entry.async('string');
}
function csvRecords(raw){
 const lines=[];let row=[],field='',quoted=false;
 for(let i=0;i<raw.length;i++){
  const c=raw[i];
  if(c==='"'){
   if(quoted&&raw[i+1]==='"'){field+='"';i++;}else quoted=!quoted;
  }else if(c===','&&!quoted){row.push(field);field='';}
  else if((c==='\n'||c==='\r')&&!quoted){
   if(c==='\r'&&raw[i+1]==='\n')i++;
   row.push(field);if(row.some(x=>x!==''))lines.push(row);
   row=[];field='';
  }else field+=c;
 }
 if(field||row.length){row.push(field);lines.push(row);}
 return lines;
}
async function getMetaqaRows(url,id){
 if(!metaqaRowsPromise){
  metaqaRowsPromise=(async()=>{
   const zip=await getMetaqaArchive(url,id);
   progress(id,'Reading MetaQA questions…');
   const csv=await archiveText(zip,'qa/metaqa_nhop.csv');
   const lines=csvRecords(csv);
   const headers=(lines.shift()||[]).map(s=>s.trim().replace(/^\ufeff/,''));
   for(const col of ['Question','Source-Entity','Answer-Entity','SplitLabel']){
    if(!headers.includes(col))throw Error('MetaQA QA file is missing '+col);
   }
   const q=headers.indexOf('Question'),s=headers.indexOf('Source-Entity'),a=headers.indexOf('Answer-Entity'),p=headers.indexOf('SplitLabel'),h=headers.indexOf('Hops');
   const rows=lines.filter(row=>row[q]&&row[s]&&row[a]).map(row=>({
    Question:row[q], 'Source-Entity':row[s], 'Answer-Entity':row[a],
    SplitLabel:row[p], Hops:h<0?'':row[h]
   }));
   if(!rows.length)throw Error('MetaQA QA file has no usable questions');
   return rows;
  })().catch(err=>{metaqaRowsPromise=null;throw err;});
 }
 return metaqaRowsPromise;
}
function normalizedSplit(s){
 const label=String(s||'').trim().toLowerCase();
 return ['dev','development','valid','validation','val'].includes(label)?'dev':label==='training'?'train':label==='testing'?'test':label;
}
async function sampleMetaqa(url,split,id){
 const rows=await getMetaqaRows(url,id);
 const eligible=split==='all'?rows:rows.filter(row=>normalizedSplit(row.SplitLabel)===split);
 if(!eligible.length)throw Error('MetaQA has no questions in the '+split+' partition');
 const row=eligible[Math.floor(Math.random()*eligible.length)];
 emit({id,type:'sample-metaqa',row,count:eligible.length});
}
async function getMetaqaLabels(url,id){
 if(!metaqaLabelsPromise){
  metaqaLabelsPromise=(async()=>{
   const zip=await getMetaqaArchive(url,id);
   progress(id,'Reading MetaQA entity and relation labels…');
   const labels=[];
   for(const [path,key] of [['metadata/node_data.csv','EID'],['metadata/relation_data.csv','RID']]){
    const lines=csvRecords(await archiveText(zip,path));
    const head=(lines.shift()||[]).map(s=>s.trim().replace(/^\ufeff/,''));
    const idCol=head.indexOf(key),nameCol=head.indexOf('Title');
    if(idCol<0||nameCol<0)throw Error('Missing '+key+' / Title metadata columns');
    labels.push(lines.filter(r=>r[idCol]&&r[nameCol]).map(r=>[r[idCol].trim(),r[nameCol].trim()]));
   }
   return {nodes:labels[0],relations:labels[1]};
  })().catch(err=>{metaqaLabelsPromise=null;throw err;});
 }
 return metaqaLabelsPromise;
}
async function loadMetaqaGraph(key,url,id){
 if(graphs.has(key)){
  const labels=await getMetaqaLabels(url,id);
  emit({id,type:'ready',key,edges:graphs.get(key).edges,vertices:graphs.get(key).index.size,cached:true,...labels});
  return;
 }
 const zip=await getMetaqaArchive(url,id);
 progress(id,'Indexing MetaQA graph and reverse actions…');
 const raw=await archiveText(zip,'kg/triplets.txt');
 const index=new Map();let edges=0;
 function add(from,relation,to){
  let rels=index.get(from);
  if(!rels){rels=new Map();index.set(from,rels);}
  let neighbors=rels.get(relation);
  if(!neighbors){neighbors=[];rels.set(relation,neighbors);}
  neighbors.push(to);edges++;
 }
 for(const line of raw.split('\n')){
  if(!line.trim())continue;
  const [head,rel,tail]=line.replace(/\r$/,'').split('\t');
  if(!head||!rel||!tail)continue;
  add(head,rel,tail);
  add(tail,'_'+rel,head);
 }
 if(!edges)throw Error('MetaQA graph contained no valid head–relation–tail triples');
 const labels=await getMetaqaLabels(url,id);
 metaqaNodeNames=new Map(labels.nodes);
 graphs.set(key,{index,edges});
 emit({id,type:'ready',key,edges,vertices:index.size,cached:false,...labels});
}

self.onmessage = async (event) => {
  const {id,type,key,node,relation,offset=0,limit=50,query='',url} = event.data;
  try {
    if (type === 'sample-metaqa') return await sampleMetaqa(url,event.data.split,id);
    if (type === 'load-metaqa') return await loadMetaqaGraph(key,url,id);
    if (type === 'load') return await loadGraph(key,url,id);
    const graph=choose(key);
    if (!graph) throw Error('Graph not loaded');
    const rels=graph.index.get(String(node)) || new Map();
    if (type === 'relations') {
      emit({id,type:'relations',node,items:[...rels].map(([relation,values]) => ({relation,count:values.length})).sort((a,b)=>a.relation.localeCompare(b.relation)),total:[...rels.values()].reduce((n,v)=>n+v.length,0)});
    } else if (type === 'neighbors') {
      let vals=rels.get(String(relation)) || [];
      const q=String(query).trim().toLowerCase();
      if(q) vals=vals.filter(n=>n.toLowerCase().includes(q)||(key==='metaqa'&&(metaqaNodeNames.get(n)||'').toLowerCase().includes(q)));
      // Sort a single relation's targets only, never the global graph.
      const total=vals.length;
      emit({id,type:'neighbors',node,relation,total,offset,items:vals.slice(offset,offset+limit)});
    }
  } catch(error) {
    if (type === 'load' || type === 'load-metaqa') graphs.delete(key);
    emit({id,type:'error',message:String(error?.message||error)});
  }
};