my-cool-model / Afrivoice_Kinyarwanda.py
TLMsp's picture
Upload folder using huggingface_hub
38ad221 verified
Raw History Blame Contribute Delete
12.5 kB
import os
import json
import ffmpeg
import tarfile
from pydub import AudioSegment
from glob import glob
import numpy as np
import datasets
import subprocess
from typing import List, Dict, Any
from huggingface_hub import list_repo_files
import librosa
import shutil
_CITATION = ""
_DESCRIPTION = "Multimodal dataset with manifests specifying metadata, audio, and image shards."
class CategoryConfig(datasets.BuilderConfig):
def __init__(self, *, category: str, **kwargs):
super().__init__(**kwargs)
self.category = category
class AfrivoiceKinyarwanda(datasets.GeneratorBasedBuilder):
BUILDER_CONFIGS = [
CategoryConfig(name=cat,
category=cat,
version=datasets.Version("1.0.0"),
description=f"{cat} category")
for cat in ["agriculture", "education", "financial", "government", "health","scripted_education"]
]
DEFAULT_CONFIG_NAME = "agriculture"
#print("*********JUST AFTER BUILDER CONFIG********")
def _info(self):
return datasets.DatasetInfo(
description=_DESCRIPTION,
features=datasets.Features({
"creator_id": datasets.Value("string"),
"raw_text": datasets.Value("string"),
"duration": datasets.Value("float"),
"lufs": datasets.Value("float"),
"image_category": datasets.Value("string"),
"image_sub_category": datasets.Value("string"),
"text": datasets.Value("string"),
"age_group": datasets.Value("string"),
"gender": datasets.Value("string"),
"location": datasets.Value("string"),
"shard_id": datasets.Value("string"),
"image_shard_id": datasets.Value("string"),
"audio_filepath": datasets.Value("string"),
"image_filepath": datasets.Value("string"),
"media": datasets.Value("binary"), # raw bytes of media
"media_type": datasets.Value("string"), # "audio" or "image"
}),
supervised_keys=None,
citation=_CITATION,
homepage="",
)
def _split_generators(self, dl_manager):
"""
For the given category (self.config.name), download only the manifests
and shards relevant to each split (train/val/test).
"""
def _list_manifest_urls(category: str, split: str):
repo_id = "DigitalUmuganda/afrivoice_kin"
revision = "main"
manifest_dir = f"{category}_{split}_tarred/sharded_manifests_with_image"
files = list_repo_files(repo_id, repo_type="dataset", revision=revision)
manifests = [f for f in files if f.startswith(manifest_dir) and f.endswith(".json")]
base_url = f"https://huggingface.co/datasets/{repo_id}/resolve/{revision}"
return [f"{base_url}/{f}" for f in manifests]
cfg_name = self.config.name # e.g. "education"
# mapping category -> repo naming convention
split_dirs = {
"train": f"{cfg_name}_train_tarred/sharded_manifests_with_image",
"val": f"{cfg_name}_val_tarred/sharded_manifests_with_image",
"test": f"{cfg_name}_test_tarred/sharded_manifests_with_image",
}
# print("-----> self.config: ",self.config)
requested_split = getattr(self.config, 'split', None)
# print(f"======> DEBUG: self.config object is: {self.config}")
# print(f"======> DEBUG: Found requested split from config: '{requested_split}'")
split_generators = []
for split_name, manifest_dir in split_dirs.items():
# Construct URL to manifests (your dataset already has them hosted)
manifest_glob_url = (
f"https://huggingface.co/datasets/DigitalUmuganda/Afrivoice_Kinyarwanda/resolve/main/"
f"{manifest_dir}/manifest_*.json"
) # Download manifests
manifest_urls = _list_manifest_urls(cfg_name, split_name) # adjust per split
# print("---->>>>> split: ",split_name," | Manifest_urls: ",manifest_urls)
manifest_paths = dl_manager.download(manifest_urls)
if isinstance(manifest_paths, str):
manifest_paths = [manifest_paths]
# Collect shard IDs from manifests
needed_audio_ids, needed_image_ids = set(), set()
for mp in manifest_paths:
with open(mp, "r", encoding="utf-8") as f:
for line in f:
if not line.strip():
continue
try:
# if split_name != 'train':
# print("****** split: ",split_name," | manifest line: ",json.loads(line)['shard_id'])
rec = json.loads(line)
needed_audio_ids.add(str(rec["shard_id"]))
needed_image_ids.add(str(rec["image_shard_id"]))
except Exception as e:
# print("Failed adding shardings: ",e)
continue
# Infer base URLs
manifest_base = manifest_dir.rsplit("/", 1)[0]
# print("*************** COnfig: ",self.config)
# print("############ Manifest_dir: ",manifest_dir)
audio_base = os.path.join(manifest_base,"sharded_manifests_with_image", "audio_shards")
image_base = os.path.join(manifest_base,"sharded_manifests_with_image", "image_shards") #manifest_base.replace("sharded_manifests_with_image", "image_shards")
# Build candidate URLs for shards
def candidate_urls(base, sid):
# print("=====> Base: ",base," | sid: ",sid)
return [
f"https://huggingface.co/datasets/DigitalUmuganda/Afrivoice_Kinyarwanda/resolve/main/{base}/{sid}.tar.xz",
f"https://huggingface.co/datasets/DigitalUmuganda/Afrivoice_Kinyarwanda/resolve/main/{base}/{sid}.tar",
]
shard_map = {}
# print("SPLIT: ",split_name," | needed_audio_ids: ",needed_audio_ids)
for sid in needed_audio_ids:
for url in candidate_urls(audio_base, "audio_"+sid):
try:
# print("URL: ",url)
lp = dl_manager.download(url)
shard_map.setdefault(sid, {})["audio"] = lp #url #lp
break
except Exception:
continue
for sid in needed_image_ids:
for url in candidate_urls(image_base, "image_"+sid):
try:
lp = dl_manager.download(url)
shard_map.setdefault(sid, {})["image"] = lp #url
break
except Exception:
continue
# print("--> SHard map: ",shard_map)
# Add SplitGenerator only for this split
split_generators.append(
datasets.SplitGenerator(
name=split_name,
gen_kwargs={
"split": split_name,
"manifest_paths": manifest_paths,
"shard_map": shard_map,
},
)
)
# print("############### SPLIT GENERATORS: #################\n",split_generators)
return split_generators
def _generate_examples(self, split, manifest_paths, shard_map=None):
"""
manifest_paths: local manifest file paths (downloaded by dl_manager in _split_generators)
shard_map: dict { shard_id: {"audio": local_path_or_None, "image": local_path_or_None} }
"""
if shard_map is None:
shard_map = {}
audio_tar_cache = {}
image_tar_cache = {}
# print("\n################################################################\nINside generate examples........................\n################################################\n")
# print("---> Manifest paths: ",manifest_paths)
# print("==> shard map: ",shard_map)
def _open_local_tar(local_path, cache):
if not local_path:
return None
if local_path not in cache:
cache[local_path] = tarfile.open(local_path, "r:*")
return cache[local_path]
example_counter = 0
for manifest_path in manifest_paths:
try:
with open(manifest_path, "r", encoding="utf-8") as mf:
for i,line in enumerate(mf):
if not line.strip():
continue
try:
rec = json.loads(line)
except Exception:
continue
audio_fp = rec.get("audio_filepath")
image_fp = rec.get("image_filepath")
shard_id = str(rec.get("shard_id")) if rec.get("shard_id") is not None else None
image_shard_id = str(rec.get("image_shard_id")) if rec.get("image_shard_id") is not None else None
audio_media_found = False
image_media_found = False
# try audio
if audio_fp and shard_id:
local_audio = shard_map.get(shard_id, {}).get("audio")
tf = _open_local_tar(local_audio, audio_tar_cache)
if tf:
try:
member = tf.getmember(audio_fp)
f = tf.extractfile(member)
f.seek(0)
if f:
audio = AudioSegment.from_file(f)
sr = audio.frame_rate
audio = np.array(audio.get_array_of_samples())
audio = audio.astype(np.float32) / 32768.0
rec["audio"] = (audio,sr)
rec["media_type"] = "audio"
audio_media_found = True
except KeyError:
pass
# try image
if not image_media_found and image_fp and image_shard_id:
local_image = shard_map.get(image_shard_id, {}).get("image")
tf = _open_local_tar(local_image, image_tar_cache)
if tf:
try:
member = tf.getmember(image_fp)
f = tf.extractfile(member)
if f:
rec["image"] = f.read()
rec["media_type"] = "image"
image_media_found = True
except KeyError:
pass
# print("########### WE ARE HERE ####################")
# example_id = rec.get("name") #("creator_id") or rec.get("shard_id") or rec.get("image_shard_id") or f"manifest:{os.path.basename(manifest_path)}:{example_counter}"
example_counter += 1
# print("example_id: ",example_id)
# print(">>> REC: ",rec)
yield i, {**rec}
except FileNotFoundError:
continue
# close open tarfiles
for t in audio_tar_cache.values():
try:
t.close()
except Exception:
pass
for t in image_tar_cache.values():
try:
t.close()
except Exception:
pass