import os import json import ffmpeg import tarfile from pydub import AudioSegment from glob import glob import numpy as np import datasets import subprocess from typing import List, Dict, Any from huggingface_hub import list_repo_files import librosa import shutil _CITATION = "" _DESCRIPTION = "Multimodal dataset with manifests specifying metadata, audio, and image shards." class CategoryConfig(datasets.BuilderConfig): def __init__(self, *, category: str, **kwargs): super().__init__(**kwargs) self.category = category class AfrivoiceKinyarwanda(datasets.GeneratorBasedBuilder): BUILDER_CONFIGS = [ CategoryConfig(name=cat, category=cat, version=datasets.Version("1.0.0"), description=f"{cat} category") for cat in ["agriculture", "education", "financial", "government", "health","scripted_education"] ] DEFAULT_CONFIG_NAME = "agriculture" #print("*********JUST AFTER BUILDER CONFIG********") def _info(self): return datasets.DatasetInfo( description=_DESCRIPTION, features=datasets.Features({ "creator_id": datasets.Value("string"), "raw_text": datasets.Value("string"), "duration": datasets.Value("float"), "lufs": datasets.Value("float"), "image_category": datasets.Value("string"), "image_sub_category": datasets.Value("string"), "text": datasets.Value("string"), "age_group": datasets.Value("string"), "gender": datasets.Value("string"), "location": datasets.Value("string"), "shard_id": datasets.Value("string"), "image_shard_id": datasets.Value("string"), "audio_filepath": datasets.Value("string"), "image_filepath": datasets.Value("string"), "media": datasets.Value("binary"), # raw bytes of media "media_type": datasets.Value("string"), # "audio" or "image" }), supervised_keys=None, citation=_CITATION, homepage="", ) def _split_generators(self, dl_manager): """ For the given category (self.config.name), download only the manifests and shards relevant to each split (train/val/test). """ def _list_manifest_urls(category: str, split: str): repo_id = "DigitalUmuganda/afrivoice_kin" revision = "main" manifest_dir = f"{category}_{split}_tarred/sharded_manifests_with_image" files = list_repo_files(repo_id, repo_type="dataset", revision=revision) manifests = [f for f in files if f.startswith(manifest_dir) and f.endswith(".json")] base_url = f"https://huggingface.co/datasets/{repo_id}/resolve/{revision}" return [f"{base_url}/{f}" for f in manifests] cfg_name = self.config.name # e.g. "education" # mapping category -> repo naming convention split_dirs = { "train": f"{cfg_name}_train_tarred/sharded_manifests_with_image", "val": f"{cfg_name}_val_tarred/sharded_manifests_with_image", "test": f"{cfg_name}_test_tarred/sharded_manifests_with_image", } # print("-----> self.config: ",self.config) requested_split = getattr(self.config, 'split', None) # print(f"======> DEBUG: self.config object is: {self.config}") # print(f"======> DEBUG: Found requested split from config: '{requested_split}'") split_generators = [] for split_name, manifest_dir in split_dirs.items(): # Construct URL to manifests (your dataset already has them hosted) manifest_glob_url = ( f"https://huggingface.co/datasets/DigitalUmuganda/Afrivoice_Kinyarwanda/resolve/main/" f"{manifest_dir}/manifest_*.json" ) # Download manifests manifest_urls = _list_manifest_urls(cfg_name, split_name) # adjust per split # print("---->>>>> split: ",split_name," | Manifest_urls: ",manifest_urls) manifest_paths = dl_manager.download(manifest_urls) if isinstance(manifest_paths, str): manifest_paths = [manifest_paths] # Collect shard IDs from manifests needed_audio_ids, needed_image_ids = set(), set() for mp in manifest_paths: with open(mp, "r", encoding="utf-8") as f: for line in f: if not line.strip(): continue try: # if split_name != 'train': # print("****** split: ",split_name," | manifest line: ",json.loads(line)['shard_id']) rec = json.loads(line) needed_audio_ids.add(str(rec["shard_id"])) needed_image_ids.add(str(rec["image_shard_id"])) except Exception as e: # print("Failed adding shardings: ",e) continue # Infer base URLs manifest_base = manifest_dir.rsplit("/", 1)[0] # print("*************** COnfig: ",self.config) # print("############ Manifest_dir: ",manifest_dir) audio_base = os.path.join(manifest_base,"sharded_manifests_with_image", "audio_shards") image_base = os.path.join(manifest_base,"sharded_manifests_with_image", "image_shards") #manifest_base.replace("sharded_manifests_with_image", "image_shards") # Build candidate URLs for shards def candidate_urls(base, sid): # print("=====> Base: ",base," | sid: ",sid) return [ f"https://huggingface.co/datasets/DigitalUmuganda/Afrivoice_Kinyarwanda/resolve/main/{base}/{sid}.tar.xz", f"https://huggingface.co/datasets/DigitalUmuganda/Afrivoice_Kinyarwanda/resolve/main/{base}/{sid}.tar", ] shard_map = {} # print("SPLIT: ",split_name," | needed_audio_ids: ",needed_audio_ids) for sid in needed_audio_ids: for url in candidate_urls(audio_base, "audio_"+sid): try: # print("URL: ",url) lp = dl_manager.download(url) shard_map.setdefault(sid, {})["audio"] = lp #url #lp break except Exception: continue for sid in needed_image_ids: for url in candidate_urls(image_base, "image_"+sid): try: lp = dl_manager.download(url) shard_map.setdefault(sid, {})["image"] = lp #url break except Exception: continue # print("--> SHard map: ",shard_map) # Add SplitGenerator only for this split split_generators.append( datasets.SplitGenerator( name=split_name, gen_kwargs={ "split": split_name, "manifest_paths": manifest_paths, "shard_map": shard_map, }, ) ) # print("############### SPLIT GENERATORS: #################\n",split_generators) return split_generators def _generate_examples(self, split, manifest_paths, shard_map=None): """ manifest_paths: local manifest file paths (downloaded by dl_manager in _split_generators) shard_map: dict { shard_id: {"audio": local_path_or_None, "image": local_path_or_None} } """ if shard_map is None: shard_map = {} audio_tar_cache = {} image_tar_cache = {} # print("\n################################################################\nINside generate examples........................\n################################################\n") # print("---> Manifest paths: ",manifest_paths) # print("==> shard map: ",shard_map) def _open_local_tar(local_path, cache): if not local_path: return None if local_path not in cache: cache[local_path] = tarfile.open(local_path, "r:*") return cache[local_path] example_counter = 0 for manifest_path in manifest_paths: try: with open(manifest_path, "r", encoding="utf-8") as mf: for i,line in enumerate(mf): if not line.strip(): continue try: rec = json.loads(line) except Exception: continue audio_fp = rec.get("audio_filepath") image_fp = rec.get("image_filepath") shard_id = str(rec.get("shard_id")) if rec.get("shard_id") is not None else None image_shard_id = str(rec.get("image_shard_id")) if rec.get("image_shard_id") is not None else None audio_media_found = False image_media_found = False # try audio if audio_fp and shard_id: local_audio = shard_map.get(shard_id, {}).get("audio") tf = _open_local_tar(local_audio, audio_tar_cache) if tf: try: member = tf.getmember(audio_fp) f = tf.extractfile(member) f.seek(0) if f: audio = AudioSegment.from_file(f) sr = audio.frame_rate audio = np.array(audio.get_array_of_samples()) audio = audio.astype(np.float32) / 32768.0 rec["audio"] = (audio,sr) rec["media_type"] = "audio" audio_media_found = True except KeyError: pass # try image if not image_media_found and image_fp and image_shard_id: local_image = shard_map.get(image_shard_id, {}).get("image") tf = _open_local_tar(local_image, image_tar_cache) if tf: try: member = tf.getmember(image_fp) f = tf.extractfile(member) if f: rec["image"] = f.read() rec["media_type"] = "image" image_media_found = True except KeyError: pass # print("########### WE ARE HERE ####################") # example_id = rec.get("name") #("creator_id") or rec.get("shard_id") or rec.get("image_shard_id") or f"manifest:{os.path.basename(manifest_path)}:{example_counter}" example_counter += 1 # print("example_id: ",example_id) # print(">>> REC: ",rec) yield i, {**rec} except FileNotFoundError: continue # close open tarfiles for t in audio_tar_cache.values(): try: t.close() except Exception: pass for t in image_tar_cache.values(): try: t.close() except Exception: pass