Download Afrivoice_Kinyarwanda.py from TLMsp/my-cool-model: direct link, hf CLI and curl.
- Browser
- Download file 12.5 kB
-
https://huggingface.co/TLMsp/my-cool-model/resolve/main/Afrivoice_Kinyarwanda.py
- Command line
-
hf download hf://TLMsp/my-cool-model/Afrivoice_Kinyarwanda.py
-
curl -L -o Afrivoice_Kinyarwanda.py https://huggingface.co/TLMsp/my-cool-model/resolve/main/Afrivoice_Kinyarwanda.py
12.5 kB
| import os | |
| import json | |
| import ffmpeg | |
| import tarfile | |
| from pydub import AudioSegment | |
| from glob import glob | |
| import numpy as np | |
| import datasets | |
| import subprocess | |
| from typing import List, Dict, Any | |
| from huggingface_hub import list_repo_files | |
| import librosa | |
| import shutil | |
| _CITATION = "" | |
| _DESCRIPTION = "Multimodal dataset with manifests specifying metadata, audio, and image shards." | |
| class CategoryConfig(datasets.BuilderConfig): | |
| def __init__(self, *, category: str, **kwargs): | |
| super().__init__(**kwargs) | |
| self.category = category | |
| class AfrivoiceKinyarwanda(datasets.GeneratorBasedBuilder): | |
| BUILDER_CONFIGS = [ | |
| CategoryConfig(name=cat, | |
| category=cat, | |
| version=datasets.Version("1.0.0"), | |
| description=f"{cat} category") | |
| for cat in ["agriculture", "education", "financial", "government", "health","scripted_education"] | |
| ] | |
| DEFAULT_CONFIG_NAME = "agriculture" | |
| #print("*********JUST AFTER BUILDER CONFIG********") | |
| def _info(self): | |
| return datasets.DatasetInfo( | |
| description=_DESCRIPTION, | |
| features=datasets.Features({ | |
| "creator_id": datasets.Value("string"), | |
| "raw_text": datasets.Value("string"), | |
| "duration": datasets.Value("float"), | |
| "lufs": datasets.Value("float"), | |
| "image_category": datasets.Value("string"), | |
| "image_sub_category": datasets.Value("string"), | |
| "text": datasets.Value("string"), | |
| "age_group": datasets.Value("string"), | |
| "gender": datasets.Value("string"), | |
| "location": datasets.Value("string"), | |
| "shard_id": datasets.Value("string"), | |
| "image_shard_id": datasets.Value("string"), | |
| "audio_filepath": datasets.Value("string"), | |
| "image_filepath": datasets.Value("string"), | |
| "media": datasets.Value("binary"), # raw bytes of media | |
| "media_type": datasets.Value("string"), # "audio" or "image" | |
| }), | |
| supervised_keys=None, | |
| citation=_CITATION, | |
| homepage="", | |
| ) | |
| def _split_generators(self, dl_manager): | |
| """ | |
| For the given category (self.config.name), download only the manifests | |
| and shards relevant to each split (train/val/test). | |
| """ | |
| def _list_manifest_urls(category: str, split: str): | |
| repo_id = "DigitalUmuganda/afrivoice_kin" | |
| revision = "main" | |
| manifest_dir = f"{category}_{split}_tarred/sharded_manifests_with_image" | |
| files = list_repo_files(repo_id, repo_type="dataset", revision=revision) | |
| manifests = [f for f in files if f.startswith(manifest_dir) and f.endswith(".json")] | |
| base_url = f"https://huggingface.co/datasets/{repo_id}/resolve/{revision}" | |
| return [f"{base_url}/{f}" for f in manifests] | |
| cfg_name = self.config.name # e.g. "education" | |
| # mapping category -> repo naming convention | |
| split_dirs = { | |
| "train": f"{cfg_name}_train_tarred/sharded_manifests_with_image", | |
| "val": f"{cfg_name}_val_tarred/sharded_manifests_with_image", | |
| "test": f"{cfg_name}_test_tarred/sharded_manifests_with_image", | |
| } | |
| # print("-----> self.config: ",self.config) | |
| requested_split = getattr(self.config, 'split', None) | |
| # print(f"======> DEBUG: self.config object is: {self.config}") | |
| # print(f"======> DEBUG: Found requested split from config: '{requested_split}'") | |
| split_generators = [] | |
| for split_name, manifest_dir in split_dirs.items(): | |
| # Construct URL to manifests (your dataset already has them hosted) | |
| manifest_glob_url = ( | |
| f"https://huggingface.co/datasets/DigitalUmuganda/Afrivoice_Kinyarwanda/resolve/main/" | |
| f"{manifest_dir}/manifest_*.json" | |
| ) # Download manifests | |
| manifest_urls = _list_manifest_urls(cfg_name, split_name) # adjust per split | |
| # print("---->>>>> split: ",split_name," | Manifest_urls: ",manifest_urls) | |
| manifest_paths = dl_manager.download(manifest_urls) | |
| if isinstance(manifest_paths, str): | |
| manifest_paths = [manifest_paths] | |
| # Collect shard IDs from manifests | |
| needed_audio_ids, needed_image_ids = set(), set() | |
| for mp in manifest_paths: | |
| with open(mp, "r", encoding="utf-8") as f: | |
| for line in f: | |
| if not line.strip(): | |
| continue | |
| try: | |
| # if split_name != 'train': | |
| # print("****** split: ",split_name," | manifest line: ",json.loads(line)['shard_id']) | |
| rec = json.loads(line) | |
| needed_audio_ids.add(str(rec["shard_id"])) | |
| needed_image_ids.add(str(rec["image_shard_id"])) | |
| except Exception as e: | |
| # print("Failed adding shardings: ",e) | |
| continue | |
| # Infer base URLs | |
| manifest_base = manifest_dir.rsplit("/", 1)[0] | |
| # print("*************** COnfig: ",self.config) | |
| # print("############ Manifest_dir: ",manifest_dir) | |
| audio_base = os.path.join(manifest_base,"sharded_manifests_with_image", "audio_shards") | |
| image_base = os.path.join(manifest_base,"sharded_manifests_with_image", "image_shards") #manifest_base.replace("sharded_manifests_with_image", "image_shards") | |
| # Build candidate URLs for shards | |
| def candidate_urls(base, sid): | |
| # print("=====> Base: ",base," | sid: ",sid) | |
| return [ | |
| f"https://huggingface.co/datasets/DigitalUmuganda/Afrivoice_Kinyarwanda/resolve/main/{base}/{sid}.tar.xz", | |
| f"https://huggingface.co/datasets/DigitalUmuganda/Afrivoice_Kinyarwanda/resolve/main/{base}/{sid}.tar", | |
| ] | |
| shard_map = {} | |
| # print("SPLIT: ",split_name," | needed_audio_ids: ",needed_audio_ids) | |
| for sid in needed_audio_ids: | |
| for url in candidate_urls(audio_base, "audio_"+sid): | |
| try: | |
| # print("URL: ",url) | |
| lp = dl_manager.download(url) | |
| shard_map.setdefault(sid, {})["audio"] = lp #url #lp | |
| break | |
| except Exception: | |
| continue | |
| for sid in needed_image_ids: | |
| for url in candidate_urls(image_base, "image_"+sid): | |
| try: | |
| lp = dl_manager.download(url) | |
| shard_map.setdefault(sid, {})["image"] = lp #url | |
| break | |
| except Exception: | |
| continue | |
| # print("--> SHard map: ",shard_map) | |
| # Add SplitGenerator only for this split | |
| split_generators.append( | |
| datasets.SplitGenerator( | |
| name=split_name, | |
| gen_kwargs={ | |
| "split": split_name, | |
| "manifest_paths": manifest_paths, | |
| "shard_map": shard_map, | |
| }, | |
| ) | |
| ) | |
| # print("############### SPLIT GENERATORS: #################\n",split_generators) | |
| return split_generators | |
| def _generate_examples(self, split, manifest_paths, shard_map=None): | |
| """ | |
| manifest_paths: local manifest file paths (downloaded by dl_manager in _split_generators) | |
| shard_map: dict { shard_id: {"audio": local_path_or_None, "image": local_path_or_None} } | |
| """ | |
| if shard_map is None: | |
| shard_map = {} | |
| audio_tar_cache = {} | |
| image_tar_cache = {} | |
| # print("\n################################################################\nINside generate examples........................\n################################################\n") | |
| # print("---> Manifest paths: ",manifest_paths) | |
| # print("==> shard map: ",shard_map) | |
| def _open_local_tar(local_path, cache): | |
| if not local_path: | |
| return None | |
| if local_path not in cache: | |
| cache[local_path] = tarfile.open(local_path, "r:*") | |
| return cache[local_path] | |
| example_counter = 0 | |
| for manifest_path in manifest_paths: | |
| try: | |
| with open(manifest_path, "r", encoding="utf-8") as mf: | |
| for i,line in enumerate(mf): | |
| if not line.strip(): | |
| continue | |
| try: | |
| rec = json.loads(line) | |
| except Exception: | |
| continue | |
| audio_fp = rec.get("audio_filepath") | |
| image_fp = rec.get("image_filepath") | |
| shard_id = str(rec.get("shard_id")) if rec.get("shard_id") is not None else None | |
| image_shard_id = str(rec.get("image_shard_id")) if rec.get("image_shard_id") is not None else None | |
| audio_media_found = False | |
| image_media_found = False | |
| # try audio | |
| if audio_fp and shard_id: | |
| local_audio = shard_map.get(shard_id, {}).get("audio") | |
| tf = _open_local_tar(local_audio, audio_tar_cache) | |
| if tf: | |
| try: | |
| member = tf.getmember(audio_fp) | |
| f = tf.extractfile(member) | |
| f.seek(0) | |
| if f: | |
| audio = AudioSegment.from_file(f) | |
| sr = audio.frame_rate | |
| audio = np.array(audio.get_array_of_samples()) | |
| audio = audio.astype(np.float32) / 32768.0 | |
| rec["audio"] = (audio,sr) | |
| rec["media_type"] = "audio" | |
| audio_media_found = True | |
| except KeyError: | |
| pass | |
| # try image | |
| if not image_media_found and image_fp and image_shard_id: | |
| local_image = shard_map.get(image_shard_id, {}).get("image") | |
| tf = _open_local_tar(local_image, image_tar_cache) | |
| if tf: | |
| try: | |
| member = tf.getmember(image_fp) | |
| f = tf.extractfile(member) | |
| if f: | |
| rec["image"] = f.read() | |
| rec["media_type"] = "image" | |
| image_media_found = True | |
| except KeyError: | |
| pass | |
| # print("########### WE ARE HERE ####################") | |
| # example_id = rec.get("name") #("creator_id") or rec.get("shard_id") or rec.get("image_shard_id") or f"manifest:{os.path.basename(manifest_path)}:{example_counter}" | |
| example_counter += 1 | |
| # print("example_id: ",example_id) | |
| # print(">>> REC: ",rec) | |
| yield i, {**rec} | |
| except FileNotFoundError: | |
| continue | |
| # close open tarfiles | |
| for t in audio_tar_cache.values(): | |
| try: | |
| t.close() | |
| except Exception: | |
| pass | |
| for t in image_tar_cache.values(): | |
| try: | |
| t.close() | |
| except Exception: | |
| pass |