Spaces:
Sleeping
Sleeping
Add pillow
Browse files- requirements.txt +1 -0
- src/galaxy_data_loader.py +24 -11
requirements.txt
CHANGED
|
@@ -3,4 +3,5 @@ dash-bootstrap-components==1.5.0
|
|
| 3 |
gunicorn==21.2.0
|
| 4 |
huggingface-hub>=0.33.5
|
| 5 |
datasets>=2.14.0
|
|
|
|
| 6 |
python-dotenv==1.1.1
|
|
|
|
| 3 |
gunicorn==21.2.0
|
| 4 |
huggingface-hub>=0.33.5
|
| 5 |
datasets>=2.14.0
|
| 6 |
+
pillow>=9.0
|
| 7 |
python-dotenv==1.1.1
|
src/galaxy_data_loader.py
CHANGED
|
@@ -143,16 +143,33 @@ def sample_pool_streaming(
|
|
| 143 |
it = _make_dataset(seed, pool_size)
|
| 144 |
sync_count = min(prefetch_images, pool_size)
|
| 145 |
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
if
|
| 151 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 152 |
else:
|
| 153 |
logger.warning("No image bytes for row %d", i)
|
| 154 |
metadata_map[i] = {k: v for k, v in row.items() if k != IMAGE_COLUMN}
|
| 155 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 156 |
logger.info("%d images cached β app ready, filling remaining %d in background",
|
| 157 |
sync_count, pool_size - sync_count)
|
| 158 |
|
|
@@ -161,11 +178,7 @@ def sample_pool_streaming(
|
|
| 161 |
def _bg():
|
| 162 |
for i in range(sync_count, pool_size):
|
| 163 |
try:
|
| 164 |
-
|
| 165 |
-
img_col = row.get(IMAGE_COLUMN)
|
| 166 |
-
if isinstance(img_col, dict) and img_col.get("bytes"):
|
| 167 |
-
image_cache.put(i, img_col["bytes"])
|
| 168 |
-
metadata_map[i] = {k: v for k, v in row.items() if k != IMAGE_COLUMN}
|
| 169 |
except StopIteration:
|
| 170 |
break
|
| 171 |
except Exception as e:
|
|
|
|
| 143 |
it = _make_dataset(seed, pool_size)
|
| 144 |
sync_count = min(prefetch_images, pool_size)
|
| 145 |
|
| 146 |
+
def _extract_bytes(img_col) -> bytes | None:
|
| 147 |
+
"""Extract raw JPEG bytes from either a bytes-dict or a PIL Image."""
|
| 148 |
+
if isinstance(img_col, dict):
|
| 149 |
+
return img_col.get("bytes")
|
| 150 |
+
if img_col is not None:
|
| 151 |
+
# cast_column didn't take effect β img_col is a PIL Image
|
| 152 |
+
try:
|
| 153 |
+
import io
|
| 154 |
+
buf = io.BytesIO()
|
| 155 |
+
img_col.save(buf, format="JPEG")
|
| 156 |
+
return buf.getvalue()
|
| 157 |
+
except Exception as e:
|
| 158 |
+
logger.warning("Failed to convert PIL image to bytes: %s", e)
|
| 159 |
+
return None
|
| 160 |
+
|
| 161 |
+
def _process_row(i: int, row: dict):
|
| 162 |
+
img_bytes = _extract_bytes(row.get(IMAGE_COLUMN))
|
| 163 |
+
if img_bytes:
|
| 164 |
+
image_cache.put(i, img_bytes)
|
| 165 |
else:
|
| 166 |
logger.warning("No image bytes for row %d", i)
|
| 167 |
metadata_map[i] = {k: v for k, v in row.items() if k != IMAGE_COLUMN}
|
| 168 |
|
| 169 |
+
# Synchronous: first sync_count rows β populate metadata + cache images
|
| 170 |
+
for i in range(sync_count):
|
| 171 |
+
_process_row(i, next(it))
|
| 172 |
+
|
| 173 |
logger.info("%d images cached β app ready, filling remaining %d in background",
|
| 174 |
sync_count, pool_size - sync_count)
|
| 175 |
|
|
|
|
| 178 |
def _bg():
|
| 179 |
for i in range(sync_count, pool_size):
|
| 180 |
try:
|
| 181 |
+
_process_row(i, next(it))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 182 |
except StopIteration:
|
| 183 |
break
|
| 184 |
except Exception as e:
|