Smith42 commited on
Commit
df55019
Β·
1 Parent(s): 0a4af2e

Add pillow

Browse files
Files changed (2) hide show
  1. requirements.txt +1 -0
  2. src/galaxy_data_loader.py +24 -11
requirements.txt CHANGED
@@ -3,4 +3,5 @@ dash-bootstrap-components==1.5.0
3
  gunicorn==21.2.0
4
  huggingface-hub>=0.33.5
5
  datasets>=2.14.0
 
6
  python-dotenv==1.1.1
 
3
  gunicorn==21.2.0
4
  huggingface-hub>=0.33.5
5
  datasets>=2.14.0
6
+ pillow>=9.0
7
  python-dotenv==1.1.1
src/galaxy_data_loader.py CHANGED
@@ -143,16 +143,33 @@ def sample_pool_streaming(
143
  it = _make_dataset(seed, pool_size)
144
  sync_count = min(prefetch_images, pool_size)
145
 
146
- # Synchronous: first sync_count rows β€” populate metadata + cache images
147
- for i in range(sync_count):
148
- row = next(it)
149
- img_col = row.get(IMAGE_COLUMN)
150
- if isinstance(img_col, dict) and img_col.get("bytes"):
151
- image_cache.put(i, img_col["bytes"])
 
 
 
 
 
 
 
 
 
 
 
 
 
152
  else:
153
  logger.warning("No image bytes for row %d", i)
154
  metadata_map[i] = {k: v for k, v in row.items() if k != IMAGE_COLUMN}
155
 
 
 
 
 
156
  logger.info("%d images cached β€” app ready, filling remaining %d in background",
157
  sync_count, pool_size - sync_count)
158
 
@@ -161,11 +178,7 @@ def sample_pool_streaming(
161
  def _bg():
162
  for i in range(sync_count, pool_size):
163
  try:
164
- row = next(it)
165
- img_col = row.get(IMAGE_COLUMN)
166
- if isinstance(img_col, dict) and img_col.get("bytes"):
167
- image_cache.put(i, img_col["bytes"])
168
- metadata_map[i] = {k: v for k, v in row.items() if k != IMAGE_COLUMN}
169
  except StopIteration:
170
  break
171
  except Exception as e:
 
143
  it = _make_dataset(seed, pool_size)
144
  sync_count = min(prefetch_images, pool_size)
145
 
146
+ def _extract_bytes(img_col) -> bytes | None:
147
+ """Extract raw JPEG bytes from either a bytes-dict or a PIL Image."""
148
+ if isinstance(img_col, dict):
149
+ return img_col.get("bytes")
150
+ if img_col is not None:
151
+ # cast_column didn't take effect β€” img_col is a PIL Image
152
+ try:
153
+ import io
154
+ buf = io.BytesIO()
155
+ img_col.save(buf, format="JPEG")
156
+ return buf.getvalue()
157
+ except Exception as e:
158
+ logger.warning("Failed to convert PIL image to bytes: %s", e)
159
+ return None
160
+
161
+ def _process_row(i: int, row: dict):
162
+ img_bytes = _extract_bytes(row.get(IMAGE_COLUMN))
163
+ if img_bytes:
164
+ image_cache.put(i, img_bytes)
165
  else:
166
  logger.warning("No image bytes for row %d", i)
167
  metadata_map[i] = {k: v for k, v in row.items() if k != IMAGE_COLUMN}
168
 
169
+ # Synchronous: first sync_count rows β€” populate metadata + cache images
170
+ for i in range(sync_count):
171
+ _process_row(i, next(it))
172
+
173
  logger.info("%d images cached β€” app ready, filling remaining %d in background",
174
  sync_count, pool_size - sync_count)
175
 
 
178
  def _bg():
179
  for i in range(sync_count, pool_size):
180
  try:
181
+ _process_row(i, next(it))
 
 
 
 
182
  except StopIteration:
183
  break
184
  except Exception as e: