FAR release bundles

README.md

@@ -1,3 +1,83 @@
 ---
-license: apache-2.0
+license: cc-by-nc-4.0
+tags:
+- world-model
+- diffusion
+- retrieval
+- navigation
+- embodied-ai
 ---
+
+# FAR checkpoints
+
+Checkpoint bundles for **FAR**, a latent-diffusion world model with a learned,
+action-conditioned retrieval memory, and its baselines, on the LoopNav,
+SoundSpaces and AI2-THOR corpora. Code: TODO repo URL. Paper: TODO.
+
+## Layout
+
+The repo mirrors the code's `models/` directory, so a download lands exactly
+where the configs, launchers and notebooks look:
+
+```
+<corpus>/<arm>/config.yaml                  # rebuilds the model (Hydra)
+<corpus>/<arm>/checkpoints/<step>.pth.tar   # EMA generator weights + memory state + step (inference only)
+oasis_500m_vit_vae.pth                      # ViT-VAE tokenizer for LoopNav latents
+agent_cell_detector.pt                      # latent agent-cell detector for the AI2-THOR-dyn probe
+bundles.json                                # this listing with sizes and SHA-256
+```
+
+Bundles hold what inference needs (EMA generator, the trained or frozen memory
+state, the training step) and cannot resume training. The SDXL VAE used for
+the SoundSpaces and AI2-THOR corpora is fetched from `madebyollin/sdxl-vae-fp16-fix`
+automatically.
+
+## Download
+
+```bash
+python scripts/download_release.py                       # everything, ~12 GB
+python scripts/download_release.py --corpus ai2thor_v3   # one corpus
+python scripts/download_release.py --arm loopnav/far_multicue
+```
+
+or `huggingface-cli download 1202kbs/FAR-Checkpoints --local-dir models`.
+
+## Bundles
+
+| Corpus | Bundle | Arm | Step | Size |
+|---|---|---|---|---|
+| AI2-THOR-dyn | `ai2thor_dyn/far_meta` | FAR -- Meta | 500k | 0.56 GB |
+| AI2-THOR-dyn | `ai2thor_dyn/far_multicue` | FAR -- Multi-Cue | 500k | 0.56 GB |
+| AI2-THOR-dyn | `ai2thor_dyn/temporal` | Temporal | 500k | 0.49 GB |
+| AI2-THOR-dyn | `ai2thor_dyn/worldmem` | WorldMem | 500k | 0.49 GB |
+| AI2-THOR v3 | `ai2thor_v3/far_multicue` | FAR -- Multi-Cue | 700k | 0.55 GB |
+| AI2-THOR v3 | `ai2thor_v3/temporal` | Temporal | 700k | 0.49 GB |
+| AI2-THOR v3 | `ai2thor_v3/worldmem` | WorldMem | 700k | 0.49 GB |
+| LoopNav | `loopnav/far_meta` | FAR -- Meta | 700k | 0.55 GB |
+| LoopNav | `loopnav/far_multicue` | FAR -- Multi-Cue | 700k | 0.55 GB |
+| LoopNav | `loopnav/far_visual` | FAR -- Visual | 700k | 0.55 GB |
+| LoopNav | `loopnav/longlive_rag` | LongLive-RAG | 700k | 0.55 GB |
+| LoopNav | `loopnav/temporal` | Temporal | 700k | 0.49 GB |
+| LoopNav | `loopnav/worldmem` | WorldMem | 700k | 0.49 GB |
+| SoundSpaces v1 | `soundspaces_v1/far_meta` | FAR -- Meta | 700k | 0.55 GB |
+| SoundSpaces v1 | `soundspaces_v1/far_multicue` | FAR -- Multi-Cue | 700k | 0.55 GB |
+| SoundSpaces v1 | `soundspaces_v1/temporal` | Temporal | 700k | 0.49 GB |
+| SoundSpaces v1 | `soundspaces_v1/worldmem` | WorldMem | 700k | 0.49 GB |
+| SoundSpaces v2 | `soundspaces_v2/far_meta` | FAR -- Meta | 700k | 0.55 GB |
+| SoundSpaces v2 | `soundspaces_v2/far_multicue` | FAR -- Multi-Cue | 700k | 0.55 GB |
+| SoundSpaces v2 | `soundspaces_v2/temporal` | Temporal | 700k | 0.49 GB |
+| SoundSpaces v2 | `soundspaces_v2/worldmem` | WorldMem | 700k | 0.49 GB |
+
+"Temporal" and "WorldMem" are the recency and field-of-view retrieval
+baselines, "LongLive-RAG" the content-query retrieval baseline; the FAR arms
+differ in the cues the retriever fuses (metadata, visual, multi-cue).
+
+## License
+
+CC BY-NC 4.0. The generator and diffusion code these weights belong to are
+adapted from Navigation World Models and DiT (Meta Platforms, CC BY-NC 4.0);
+see the code repository's THIRD_PARTY_NOTICES.md.
+
+## Citation
+
+TODO

agent_cell_detector.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d2a3eeec72e0f3707fdcd5543c4f917b5344d9ff4ec9a1b0e193989e23182d5
+size 177781

ai2thor_dyn/far_meta/checkpoints/0500000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0df9561e2e2922b5f576c29f625102681c0ad381dfd36d68419b8b1b3f5f5be4
+size 596752547

ai2thor_dyn/far_meta/config.yaml

@@ -0,0 +1,994 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys.npy
+      standard_frame: false
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+soundspaces_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces/soundspaces_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_train.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal}
+      transform: ${soundspaces_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_test.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal_test}
+      transform: ${soundspaces_latent.shared.transform}
+soundspaces_v2_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces_v2/soundspaces_v2_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_v2_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces_v2/soundspaces_v2_latent_train.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal}
+      transform: ${soundspaces_v2_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces_v2/soundspaces_v2_latent_test.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal_test}
+      transform: ${soundspaces_v2_latent.shared.transform}
+ai2thor_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/ai2thor/ai2thor_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor/ai2thor_latent_train.json
+      backend: ${ai2thor_latent.shared.backend}
+      sampler: ${ai2thor_latent.sampler.context_goal}
+      transform: ${ai2thor_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor/ai2thor_latent_test.json
+      backend: ${ai2thor_latent.shared.backend}
+      sampler: ${ai2thor_latent.sampler.context_goal_test}
+      transform: ${ai2thor_latent.shared.transform}
+ai2thor_v2_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: results/manifests/ai2thor_v2/ai2thor_v2_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+      mask_suffix: null
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+      event_bias_p: 0.0
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - -3
+      - 5
+      event_post:
+      - 0
+      - 8
+      event_goal_anchor: start
+      pool_phase: pre_eval
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+      pool_phase: pre_eval
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      pool_phase: pre_eval
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_v2_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v2_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_train.json
+      backend: ${ai2thor_v2_latent.shared.backend}
+      sampler: ${ai2thor_v2_latent.sampler.context_goal}
+      transform: ${ai2thor_v2_latent.shared.transform}
+      event_index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_train_events.json
+      event_frac: null
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v2_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_test.json
+      backend: ${ai2thor_v2_latent.shared.backend}
+      sampler: ${ai2thor_v2_latent.sampler.context_goal_test}
+      transform: ${ai2thor_v2_latent.shared.transform}
+ai2thor_v3_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/ai2thor_v3/ai2thor_v3_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+      mask_suffix: null
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+      event_bias_p: 0.0
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - -3
+      - 5
+      event_post:
+      - 0
+      - 8
+      event_goal_anchor: start
+      pool_phase: pre_eval
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+      pool_phase: pre_eval
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      pool_phase: pre_eval
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_v3_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v3_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v3_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_v3/ai2thor_v3_latent_train.json
+      backend: ${ai2thor_v3_latent.shared.backend}
+      sampler: ${ai2thor_v3_latent.sampler.context_goal}
+      transform: ${ai2thor_v3_latent.shared.transform}
+      event_index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_v3/ai2thor_v3_latent_train_events.json
+      event_frac: null
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v3_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v3_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_v3/ai2thor_v3_latent_test.json
+      backend: ${ai2thor_v3_latent.shared.backend}
+      sampler: ${ai2thor_v3_latent.sampler.context_goal_test}
+      transform: ${ai2thor_v3_latent.shared.transform}
+ai2thor_dyn_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/ai2thor_dyn/ai2thor_dyn_latent.json
+    backend:
+      _target_: far.data.backends.THORDynBackend
+      reveal_anchor: true
+      reveal_start_offset: 12
+      agent_state: true
+      agent_state_pair_s: 1.5
+      agent_state_px_min: 100
+      agent_state_noise: 0.0
+      key_suffix: .keys_obj_v1.npy
+      mask_suffix: .agent.npy
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+      event_bias_p: 0.0
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - 0
+      - 10
+      event_post:
+      - 8
+      - 30
+      event_goal_anchor: end
+      event_len_traj_pred: 135
+      pool_phase: pre_eval
+      long_horizon_p: 0.0
+      long_horizon_min_past: 50
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+      pool_phase: pre_eval
+      event_bias_p: 0.35
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - 0
+      - 10
+      event_post:
+      - 8
+      - 30
+      event_goal_anchor: end
+      event_len_traj_pred: 135
+      long_horizon_p: 0.0
+      long_horizon_min_past: 50
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      pool_phase: pre_eval
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_dyn_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_dyn_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_dyn_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_dyn/ai2thor_dyn_latent_train.json
+      backend: ${ai2thor_dyn_latent.shared.backend}
+      sampler: ${ai2thor_dyn_latent.sampler.context_goal}
+      transform: ${ai2thor_dyn_latent.shared.transform}
+      event_index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_dyn/ai2thor_dyn_latent_train_events.json
+      event_frac: 0.35
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_dyn_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_dyn_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_dyn/ai2thor_dyn_latent_test.json
+      backend: ${ai2thor_dyn_latent.shared.backend}
+      sampler: ${ai2thor_dyn_latent.sampler.context_goal_test}
+      transform: ${ai2thor_dyn_latent.shared.transform}
+model:
+  tokenizer:
+    _target_: far.tokenizers.sd_vae.SD_VAE
+    image_height: ${image_height}
+    image_width: ${image_width}
+    vae_scale: 0.13025
+    ckpt_path: models/sdxl-vae-fp16-fix
+  generator:
+    _target_: far.generators.agent_state.AgentStateCDiT
+    depth: 12
+    hidden_size: 768
+    patch_size: 2
+    num_heads: 12
+    context_size: 4
+    ckpt_path: null
+    action_dim: ${action_dim}
+    pose_dim: ${pose_dim}
+    fov_half_h: 45.0
+    fov_half_v: 45.0
+    local_pos_stats: ${ai2thor_dyn_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+    interact_dim: 5
+    predictor_dim: 128
+    predictor_heads: 4
+    pos_loss_weight: 0.05
+    state_dim: 6
+  memory:
+    _target_: far.memories.pnp_cl.PnPRetrieval
+    ckpt_path: null
+    chunk_size: 30
+    num_sets: 5
+    softmax_temperature: 0.1
+    scoring: meta
+    metadata_hidden: 128
+    metadata_layers: 3
+    local_pos_stats: ${ai2thor_dyn_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+    metadata_agent_state: false
+    encoder_mode: frozen
+    adapter_hidden: 384
+    query_source: frame
+    query_conditioning: action
+    gate_eps: 0.05
+    query_vit:
+      _target_: far.memories.pnp_cl.QueryViT
+      img_height: 32
+      img_width: 32
+      patch_size: 2
+      in_chans: 4
+      embed_dim: 384
+      depth: 6
+      num_heads: 12
+      action_dim: ${action_dim}
+      key_dim: 256
+      mlp_ratio: 4.0
+      interact_dim: 4
+  _target_: far.builders.models.build_ldwm
+  _recursive_: false
+  condition_on_context_time: true
+  condition_on_context_pose: true
+  retriever_loss_weight: 1.0
+  mask_loss_lambda: 5.0
+  mask_max_frac: 0.15
+  rag_mask_loss_lambda: 5.0
+  agent_loss_weight: 1.0
+  agent_reward_weight: 1.0
+  agent_cond_binary: true
+  diffusion:
+    _target_: far.generators.diffusion.create_diffusion
+    timestep_respacing: null
+    noise_schedule: linear
+    diffusion_steps: 1000
+    learn_sigma: true
+    predict_xstart: false
+    sigma_small: false
+    use_kl: false
+    rescale_learned_sigmas: false
+wandb:
+  project: PMem-THOR
+  name: null
+  entity: null
+  mode: online
+  group: ai2thor_dyn_latent
+  tags:
+  - ai2thor_dyn_latent
+run_name: thordyn_cdit_b_pnp_meta_posefa_A_nocue_ctx4_rev_agent
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 256
+image_width: 256
+len_traj_pred: 32
+goals_per_obs: 4
+lr: 0.0001
+lr_mem: 0.0001
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 5000000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 100000
+latest_every: 2500
+eval_every: 2500
+rag_every: 7
+rag_noise_levels: 4
+rag_mode: diffusion
+rag_ddim_steps: 8
+rag_mse_weight: 1.0
+local_rag: true
+local_rag_weight: 1.0
+rag_anchored_only: true
+bfloat16: false
+torch_compile: true
+batch_size: 16
+eval_batch_size: 32
+num_workers: 12
+datasets:
+- ai2thor_dyn_latent

ai2thor_dyn/far_multicue/checkpoints/0500000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ac646018deaae9c66cc19b2b7c4e1714641eeaed01f40a1f0eb9ee815afa6b2
+size 596759399

ai2thor_dyn/far_multicue/config.yaml

@@ -0,0 +1,994 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys.npy
+      standard_frame: false
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+soundspaces_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces/soundspaces_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_train.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal}
+      transform: ${soundspaces_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_test.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal_test}
+      transform: ${soundspaces_latent.shared.transform}
+soundspaces_v2_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces_v2/soundspaces_v2_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_v2_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces_v2/soundspaces_v2_latent_train.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal}
+      transform: ${soundspaces_v2_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces_v2/soundspaces_v2_latent_test.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal_test}
+      transform: ${soundspaces_v2_latent.shared.transform}
+ai2thor_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/ai2thor/ai2thor_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor/ai2thor_latent_train.json
+      backend: ${ai2thor_latent.shared.backend}
+      sampler: ${ai2thor_latent.sampler.context_goal}
+      transform: ${ai2thor_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor/ai2thor_latent_test.json
+      backend: ${ai2thor_latent.shared.backend}
+      sampler: ${ai2thor_latent.sampler.context_goal_test}
+      transform: ${ai2thor_latent.shared.transform}
+ai2thor_v2_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: results/manifests/ai2thor_v2/ai2thor_v2_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+      mask_suffix: null
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+      event_bias_p: 0.0
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - -3
+      - 5
+      event_post:
+      - 0
+      - 8
+      event_goal_anchor: start
+      pool_phase: pre_eval
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+      pool_phase: pre_eval
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      pool_phase: pre_eval
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_v2_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v2_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_train.json
+      backend: ${ai2thor_v2_latent.shared.backend}
+      sampler: ${ai2thor_v2_latent.sampler.context_goal}
+      transform: ${ai2thor_v2_latent.shared.transform}
+      event_index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_train_events.json
+      event_frac: null
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v2_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_test.json
+      backend: ${ai2thor_v2_latent.shared.backend}
+      sampler: ${ai2thor_v2_latent.sampler.context_goal_test}
+      transform: ${ai2thor_v2_latent.shared.transform}
+ai2thor_v3_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/ai2thor_v3/ai2thor_v3_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+      mask_suffix: null
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+      event_bias_p: 0.0
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - -3
+      - 5
+      event_post:
+      - 0
+      - 8
+      event_goal_anchor: start
+      pool_phase: pre_eval
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+      pool_phase: pre_eval
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      pool_phase: pre_eval
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_v3_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v3_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v3_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_v3/ai2thor_v3_latent_train.json
+      backend: ${ai2thor_v3_latent.shared.backend}
+      sampler: ${ai2thor_v3_latent.sampler.context_goal}
+      transform: ${ai2thor_v3_latent.shared.transform}
+      event_index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_v3/ai2thor_v3_latent_train_events.json
+      event_frac: null
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v3_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v3_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_v3/ai2thor_v3_latent_test.json
+      backend: ${ai2thor_v3_latent.shared.backend}
+      sampler: ${ai2thor_v3_latent.sampler.context_goal_test}
+      transform: ${ai2thor_v3_latent.shared.transform}
+ai2thor_dyn_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/ai2thor_dyn/ai2thor_dyn_latent.json
+    backend:
+      _target_: far.data.backends.THORDynBackend
+      reveal_anchor: true
+      reveal_start_offset: 12
+      agent_state: true
+      agent_state_pair_s: 1.5
+      agent_state_px_min: 100
+      agent_state_noise: 0.0
+      key_suffix: .keys_obj_v1.npy
+      mask_suffix: .agent.npy
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+      event_bias_p: 0.0
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - 0
+      - 10
+      event_post:
+      - 8
+      - 30
+      event_goal_anchor: end
+      event_len_traj_pred: 135
+      pool_phase: pre_eval
+      long_horizon_p: 0.0
+      long_horizon_min_past: 50
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+      pool_phase: pre_eval
+      event_bias_p: 0.35
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - 0
+      - 10
+      event_post:
+      - 8
+      - 30
+      event_goal_anchor: end
+      event_len_traj_pred: 135
+      long_horizon_p: 0.0
+      long_horizon_min_past: 50
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      pool_phase: pre_eval
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_dyn_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_dyn_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_dyn_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_dyn/ai2thor_dyn_latent_train.json
+      backend: ${ai2thor_dyn_latent.shared.backend}
+      sampler: ${ai2thor_dyn_latent.sampler.context_goal}
+      transform: ${ai2thor_dyn_latent.shared.transform}
+      event_index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_dyn/ai2thor_dyn_latent_train_events.json
+      event_frac: 0.35
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_dyn_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_dyn_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_dyn/ai2thor_dyn_latent_test.json
+      backend: ${ai2thor_dyn_latent.shared.backend}
+      sampler: ${ai2thor_dyn_latent.sampler.context_goal_test}
+      transform: ${ai2thor_dyn_latent.shared.transform}
+model:
+  tokenizer:
+    _target_: far.tokenizers.sd_vae.SD_VAE
+    image_height: ${image_height}
+    image_width: ${image_width}
+    vae_scale: 0.13025
+    ckpt_path: models/sdxl-vae-fp16-fix
+  generator:
+    _target_: far.generators.agent_state.AgentStateCDiT
+    depth: 12
+    hidden_size: 768
+    patch_size: 2
+    num_heads: 12
+    context_size: 4
+    ckpt_path: null
+    action_dim: ${action_dim}
+    pose_dim: ${pose_dim}
+    fov_half_h: 45.0
+    fov_half_v: 45.0
+    local_pos_stats: ${ai2thor_dyn_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+    interact_dim: 5
+    predictor_dim: 128
+    predictor_heads: 4
+    pos_loss_weight: 0.05
+    state_dim: 6
+  memory:
+    _target_: far.memories.pnp_cl.PnPRetrieval
+    ckpt_path: null
+    chunk_size: 30
+    num_sets: 5
+    softmax_temperature: 0.1
+    scoring: meta
+    metadata_hidden: 128
+    metadata_layers: 3
+    local_pos_stats: ${ai2thor_dyn_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+    metadata_agent_state: true
+    encoder_mode: frozen
+    adapter_hidden: 384
+    query_source: frame
+    query_conditioning: action
+    gate_eps: 0.05
+    query_vit:
+      _target_: far.memories.pnp_cl.QueryViT
+      img_height: 32
+      img_width: 32
+      patch_size: 2
+      in_chans: 4
+      embed_dim: 384
+      depth: 6
+      num_heads: 12
+      action_dim: ${action_dim}
+      key_dim: 256
+      mlp_ratio: 4.0
+      interact_dim: 4
+  _target_: far.builders.models.build_ldwm
+  _recursive_: false
+  condition_on_context_time: true
+  condition_on_context_pose: true
+  retriever_loss_weight: 1.0
+  mask_loss_lambda: 5.0
+  mask_max_frac: 0.15
+  rag_mask_loss_lambda: 5.0
+  agent_loss_weight: 1.0
+  agent_reward_weight: 1.0
+  agent_cond_binary: true
+  diffusion:
+    _target_: far.generators.diffusion.create_diffusion
+    timestep_respacing: null
+    noise_schedule: linear
+    diffusion_steps: 1000
+    learn_sigma: true
+    predict_xstart: false
+    sigma_small: false
+    use_kl: false
+    rescale_learned_sigmas: false
+wandb:
+  project: PMem-THOR
+  name: null
+  entity: null
+  mode: online
+  group: ai2thor_dyn_latent
+  tags:
+  - ai2thor_dyn_latent
+run_name: thordyn_cdit_b_pnp_meta_posefa_B_rag_ctx4_rev_agent
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 256
+image_width: 256
+len_traj_pred: 32
+goals_per_obs: 4
+lr: 0.0001
+lr_mem: 0.0001
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 5000000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 100000
+latest_every: 2500
+eval_every: 2500
+rag_every: 7
+rag_noise_levels: 4
+rag_mode: diffusion
+rag_ddim_steps: 8
+rag_mse_weight: 1.0
+local_rag: true
+local_rag_weight: 1.0
+rag_anchored_only: true
+bfloat16: false
+torch_compile: true
+batch_size: 16
+eval_batch_size: 32
+num_workers: 12
+datasets:
+- ai2thor_dyn_latent

ai2thor_dyn/temporal/checkpoints/0500000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:672a67249b9039f04dbecea63c432f3048c0919c2da3749529e857c2ff105241
+size 530185339

ai2thor_dyn/temporal/config.yaml

@@ -0,0 +1,967 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys.npy
+      standard_frame: false
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+soundspaces_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces/soundspaces_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_train.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal}
+      transform: ${soundspaces_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_test.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal_test}
+      transform: ${soundspaces_latent.shared.transform}
+soundspaces_v2_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces_v2/soundspaces_v2_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_v2_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces_v2/soundspaces_v2_latent_train.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal}
+      transform: ${soundspaces_v2_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces_v2/soundspaces_v2_latent_test.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal_test}
+      transform: ${soundspaces_v2_latent.shared.transform}
+ai2thor_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/ai2thor/ai2thor_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor/ai2thor_latent_train.json
+      backend: ${ai2thor_latent.shared.backend}
+      sampler: ${ai2thor_latent.sampler.context_goal}
+      transform: ${ai2thor_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor/ai2thor_latent_test.json
+      backend: ${ai2thor_latent.shared.backend}
+      sampler: ${ai2thor_latent.sampler.context_goal_test}
+      transform: ${ai2thor_latent.shared.transform}
+ai2thor_v2_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: results/manifests/ai2thor_v2/ai2thor_v2_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+      mask_suffix: null
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+      event_bias_p: 0.0
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - -3
+      - 5
+      event_post:
+      - 0
+      - 8
+      event_goal_anchor: start
+      pool_phase: pre_eval
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+      pool_phase: pre_eval
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      pool_phase: pre_eval
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_v2_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v2_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_train.json
+      backend: ${ai2thor_v2_latent.shared.backend}
+      sampler: ${ai2thor_v2_latent.sampler.context_goal}
+      transform: ${ai2thor_v2_latent.shared.transform}
+      event_index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_train_events.json
+      event_frac: null
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v2_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_test.json
+      backend: ${ai2thor_v2_latent.shared.backend}
+      sampler: ${ai2thor_v2_latent.sampler.context_goal_test}
+      transform: ${ai2thor_v2_latent.shared.transform}
+ai2thor_v3_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/ai2thor_v3/ai2thor_v3_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+      mask_suffix: null
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+      event_bias_p: 0.0
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - -3
+      - 5
+      event_post:
+      - 0
+      - 8
+      event_goal_anchor: start
+      pool_phase: pre_eval
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+      pool_phase: pre_eval
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      pool_phase: pre_eval
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_v3_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v3_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v3_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_v3/ai2thor_v3_latent_train.json
+      backend: ${ai2thor_v3_latent.shared.backend}
+      sampler: ${ai2thor_v3_latent.sampler.context_goal}
+      transform: ${ai2thor_v3_latent.shared.transform}
+      event_index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_v3/ai2thor_v3_latent_train_events.json
+      event_frac: null
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v3_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v3_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_v3/ai2thor_v3_latent_test.json
+      backend: ${ai2thor_v3_latent.shared.backend}
+      sampler: ${ai2thor_v3_latent.sampler.context_goal_test}
+      transform: ${ai2thor_v3_latent.shared.transform}
+ai2thor_dyn_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/ai2thor_dyn/ai2thor_dyn_latent.json
+    backend:
+      _target_: far.data.backends.THORDynBackend
+      reveal_anchor: true
+      reveal_start_offset: 12
+      agent_state: true
+      agent_state_pair_s: 1.5
+      agent_state_px_min: 100
+      agent_state_noise: 0.0
+      key_suffix: null
+      mask_suffix: .agent.npy
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+      event_bias_p: 0.0
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - 0
+      - 10
+      event_post:
+      - 8
+      - 30
+      event_goal_anchor: end
+      event_len_traj_pred: 135
+      pool_phase: pre_eval
+      long_horizon_p: 0.0
+      long_horizon_min_past: 50
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+      pool_phase: pre_eval
+      event_bias_p: 0.35
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - 0
+      - 10
+      event_post:
+      - 8
+      - 30
+      event_goal_anchor: end
+      event_len_traj_pred: 135
+      long_horizon_p: 0.0
+      long_horizon_min_past: 50
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      pool_phase: pre_eval
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_dyn_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_dyn_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_dyn_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_dyn/ai2thor_dyn_latent_train.json
+      backend: ${ai2thor_dyn_latent.shared.backend}
+      sampler: ${ai2thor_dyn_latent.sampler.context_goal}
+      transform: ${ai2thor_dyn_latent.shared.transform}
+      event_index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_dyn/ai2thor_dyn_latent_train_events.json
+      event_frac: 0.35
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_dyn_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_dyn_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_dyn/ai2thor_dyn_latent_test.json
+      backend: ${ai2thor_dyn_latent.shared.backend}
+      sampler: ${ai2thor_dyn_latent.sampler.context_goal_test}
+      transform: ${ai2thor_dyn_latent.shared.transform}
+model:
+  tokenizer:
+    _target_: far.tokenizers.sd_vae.SD_VAE
+    image_height: ${image_height}
+    image_width: ${image_width}
+    vae_scale: 0.13025
+    ckpt_path: models/sdxl-vae-fp16-fix
+  generator:
+    _target_: far.generators.agent_state.AgentStateCDiT
+    depth: 12
+    hidden_size: 768
+    patch_size: 2
+    num_heads: 12
+    context_size: 4
+    ckpt_path: null
+    action_dim: ${action_dim}
+    pose_dim: ${pose_dim}
+    fov_half_h: 45.0
+    fov_half_v: 45.0
+    local_pos_stats: ${ai2thor_dyn_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+    interact_dim: 5
+    predictor_dim: 128
+    predictor_heads: 4
+    pos_loss_weight: 0.05
+    state_dim: 6
+  memory:
+    _target_: far.memories.baselines.TemporalRetrieval
+    interval: 1.0
+  _target_: far.builders.models.build_ldwm
+  _recursive_: false
+  condition_on_context_time: true
+  condition_on_context_pose: true
+  retriever_loss_weight: 1.0
+  mask_loss_lambda: 5.0
+  mask_max_frac: 0.15
+  rag_mask_loss_lambda: null
+  agent_loss_weight: 1.0
+  agent_reward_weight: 1.0
+  agent_cond_binary: true
+  diffusion:
+    _target_: far.generators.diffusion.create_diffusion
+    timestep_respacing: null
+    noise_schedule: linear
+    diffusion_steps: 1000
+    learn_sigma: true
+    predict_xstart: false
+    sigma_small: false
+    use_kl: false
+    rescale_learned_sigmas: false
+wandb:
+  project: PMem-THOR
+  name: null
+  entity: null
+  mode: online
+  group: ai2thor_dyn_latent
+  tags:
+  - ai2thor_dyn_latent
+run_name: thordyn_cdit_b_temporal_posefa_rev_agent
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 256
+image_width: 256
+len_traj_pred: 32
+goals_per_obs: 4
+lr: 0.0001
+lr_mem: null
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 5000000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 100000
+latest_every: 2500
+eval_every: 2500
+rag_every: 0
+rag_noise_levels: 4
+rag_mode: diffusion
+rag_ddim_steps: 8
+rag_mse_weight: 1.0
+local_rag: false
+local_rag_weight: 1.0
+rag_anchored_only: false
+bfloat16: false
+torch_compile: true
+batch_size: 16
+eval_batch_size: 32
+num_workers: 12
+datasets:
+- ai2thor_dyn_latent

ai2thor_dyn/worldmem/checkpoints/0500000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b26452f5ef3b4dbf16b60e6bd00f3f06b99f9998be475273fad22f41c8c7d9a
+size 530185339

ai2thor_dyn/worldmem/config.yaml

@@ -0,0 +1,972 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys.npy
+      standard_frame: false
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+soundspaces_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces/soundspaces_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_train.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal}
+      transform: ${soundspaces_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_test.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal_test}
+      transform: ${soundspaces_latent.shared.transform}
+soundspaces_v2_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces_v2/soundspaces_v2_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_v2_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces_v2/soundspaces_v2_latent_train.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal}
+      transform: ${soundspaces_v2_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces_v2/soundspaces_v2_latent_test.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal_test}
+      transform: ${soundspaces_v2_latent.shared.transform}
+ai2thor_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/ai2thor/ai2thor_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor/ai2thor_latent_train.json
+      backend: ${ai2thor_latent.shared.backend}
+      sampler: ${ai2thor_latent.sampler.context_goal}
+      transform: ${ai2thor_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor/ai2thor_latent_test.json
+      backend: ${ai2thor_latent.shared.backend}
+      sampler: ${ai2thor_latent.sampler.context_goal_test}
+      transform: ${ai2thor_latent.shared.transform}
+ai2thor_v2_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: results/manifests/ai2thor_v2/ai2thor_v2_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+      mask_suffix: null
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+      event_bias_p: 0.0
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - -3
+      - 5
+      event_post:
+      - 0
+      - 8
+      event_goal_anchor: start
+      pool_phase: pre_eval
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+      pool_phase: pre_eval
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      pool_phase: pre_eval
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_v2_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v2_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_train.json
+      backend: ${ai2thor_v2_latent.shared.backend}
+      sampler: ${ai2thor_v2_latent.sampler.context_goal}
+      transform: ${ai2thor_v2_latent.shared.transform}
+      event_index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_train_events.json
+      event_frac: null
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v2_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_test.json
+      backend: ${ai2thor_v2_latent.shared.backend}
+      sampler: ${ai2thor_v2_latent.sampler.context_goal_test}
+      transform: ${ai2thor_v2_latent.shared.transform}
+ai2thor_v3_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/ai2thor_v3/ai2thor_v3_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+      mask_suffix: null
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+      event_bias_p: 0.0
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - -3
+      - 5
+      event_post:
+      - 0
+      - 8
+      event_goal_anchor: start
+      pool_phase: pre_eval
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+      pool_phase: pre_eval
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      pool_phase: pre_eval
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_v3_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v3_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v3_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_v3/ai2thor_v3_latent_train.json
+      backend: ${ai2thor_v3_latent.shared.backend}
+      sampler: ${ai2thor_v3_latent.sampler.context_goal}
+      transform: ${ai2thor_v3_latent.shared.transform}
+      event_index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_v3/ai2thor_v3_latent_train_events.json
+      event_frac: null
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v3_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v3_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_v3/ai2thor_v3_latent_test.json
+      backend: ${ai2thor_v3_latent.shared.backend}
+      sampler: ${ai2thor_v3_latent.sampler.context_goal_test}
+      transform: ${ai2thor_v3_latent.shared.transform}
+ai2thor_dyn_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/ai2thor_dyn/ai2thor_dyn_latent.json
+    backend:
+      _target_: far.data.backends.THORDynBackend
+      reveal_anchor: true
+      reveal_start_offset: 12
+      agent_state: true
+      agent_state_pair_s: 1.5
+      agent_state_px_min: 100
+      agent_state_noise: 0.0
+      key_suffix: null
+      mask_suffix: .agent.npy
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+      event_bias_p: 0.0
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - 0
+      - 10
+      event_post:
+      - 8
+      - 30
+      event_goal_anchor: end
+      event_len_traj_pred: 135
+      pool_phase: pre_eval
+      long_horizon_p: 0.0
+      long_horizon_min_past: 50
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+      pool_phase: pre_eval
+      event_bias_p: 0.35
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - 0
+      - 10
+      event_post:
+      - 8
+      - 30
+      event_goal_anchor: end
+      event_len_traj_pred: 135
+      long_horizon_p: 0.0
+      long_horizon_min_past: 50
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      pool_phase: pre_eval
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_dyn_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_dyn_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_dyn_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_dyn/ai2thor_dyn_latent_train.json
+      backend: ${ai2thor_dyn_latent.shared.backend}
+      sampler: ${ai2thor_dyn_latent.sampler.context_goal}
+      transform: ${ai2thor_dyn_latent.shared.transform}
+      event_index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_dyn/ai2thor_dyn_latent_train_events.json
+      event_frac: 0.35
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_dyn_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_dyn_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_dyn/ai2thor_dyn_latent_test.json
+      backend: ${ai2thor_dyn_latent.shared.backend}
+      sampler: ${ai2thor_dyn_latent.sampler.context_goal_test}
+      transform: ${ai2thor_dyn_latent.shared.transform}
+model:
+  tokenizer:
+    _target_: far.tokenizers.sd_vae.SD_VAE
+    image_height: ${image_height}
+    image_width: ${image_width}
+    vae_scale: 0.13025
+    ckpt_path: models/sdxl-vae-fp16-fix
+  generator:
+    _target_: far.generators.agent_state.AgentStateCDiT
+    depth: 12
+    hidden_size: 768
+    patch_size: 2
+    num_heads: 12
+    context_size: 4
+    ckpt_path: null
+    action_dim: ${action_dim}
+    pose_dim: ${pose_dim}
+    fov_half_h: 45.0
+    fov_half_v: 45.0
+    local_pos_stats: ${ai2thor_dyn_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+    interact_dim: 5
+    predictor_dim: 128
+    predictor_heads: 4
+    pos_loss_weight: 0.05
+    state_dim: 6
+  memory:
+    _target_: far.memories.worldmem.WorldMemRetrieval
+    num_samples: 10000
+    radius: 2.0
+    fov_half_h: 45.0
+    fov_half_v: 45.0
+    recency_factor: 0.2
+    local_pos_stats: ${ai2thor_dyn_latent.spec.local_pos_stats}
+  _target_: far.builders.models.build_ldwm
+  _recursive_: false
+  condition_on_context_time: true
+  condition_on_context_pose: true
+  retriever_loss_weight: 1.0
+  mask_loss_lambda: 5.0
+  mask_max_frac: 0.15
+  rag_mask_loss_lambda: null
+  agent_loss_weight: 1.0
+  agent_reward_weight: 1.0
+  agent_cond_binary: true
+  diffusion:
+    _target_: far.generators.diffusion.create_diffusion
+    timestep_respacing: null
+    noise_schedule: linear
+    diffusion_steps: 1000
+    learn_sigma: true
+    predict_xstart: false
+    sigma_small: false
+    use_kl: false
+    rescale_learned_sigmas: false
+wandb:
+  project: PMem-THOR
+  name: null
+  entity: null
+  mode: online
+  group: ai2thor_dyn_latent
+  tags:
+  - ai2thor_dyn_latent
+run_name: thordyn_cdit_b_worldmem_posefa_rev_agent
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 256
+image_width: 256
+len_traj_pred: 32
+goals_per_obs: 4
+lr: 0.0001
+lr_mem: null
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 5000000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 100000
+latest_every: 2500
+eval_every: 2500
+rag_every: 0
+rag_noise_levels: 4
+rag_mode: diffusion
+rag_ddim_steps: 8
+rag_mse_weight: 1.0
+local_rag: false
+local_rag_weight: 1.0
+rag_anchored_only: false
+bfloat16: false
+torch_compile: true
+batch_size: 16
+eval_batch_size: 32
+num_workers: 12
+datasets:
+- ai2thor_dyn_latent

ai2thor_v3/far_multicue/checkpoints/0700000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd66115fa55d57c005c23c4ae3236797d257e20c977da34069c91248ae05af60
+size 594316275

ai2thor_v3/far_multicue/config.yaml

@@ -0,0 +1,858 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys.npy
+      standard_frame: false
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+soundspaces_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces/soundspaces_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_train.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal}
+      transform: ${soundspaces_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_test.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal_test}
+      transform: ${soundspaces_latent.shared.transform}
+soundspaces_v2_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces_v2/soundspaces_v2_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_v2_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces_v2/soundspaces_v2_latent_train.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal}
+      transform: ${soundspaces_v2_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces_v2/soundspaces_v2_latent_test.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal_test}
+      transform: ${soundspaces_v2_latent.shared.transform}
+ai2thor_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/ai2thor/ai2thor_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor/ai2thor_latent_train.json
+      backend: ${ai2thor_latent.shared.backend}
+      sampler: ${ai2thor_latent.sampler.context_goal}
+      transform: ${ai2thor_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor/ai2thor_latent_test.json
+      backend: ${ai2thor_latent.shared.backend}
+      sampler: ${ai2thor_latent.sampler.context_goal_test}
+      transform: ${ai2thor_latent.shared.transform}
+ai2thor_v2_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: results/manifests/ai2thor_v2/ai2thor_v2_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+      mask_suffix: null
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+      event_bias_p: 0.0
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - -3
+      - 5
+      event_post:
+      - 0
+      - 8
+      event_goal_anchor: start
+      pool_phase: pre_eval
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+      pool_phase: pre_eval
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      pool_phase: pre_eval
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_v2_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v2_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_train.json
+      backend: ${ai2thor_v2_latent.shared.backend}
+      sampler: ${ai2thor_v2_latent.sampler.context_goal}
+      transform: ${ai2thor_v2_latent.shared.transform}
+      event_index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_train_events.json
+      event_frac: null
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v2_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_test.json
+      backend: ${ai2thor_v2_latent.shared.backend}
+      sampler: ${ai2thor_v2_latent.sampler.context_goal_test}
+      transform: ${ai2thor_v2_latent.shared.transform}
+ai2thor_v3_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/ai2thor_v3/ai2thor_v3_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: .keys_thor_v1.npy
+      mask_suffix: .change_interact.npy
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+      event_bias_p: 0.0
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - -5
+      - 5
+      event_post:
+      - 0
+      - 14
+      event_goal_anchor: start
+      pool_phase: pre_eval
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+      pool_phase: pre_eval
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      pool_phase: pre_eval
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_v3_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v3_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v3_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_v3/ai2thor_v3_latent_train.json
+      backend: ${ai2thor_v3_latent.shared.backend}
+      sampler: ${ai2thor_v3_latent.sampler.context_goal}
+      transform: ${ai2thor_v3_latent.shared.transform}
+      event_index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_v3/ai2thor_v3_latent_train_events.json
+      event_frac: 0.3
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v3_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v3_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_v3/ai2thor_v3_latent_test.json
+      backend: ${ai2thor_v3_latent.shared.backend}
+      sampler: ${ai2thor_v3_latent.sampler.context_goal_test}
+      transform: ${ai2thor_v3_latent.shared.transform}
+model:
+  tokenizer:
+    _target_: far.tokenizers.sd_vae.SD_VAE
+    image_height: ${image_height}
+    image_width: ${image_width}
+    vae_scale: 0.13025
+    ckpt_path: models/sdxl-vae-fp16-fix
+  generator:
+    _target_: far.generators.cdit_mod_pose_fa.CDiT
+    depth: 12
+    hidden_size: 768
+    patch_size: 2
+    num_heads: 12
+    context_size: 4
+    ckpt_path: null
+    action_dim: ${action_dim}
+    pose_dim: ${pose_dim}
+    fov_half_h: 45.0
+    fov_half_v: 45.0
+    local_pos_stats: ${ai2thor_v3_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+    interact_dim: 5
+  memory:
+    _target_: far.memories.pnp_cl.PnPRetrieval
+    ckpt_path: null
+    chunk_size: 10
+    num_sets: 5
+    softmax_temperature: 1.0
+    scoring: visual+meta_z
+    metadata_hidden: 128
+    metadata_layers: 3
+    local_pos_stats: ${ai2thor_v3_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+    encoder_mode: adapter
+    adapter_hidden: 384
+    query_source: frame
+    query_conditioning: action
+    gate_eps: 0.05
+    query_vit:
+      _target_: far.memories.pnp_cl.QueryViT
+      img_height: 32
+      img_width: 32
+      patch_size: 2
+      in_chans: 4
+      embed_dim: 384
+      depth: 6
+      num_heads: 12
+      action_dim: ${action_dim}
+      key_dim: 256
+      mlp_ratio: 4.0
+      interact_dim: 4
+  _target_: far.builders.models.build_ldwm
+  _recursive_: false
+  condition_on_context_time: true
+  condition_on_context_pose: true
+  retriever_loss_weight: 1.0
+  mask_loss_lambda: 0.3
+  mask_max_frac: 0.15
+  rag_mask_loss_lambda: null
+  diffusion:
+    _target_: far.generators.diffusion.create_diffusion
+    timestep_respacing: null
+    noise_schedule: linear
+    diffusion_steps: 1000
+    learn_sigma: true
+    predict_xstart: false
+    sigma_small: false
+    use_kl: false
+    rescale_learned_sigmas: false
+wandb:
+  project: PMem-THOR
+  name: null
+  entity: null
+  mode: online
+  group: ai2thor_v3_latent
+  tags:
+  - ai2thor_v3_latent
+run_name: thor3_cdit_b_pnp_vmz_posefa
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 256
+image_width: 256
+len_traj_pred: 32
+goals_per_obs: 4
+lr: 0.0001
+lr_mem: 0.0001
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 5000000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 100000
+latest_every: 2500
+eval_every: 2500
+rag_every: 20
+rag_noise_levels: 4
+rag_mode: diffusion
+rag_ddim_steps: 8
+rag_mse_weight: 1.0
+local_rag: true
+local_rag_weight: 1.0
+bfloat16: false
+torch_compile: true
+batch_size: 16
+eval_batch_size: 32
+num_workers: 12
+datasets:
+- ai2thor_v3_latent

ai2thor_v3/temporal/checkpoints/0700000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe9512b34ba3a2f124c284525f45084df115ab3a8244b8a8dbe728693ef0da89
+size 526958337

ai2thor_v3/temporal/config.yaml

@@ -0,0 +1,832 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys.npy
+      standard_frame: false
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+soundspaces_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces/soundspaces_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_train.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal}
+      transform: ${soundspaces_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_test.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal_test}
+      transform: ${soundspaces_latent.shared.transform}
+soundspaces_v2_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces_v2/soundspaces_v2_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_v2_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces_v2/soundspaces_v2_latent_train.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal}
+      transform: ${soundspaces_v2_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces_v2/soundspaces_v2_latent_test.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal_test}
+      transform: ${soundspaces_v2_latent.shared.transform}
+ai2thor_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/ai2thor/ai2thor_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor/ai2thor_latent_train.json
+      backend: ${ai2thor_latent.shared.backend}
+      sampler: ${ai2thor_latent.sampler.context_goal}
+      transform: ${ai2thor_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor/ai2thor_latent_test.json
+      backend: ${ai2thor_latent.shared.backend}
+      sampler: ${ai2thor_latent.sampler.context_goal_test}
+      transform: ${ai2thor_latent.shared.transform}
+ai2thor_v2_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: results/manifests/ai2thor_v2/ai2thor_v2_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+      mask_suffix: null
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+      event_bias_p: 0.0
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - -3
+      - 5
+      event_post:
+      - 0
+      - 8
+      event_goal_anchor: start
+      pool_phase: pre_eval
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+      pool_phase: pre_eval
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      pool_phase: pre_eval
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_v2_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v2_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_train.json
+      backend: ${ai2thor_v2_latent.shared.backend}
+      sampler: ${ai2thor_v2_latent.sampler.context_goal}
+      transform: ${ai2thor_v2_latent.shared.transform}
+      event_index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_train_events.json
+      event_frac: null
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v2_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_test.json
+      backend: ${ai2thor_v2_latent.shared.backend}
+      sampler: ${ai2thor_v2_latent.sampler.context_goal_test}
+      transform: ${ai2thor_v2_latent.shared.transform}
+ai2thor_v3_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/ai2thor_v3/ai2thor_v3_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+      mask_suffix: .change_interact.npy
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+      event_bias_p: 0.0
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - -5
+      - 5
+      event_post:
+      - 0
+      - 14
+      event_goal_anchor: start
+      pool_phase: pre_eval
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+      pool_phase: pre_eval
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      pool_phase: pre_eval
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_v3_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v3_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v3_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_v3/ai2thor_v3_latent_train.json
+      backend: ${ai2thor_v3_latent.shared.backend}
+      sampler: ${ai2thor_v3_latent.sampler.context_goal}
+      transform: ${ai2thor_v3_latent.shared.transform}
+      event_index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_v3/ai2thor_v3_latent_train_events.json
+      event_frac: 0.3
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v3_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v3_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_v3/ai2thor_v3_latent_test.json
+      backend: ${ai2thor_v3_latent.shared.backend}
+      sampler: ${ai2thor_v3_latent.sampler.context_goal_test}
+      transform: ${ai2thor_v3_latent.shared.transform}
+model:
+  tokenizer:
+    _target_: far.tokenizers.sd_vae.SD_VAE
+    image_height: ${image_height}
+    image_width: ${image_width}
+    vae_scale: 0.13025
+    ckpt_path: models/sdxl-vae-fp16-fix
+  generator:
+    _target_: far.generators.cdit_mod_pose_fa.CDiT
+    depth: 12
+    hidden_size: 768
+    patch_size: 2
+    num_heads: 12
+    context_size: 4
+    ckpt_path: null
+    action_dim: ${action_dim}
+    pose_dim: ${pose_dim}
+    fov_half_h: 45.0
+    fov_half_v: 45.0
+    local_pos_stats: ${ai2thor_v3_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+    interact_dim: 5
+  memory:
+    _target_: far.memories.baselines.TemporalRetrieval
+    interval: 1.0
+  _target_: far.builders.models.build_ldwm
+  _recursive_: false
+  condition_on_context_time: true
+  condition_on_context_pose: true
+  retriever_loss_weight: 1.0
+  mask_loss_lambda: 0.3
+  mask_max_frac: 0.15
+  rag_mask_loss_lambda: null
+  diffusion:
+    _target_: far.generators.diffusion.create_diffusion
+    timestep_respacing: null
+    noise_schedule: linear
+    diffusion_steps: 1000
+    learn_sigma: true
+    predict_xstart: false
+    sigma_small: false
+    use_kl: false
+    rescale_learned_sigmas: false
+wandb:
+  project: PMem-THOR
+  name: null
+  entity: null
+  mode: online
+  group: ai2thor_v3_latent
+  tags:
+  - ai2thor_v3_latent
+run_name: thor3_cdit_b_temporal_posefa_contin2
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 256
+image_width: 256
+len_traj_pred: 32
+goals_per_obs: 4
+lr: 0.0001
+lr_mem: null
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 5000000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 100000
+latest_every: 2500
+eval_every: 2500
+rag_every: 0
+rag_noise_levels: 4
+rag_mode: diffusion
+rag_ddim_steps: 8
+rag_mse_weight: 1.0
+local_rag: false
+local_rag_weight: 1.0
+bfloat16: false
+torch_compile: true
+batch_size: 16
+eval_batch_size: 32
+num_workers: 12
+datasets:
+- ai2thor_v3_latent

ai2thor_v3/worldmem/checkpoints/0700000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d946f4ef956ed2eb34921c0ba26129960852a9ab0c3212c03a0f4fb5e1b09fdc
+size 526958337

ai2thor_v3/worldmem/config.yaml

@@ -0,0 +1,837 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys.npy
+      standard_frame: false
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+soundspaces_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces/soundspaces_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_train.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal}
+      transform: ${soundspaces_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_test.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal_test}
+      transform: ${soundspaces_latent.shared.transform}
+soundspaces_v2_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces_v2/soundspaces_v2_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_v2_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces_v2/soundspaces_v2_latent_train.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal}
+      transform: ${soundspaces_v2_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces_v2/soundspaces_v2_latent_test.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal_test}
+      transform: ${soundspaces_v2_latent.shared.transform}
+ai2thor_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/ai2thor/ai2thor_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor/ai2thor_latent_train.json
+      backend: ${ai2thor_latent.shared.backend}
+      sampler: ${ai2thor_latent.sampler.context_goal}
+      transform: ${ai2thor_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor/ai2thor_latent_test.json
+      backend: ${ai2thor_latent.shared.backend}
+      sampler: ${ai2thor_latent.sampler.context_goal_test}
+      transform: ${ai2thor_latent.shared.transform}
+ai2thor_v2_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: results/manifests/ai2thor_v2/ai2thor_v2_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+      mask_suffix: null
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+      event_bias_p: 0.0
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - -3
+      - 5
+      event_post:
+      - 0
+      - 8
+      event_goal_anchor: start
+      pool_phase: pre_eval
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+      pool_phase: pre_eval
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      pool_phase: pre_eval
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_v2_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v2_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_train.json
+      backend: ${ai2thor_v2_latent.shared.backend}
+      sampler: ${ai2thor_v2_latent.sampler.context_goal}
+      transform: ${ai2thor_v2_latent.shared.transform}
+      event_index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_train_events.json
+      event_frac: null
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v2_latent.shared.manifest_path}
+      index_path: results/indices/ai2thor_v2/ai2thor_v2_latent_test.json
+      backend: ${ai2thor_v2_latent.shared.backend}
+      sampler: ${ai2thor_v2_latent.sampler.context_goal_test}
+      transform: ${ai2thor_v2_latent.shared.transform}
+ai2thor_v3_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/ai2thor_v3/ai2thor_v3_latent.json
+    backend:
+      _target_: far.data.backends.THORBackend
+      key_suffix: null
+      mask_suffix: .change_interact.npy
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+      event_bias_p: 0.0
+      event_phase: eval
+      event_codes: null
+      event_pre:
+      - -5
+      - 5
+      event_post:
+      - 0
+      - 14
+      event_goal_anchor: start
+      pool_phase: pre_eval
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+      pool_phase: pre_eval
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      pool_phase: pre_eval
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${ai2thor_v3_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v3_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v3_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_v3/ai2thor_v3_latent_train.json
+      backend: ${ai2thor_v3_latent.shared.backend}
+      sampler: ${ai2thor_v3_latent.sampler.context_goal}
+      transform: ${ai2thor_v3_latent.shared.transform}
+      event_index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_v3/ai2thor_v3_latent_train_events.json
+      event_frac: 0.3
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${ai2thor_v3_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${ai2thor_v3_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/ai2thor_v3/ai2thor_v3_latent_test.json
+      backend: ${ai2thor_v3_latent.shared.backend}
+      sampler: ${ai2thor_v3_latent.sampler.context_goal_test}
+      transform: ${ai2thor_v3_latent.shared.transform}
+model:
+  tokenizer:
+    _target_: far.tokenizers.sd_vae.SD_VAE
+    image_height: ${image_height}
+    image_width: ${image_width}
+    vae_scale: 0.13025
+    ckpt_path: models/sdxl-vae-fp16-fix
+  generator:
+    _target_: far.generators.cdit_mod_pose_fa.CDiT
+    depth: 12
+    hidden_size: 768
+    patch_size: 2
+    num_heads: 12
+    context_size: 4
+    ckpt_path: null
+    action_dim: ${action_dim}
+    pose_dim: ${pose_dim}
+    fov_half_h: 45.0
+    fov_half_v: 45.0
+    local_pos_stats: ${ai2thor_v3_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+    interact_dim: 5
+  memory:
+    _target_: far.memories.worldmem.WorldMemRetrieval
+    num_samples: 10000
+    radius: 2.0
+    fov_half_h: 45.0
+    fov_half_v: 45.0
+    recency_factor: 0.2
+    local_pos_stats: ${ai2thor_v3_latent.spec.local_pos_stats}
+  _target_: far.builders.models.build_ldwm
+  _recursive_: false
+  condition_on_context_time: true
+  condition_on_context_pose: true
+  retriever_loss_weight: 1.0
+  mask_loss_lambda: 0.3
+  mask_max_frac: 0.15
+  rag_mask_loss_lambda: null
+  diffusion:
+    _target_: far.generators.diffusion.create_diffusion
+    timestep_respacing: null
+    noise_schedule: linear
+    diffusion_steps: 1000
+    learn_sigma: true
+    predict_xstart: false
+    sigma_small: false
+    use_kl: false
+    rescale_learned_sigmas: false
+wandb:
+  project: PMem-THOR
+  name: null
+  entity: null
+  mode: online
+  group: ai2thor_v3_latent
+  tags:
+  - ai2thor_v3_latent
+run_name: thor3_cdit_b_worldmem_posefa
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 256
+image_width: 256
+len_traj_pred: 32
+goals_per_obs: 4
+lr: 0.0001
+lr_mem: null
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 5000000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 100000
+latest_every: 2500
+eval_every: 2500
+rag_every: 0
+rag_noise_levels: 4
+rag_mode: diffusion
+rag_ddim_steps: 8
+rag_mse_weight: 1.0
+local_rag: false
+local_rag_weight: 1.0
+bfloat16: false
+torch_compile: true
+batch_size: 16
+eval_batch_size: 32
+num_workers: 12
+datasets:
+- ai2thor_v3_latent

bundles.json

@@ -0,0 +1,363 @@
+{
+ "layout": "models/<corpus>/<arm>/{config.yaml,checkpoints/<step>.pth.tar}",
+ "bundles": [
+  {
+   "corpus": "ai2thor_dyn",
+   "arm": "far_meta",
+   "step": "0500000",
+   "role": "ours",
+   "release": true,
+   "path": "ai2thor_dyn/far_meta/checkpoints/0500000.pth.tar",
+   "size_bytes": 596752547,
+   "sha256": "0df9561e2e2922b5f576c29f625102681c0ad381dfd36d68419b8b1b3f5f5be4",
+   "keys": [
+    "ema",
+    "memory",
+    "train_steps",
+    "core_buffers"
+   ]
+  },
+  {
+   "corpus": "ai2thor_dyn",
+   "arm": "far_multicue",
+   "step": "0500000",
+   "role": "ours",
+   "release": true,
+   "path": "ai2thor_dyn/far_multicue/checkpoints/0500000.pth.tar",
+   "size_bytes": 596759399,
+   "sha256": "5ac646018deaae9c66cc19b2b7c4e1714641eeaed01f40a1f0eb9ee815afa6b2",
+   "keys": [
+    "ema",
+    "memory",
+    "train_steps",
+    "core_buffers"
+   ]
+  },
+  {
+   "corpus": "ai2thor_dyn",
+   "arm": "temporal",
+   "step": "0500000",
+   "role": "baseline",
+   "release": true,
+   "path": "ai2thor_dyn/temporal/checkpoints/0500000.pth.tar",
+   "size_bytes": 530185339,
+   "sha256": "672a67249b9039f04dbecea63c432f3048c0919c2da3749529e857c2ff105241",
+   "keys": [
+    "ema",
+    "memory",
+    "train_steps",
+    "core_buffers"
+   ]
+  },
+  {
+   "corpus": "ai2thor_dyn",
+   "arm": "worldmem",
+   "step": "0500000",
+   "role": "baseline",
+   "release": true,
+   "path": "ai2thor_dyn/worldmem/checkpoints/0500000.pth.tar",
+   "size_bytes": 530185339,
+   "sha256": "4b26452f5ef3b4dbf16b60e6bd00f3f06b99f9998be475273fad22f41c8c7d9a",
+   "keys": [
+    "ema",
+    "memory",
+    "train_steps",
+    "core_buffers"
+   ]
+  },
+  {
+   "corpus": "ai2thor_v3",
+   "arm": "far_multicue",
+   "step": "0700000",
+   "role": "ours",
+   "release": true,
+   "path": "ai2thor_v3/far_multicue/checkpoints/0700000.pth.tar",
+   "size_bytes": 594316275,
+   "sha256": "bd66115fa55d57c005c23c4ae3236797d257e20c977da34069c91248ae05af60",
+   "keys": [
+    "ema",
+    "memory",
+    "train_steps",
+    "core_buffers"
+   ]
+  },
+  {
+   "corpus": "ai2thor_v3",
+   "arm": "temporal",
+   "step": "0700000",
+   "role": "baseline",
+   "release": true,
+   "path": "ai2thor_v3/temporal/checkpoints/0700000.pth.tar",
+   "size_bytes": 526958337,
+   "sha256": "fe9512b34ba3a2f124c284525f45084df115ab3a8244b8a8dbe728693ef0da89",
+   "keys": [
+    "ema",
+    "memory",
+    "train_steps",
+    "core_buffers"
+   ]
+  },
+  {
+   "corpus": "ai2thor_v3",
+   "arm": "worldmem",
+   "step": "0700000",
+   "role": "baseline",
+   "release": true,
+   "path": "ai2thor_v3/worldmem/checkpoints/0700000.pth.tar",
+   "size_bytes": 526958337,
+   "sha256": "d946f4ef956ed2eb34921c0ba26129960852a9ab0c3212c03a0f4fb5e1b09fdc",
+   "keys": [
+    "ema",
+    "memory",
+    "train_steps",
+    "core_buffers"
+   ]
+  },
+  {
+   "corpus": "loopnav",
+   "arm": "far_meta",
+   "step": "0700000",
+   "role": "ours",
+   "release": true,
+   "path": "loopnav/far_meta/checkpoints/0700000.pth.tar",
+   "size_bytes": 590646031,
+   "sha256": "4c2eecd804083576df9e5d707871fd0a487a177a3feb8935469569a54bb70ec3",
+   "keys": [
+    "ema",
+    "memory",
+    "train_steps",
+    "core_buffers"
+   ]
+  },
+  {
+   "corpus": "loopnav",
+   "arm": "far_multicue",
+   "step": "0700000",
+   "role": "ours",
+   "release": true,
+   "path": "loopnav/far_multicue/checkpoints/0700000.pth.tar",
+   "size_bytes": 591437405,
+   "sha256": "9933e9410a7bfe4c244756602b1b96ad317db86f2dbb37ff9d803f86290f65af",
+   "keys": [
+    "ema",
+    "memory",
+    "train_steps",
+    "core_buffers"
+   ]
+  },
+  {
+   "corpus": "loopnav",
+   "arm": "far_visual",
+   "step": "0700000",
+   "role": "ablation",
+   "release": true,
+   "path": "loopnav/far_visual/checkpoints/0700000.pth.tar",
+   "size_bytes": 591367375,
+   "sha256": "ef0f1e473ce585d877606d64f0bae04099afc9108bf7dca564bf4a279d449705",
+   "keys": [
+    "ema",
+    "memory",
+    "train_steps",
+    "core_buffers"
+   ]
+  },
+  {
+   "corpus": "loopnav",
+   "arm": "longlive_rag",
+   "step": "0700000",
+   "role": "baseline",
+   "release": true,
+   "path": "loopnav/longlive_rag/checkpoints/0700000.pth.tar",
+   "size_bytes": 590567691,
+   "sha256": "486b0089f14940c0d90f31b8244ce198d3258380628c0098beff7577de92767f",
+   "keys": [
+    "ema",
+    "memory",
+    "train_steps",
+    "core_buffers"
+   ]
+  },
+  {
+   "corpus": "loopnav",
+   "arm": "temporal",
+   "step": "0700000",
+   "role": "baseline",
+   "release": true,
+   "path": "loopnav/temporal/checkpoints/0700000.pth.tar",
+   "size_bytes": 524674293,
+   "sha256": "7559efc7408a420264b6fd5fdb000ed57e1efaf3a2620464d8865e25737386a7",
+   "keys": [
+    "ema",
+    "memory",
+    "train_steps",
+    "core_buffers"
+   ]
+  },
+  {
+   "corpus": "loopnav",
+   "arm": "worldmem",
+   "step": "0700000",
+   "role": "baseline",
+   "release": true,
+   "path": "loopnav/worldmem/checkpoints/0700000.pth.tar",
+   "size_bytes": 524674293,
+   "sha256": "d3b0ccf8ad353c1267a07c039f893024042c6b4c2a52cb2396934741241a692a",
+   "keys": [
+    "ema",
+    "memory",
+    "train_steps",
+    "core_buffers"
+   ]
+  },
+  {
+   "corpus": "soundspaces_v1",
+   "arm": "far_meta",
+   "step": "0700000",
+   "role": "ours",
+   "release": true,
+   "path": "soundspaces_v1/far_meta/checkpoints/0700000.pth.tar",
+   "size_bytes": 590474195,
+   "sha256": "299fa9ff69f8374acc4204a2a25a457752002b557ee166caa9feb50f02cb51c5",
+   "keys": [
+    "ema",
+    "memory",
+    "train_steps",
+    "core_buffers"
+   ]
+  },
+  {
+   "corpus": "soundspaces_v1",
+   "arm": "far_multicue",
+   "step": "0700000",
+   "role": "ours",
+   "release": true,
+   "path": "soundspaces_v1/far_multicue/checkpoints/0700000.pth.tar",
+   "size_bytes": 591338717,
+   "sha256": "9bbaf89285ebd73300b4057fd1a1f44d52203a1767751ece35ea4613ae9f42ea",
+   "keys": [
+    "ema",
+    "memory",
+    "train_steps",
+    "core_buffers"
+   ]
+  },
+  {
+   "corpus": "soundspaces_v1",
+   "arm": "temporal",
+   "step": "0700000",
+   "role": "baseline",
+   "release": true,
+   "path": "soundspaces_v1/temporal/checkpoints/0700000.pth.tar",
+   "size_bytes": 524576185,
+   "sha256": "323116a1764a44b0c53c055085239dcb095647809131c9bc5f77204ee789955b",
+   "keys": [
+    "ema",
+    "memory",
+    "train_steps",
+    "core_buffers"
+   ]
+  },
+  {
+   "corpus": "soundspaces_v1",
+   "arm": "worldmem",
+   "step": "0700000",
+   "role": "baseline",
+   "release": true,
+   "path": "soundspaces_v1/worldmem/checkpoints/0700000.pth.tar",
+   "size_bytes": 524576185,
+   "sha256": "b4a25b3c10e25baa68360ffe6e088f063a1f79112f58add499d7ce0ca23c4c22",
+   "keys": [
+    "ema",
+    "memory",
+    "train_steps",
+    "core_buffers"
+   ]
+  },
+  {
+   "corpus": "soundspaces_v2",
+   "arm": "far_meta",
+   "step": "0700000",
+   "role": "ours",
+   "release": true,
+   "path": "soundspaces_v2/far_meta/checkpoints/0700000.pth.tar",
+   "size_bytes": 590474195,
+   "sha256": "c521386dc7045c4aa24d555dfbc64d18f0760adfb95cb3a6b6d141b66eaf5cc6",
+   "keys": [
+    "ema",
+    "memory",
+    "train_steps",
+    "core_buffers"
+   ]
+  },
+  {
+   "corpus": "soundspaces_v2",
+   "arm": "far_multicue",
+   "step": "0700000",
+   "role": "ours",
+   "release": true,
+   "path": "soundspaces_v2/far_multicue/checkpoints/0700000.pth.tar",
+   "size_bytes": 591338717,
+   "sha256": "32409877c67f1a8658364e1b2360e6cc73749979c80c6fd7afeba9a2a5c4e9cf",
+   "keys": [
+    "ema",
+    "memory",
+    "train_steps",
+    "core_buffers"
+   ]
+  },
+  {
+   "corpus": "soundspaces_v2",
+   "arm": "temporal",
+   "step": "0700000",
+   "role": "baseline",
+   "release": true,
+   "path": "soundspaces_v2/temporal/checkpoints/0700000.pth.tar",
+   "size_bytes": 524576185,
+   "sha256": "a21bbe5a81d3b61eebc1ae2def893dc1cd3fc37ee72dda8594980301c64bf321",
+   "keys": [
+    "ema",
+    "memory",
+    "train_steps",
+    "core_buffers"
+   ]
+  },
+  {
+   "corpus": "soundspaces_v2",
+   "arm": "worldmem",
+   "step": "0700000",
+   "role": "baseline",
+   "release": true,
+   "path": "soundspaces_v2/worldmem/checkpoints/0700000.pth.tar",
+   "size_bytes": 524576185,
+   "sha256": "4ef717aa64f35123d4567e204027d054534129fac18c527d3c266b4a24f0762f",
+   "keys": [
+    "ema",
+    "memory",
+    "train_steps",
+    "core_buffers"
+   ]
+  },
+  {
+   "corpus": null,
+   "arm": null,
+   "step": null,
+   "role": "asset",
+   "release": true,
+   "path": "oasis_500m_vit_vae.pth",
+   "size_bytes": 917069061,
+   "sha256": "9f2e2a1645596e5b0dc2ef54f762c70eac6515226632270d58d6fa6c8387ce33",
+   "keys": null
+  },
+  {
+   "corpus": null,
+   "arm": null,
+   "step": null,
+   "role": "asset",
+   "release": true,
+   "path": "agent_cell_detector.pt",
+   "size_bytes": 177781,
+   "sha256": "5d2a3eeec72e0f3707fdcd5543c4f917b5344d9ff4ec9a1b0e193989e23182d5",
+   "keys": null
+  }
+ ]
+}

loopnav/far_meta/checkpoints/0700000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c2eecd804083576df9e5d707871fd0a487a177a3feb8935469569a54bb70ec3
+size 590646031

loopnav/far_meta/config.yaml

@@ -0,0 +1,384 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys_jepa.npy
+      standard_frame: true
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+model:
+  tokenizer:
+    _target_: far.tokenizers.vit_vae.ViTVAE
+    kl_weight: 1.0e-06
+    lpips_weight: 0.0
+    ckpt_path: ${oc.env:FAR_MODELS_ROOT}/oasis_500m_vit_vae.pth
+    encoder:
+      _target_: far.tokenizers.vit_vae.Encoder
+      input_height: ${image_height}
+      input_width: ${image_width}
+      patch_size: 20
+      dim: 1024
+      depth: 6
+      heads: 16
+      latent_dim: 16
+      use_variational: true
+      mlp_ratio: 4.0
+    decoder:
+      _target_: far.tokenizers.vit_vae.Decoder
+      input_height: ${image_height}
+      input_width: ${image_width}
+      patch_size: 20
+      dim: 1024
+      depth: 12
+      heads: 16
+      latent_dim: 16
+      mlp_ratio: 4.0
+  generator:
+    _target_: far.generators.cdit_mod_pose_fa.CDiT
+    depth: 12
+    hidden_size: 768
+    patch_size: 2
+    num_heads: 12
+    context_size: 4
+    ckpt_path: null
+    action_dim: ${action_dim}
+    pose_dim: ${pose_dim}
+    fov_half_h: 52.5
+    fov_half_v: 37.5
+    local_pos_stats: ${loopnav_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+  memory:
+    _target_: far.memories.pnp_cl.PnPRetrieval
+    ckpt_path: null
+    chunk_size: 10
+    num_sets: 5
+    softmax_temperature: 1.0
+    scoring: meta
+    metadata_hidden: 128
+    local_pos_stats: ${loopnav_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+    encoder_mode: frozen
+    adapter_hidden: 384
+    query_vit:
+      _target_: far.memories.pnp_cl.QueryViT
+      img_height: 18
+      img_width: 32
+      patch_size: 2
+      in_chans: 16
+      embed_dim: 384
+      depth: 6
+      num_heads: 12
+      action_dim: ${action_dim}
+      key_dim: 256
+      mlp_ratio: 4.0
+  _target_: far.builders.models.build_ldwm
+  _recursive_: false
+  condition_on_context_time: true
+  condition_on_context_pose: true
+  retriever_loss_weight: 1.0
+  diffusion:
+    _target_: far.generators.diffusion.create_diffusion
+    timestep_respacing: null
+    noise_schedule: linear
+    diffusion_steps: 1000
+    learn_sigma: true
+    predict_xstart: false
+    sigma_small: false
+    use_kl: false
+    rescale_learned_sigmas: false
+wandb:
+  project: PMem-World
+  name: null
+  entity: null
+  mode: online
+  group: loopnav_latent
+  tags:
+  - loopnav_latent
+run_name: nwm_cdit_b_pnp_meta_posefa_l1_ddim20
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 360
+image_width: 640
+len_traj_pred: 128
+goals_per_obs: 4
+lr: 0.0001
+lr_mem: 0.0001
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 5000000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 100000
+eval_every: 2500
+rag_every: 20
+rag_noise_levels: 4
+local_rag: true
+local_rag_weight: 1.0
+bfloat16: false
+torch_compile: true
+batch_size: 32
+eval_batch_size: 64
+num_workers: 12
+datasets:
+- loopnav_latent

loopnav/far_multicue/checkpoints/0700000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9933e9410a7bfe4c244756602b1b96ad317db86f2dbb37ff9d803f86290f65af
+size 591437405

loopnav/far_multicue/config.yaml

@@ -0,0 +1,385 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys_jepa.npy
+      standard_frame: true
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: results/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: results/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+model:
+  tokenizer:
+    _target_: far.tokenizers.vit_vae.ViTVAE
+    kl_weight: 1.0e-06
+    lpips_weight: 0.0
+    ckpt_path: models/oasis_500m_vit_vae.pth
+    encoder:
+      _target_: far.tokenizers.vit_vae.Encoder
+      input_height: ${image_height}
+      input_width: ${image_width}
+      patch_size: 20
+      dim: 1024
+      depth: 6
+      heads: 16
+      latent_dim: 16
+      use_variational: true
+      mlp_ratio: 4.0
+    decoder:
+      _target_: far.tokenizers.vit_vae.Decoder
+      input_height: ${image_height}
+      input_width: ${image_width}
+      patch_size: 20
+      dim: 1024
+      depth: 12
+      heads: 16
+      latent_dim: 16
+      mlp_ratio: 4.0
+  generator:
+    _target_: far.generators.cdit_mod_pose_fa.CDiT
+    depth: 12
+    hidden_size: 768
+    patch_size: 2
+    num_heads: 12
+    context_size: 4
+    ckpt_path: null
+    action_dim: ${action_dim}
+    pose_dim: ${pose_dim}
+    fov_half_h: 52.5
+    fov_half_v: 37.5
+    local_pos_stats: ${loopnav_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+  memory:
+    _target_: far.memories.pnp_cl.PnPRetrieval
+    ckpt_path: null
+    chunk_size: 10
+    num_sets: 5
+    softmax_temperature: 1.0
+    scoring: visual+meta_z
+    metadata_hidden: 128
+    local_pos_stats: ${loopnav_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+    encoder_mode: adapter
+    adapter_hidden: 384
+    gate_eps: 0.05
+    query_vit:
+      _target_: far.memories.pnp_cl.QueryViT
+      img_height: 18
+      img_width: 32
+      patch_size: 2
+      in_chans: 16
+      embed_dim: 384
+      depth: 6
+      num_heads: 12
+      action_dim: ${action_dim}
+      key_dim: 256
+      mlp_ratio: 4.0
+  _target_: far.builders.models.build_ldwm
+  _recursive_: false
+  condition_on_context_time: true
+  condition_on_context_pose: true
+  retriever_loss_weight: 1.0
+  diffusion:
+    _target_: far.generators.diffusion.create_diffusion
+    timestep_respacing: null
+    noise_schedule: linear
+    diffusion_steps: 1000
+    learn_sigma: true
+    predict_xstart: false
+    sigma_small: false
+    use_kl: false
+    rescale_learned_sigmas: false
+wandb:
+  project: PMem
+  name: null
+  entity: null
+  mode: online
+  group: null
+  tags: null
+run_name: nwm_cdit_b_pnp_vmz_posefa_l1_ddim20
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 360
+image_width: 640
+len_traj_pred: 128
+goals_per_obs: 4
+lr: 0.0001
+lr_mem: 0.0001
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 5000000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 100000
+latest_every: 10000
+eval_every: 2500
+rag_every: 20
+rag_noise_levels: 4
+local_rag: true
+local_rag_weight: 1.0
+bfloat16: false
+torch_compile: true
+batch_size: 32
+eval_batch_size: 64
+num_workers: 12
+datasets:
+- loopnav_latent

loopnav/far_visual/checkpoints/0700000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef0f1e473ce585d877606d64f0bae04099afc9108bf7dca564bf4a279d449705
+size 591367375

loopnav/far_visual/config.yaml

@@ -0,0 +1,384 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys_jepa.npy
+      standard_frame: true
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+model:
+  tokenizer:
+    _target_: far.tokenizers.vit_vae.ViTVAE
+    kl_weight: 1.0e-06
+    lpips_weight: 0.0
+    ckpt_path: ${oc.env:FAR_MODELS_ROOT}/oasis_500m_vit_vae.pth
+    encoder:
+      _target_: far.tokenizers.vit_vae.Encoder
+      input_height: ${image_height}
+      input_width: ${image_width}
+      patch_size: 20
+      dim: 1024
+      depth: 6
+      heads: 16
+      latent_dim: 16
+      use_variational: true
+      mlp_ratio: 4.0
+    decoder:
+      _target_: far.tokenizers.vit_vae.Decoder
+      input_height: ${image_height}
+      input_width: ${image_width}
+      patch_size: 20
+      dim: 1024
+      depth: 12
+      heads: 16
+      latent_dim: 16
+      mlp_ratio: 4.0
+  generator:
+    _target_: far.generators.cdit_mod_pose_fa.CDiT
+    depth: 12
+    hidden_size: 768
+    patch_size: 2
+    num_heads: 12
+    context_size: 4
+    ckpt_path: null
+    action_dim: ${action_dim}
+    pose_dim: ${pose_dim}
+    fov_half_h: 52.5
+    fov_half_v: 37.5
+    local_pos_stats: ${loopnav_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+  memory:
+    _target_: far.memories.pnp_cl.PnPRetrieval
+    ckpt_path: null
+    chunk_size: 10
+    num_sets: 5
+    softmax_temperature: 1.0
+    scoring: visual
+    metadata_hidden: 128
+    local_pos_stats: ${loopnav_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+    encoder_mode: adapter
+    adapter_hidden: 384
+    query_vit:
+      _target_: far.memories.pnp_cl.QueryViT
+      img_height: 18
+      img_width: 32
+      patch_size: 2
+      in_chans: 16
+      embed_dim: 384
+      depth: 6
+      num_heads: 12
+      action_dim: ${action_dim}
+      key_dim: 256
+      mlp_ratio: 4.0
+  _target_: far.builders.models.build_ldwm
+  _recursive_: false
+  condition_on_context_time: true
+  condition_on_context_pose: true
+  retriever_loss_weight: 1.0
+  diffusion:
+    _target_: far.generators.diffusion.create_diffusion
+    timestep_respacing: null
+    noise_schedule: linear
+    diffusion_steps: 1000
+    learn_sigma: true
+    predict_xstart: false
+    sigma_small: false
+    use_kl: false
+    rescale_learned_sigmas: false
+wandb:
+  project: PMem-World
+  name: null
+  entity: null
+  mode: online
+  group: loopnav_latent
+  tags:
+  - loopnav_latent
+run_name: nwm_cdit_b_pnp_visual_posefa_l1_ddim20
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 360
+image_width: 640
+len_traj_pred: 128
+goals_per_obs: 4
+lr: 0.0001
+lr_mem: 0.0001
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 5000000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 100000
+eval_every: 2500
+rag_every: 20
+rag_noise_levels: 4
+local_rag: true
+local_rag_weight: 1.0
+bfloat16: false
+torch_compile: true
+batch_size: 32
+eval_batch_size: 64
+num_workers: 12
+datasets:
+- loopnav_latent

loopnav/longlive_rag/checkpoints/0700000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:486b0089f14940c0d90f31b8244ce198d3258380628c0098beff7577de92767f
+size 590567691

loopnav/longlive_rag/config.yaml

@@ -0,0 +1,570 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys_longlive.npy
+      standard_frame: true
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: results/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: results/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+soundspaces_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: results/manifests/soundspaces/soundspaces_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: results/indices/soundspaces/soundspaces_latent_train.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal}
+      transform: ${soundspaces_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: results/indices/soundspaces/soundspaces_latent_test.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal_test}
+      transform: ${soundspaces_latent.shared.transform}
+soundspaces_v2_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: results/manifests/soundspaces_v2/soundspaces_v2_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_v2_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: results/indices/soundspaces_v2/soundspaces_latent_train.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal}
+      transform: ${soundspaces_v2_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: results/indices/soundspaces_v2/soundspaces_latent_test.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal_test}
+      transform: ${soundspaces_v2_latent.shared.transform}
+model:
+  tokenizer:
+    _target_: far.tokenizers.vit_vae.ViTVAE
+    kl_weight: 1.0e-06
+    lpips_weight: 0.0
+    ckpt_path: models/oasis_500m_vit_vae.pth
+    encoder:
+      _target_: far.tokenizers.vit_vae.Encoder
+      input_height: ${image_height}
+      input_width: ${image_width}
+      patch_size: 20
+      dim: 1024
+      depth: 6
+      heads: 16
+      latent_dim: 16
+      use_variational: true
+      mlp_ratio: 4.0
+    decoder:
+      _target_: far.tokenizers.vit_vae.Decoder
+      input_height: ${image_height}
+      input_width: ${image_width}
+      patch_size: 20
+      dim: 1024
+      depth: 12
+      heads: 16
+      latent_dim: 16
+      mlp_ratio: 4.0
+  generator:
+    _target_: far.generators.cdit_mod_pose_fa.CDiT
+    depth: 12
+    hidden_size: 768
+    patch_size: 2
+    num_heads: 12
+    context_size: 4
+    ckpt_path: null
+    action_dim: ${action_dim}
+    pose_dim: ${pose_dim}
+    fov_half_h: 52.5
+    fov_half_v: 37.5
+    local_pos_stats: ${loopnav_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+  memory:
+    _target_: far.memories.pnp_cl.PnPRetrieval
+    ckpt_path: null
+    chunk_size: 10
+    num_sets: 5
+    softmax_temperature: 1.0
+    scoring: visual
+    metadata_hidden: 128
+    local_pos_stats: ${loopnav_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+    encoder_mode: frozen
+    adapter_hidden: 384
+    query_source: frame
+    query_conditioning: none
+    gate_eps: 0.05
+    query_vit:
+      _target_: far.memories.pnp_cl.QueryViT
+      img_height: 18
+      img_width: 32
+      patch_size: 2
+      in_chans: 16
+      embed_dim: 384
+      depth: 6
+      num_heads: 12
+      action_dim: ${action_dim}
+      key_dim: 256
+      mlp_ratio: 4.0
+  _target_: far.builders.models.build_ldwm
+  _recursive_: false
+  condition_on_context_time: true
+  condition_on_context_pose: true
+  retriever_loss_weight: 0.0
+  diffusion:
+    _target_: far.generators.diffusion.create_diffusion
+    timestep_respacing: null
+    noise_schedule: linear
+    diffusion_steps: 1000
+    learn_sigma: true
+    predict_xstart: false
+    sigma_small: false
+    use_kl: false
+    rescale_learned_sigmas: false
+wandb:
+  project: PMem-World
+  name: null
+  entity: null
+  mode: online
+  group: loopnav_latent
+  tags:
+  - loopnav_latent
+run_name: nwm_cdit_b_pnp_longlive_posefa_l0
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 360
+image_width: 640
+len_traj_pred: 128
+goals_per_obs: 4
+lr: 0.0001
+lr_mem: null
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 5000000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 100000
+latest_every: 10000
+eval_every: 2500
+rag_every: 0
+rag_noise_levels: 4
+local_rag: false
+local_rag_weight: 1.0
+bfloat16: false
+torch_compile: true
+batch_size: 32
+eval_batch_size: 64
+num_workers: 12
+datasets:
+- loopnav_latent

loopnav/temporal/checkpoints/0700000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7559efc7408a420264b6fd5fdb000ed57e1efaf3a2620464d8865e25737386a7
+size 524674293

loopnav/temporal/config.yaml

@@ -0,0 +1,363 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys.npy
+      standard_frame: true
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+model:
+  tokenizer:
+    _target_: far.tokenizers.vit_vae.ViTVAE
+    kl_weight: 1.0e-06
+    lpips_weight: 0.0
+    ckpt_path: ${oc.env:FAR_MODELS_ROOT}/oasis_500m_vit_vae.pth
+    encoder:
+      _target_: far.tokenizers.vit_vae.Encoder
+      input_height: ${image_height}
+      input_width: ${image_width}
+      patch_size: 20
+      dim: 1024
+      depth: 6
+      heads: 16
+      latent_dim: 16
+      use_variational: true
+      mlp_ratio: 4.0
+    decoder:
+      _target_: far.tokenizers.vit_vae.Decoder
+      input_height: ${image_height}
+      input_width: ${image_width}
+      patch_size: 20
+      dim: 1024
+      depth: 12
+      heads: 16
+      latent_dim: 16
+      mlp_ratio: 4.0
+  generator:
+    _target_: far.generators.cdit_mod_pose_fa.CDiT
+    depth: 12
+    hidden_size: 768
+    patch_size: 2
+    num_heads: 12
+    context_size: 4
+    ckpt_path: null
+    action_dim: ${action_dim}
+    pose_dim: ${pose_dim}
+    fov_half_h: 52.5
+    fov_half_v: 37.5
+    local_pos_stats: ${loopnav_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+  memory:
+    _target_: far.memories.baselines.TemporalRetrieval
+    interval: 1.0
+  _target_: far.builders.models.build_ldwm
+  _recursive_: false
+  condition_on_context_time: true
+  condition_on_context_pose: true
+  retriever_loss_weight: 1.0
+  diffusion:
+    _target_: far.generators.diffusion.create_diffusion
+    timestep_respacing: null
+    noise_schedule: linear
+    diffusion_steps: 1000
+    learn_sigma: true
+    predict_xstart: false
+    sigma_small: false
+    use_kl: false
+    rescale_learned_sigmas: false
+wandb:
+  project: PMem-World
+  name: null
+  entity: null
+  mode: online
+  group: loopnav_latent
+  tags:
+  - loopnav_latent
+run_name: nwm_cdit_b_temporal_posefa_ddim20
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 360
+image_width: 640
+len_traj_pred: 128
+goals_per_obs: 4
+lr: 0.0001
+lr_mem: null
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 5000000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 100000
+eval_every: 2500
+rag_every: 0
+rag_noise_levels: 4
+local_rag: false
+local_rag_weight: 1.0
+bfloat16: false
+torch_compile: true
+batch_size: 16
+eval_batch_size: 32
+num_workers: 12
+datasets:
+- loopnav_latent

loopnav/worldmem/checkpoints/0700000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3b0ccf8ad353c1267a07c039f893024042c6b4c2a52cb2396934741241a692a
+size 524674293

loopnav/worldmem/config.yaml

@@ -0,0 +1,368 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys.npy
+      standard_frame: true
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: results/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: results/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+model:
+  tokenizer:
+    _target_: far.tokenizers.vit_vae.ViTVAE
+    kl_weight: 1.0e-06
+    lpips_weight: 0.0
+    ckpt_path: models/oasis_500m_vit_vae.pth
+    encoder:
+      _target_: far.tokenizers.vit_vae.Encoder
+      input_height: ${image_height}
+      input_width: ${image_width}
+      patch_size: 20
+      dim: 1024
+      depth: 6
+      heads: 16
+      latent_dim: 16
+      use_variational: true
+      mlp_ratio: 4.0
+    decoder:
+      _target_: far.tokenizers.vit_vae.Decoder
+      input_height: ${image_height}
+      input_width: ${image_width}
+      patch_size: 20
+      dim: 1024
+      depth: 12
+      heads: 16
+      latent_dim: 16
+      mlp_ratio: 4.0
+  generator:
+    _target_: far.generators.cdit_mod_pose_fa.CDiT
+    depth: 12
+    hidden_size: 768
+    patch_size: 2
+    num_heads: 12
+    context_size: 4
+    ckpt_path: null
+    action_dim: ${action_dim}
+    pose_dim: ${pose_dim}
+    fov_half_h: 52.5
+    fov_half_v: 37.5
+    local_pos_stats: ${loopnav_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+  memory:
+    _target_: far.memories.worldmem.WorldMemRetrieval
+    num_samples: 10000
+    radius: 30.0
+    fov_half_h: 52.5
+    fov_half_v: 37.5
+    recency_factor: 0.2
+    local_pos_stats: ${loopnav_latent.spec.local_pos_stats}
+  _target_: far.builders.models.build_ldwm
+  _recursive_: false
+  condition_on_context_time: true
+  condition_on_context_pose: true
+  retriever_loss_weight: 1.0
+  diffusion:
+    _target_: far.generators.diffusion.create_diffusion
+    timestep_respacing: null
+    noise_schedule: linear
+    diffusion_steps: 1000
+    learn_sigma: true
+    predict_xstart: false
+    sigma_small: false
+    use_kl: false
+    rescale_learned_sigmas: false
+wandb:
+  project: PMem-World
+  name: null
+  entity: null
+  mode: online
+  group: loopnav_latent
+  tags:
+  - loopnav_latent
+run_name: nwm_cdit_b_worldmem_posefa_ddim20
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 360
+image_width: 640
+len_traj_pred: 128
+goals_per_obs: 4
+lr: 0.0001
+lr_mem: null
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 5000000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 100000
+eval_every: 2500
+rag_every: 0
+rag_noise_levels: 4
+local_rag: false
+local_rag_weight: 1.0
+bfloat16: false
+torch_compile: true
+batch_size: 32
+eval_batch_size: 64
+num_workers: 12
+datasets:
+- loopnav_latent

oasis_500m_vit_vae.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f2e2a1645596e5b0dc2ef54f762c70eac6515226632270d58d6fa6c8387ce33
+size 917069061

soundspaces_v1/far_meta/checkpoints/0700000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:299fa9ff69f8374acc4204a2a25a457752002b557ee166caa9feb50f02cb51c5
+size 590474195

soundspaces_v1/far_meta/config.yaml

@@ -0,0 +1,458 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys.npy
+      standard_frame: false
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+soundspaces_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces/soundspaces_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: .keys_audio.npy
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_train.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal}
+      transform: ${soundspaces_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_test.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal_test}
+      transform: ${soundspaces_latent.shared.transform}
+model:
+  tokenizer:
+    _target_: far.tokenizers.sd_vae.SD_VAE
+    image_height: ${image_height}
+    image_width: ${image_width}
+    vae_scale: 0.13025
+    ckpt_path: models/sdxl-vae-fp16-fix
+  generator:
+    _target_: far.generators.cdit_mod_pose_fa.CDiT
+    depth: 12
+    hidden_size: 768
+    patch_size: 2
+    num_heads: 12
+    context_size: 4
+    ckpt_path: null
+    action_dim: ${action_dim}
+    pose_dim: ${pose_dim}
+    fov_half_h: 45.0
+    fov_half_v: 45.0
+    local_pos_stats: ${soundspaces_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+  memory:
+    _target_: far.memories.pnp_cl.PnPRetrieval
+    ckpt_path: null
+    chunk_size: 10
+    num_sets: 5
+    softmax_temperature: 1.0
+    scoring: meta
+    metadata_hidden: 128
+    local_pos_stats: ${soundspaces_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+    encoder_mode: frozen
+    adapter_hidden: 384
+    query_source: frame
+    gate_eps: 0.05
+    query_vit:
+      _target_: far.memories.pnp_cl.QueryViT
+      img_height: 32
+      img_width: 32
+      patch_size: 2
+      in_chans: 4
+      embed_dim: 384
+      depth: 6
+      num_heads: 12
+      action_dim: ${action_dim}
+      key_dim: 256
+      mlp_ratio: 4.0
+  _target_: far.builders.models.build_ldwm
+  _recursive_: false
+  condition_on_context_time: true
+  condition_on_context_pose: true
+  retriever_loss_weight: 1.0
+  diffusion:
+    _target_: far.generators.diffusion.create_diffusion
+    timestep_respacing: null
+    noise_schedule: linear
+    diffusion_steps: 1000
+    learn_sigma: true
+    predict_xstart: false
+    sigma_small: false
+    use_kl: false
+    rescale_learned_sigmas: false
+wandb:
+  project: PMem-SS
+  name: null
+  entity: null
+  mode: online
+  group: soundspaces_latent
+  tags:
+  - soundspaces_latent
+run_name: ss_cdit_b_pnp_meta_posefa_ps2
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 256
+image_width: 256
+len_traj_pred: 128
+goals_per_obs: 4
+lr: 0.0001
+lr_mem: 0.0001
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 5000000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 100000
+latest_every: 10000
+eval_every: 2500
+rag_every: 20
+rag_noise_levels: 4
+local_rag: true
+local_rag_weight: 1.0
+bfloat16: false
+torch_compile: true
+batch_size: 16
+eval_batch_size: 32
+num_workers: 12
+datasets:
+- soundspaces_latent

soundspaces_v1/far_multicue/checkpoints/0700000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9bbaf89285ebd73300b4057fd1a1f44d52203a1767751ece35ea4613ae9f42ea
+size 591338717

soundspaces_v1/far_multicue/config.yaml

@@ -0,0 +1,458 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys.npy
+      standard_frame: false
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+soundspaces_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces/soundspaces_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: .keys_audio.npy
+      audio_mode: cue
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_train.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal}
+      transform: ${soundspaces_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_test.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal_test}
+      transform: ${soundspaces_latent.shared.transform}
+model:
+  tokenizer:
+    _target_: far.tokenizers.sd_vae.SD_VAE
+    image_height: ${image_height}
+    image_width: ${image_width}
+    vae_scale: 0.13025
+    ckpt_path: models/sdxl-vae-fp16-fix
+  generator:
+    _target_: far.generators.cdit_mod_pose_fa.CDiT
+    depth: 12
+    hidden_size: 768
+    patch_size: 2
+    num_heads: 12
+    context_size: 4
+    ckpt_path: null
+    action_dim: ${action_dim}
+    pose_dim: ${pose_dim}
+    fov_half_h: 45.0
+    fov_half_v: 45.0
+    local_pos_stats: ${soundspaces_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+  memory:
+    _target_: far.memories.pnp_cl.PnPRetrieval
+    ckpt_path: null
+    chunk_size: 10
+    num_sets: 5
+    softmax_temperature: 1.0
+    scoring: visual+meta_z
+    metadata_hidden: 128
+    local_pos_stats: ${soundspaces_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+    encoder_mode: adapter
+    adapter_hidden: 384
+    query_source: cue
+    gate_eps: 0.05
+    query_vit:
+      _target_: far.memories.pnp_cl.QueryViT
+      img_height: 64
+      img_width: 48
+      patch_size: 4
+      in_chans: 4
+      embed_dim: 384
+      depth: 6
+      num_heads: 12
+      action_dim: ${action_dim}
+      key_dim: 256
+      mlp_ratio: 4.0
+  _target_: far.builders.models.build_ldwm
+  _recursive_: false
+  condition_on_context_time: true
+  condition_on_context_pose: true
+  retriever_loss_weight: 1.0
+  diffusion:
+    _target_: far.generators.diffusion.create_diffusion
+    timestep_respacing: null
+    noise_schedule: linear
+    diffusion_steps: 1000
+    learn_sigma: true
+    predict_xstart: false
+    sigma_small: false
+    use_kl: false
+    rescale_learned_sigmas: false
+wandb:
+  project: PMem-SS
+  name: null
+  entity: null
+  mode: online
+  group: soundspaces_latent
+  tags:
+  - soundspaces_latent
+run_name: ss_cdit_b_pnp_amz_posefa_ps2
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 256
+image_width: 256
+len_traj_pred: 128
+goals_per_obs: 4
+lr: 0.0001
+lr_mem: 0.0001
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 5000000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 100000
+latest_every: 10000
+eval_every: 2500
+rag_every: 20
+rag_noise_levels: 4
+local_rag: true
+local_rag_weight: 1.0
+bfloat16: false
+torch_compile: true
+batch_size: 16
+eval_batch_size: 32
+num_workers: 12
+datasets:
+- soundspaces_latent

soundspaces_v1/temporal/checkpoints/0700000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:323116a1764a44b0c53c055085239dcb095647809131c9bc5f77204ee789955b
+size 524576185

soundspaces_v1/temporal/config.yaml

@@ -0,0 +1,435 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys.npy
+      standard_frame: false
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+soundspaces_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces/soundspaces_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_train.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal}
+      transform: ${soundspaces_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_test.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal_test}
+      transform: ${soundspaces_latent.shared.transform}
+model:
+  tokenizer:
+    _target_: far.tokenizers.sd_vae.SD_VAE
+    image_height: ${image_height}
+    image_width: ${image_width}
+    vae_scale: 0.13025
+    ckpt_path: models/sdxl-vae-fp16-fix
+  generator:
+    _target_: far.generators.cdit_mod_pose_fa.CDiT
+    depth: 12
+    hidden_size: 768
+    patch_size: 2
+    num_heads: 12
+    context_size: 4
+    ckpt_path: null
+    action_dim: ${action_dim}
+    pose_dim: ${pose_dim}
+    fov_half_h: 45.0
+    fov_half_v: 45.0
+    local_pos_stats: ${soundspaces_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+  memory:
+    _target_: far.memories.baselines.TemporalRetrieval
+    interval: 1.0
+  _target_: far.builders.models.build_ldwm
+  _recursive_: false
+  condition_on_context_time: true
+  condition_on_context_pose: true
+  retriever_loss_weight: 1.0
+  diffusion:
+    _target_: far.generators.diffusion.create_diffusion
+    timestep_respacing: null
+    noise_schedule: linear
+    diffusion_steps: 1000
+    learn_sigma: true
+    predict_xstart: false
+    sigma_small: false
+    use_kl: false
+    rescale_learned_sigmas: false
+wandb:
+  project: PMem-SS
+  name: null
+  entity: null
+  mode: online
+  group: soundspaces_latent
+  tags:
+  - soundspaces_latent
+run_name: ss_cdit_b_temporal_posefa_ps2
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 256
+image_width: 256
+len_traj_pred: 128
+goals_per_obs: 4
+lr: 0.0001
+lr_mem: null
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 5000000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 100000
+latest_every: 10000
+eval_every: 2500
+rag_every: 0
+rag_noise_levels: 4
+local_rag: false
+local_rag_weight: 1.0
+bfloat16: false
+torch_compile: true
+batch_size: 9
+eval_batch_size: 16
+num_workers: 12
+datasets:
+- soundspaces_latent

soundspaces_v1/worldmem/checkpoints/0700000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4a25b3c10e25baa68360ffe6e088f063a1f79112f58add499d7ce0ca23c4c22
+size 524576185

soundspaces_v1/worldmem/config.yaml

@@ -0,0 +1,440 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys.npy
+      standard_frame: false
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: results/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: results/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+soundspaces_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: results/manifests/soundspaces/soundspaces_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: results/indices/soundspaces/soundspaces_latent_train.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal}
+      transform: ${soundspaces_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: results/indices/soundspaces/soundspaces_latent_test.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal_test}
+      transform: ${soundspaces_latent.shared.transform}
+model:
+  tokenizer:
+    _target_: far.tokenizers.sd_vae.SD_VAE
+    image_height: ${image_height}
+    image_width: ${image_width}
+    vae_scale: 0.13025
+    ckpt_path: madebyollin/sdxl-vae-fp16-fix
+  generator:
+    _target_: far.generators.cdit_mod_pose_fa.CDiT
+    depth: 12
+    hidden_size: 768
+    patch_size: 2
+    num_heads: 12
+    context_size: 4
+    ckpt_path: null
+    action_dim: ${action_dim}
+    pose_dim: ${pose_dim}
+    fov_half_h: 45.0
+    fov_half_v: 45.0
+    local_pos_stats: ${soundspaces_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+  memory:
+    _target_: far.memories.worldmem.WorldMemRetrieval
+    num_samples: 10000
+    radius: 5.0
+    fov_half_h: 45.0
+    fov_half_v: 45.0
+    recency_factor: 0.2
+    local_pos_stats: ${soundspaces_latent.spec.local_pos_stats}
+  _target_: far.builders.models.build_ldwm
+  _recursive_: false
+  condition_on_context_time: true
+  condition_on_context_pose: true
+  retriever_loss_weight: 1.0
+  diffusion:
+    _target_: far.generators.diffusion.create_diffusion
+    timestep_respacing: null
+    noise_schedule: linear
+    diffusion_steps: 1000
+    learn_sigma: true
+    predict_xstart: false
+    sigma_small: false
+    use_kl: false
+    rescale_learned_sigmas: false
+wandb:
+  project: PMem-SS
+  name: null
+  entity: null
+  mode: online
+  group: soundspaces_latent
+  tags:
+  - soundspaces_latent
+run_name: ss_cdit_b_worldmem_posefa_ps2
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 256
+image_width: 256
+len_traj_pred: 128
+goals_per_obs: 4
+lr: 0.0001
+lr_mem: null
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 5000000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 100000
+latest_every: 10000
+eval_every: 2500
+rag_every: 0
+rag_noise_levels: 4
+local_rag: false
+local_rag_weight: 1.0
+bfloat16: false
+torch_compile: true
+batch_size: 16
+eval_batch_size: 32
+num_workers: 12
+datasets:
+- soundspaces_latent

soundspaces_v2/far_meta/checkpoints/0700000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c521386dc7045c4aa24d555dfbc64d18f0760adfb95cb3a6b6d141b66eaf5cc6
+size 590474195

soundspaces_v2/far_meta/config.yaml

@@ -0,0 +1,550 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys.npy
+      standard_frame: false
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+soundspaces_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces/soundspaces_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_train.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal}
+      transform: ${soundspaces_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_test.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal_test}
+      transform: ${soundspaces_latent.shared.transform}
+soundspaces_v2_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces_v2/soundspaces_v2_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: .keys_audio.npy
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_v2_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces_v2/soundspaces_v2_latent_train.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal}
+      transform: ${soundspaces_v2_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces_v2/soundspaces_v2_latent_test.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal_test}
+      transform: ${soundspaces_v2_latent.shared.transform}
+model:
+  tokenizer:
+    _target_: far.tokenizers.sd_vae.SD_VAE
+    image_height: ${image_height}
+    image_width: ${image_width}
+    vae_scale: 0.13025
+    ckpt_path: models/sdxl-vae-fp16-fix
+  generator:
+    _target_: far.generators.cdit_mod_pose_fa.CDiT
+    depth: 12
+    hidden_size: 768
+    patch_size: 2
+    num_heads: 12
+    context_size: 4
+    ckpt_path: null
+    action_dim: ${action_dim}
+    pose_dim: ${pose_dim}
+    fov_half_h: 45.0
+    fov_half_v: 45.0
+    local_pos_stats: ${soundspaces_v2_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+  memory:
+    _target_: far.memories.pnp_cl.PnPRetrieval
+    ckpt_path: null
+    chunk_size: 10
+    num_sets: 5
+    softmax_temperature: 1.0
+    scoring: meta
+    metadata_hidden: 128
+    local_pos_stats: ${soundspaces_v2_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+    encoder_mode: frozen
+    adapter_hidden: 384
+    query_source: frame
+    query_conditioning: action
+    gate_eps: 0.05
+    query_vit:
+      _target_: far.memories.pnp_cl.QueryViT
+      img_height: 32
+      img_width: 32
+      patch_size: 2
+      in_chans: 4
+      embed_dim: 384
+      depth: 6
+      num_heads: 12
+      action_dim: ${action_dim}
+      key_dim: 256
+      mlp_ratio: 4.0
+  _target_: far.builders.models.build_ldwm
+  _recursive_: false
+  condition_on_context_time: true
+  condition_on_context_pose: true
+  retriever_loss_weight: 1.0
+  diffusion:
+    _target_: far.generators.diffusion.create_diffusion
+    timestep_respacing: null
+    noise_schedule: linear
+    diffusion_steps: 1000
+    learn_sigma: true
+    predict_xstart: false
+    sigma_small: false
+    use_kl: false
+    rescale_learned_sigmas: false
+wandb:
+  project: PMem-SS
+  name: null
+  entity: null
+  mode: online
+  group: soundspaces_v2_latent
+  tags:
+  - soundspaces_v2_latent
+run_name: ssv2_cdit_b_pnp_meta_posefa_ps2
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 256
+image_width: 256
+len_traj_pred: 128
+goals_per_obs: 4
+lr: 0.0001
+lr_mem: 0.0001
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 5000000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 100000
+latest_every: 10000
+eval_every: 2500
+rag_every: 20
+rag_noise_levels: 4
+local_rag: true
+local_rag_weight: 1.0
+bfloat16: false
+torch_compile: true
+batch_size: 16
+eval_batch_size: 32
+num_workers: 12
+datasets:
+- soundspaces_v2_latent

soundspaces_v2/far_multicue/checkpoints/0700000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32409877c67f1a8658364e1b2360e6cc73749979c80c6fd7afeba9a2a5c4e9cf
+size 591338717

soundspaces_v2/far_multicue/config.yaml

@@ -0,0 +1,550 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys.npy
+      standard_frame: false
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+soundspaces_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces/soundspaces_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_train.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal}
+      transform: ${soundspaces_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_test.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal_test}
+      transform: ${soundspaces_latent.shared.transform}
+soundspaces_v2_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces_v2/soundspaces_v2_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: .keys_audio.npy
+      audio_mode: cue
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_v2_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces_v2/soundspaces_v2_latent_train.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal}
+      transform: ${soundspaces_v2_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces_v2/soundspaces_v2_latent_test.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal_test}
+      transform: ${soundspaces_v2_latent.shared.transform}
+model:
+  tokenizer:
+    _target_: far.tokenizers.sd_vae.SD_VAE
+    image_height: ${image_height}
+    image_width: ${image_width}
+    vae_scale: 0.13025
+    ckpt_path: models/sdxl-vae-fp16-fix
+  generator:
+    _target_: far.generators.cdit_mod_pose_fa.CDiT
+    depth: 12
+    hidden_size: 768
+    patch_size: 2
+    num_heads: 12
+    context_size: 4
+    ckpt_path: null
+    action_dim: ${action_dim}
+    pose_dim: ${pose_dim}
+    fov_half_h: 45.0
+    fov_half_v: 45.0
+    local_pos_stats: ${soundspaces_v2_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+  memory:
+    _target_: far.memories.pnp_cl.PnPRetrieval
+    ckpt_path: null
+    chunk_size: 10
+    num_sets: 5
+    softmax_temperature: 1.0
+    scoring: visual+meta_z
+    metadata_hidden: 128
+    local_pos_stats: ${soundspaces_v2_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+    encoder_mode: adapter
+    adapter_hidden: 384
+    query_source: cue
+    query_conditioning: action
+    gate_eps: 0.05
+    query_vit:
+      _target_: far.memories.pnp_cl.QueryViT
+      img_height: 64
+      img_width: 48
+      patch_size: 4
+      in_chans: 4
+      embed_dim: 384
+      depth: 6
+      num_heads: 12
+      action_dim: ${action_dim}
+      key_dim: 256
+      mlp_ratio: 4.0
+  _target_: far.builders.models.build_ldwm
+  _recursive_: false
+  condition_on_context_time: true
+  condition_on_context_pose: true
+  retriever_loss_weight: 1.0
+  diffusion:
+    _target_: far.generators.diffusion.create_diffusion
+    timestep_respacing: null
+    noise_schedule: linear
+    diffusion_steps: 1000
+    learn_sigma: true
+    predict_xstart: false
+    sigma_small: false
+    use_kl: false
+    rescale_learned_sigmas: false
+wandb:
+  project: PMem-SS
+  name: null
+  entity: null
+  mode: online
+  group: soundspaces_v2_latent
+  tags:
+  - soundspaces_v2_latent
+run_name: ssv2_cdit_b_pnp_amz_posefa_ps2
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 256
+image_width: 256
+len_traj_pred: 128
+goals_per_obs: 4
+lr: 0.0001
+lr_mem: 0.0001
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 5000000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 100000
+latest_every: 10000
+eval_every: 2500
+rag_every: 20
+rag_noise_levels: 4
+local_rag: true
+local_rag_weight: 1.0
+bfloat16: false
+torch_compile: true
+batch_size: 16
+eval_batch_size: 32
+num_workers: 12
+datasets:
+- soundspaces_v2_latent

soundspaces_v2/temporal/checkpoints/0700000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a21bbe5a81d3b61eebc1ae2def893dc1cd3fc37ee72dda8594980301c64bf321
+size 524576185

soundspaces_v2/temporal/config.yaml

@@ -0,0 +1,526 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys.npy
+      standard_frame: false
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+soundspaces_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces/soundspaces_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_train.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal}
+      transform: ${soundspaces_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_test.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal_test}
+      transform: ${soundspaces_latent.shared.transform}
+soundspaces_v2_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces_v2/soundspaces_v2_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_v2_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces_v2/soundspaces_v2_latent_train.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal}
+      transform: ${soundspaces_v2_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces_v2/soundspaces_v2_latent_test.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal_test}
+      transform: ${soundspaces_v2_latent.shared.transform}
+model:
+  tokenizer:
+    _target_: far.tokenizers.sd_vae.SD_VAE
+    image_height: ${image_height}
+    image_width: ${image_width}
+    vae_scale: 0.13025
+    ckpt_path: models/sdxl-vae-fp16-fix
+  generator:
+    _target_: far.generators.cdit_mod_pose_fa.CDiT
+    depth: 12
+    hidden_size: 768
+    patch_size: 2
+    num_heads: 12
+    context_size: 4
+    ckpt_path: null
+    action_dim: ${action_dim}
+    pose_dim: ${pose_dim}
+    fov_half_h: 45.0
+    fov_half_v: 45.0
+    local_pos_stats: ${soundspaces_v2_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+  memory:
+    _target_: far.memories.baselines.TemporalRetrieval
+    interval: 1.0
+  _target_: far.builders.models.build_ldwm
+  _recursive_: false
+  condition_on_context_time: true
+  condition_on_context_pose: true
+  retriever_loss_weight: 1.0
+  diffusion:
+    _target_: far.generators.diffusion.create_diffusion
+    timestep_respacing: null
+    noise_schedule: linear
+    diffusion_steps: 1000
+    learn_sigma: true
+    predict_xstart: false
+    sigma_small: false
+    use_kl: false
+    rescale_learned_sigmas: false
+wandb:
+  project: PMem-SS
+  name: null
+  entity: null
+  mode: online
+  group: soundspaces_v2_latent
+  tags:
+  - soundspaces_v2_latent
+run_name: ssv2_cdit_b_temporal_posefa_ps2
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 256
+image_width: 256
+len_traj_pred: 128
+goals_per_obs: 4
+lr: 0.0001
+lr_mem: null
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 5000000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 100000
+latest_every: 10000
+eval_every: 2500
+rag_every: 0
+rag_noise_levels: 4
+local_rag: false
+local_rag_weight: 1.0
+bfloat16: false
+torch_compile: true
+batch_size: 9
+eval_batch_size: 16
+num_workers: 12
+datasets:
+- soundspaces_v2_latent

soundspaces_v2/worldmem/checkpoints/0700000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ef717aa64f35123d4567e204027d054534129fac18c527d3c266b4a24f0762f
+size 524576185

soundspaces_v2/worldmem/config.yaml

@@ -0,0 +1,531 @@
+recon:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      max:
+      - 5
+      - 4
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 2
+    angle_dim: 1
+    pose_mode: 2d
+    fps: 4.0
+  shared:
+    manifest_path: results/manifests/recon/recon_split.json
+    backend:
+      _target_: far.data.backends.NavigationBackend
+    transform:
+      _target_: far.data.transforms.make_centercrop_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 12
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter2D
+      spec: ${recon.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_train.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_goal}
+      transform: ${recon.shared.transform}
+  eval:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_time.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+  eval_traj:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${recon.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${recon.shared.manifest_path}
+      index_path: results/indices/recon/recon_test_plan.json
+      backend: ${recon.shared.backend}
+      sampler: ${recon.sampler.context_traj}
+      transform: ${recon.shared.transform}
+loopnav:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - -2.5
+      - -4
+      - 0
+      max:
+      - 5
+      - 4
+      - 1
+    meters_per_waypoint: 0.25
+    max_frame_offset: 128
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: results/manifests/loopnav/loopnav_split.json
+    backend:
+      _target_: far.data.backends.LoopNavBackend
+      standard_frame: false
+    transform:
+      _target_: far.data.transforms.make_resize
+      image_height: ${image_height}
+      image_width: ${image_width}
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 100
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav.spec}
+  train_tokenizer:
+    _target_: far.data.datasets.FormattedIterableDataset
+    formatter: ${loopnav.formatter.frame}
+    base_dataset:
+      _target_: far.data.datasets.WebVideoIterableDataset
+      manifest_path: results/manifests/loopnav/loopnav.json
+      index_path: results/indices/loopnav/loopnav_train_ABCA.json
+      backend:
+        _target_: far.data.backends.WebDatasetBackend
+      sampler: ${loopnav.sampler.random_clip}
+      transform: ${loopnav.shared.transform}
+      cache_dir: null
+      cache_size: 100
+      shuffle_shards: 32
+      shuffle_samples: 100
+loopnav_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 20.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 20.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/loopnav/loopnav_latent.json
+    backend:
+      _target_: far.data.backends.LatentLoopNavBackend
+      key_suffix: .keys.npy
+      standard_frame: false
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 10
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${loopnav_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_train.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal}
+      transform: ${loopnav_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${loopnav_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${loopnav_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_test.json
+      backend: ${loopnav_latent.shared.backend}
+      sampler: ${loopnav_latent.sampler.context_goal_test}
+      transform: ${loopnav_latent.shared.transform}
+soundspaces_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces/soundspaces_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_train.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal}
+      transform: ${soundspaces_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces/soundspaces_latent_test.json
+      backend: ${soundspaces_latent.shared.backend}
+      sampler: ${soundspaces_latent.sampler.context_goal_test}
+      transform: ${soundspaces_latent.shared.transform}
+soundspaces_v2_latent:
+  spec:
+    _target_: far.builders.data.build_dataspec
+    local_pos_stats:
+      min:
+      - 0
+      - 0
+      - 0
+      max:
+      - 1
+      - 1
+      - 1
+    meters_per_waypoint: 1.0
+    max_frame_offset: 10.0
+    len_traj_pred: ${len_traj_pred}
+    planner_mu_init:
+    - -0.1
+    - 0
+    - 0
+    - 0
+    - 0
+    planner_sigma_init:
+    - 0.02
+    - 0.1
+    - 0
+    - 0
+    - 0.3142
+    pose_fields:
+    - x
+    - 'y'
+    - z
+    - pitch
+    - yaw
+    pose_dim: 5
+    pos_dim: 3
+    angle_dim: 2
+    pose_mode: 3d
+    fps: 10.0
+  shared:
+    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/soundspaces_v2/soundspaces_v2_latent.json
+    backend:
+      _target_: far.data.backends.SoundSpacesBackend
+      key_suffix: null
+      audio_mode: null
+      audio_window_hops: 48
+    transform: null
+  sampler:
+    random_clip:
+      _target_: far.data.samplers.RandomClipSampler
+      clip_len: 16
+      frame_stride: 5
+    context_goal:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 400
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: ${goals_per_obs}
+    context_goal_test:
+      _target_: far.data.samplers.ContextGoalSampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+      goals_per_obs: 4
+    context_traj:
+      _target_: far.data.samplers.ContextTrajectorySampler
+      context_pool_size: 200
+      len_traj_pred: ${len_traj_pred}
+  formatter:
+    frame:
+      _target_: far.data.formatters.FrameFormatter
+    context_pose:
+      _target_: far.data.formatters.ContextPoseFormatter
+      spec: ${soundspaces_v2_latent.spec}
+  train:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces_v2/soundspaces_v2_latent_train.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal}
+      transform: ${soundspaces_v2_latent.shared.transform}
+  test:
+    _target_: far.data.datasets.FormattedVideoDataset
+    formatter: ${soundspaces_v2_latent.formatter.context_pose}
+    base_dataset:
+      _target_: far.data.datasets.VideoDataset
+      manifest_path: ${soundspaces_v2_latent.shared.manifest_path}
+      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/soundspaces_v2/soundspaces_v2_latent_test.json
+      backend: ${soundspaces_v2_latent.shared.backend}
+      sampler: ${soundspaces_v2_latent.sampler.context_goal_test}
+      transform: ${soundspaces_v2_latent.shared.transform}
+model:
+  tokenizer:
+    _target_: far.tokenizers.sd_vae.SD_VAE
+    image_height: ${image_height}
+    image_width: ${image_width}
+    vae_scale: 0.13025
+    ckpt_path: models/sdxl-vae-fp16-fix
+  generator:
+    _target_: far.generators.cdit_mod_pose_fa.CDiT
+    depth: 12
+    hidden_size: 768
+    patch_size: 2
+    num_heads: 12
+    context_size: 4
+    ckpt_path: null
+    action_dim: ${action_dim}
+    pose_dim: ${pose_dim}
+    fov_half_h: 45.0
+    fov_half_v: 45.0
+    local_pos_stats: ${soundspaces_v2_latent.spec.local_pos_stats}
+    stride_to_seconds: 1.0
+  memory:
+    _target_: far.memories.worldmem.WorldMemRetrieval
+    num_samples: 10000
+    radius: 5.0
+    fov_half_h: 45.0
+    fov_half_v: 45.0
+    recency_factor: 0.2
+    local_pos_stats: ${soundspaces_v2_latent.spec.local_pos_stats}
+  _target_: far.builders.models.build_ldwm
+  _recursive_: false
+  condition_on_context_time: true
+  condition_on_context_pose: true
+  retriever_loss_weight: 1.0
+  diffusion:
+    _target_: far.generators.diffusion.create_diffusion
+    timestep_respacing: null
+    noise_schedule: linear
+    diffusion_steps: 1000
+    learn_sigma: true
+    predict_xstart: false
+    sigma_small: false
+    use_kl: false
+    rescale_learned_sigmas: false
+wandb:
+  project: PMem-SS
+  name: null
+  entity: null
+  mode: online
+  group: soundspaces_v2_latent
+  tags:
+  - soundspaces_v2_latent
+run_name: ssv2_cdit_b_worldmem_posefa_ps2
+context_size: 4
+action_dim: 5
+pose_dim: 5
+image_height: 256
+image_width: 256
+len_traj_pred: 128
+goals_per_obs: 4
+lr: 0.0001
+lr_mem: null
+weight_decay: 0.0
+grad_clip_val: 1.0
+iterations: 5000000
+warmup_steps: 5000
+from_checkpoint: null
+global_seed: 0
+log_every: 100
+ckpt_every: 100000
+latest_every: 10000
+eval_every: 2500
+rag_every: 0
+rag_noise_levels: 4
+local_rag: false
+local_rag_weight: 1.0
+bfloat16: false
+torch_compile: true
+batch_size: 16
+eval_batch_size: 32
+num_workers: 12
+datasets:
+- soundspaces_v2_latent