Add a foundation-stereo option for warping detections between stereo cameras

client/dive-common/components/TrackSettingsPanel.vue

@@ -7,6 +7,7 @@ import {
   computed,
 } from 'vue';
 import { clientSettings } from 'dive-common/store/settings';
+import { STEREO_MATCH_METHODS } from 'dive-common/use/stereo/stereoMatcher';
 import isDesktopRuntime from 'dive-common/isDesktopRuntime';
 
 export default defineComponent({
@@ -40,6 +41,7 @@ export default defineComponent({
       showMultiCamToolbar: 'Show multi-camera tools in the top toolbar when a track is selected',
       stereoUpdateLengths: 'When a line annotation is modified on a detection that is linked across both cameras, recompute its stereo measurement (length, midpoint, range, RMS) automatically.',
       stereoAutoCompute: 'When an annotation is drawn on one camera and the other camera has no detection for it yet, automatically warp it to the other camera using stereo disparity.',
+      stereoMatchMethod: 'How a point is matched on the other camera. Template matching correlates the source patch along the epipolar line. Foundation stereo runs a dense disparity network over the pair once, which is steadier where the patch is hard to match but needs its model to be configured.',
     });
     const modes = ref(['Track', 'Detection']);
     // Add unknown as the default type to the typeList
@@ -52,6 +54,7 @@ export default defineComponent({
       help,
       modes,
       typeList,
+      stereoMatchMethods: STEREO_MATCH_METHODS,
     };
   },
 });
@@ -445,6 +448,42 @@ export default defineComponent({
             </v-tooltip>
           </v-col>
         </v-row>
+        <v-row
+          align="end"
+          dense
+        >
+          <v-col class="py-1">
+            <v-select
+              v-model="clientSettings.stereoSettings.matchMethod"
+              :items="stereoMatchMethods"
+              class="my-0 ml-1 pt-0"
+              dense
+              hide-details
+              label="Correspondence method"
+            />
+          </v-col>
+          <v-col
+            cols="2"
+            class="py-1"
+            align="right"
+          >
+            <v-tooltip
+              open-delay="200"
+              max-width="200"
+              bottom
+            >
+              <template #activator="{ on }">
+                <v-icon
+                  small
+                  v-on="on"
+                >
+                  mdi-help
+                </v-icon>
+              </template>
+              <span>{{ help.stereoMatchMethod }}</span>
+            </v-tooltip>
+          </v-col>
+        </v-row>
       </template>
     </v-card>
   </div>

client/dive-common/store/settings.ts

@@ -2,6 +2,8 @@ import { Ref, watch, reactive } from 'vue';
 import { cloneDeep, merge } from 'lodash';
 import { AnnotatorPreferences } from 'vue-media-annotator/types';
 import isDesktopRuntime from 'dive-common/isDesktopRuntime';
+import { DEFAULT_STEREO_MATCH_METHOD } from 'dive-common/use/stereo/stereoMatcher';
+import type { StereoMatchMethod } from 'dive-common/use/stereo/stereoMatcher';
 
 interface ColumnVisibilitySettings {
   type: boolean;
@@ -94,6 +96,9 @@ interface AnnotationSettings {
     // Warp an annotation drawn on one camera to the other camera when that
     // camera has no detection for it yet.
     autoComputeOtherCamera: boolean;
+    // Which correspondence method the warp uses: 'ncc' template matching or
+    // 'foundation' dense disparity.
+    matchMethod: StereoMatchMethod;
     loading: boolean;
     loadingMessage: string;
   };
@@ -191,6 +196,7 @@ const defaultSettings: AnnotationSettings = {
     clearLengthOnCameraFileLoad: true,
     updateLengthsOnModify: true,
     autoComputeOtherCamera: false,
+    matchMethod: DEFAULT_STEREO_MATCH_METHOD,
     loading: false,
     loadingMessage: '',
   },

client/dive-common/use/stereo/README.md

@@ -1,9 +1,20 @@
 # Client-side stereo transfer and measurement (ONNX)
 
 Warp a detection annotated on one camera onto the other camera and measure its
-length, entirely in the browser / Electron renderer — no backend — using VIAME's
-epipolar template-matching model (stereo measurement "method 1") exported to
-ONNX and run with `onnxruntime-web`.
+length, entirely in the browser / Electron renderer — no backend — running the
+correspondence model with `onnxruntime-web`.
+
+Two correspondence methods are available, chosen from **Track Settings → Stereo
+Settings → Correspondence method**:
+
+| Method | Model | How it matches |
+| --- | --- | --- |
+| **Template matching (NCC)** — default | VIAME's epipolar template-matching model (stereo measurement "method 1"), bundled | Per point: generate epipolar candidates, NCC the source patch along that curve |
+| **Foundation stereo (disparity)** | A Fast-FoundationStereo ONNX export, **not bundled** | Once per frame: rectify the pair, run a dense disparity network, read each point's shift out of the map |
+
+They are interchangeable behind the `StereoMatcher` interface, so everything
+downstream — box/line/polygon warping, measurement, bulk transfer — is identical
+either way.
 
 This is the client counterpart to the desktop backend stereo service: the
 desktop `ViewerLoader` warps and measures via native IPC (`stereoTransferLine` /
@@ -15,6 +26,9 @@ work client-side so it also works on the web.
 | File | Role |
 | --- | --- |
 | `StereoOnnxMatcher.ts` | Loads the `match` ONNX model and warps source points → target points via NCC along the epipolar curve. |
+| `StereoFoundationMatcher.ts` | Loads a Fast-FoundationStereo ONNX export, rectifies the pair into the network's input resolution, and reads each point's correspondence from the dense disparity map. |
+| `stereoMatcher.ts` | The `StereoMatcher` contract both matchers satisfy, the `StereoMatchMethod` union, and the dropdown's labels. |
+| `rectify.ts` | Stereo rectification ported from OpenCV `cvStereoRectify` (Rodrigues, rectifying rotations, point rectify/unrectify, and the inverse map used to sample a rectified image). Only the foundation method needs it. |
 | `calibration.ts` | `StereoRig` + loaders (`rigFromNpz`, `rigFromJson`) mirroring VIAME's `read_stereo_rig`; `invertRig` to swap the source/target camera. |
 | `npz.ts` | Minimal `.npz`/`.npy` reader (calibration files are NumPy archives). |
 | `image.ts` | RGBA → BT.601 grayscale (matches OpenCV `BGR2GRAY` used by the C++ NCC). |
@@ -107,3 +121,71 @@ The disparity range is scene-dependent — VIAME's batch measurement pipes ship
   binding, calibration download, and the GeoJS frame-pixel read in
   `frameSource.geoViewerToImageElement`) is type-checked and lint-clean but has
   not been exercised in a running web viewer with a real stereo dataset.
+
+
+## Foundation stereo method
+
+### Why a second method
+
+The NCC matcher needs the source patch to be photometrically matchable in the
+other view. Where that fails — obstructed viewpoints, repetitive substrate, low
+contrast — it either mismatches or declines. A dense disparity network does not
+depend on patch correlation, and it costs one network pass per frame no matter
+how many points are warped, so bulk-warping a whole camera amortises well.
+
+Its trade is setup: the model is large and must be supplied.
+
+### Supplying the model
+
+Unlike the NCC graph (small, committed at `client/public/models/stereo_match.onnx`),
+Fast-FoundationStereo exports run ~100 MB and are **not** committed. Obtain an
+export from the Fast-FoundationStereo release, serve it, and point the web glue
+at it:
+
+```ts
+useStereoOnnxWeb({
+  ...,
+  foundationModelUrl: '/models/stereo_foundation.onnx',
+  foundationModelSpec: { height: 576, width: 960 },  // the export's sidecar image_size
+});
+```
+
+The default URL is `/models/stereo_foundation.onnx` and the default spec is
+576×960. `foundationModelSpec` **must** match the export: the graph fixes its
+input resolution, and the sidecar `.yaml` shipped beside each export gives it as
+`image_size: [H, W]`. With no model served, selecting the method reports that it
+could not load and the warp no-ops — the same way a missing calibration does.
+
+### How it works
+
+1. Solve the rectifying rotations for the rig once per calibration
+   (`computeRectification`), sized to the network's input resolution.
+2. Build the rectified pair by inverse-mapping each output pixel back to its
+   source pixel and bilinear-sampling. Rectify and resize are fused, so the cost
+   is the network's resolution rather than the frame's.
+3. Run the network to get dense disparity in rectified pixels.
+4. Per point: rectify it, pool the disparities in a small window by median,
+   shift `x` by that disparity, and unrectify into the target image.
+
+Step 4 pools rather than sampling the single pixel deliberately. A head or tail
+tip is a couple of pixels wide at the network's working resolution, so the
+disparity exactly at the tip is frequently the background's; the median over a
+small window rejects that without dragging the estimate off the animal.
+
+The network emits no confidence channel, so the reported `score` is the fraction
+of the pooled window carrying a finite positive disparity, and a match is
+accepted when that clears `DEFAULT_MIN_VALID_FRACTION` **and** the implied
+disparity falls inside the configured search range — the same range that bounds
+the NCC search.
+
+### Testing status
+
+- **Tested** (`tests/rectify.spec.ts`): Rodrigues round-trip, orthonormality of
+  the rectifying rotations, the defining rectification property (a 3D point
+  lands on the same row in both rectified views), disparity positive and
+  decreasing with range, and pixel round-trip through rectify/unrectify with and
+  without distortion.
+- **Not tested**: `StereoFoundationMatcher` end-to-end, which needs a ~100 MB
+  model the repo does not carry. The geometry it depends on is covered above;
+  the network call, disparity pooling and the settings dropdown have not been
+  exercised against a real export in a running viewer.