Apply the interpolation options to videos geotagged from a GPX

README.md

@@ -375,6 +375,10 @@ mapillary_tools process MY_IMAGE_DIR \
     --interpolation_offset_time -28800
 ```
 
+The offset applies to videos geotagged from the GPX as well. In a directory that mixes images and videos, an offset
+that corrects the camera clock of the images also moves the track of each video, and a video that the moved track
+misses fails. To avoid that, process the images on their own with `--file_types image`.
+
 Another option `--interpolation_use_gpx_start_time` moves your images to align with the beginning of the GPS track.
 This is useful when you can confirm that you start GPS recording and capturing at the same time, or with a known delay.
 For a video without a GPS clock, a track that lasts more than twice as long as the video spans at its
@@ -472,9 +476,20 @@ VIDEO_DIR/
 
 ### Limitations of `--video_geotag_source`
 
-**External geolocation sources will be aligned with the start of video, there is
-currently no way of applying offsets or scaling the time.** This means, for instance, that GPX tracks must begin precisely
-at the instant of the video, and that timelapse videos are supported only for sources `camm, gopro, blackvue`.
+**A GPX track is synced by the GPS clock of the video, if the video has GPS of its own, and otherwise aligned with
+the start of the video.** In `process`, a timelapse video is supported for a GPX only if it has GPS of its own.
+
+As with images, `--interpolation_offset_time N` adds N seconds to the time of the video to find its positions along
+the GPX track, and `--interpolation_use_gpx_start_time` aligns the start of the video with the start of the track, even
+if the video has GPS:
+
+```sh
+# Start recording the video 2.5 seconds after starting the GPS logger
+mapillary_tools process VIDEO_DIR/ \
+    --video_geotag_source gpx \
+    --interpolation_use_gpx_start_time \
+    --interpolation_offset_time 2.5
+```
 
 
 ## Authenticate

mapillary_tools/commands/process.py

@@ -149,15 +149,15 @@ def add_basic_arguments(self, parser: argparse.ArgumentParser):
         )
         group_geotagging.add_argument(
             "--interpolation_use_gpx_start_time",
-            help=f"If supplied, the first image will use the first GPX point time for interpolation, which means the image location will be interpolated to the first GPX point too. Only works for geotagging from {', '.join(geotag_gpx_based_sources)}.",
+            help=f"If supplied, the first image, or the start of a video, will use the first GPX point time for interpolation, which means the image location will be interpolated to the first GPX point too. Only works for geotagging from {', '.join(geotag_gpx_based_sources)}.",
             action="store_true",
             required=False,
         )
         group_geotagging.add_argument(
             "--interpolation_offset_time",
             default=0.0,
             type=float,
-            help=f"Time offset, in seconds, that will be added for GPX interpolation, which affects image locations. Note that it is applied after --interpolation_use_gpx_start_time. Only works for geotagging from {', '.join(geotag_gpx_based_sources)}. [default %(default)s]",
+            help=f"Time offset, in seconds, that will be added for GPX interpolation, which affects image and video locations. Note that it is applied after --interpolation_use_gpx_start_time. Only works for geotagging from {', '.join(geotag_gpx_based_sources)}. [default %(default)s]",
             required=False,
         )
         group_geotagging.add_argument(

mapillary_tools/geotag/factory.py

@@ -463,6 +463,7 @@ def _build_video_geotag(option: SourceOption) -> base.GeotagVideosFromGeneric |
             source_path=option.source_path,
             num_processes=option.num_processes,
             video_sync=option.video_sync,
+            interpolation=option.interpolation,
         )
 
     elif option.source is SourceType.NMEA:

mapillary_tools/geotag/geotag_videos_from_gpx.py

@@ -30,12 +30,14 @@ def __init__(
         source_path: options.SourcePathOption | None = None,
         num_processes: int | None = None,
         video_sync: options.VideoSyncOption | None = None,
+        interpolation: options.InterpolationOption | None = None,
     ):
         super().__init__(num_processes=num_processes)
         if source_path is None:
             source_path = options.SourcePathOption(pattern="%g.gpx")
         self.source_path = source_path
         self.video_sync = video_sync
+        self.interpolation = interpolation
 
     @override
     def _generate_video_extractors(
@@ -50,6 +52,7 @@ def _generate_video_extractors(
                         video_path,
                         source_path,
                         video_sync=self.video_sync,
+                        interpolation=self.interpolation,
                         # The track that process uploads with the video.
                         # Sampling reads its track only within the video
                         clip_to_video=self.video_sync is None,

mapillary_tools/geotag/options.py

@@ -151,21 +151,14 @@ class VideoSyncOption:
     """
     How to line up a GPX track with a video that is being sampled, so that its
     frames are picked where the geotagging stage of video_process will place
-    them. process geotags videos without it, and places a GPX as it always has.
+    them. The interpolation options, which process applies to videos as well,
+    are in SourceOption.interpolation.
     """
 
     # Unix time of the first frame of the video (--video_start_time), to place
     # a GPX by when the video has no GPS of its own to sync it with
     start_time: float | None = None
 
-    # --interpolation_offset_time: seconds the geotagging stage adds to the
-    # capture time of a frame to find its position along the GPX
-    offset_time: float = 0.0
-
-    # --interpolation_use_gpx_start_time: the geotagging stage puts the first
-    # frame at the start of the GPX
-    use_gpx_start_time: bool = False
-
     # --video_duration_ratio: GPS seconds per second of a video without a GPS
     # clock of its own to tell, such as a timelapse
     duration_ratio: float = 1.0

mapillary_tools/geotag/video_extractors/gpx.py

@@ -105,16 +105,18 @@ def __init__(
         gpx_path: Path,
         sync_mode: SyncMode = SyncMode.SYNC,
         video_sync: options.VideoSyncOption | None = None,
+        interpolation: options.InterpolationOption | None = None,
         clip_to_video: bool = False,
     ):
         self.video_path = video_path
         self.gpx_path = gpx_path
         self.sync_mode = sync_mode
         self.video_sync = video_sync
+        self.interpolation = interpolation
         # Clip the track to the video when a clock gives its pace along the
         # video (see _clip_to_video)
         self.clip_to_video = clip_to_video
-        if video_sync is not None and video_sync.use_gpx_start_time:
+        if interpolation is not None and interpolation.use_gpx_start_time:
             self.sync_mode = SyncMode.REBASE
 
     @override
@@ -314,23 +316,29 @@ def _apply_offset_time(
         self, gpx_points: T.Sequence[geo.Point], rate: float = 1.0
     ) -> None:
         """
-        Shift the track back by --interpolation_offset_time, which the
-        geotagging stage adds to the capture time of a frame to find its
-        position along the GPX. The offset is in GPS seconds, of which a
-        timelapse packs **rate** into each second of video.
+        Shift the track back by --interpolation_offset_time, which is added to
+        the capture time of a frame to find its position along the GPX. The
+        offset is in GPS seconds, of which a timelapse packs **rate** into each
+        second of video.
 
-        The absolute times shift with the track, so that a frame is stamped
-        with its capture time, and the geotagging stage, adding the offset back,
+        The absolute times stay those of the GPX, just as an image geotagged
+        with the offset takes the time of its position along the GPX. When
+        sampling, they shift with the track, so that a frame is stamped with
+        its capture time, and the geotagging stage, adding the offset back,
         finds the position the frame was picked by.
         """
-        if self.video_sync is None or not self.video_sync.offset_time:
+        if self.interpolation is None or not self.interpolation.offset_time:
             return
 
-        offset_time = self.video_sync.offset_time
+        offset_time = self.interpolation.offset_time
         for p in gpx_points:
             p.time -= offset_time / rate
             # parse_gpx() returns CAMMGPSPoint, with epoch_time 0 for no time
-            if isinstance(p, telemetry.CAMMGPSPoint) and 0 < p.epoch_time:
+            if (
+                self.video_sync is not None
+                and isinstance(p, telemetry.CAMMGPSPoint)
+                and 0 < p.epoch_time
+            ):
                 p.epoch_time -= offset_time
 
     def _check_coverage(
@@ -347,14 +355,16 @@ def _check_coverage(
         spans **rate** GPS seconds.
 
         A track that misses the video has no position for any moment of it, so
-        it raises when the clock it was placed by says so. A track that covers
-        only part of the video warns, and one that covers all of it is silent.
+        it raises when its placement says so: the clock it was placed by, or
+        --interpolation_offset_time. A track that covers only part of the video
+        warns, and one that covers all of it is silent.
 
         Without the duration of the video, the video is known only up to its
         last GPS point, and a track that starts after it may still overlap the
         video. Then a gap only warns, unless it is too large for any clock or
-        time zone mistake to explain. A track assumed to start with the video
-        has no clock to say that it misses the video, so it only warns.
+        time zone mistake to explain. A track placed at the start of the video,
+        whether assumed to start with it or by --interpolation_use_gpx_start_time,
+        can miss it only once --interpolation_offset_time moves it.
         """
         if not gpx_points or gpx_unix_times is None:
             return
@@ -368,12 +378,30 @@ def _check_coverage(
 
         gpx_first = min(p.time for p in gpx_points)
         gpx_last = max(p.time for p in gpx_points)
+        offset_time = (
+            0.0 if self.interpolation is None else self.interpolation.offset_time
+        )
+
+        # Where the start of the video is looked up along the track, in its
+        # Unix time: the offset is added to the time of every frame, as it is
+        # to the capture time of an image
+        lookup_start = (
+            gpx_unix_times[0] if clock is None else clock.to_unix_time(0)
+        ) + offset_time
 
         gpx_track = f"The GPX track in {self.gpx_path} ({_isoformat(gpx_unix_times[0])} to {_isoformat(gpx_unix_times[1])})"
         if clock is None:
             video = f"the video {self.video_path}"
         else:
             video = f"the video {self.video_path} ({_isoformat(clock.to_unix_time(0))} to {_isoformat(clock.to_unix_time(video_end))})"
+        # Without it, the times of the video read as if the track missed it
+        # by more or less than it does
+        offset_clause = ""
+        if offset_time:
+            offset_clause = (
+                f", as --interpolation_offset_time {offset_time:g} looks the video up along it "
+                f"({_isoformat(lookup_start)} to {_isoformat(lookup_start + video_end * rate)})"
+            )
 
         # In GPS seconds, which is where clock and time zone mistakes are made
         gap = max(gpx_first - video_end, -gpx_last) * rate
@@ -382,23 +410,36 @@ def _check_coverage(
             uncovered = max(gpx_first, 0.0) + max(video_end - gpx_last, 0.0)
             if _LOGGER_TOLERANCE_SECONDS < uncovered * rate:
                 LOG.warning(
-                    f"{gpx_track} covers only part of {video}: "
+                    f"{gpx_track} covers only part of {video}{offset_clause}: "
                     f"{_format_duration(uncovered)} of the video fall outside the track"
                 )
             return
 
+        if clock is None:
+            # Placed at the start of the video, only the offset moved it off
+            advice = "Check --interpolation_offset_time"
+        elif offset_time:
+            advice = (
+                "Check --interpolation_offset_time, that the GPX file belongs to "
+                "this video, then the camera clock and the time zone of the GPX "
+                "timestamps"
+            )
+        else:
+            advice = (
+                "Check that the GPX file belongs to this video, then the camera "
+                "clock and the time zone of the GPX timestamps"
+            )
         message = (
-            f"{gpx_track} misses {video} by {_format_duration(gap)}. Check that "
-            "the GPX file belongs to this video, then the camera clock and the "
-            "time zone of the GPX timestamps"
+            f"{gpx_track} misses {video} by {_format_duration(gap)}{offset_clause}. "
+            f"{advice}"
         )
 
-        if clock is not None and (
+        if (clock is not None or offset_time) and (
             video_duration is not None or _IMPLAUSIBLE_OFFSET_SECONDS < gap
         ):
             raise exceptions.MapillaryOutsideGPXTrackError(
                 message,
-                image_time=build_capture_time(clock.to_unix_time(0)),
+                image_time=build_capture_time(lookup_start),
                 gpx_start_time=build_capture_time(gpx_unix_times[0]),
                 gpx_end_time=build_capture_time(gpx_unix_times[1]),
             )

mapillary_tools/sample_video.py

@@ -30,7 +30,13 @@
 from .exif_write import ExifEdit
 from .geotag import factory
 from .geotag.geotag_images_from_video import GPSTimeline
-from .geotag.options import SourceOption, SourcePathOption, SourceType, VideoSyncOption
+from .geotag.options import (
+    InterpolationOption,
+    SourceOption,
+    SourcePathOption,
+    SourceType,
+    VideoSyncOption,
+)
 from .geotag.video_extractors.gpx import GPXVideoExtractor
 from .geotag.video_extractors.native import NativeVideoExtractor
 from .mp4 import mp4_sample_parser
@@ -72,6 +78,7 @@ def _parse_geotag_options(
     video_geotag_source: list[str] | None,
     video_import_path: Path,
     video_sync: VideoSyncOption | None = None,
+    interpolation: InterpolationOption | None = None,
 ) -> list[SourceOption]:
     """
     Resolve which sources distance sampling may take its positions from.
@@ -82,15 +89,18 @@ def _parse_geotag_options(
     cannot see every field the noise filter rejects on, so routing sampling
     through it would accept tracks the native parser refuses.
 
-    Every option carries video_sync, for a GPX source to place its track where
-    the geotagging stage will place the frames.
+    Every option carries video_sync and the interpolation options, for a GPX
+    source to place its track where the geotagging stage will place the frames.
+    As there, the interpolation options of a source override the global ones.
     """
     options = _parse_video_sources(
         geotag_source, geotag_source_path, video_geotag_source, video_import_path
     )
 
     for option in options:
         option.video_sync = video_sync
+        if option.interpolation is None:
+            option.interpolation = interpolation
 
     return options
 
@@ -185,8 +195,6 @@ def sample_video(
         start_time=(
             video_start_time_dt.timestamp() if video_start_time_dt is not None else None
         ),
-        offset_time=interpolation_offset_time,
-        use_gpx_start_time=interpolation_use_gpx_start_time,
         duration_ratio=video_duration_ratio,
     )
 
@@ -196,6 +204,10 @@ def sample_video(
         video_geotag_source,
         video_import_path,
         video_sync,
+        InterpolationOption(
+            offset_time=interpolation_offset_time,
+            use_gpx_start_time=interpolation_use_gpx_start_time,
+        ),
     )
 
     if rerun:
@@ -610,10 +622,17 @@ def _check_gpx_start_by_interval(
     for option in geotag_options:
         if option.source is not SourceType.GPX or option.source_path is None:
             continue
+        if option.interpolation is None or not option.interpolation.use_gpx_start_time:
+            continue
         gpx_path = option.source_path.resolve(video_path)
         if not gpx_path.is_file():
             continue
-        extractor = GPXVideoExtractor(video_path, gpx_path, video_sync=video_sync)
+        extractor = GPXVideoExtractor(
+            video_path,
+            gpx_path,
+            video_sync=video_sync,
+            interpolation=option.interpolation,
+        )
         if _gpx_sync_reads_a_clock(extractor):
             continue
         gpx_points = extractor._read_gpx_points()
@@ -691,7 +710,7 @@ def _sample_single_video_by_interval(
             # times the video's track to place the samples along it
             timeline = GPSTimeline.read(telemetry_points, clock_rate)
 
-    if clock_rate is None and video_sync is not None and video_sync.use_gpx_start_time:
+    if clock_rate is None and video_sync is not None:
         _check_gpx_start_by_interval(video_path, geotag_options, video_sync)
 
     probe = ffmpeglib.Probe(ffmpeg.probe_format_and_streams(video_path))