diff --git a/docs/Deployment-AssetStore-Import.md b/docs/Deployment-AssetStore-Import.md index 1c420c2dd..6283391fa 100644 --- a/docs/Deployment-AssetStore-Import.md +++ b/docs/Deployment-AssetStore-Import.md @@ -42,6 +42,7 @@ Place the video file (and optional siblings) in a parent directory. Import creat cruise_a/ reef.mp4 reef.csv ← annotations (same stem as the video) + reef_tracks.json ← also accepted: {stem}_tracks or {stem}_detections reef_metadata.csv ← frame metadata (video-paired name; see below) lagoon.mp4 lagoon.json @@ -80,7 +81,7 @@ Annotation formats are the same as elsewhere in DIVE ([Data Formats](DataFormats | Media | How to name / place annotations | |-------|----------------------------------| -| **Video** | Same basename as the video, different extension: `reef.mp4` → `reef.csv` or `reef.json`. The sidecar may sit next to the video; import moves it into the video dataset folder. | +| **Video** | Same basename as the video, different extension: `reef.mp4` → `reef.csv` or `reef.json`. Optional `_tracks` or `_detections` suffixes are also accepted: `reef_tracks.csv`, `reef_detections.json`. The sidecar may sit next to the video; import moves it into the video dataset folder. | | **Image sequence** | Any `.csv` or `.json` file inside the sequence folder. | !!! note @@ -140,7 +141,7 @@ FPS defaults: ## Checklist * One dataset per video file or per image-sequence folder -* Video annotations share the video stem (`video.mp4` ↔ `video.csv`) +* Video annotations share the video stem (`video.mp4` ↔ `video.csv`, or `video_tracks.csv` / `video_detections.json`) * Image-sequence annotations live in the same folder as the frames * At most one frame-metadata file per dataset * Videos: use `{stem}_metadata.{ext}` beside the video, **or** `frame_metadata.{ext}` inside the video folder after layout diff --git a/docs/Deployment-Storage.md b/docs/Deployment-Storage.md index 25ec88ab1..ef4167d91 100644 --- a/docs/Deployment-Storage.md +++ b/docs/Deployment-Storage.md @@ -81,7 +81,7 @@ For the full bucket layout — videos, image sequences, annotation pairing, and During import, annotations associated with image sequences or videos are discovered automatically. Summary: -* **Video** — annotation file (CSV or JSON) must share the video basename: `video.mp4` → `video.csv` or `video.json`. +* **Video** — annotation file (CSV or JSON) must share the video basename: `video.mp4` → `video.csv` or `video.json`. Optional suffixes `_tracks` or `_detections` are also accepted (e.g. `video_tracks.csv`, `video_detections.json`). * **Image sequence** — any CSV or JSON in the same folder as the frames is imported as annotations. Optional **frame metadata** attachments (flight logs and similar) can also be discovered by reserved or video-paired filenames. Details, examples, and post-import behavior are in [AssetStore Importing and Data Structure](Deployment-AssetStore-Import.md). diff --git a/samples/scripts/assetStoreImport/generateSampleData.py b/samples/scripts/assetStoreImport/generateSampleData.py index 8e57b3f9f..46a7ff6a5 100644 --- a/samples/scripts/assetStoreImport/generateSampleData.py +++ b/samples/scripts/assetStoreImport/generateSampleData.py @@ -458,6 +458,11 @@ def write_annotations( generate_annotation_json(tracks, output_file) +def _random_annotation_suffix() -> str: + """Optional ``_tracks`` / ``_detections`` suffix for assetstore pairing tests.""" + return random.choice(["", "_tracks", "_detections"]) + + def _annotation_extension(annotation_format: str) -> str: return ".csv" if annotation_format == "viame-csv" else ".json" @@ -544,9 +549,10 @@ def create_video_content( counter["count"] += 1 ext = _annotation_extension(annotation_format) + annotation_suffix = _random_annotation_suffix() write_annotations( duration * annotation_fps, - video_path.with_suffix(ext), + base_dir / f"{stem}{annotation_suffix}{ext}", annotation_format=annotation_format, dataset_name=video_path.stem, fps=annotation_fps, diff --git a/server/dive_server/event.py b/server/dive_server/event.py index 1c961fa99..6cd17782c 100644 --- a/server/dive_server/event.py +++ b/server/dive_server/event.py @@ -18,6 +18,7 @@ from dive_tasks.dive_batch_postprocess import DIVEBatchPostprocessTaskParams from dive_utils import asbool, frame_metadata, fromMeta +from dive_utils.assetstore_import import annotation_media_stem from dive_utils.constants import ( AnnotationFileFutureProcessMarker, AssetstoreSourceMarker, @@ -212,8 +213,7 @@ def process_assetstore_import(event, meta: dict): parentFolder = Folder().findOne({"_id": item["folderId"]}) userId = parentFolder['creatorId'] or parentFolder['baseParentId'] user = User().findOne({'_id': ObjectId(userId)}) - base_name = os.path.splitext(item['name'])[0] - foldername = base_name + foldername = annotation_media_stem(item['name']) # check if folder with foldername exists in the parentFolder location; # this would be a video folder then possible_video_folder = Folder().findOne( @@ -290,9 +290,9 @@ def process_dangling_annotation_files(folder, user): # to the plain annotation path, the same resolution process_assetstore_import uses. # Check if the corresponding video folder exists - base_name = os.path.splitext(item['name'])[0] + media_stem = annotation_media_stem(item['name']) video_folder = Folder().findOne( - {'parentId': parent_folder_id, 'name': base_name, f'meta.{TypeMarker}': VideoType} + {'parentId': parent_folder_id, 'name': media_stem, f'meta.{TypeMarker}': VideoType} ) if video_folder is not None: # Move the annotation file into the video folder diff --git a/server/dive_utils/assetstore_import.py b/server/dive_utils/assetstore_import.py new file mode 100644 index 000000000..1fef8bbe2 --- /dev/null +++ b/server/dive_utils/assetstore_import.py @@ -0,0 +1,22 @@ +"""Helpers for pairing assetstore-imported files with DIVE datasets.""" + +from __future__ import annotations + +import os + +# Optional annotation filename suffixes stripped when pairing with a video stem. +ANNOTATION_MEDIA_SUFFIXES = ('_detections', '_tracks') + + +def annotation_media_stem(annotation_filename: str) -> str: + """Return the media stem used to pair an annotation with a video dataset. + + ``reef.csv``, ``reef_tracks.json``, and ``reef_detections.csv`` all resolve + to ``reef``. + """ + stem = os.path.splitext(os.path.basename(annotation_filename))[0] + lower = stem.lower() + for suffix in ANNOTATION_MEDIA_SUFFIXES: + if lower.endswith(suffix): + return stem[: -len(suffix)] + return stem diff --git a/server/tests/test_event_frame_metadata.py b/server/tests/test_event_frame_metadata.py index f74cab5d9..8b6ab7166 100644 --- a/server/tests/test_event_frame_metadata.py +++ b/server/tests/test_event_frame_metadata.py @@ -2,6 +2,7 @@ from unittest.mock import patch from dive_server import event +from dive_utils.assetstore_import import annotation_media_stem from dive_utils.constants import ( AnnotationFileFutureProcessMarker, FrameMetadataFileMarker, @@ -87,6 +88,44 @@ def test_parse_video_paired_metadata_name(): assert canonical_frame_metadata_name('CSV') == 'frame_metadata.csv' +def test_annotation_media_stem(): + assert annotation_media_stem('reef.csv') == 'reef' + assert annotation_media_stem('reef_tracks.json') == 'reef' + assert annotation_media_stem('reef_detections.CSV') == 'reef' + assert annotation_media_stem('path/to/clip_tracks.json') == 'clip' + assert annotation_media_stem('clip_metadata.csv') == 'clip_metadata' + + +@patch('dive_server.event.User') +@patch('dive_server.event.Folder') +@patch('dive_server.event.Item') +def test_assetstore_import_moves_tracks_annotation_to_video_folder(item_cls, folder_cls, user_cls): + item = {'_id': 'i1', 'name': 'reef_tracks.csv', 'meta': {}, 'folderId': 'parent'} + item_model = item_cls.return_value + item_model.findOne.return_value = item + parent_folder = {'_id': 'parent', 'creatorId': _OWNER_ID, 'baseParentId': None, 'meta': {}} + video_folder = {'_id': 'video', 'name': 'reef', 'meta': {TypeMarker: VideoType}} + folder_model = folder_cls.return_value + + def find_one(query): + if query == {'_id': 'parent'}: + return parent_folder + if query == {'parentId': 'parent', 'name': 'reef'}: + return video_folder + return None + + folder_model.findOne.side_effect = find_one + user_cls.return_value.findOne.return_value = {'_id': _OWNER_ID} + + event.process_assetstore_import( + _event({'type': 'item', 'importPath': '/data/reef_tracks.csv', 'id': 'i1'}), + {}, + ) + + item_model.move.assert_called_once_with(item, video_folder) + assert AnnotationFileFutureProcessMarker not in item['meta'] + + @patch('dive_server.event.File') @patch('dive_server.event.User') @patch('dive_server.event.Folder') @@ -397,3 +436,39 @@ def find_one(query): assert item['meta'][AnnotationFileFutureProcessMarker] is False item_model.move.assert_called_once_with(item, video_folder) assert MetadataFileItemIdMarker not in video_folder['meta'] + + +@patch('dive_server.event.Folder') +@patch('dive_server.event.Item') +def test_dangling_moves_detections_annotation_to_video_folder(item_cls, folder_cls): + item = { + '_id': 'i1', + 'name': 'reef_detections.json', + 'meta': {AnnotationFileFutureProcessMarker: True}, + 'folderId': 'parent', + } + item_model = item_cls.return_value + item_model.find.return_value = [item] + item_model.save.side_effect = lambda doc: doc + parent_folder = {'_id': 'parent', 'meta': {}} + video_folder = {'_id': 'video', 'name': 'reef', 'meta': {TypeMarker: VideoType}} + folder_model = folder_cls.return_value + + def find_one(query): + if query == {'_id': 'parent'}: + return parent_folder + if query == { + 'parentId': 'parent', + 'name': 'reef', + f'meta.{TypeMarker}': VideoType, + }: + return video_folder + return None + + folder_model.findOne.side_effect = find_one + folder_model.childFolders.return_value = [] + + event.process_dangling_annotation_files({'_id': 'parent'}, {'_id': _OWNER_ID}) + + assert item['meta'][AnnotationFileFutureProcessMarker] is False + item_model.move.assert_called_once_with(item, video_folder)