Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions FlagEmbedding/inference/auto_embedder.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ def from_finetuned(
:attr:`query_instruction_format`. Defaults to :data:`None`.
devices (Optional[Union[str, List[str]]], optional): Devices to use for model inference. Defaults to :data:`None`.
pooling_method (Optional[str], optional): Pooling method to get embedding vector from the last hidden state. Defaults to :data:`None`.
trust_remote_code (Optional[bool], optional): trust_remote_code for HF datasets or models. Defaults to :data:`None`.
trust_remote_code (Optional[bool], optional): Whether to trust custom model code. Defaults to :data:`False`
when omitted. Review the model code before explicitly setting this to :data:`True`.
query_instruction_format (Optional[str], optional): The template for :attr:`query_instruction_for_retrieval`. Defaults to :data:`None`.

Raises:
Expand Down Expand Up @@ -96,7 +97,12 @@ def from_finetuned(
if pooling_method is None:
pooling_method = model_config.pooling_method.value
if trust_remote_code is None:
trust_remote_code = model_config.trust_remote_code
trust_remote_code = False
if model_config.trust_remote_code:
logger.warning(
"This model is configured to use custom code. `trust_remote_code` defaults to False; "
"review the model code and explicitly pass `trust_remote_code=True` if you trust it."
)
if query_instruction_format is None:
query_instruction_format = model_config.query_instruction_format

Expand Down
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,10 @@ model = FlagAutoModel.from_finetuned('BAAI/bge-base-en-v1.5',
query_instruction_for_retrieval="Represent this sentence for searching relevant passages:",
use_fp16=True)
```
`FlagAutoModel` defaults to `trust_remote_code=False`, including for recognized model names.
Models that require custom code (for example, `BAAI/bge-code-v1`) now require explicit opt-in:
review the model code before passing `trust_remote_code=True` to `from_finetuned`.

Then, feed some sentences to the model and get their embeddings:
```
sentences_1 = ["I love NLP", "I love machine learning"]
Expand Down
142 changes: 142 additions & 0 deletions tests/test_auto_embedder_trust.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
"""Exercise the public dispatcher without loading model or tokenizer code."""

from dataclasses import replace

import pytest

from FlagEmbedding import FlagAutoModel
from FlagEmbedding.inference.embedder.model_mapping import (
AUTO_EMBEDDER_MAPPING,
EMBEDDER_CLASS_MAPPING,
EmbedderModelClass,
)


TRUE_REMOTE_CODE_MODELS = [
name for name, config in AUTO_EMBEDDER_MAPPING.items() if config.trust_remote_code
]
MODEL_ALIASES = [
"BAAI/bge-code-v1",
"another-namespace/bge-code-v1",
"/shared/models/bge-code-v1",
"/shared/models/bge-code-v1/checkpoint-100",
]


def capture_constructor(model_class):
"""Replace only the downstream constructor, leaving dispatch unmodified."""
class Capture:
DEFAULT_POOLING_METHOD = model_class.DEFAULT_POOLING_METHOD
selected_model_class = model_class

def __init__(self, model_name_or_path, **kwargs):
self.model_name_or_path = model_name_or_path
self.kwargs = kwargs

return Capture


@pytest.fixture
def capture_registry(monkeypatch):
def capture(name):
config = AUTO_EMBEDDER_MAPPING[name]
monkeypatch.setitem(
AUTO_EMBEDDER_MAPPING,
name,
replace(config, model_class=capture_constructor(config.model_class)),
)
return config

return capture


@pytest.mark.parametrize(
"model_name_or_path,registry_name",
[(f"another-namespace/{name}", name) for name in TRUE_REMOTE_CODE_MODELS]
+ [(path, "bge-code-v1") for path in MODEL_ALIASES],
)
def test_registry_never_implicitly_trusts_custom_code(
capture_registry, caplog, model_name_or_path, registry_name
):
config = capture_registry(registry_name)

model = FlagAutoModel.from_finetuned(model_name_or_path)

assert model.kwargs["trust_remote_code"] is False
assert model.selected_model_class is config.model_class
assert model.model_name_or_path == model_name_or_path
assert "trust_remote_code=True" in caplog.text
assert "review" in caplog.text.lower()


@pytest.mark.parametrize("model_name_or_path", MODEL_ALIASES)
@pytest.mark.parametrize("trust_remote_code", [False, True])
def test_explicit_trust_is_forwarded(
capture_registry, model_name_or_path, trust_remote_code
):
capture_registry("bge-code-v1")

model = FlagAutoModel.from_finetuned(
model_name_or_path, trust_remote_code=trust_remote_code
)

assert model.kwargs["trust_remote_code"] is trust_remote_code


@pytest.mark.parametrize(
"name", ["bge-code-v1", "bge-large-en-v1.5", "e5-large-v2"]
)
def test_registry_class_pooling_and_query_format_are_preserved(capture_registry, name):
config = capture_registry(name)

model = FlagAutoModel.from_finetuned(f"namespace/{name}")

assert model.selected_model_class is config.model_class
assert model.kwargs["pooling_method"] == config.pooling_method.value
assert model.kwargs["query_instruction_format"] == config.query_instruction_format
assert model.kwargs["trust_remote_code"] is False


def test_explicit_dispatch_options_are_preserved(capture_registry):
capture_registry("bge-code-v1")
options = {
"trust_remote_code": False,
"pooling_method": "mean",
"query_instruction_format": "Query: {}|{}",
"query_instruction_for_retrieval": "retrieve",
"normalize_embeddings": False,
"use_fp16": False,
"devices": "cpu",
"truncate_dim": 32,
"batch_size": 2,
}

model = FlagAutoModel.from_finetuned("namespace/bge-code-v1", **options)

for name, value in options.items():
assert model.kwargs[name] == value


@pytest.mark.parametrize("trust_remote_code", [None, False, True])
def test_direct_model_class_preserves_explicit_opt_in(monkeypatch, trust_remote_code):
model_class = EmbedderModelClass.DECODER_ONLY_BASE
original_class = EMBEDDER_CLASS_MAPPING[model_class]
monkeypatch.setitem(
EMBEDDER_CLASS_MAPPING, model_class, capture_constructor(original_class)
)

model = FlagAutoModel.from_finetuned(
"namespace/unregistered-model",
model_class=model_class.value,
trust_remote_code=trust_remote_code,
)

assert model.kwargs["trust_remote_code"] is (trust_remote_code is True)
assert model.selected_model_class is original_class
assert model.kwargs["pooling_method"] == original_class.DEFAULT_POOLING_METHOD
assert model.kwargs["query_instruction_format"] == "{}{}"


def test_unknown_model_still_requires_a_model_class():
with pytest.raises(ValueError, match="not found in the model mapping"):
FlagAutoModel.from_finetuned("namespace/unregistered-model")