diff --git a/FlagEmbedding/abc/inference/AbsEmbedder.py b/FlagEmbedding/abc/inference/AbsEmbedder.py index 32bf59ca0..b877e06b3 100644 --- a/FlagEmbedding/abc/inference/AbsEmbedder.py +++ b/FlagEmbedding/abc/inference/AbsEmbedder.py @@ -1,9 +1,8 @@ import logging -from tqdm import tqdm, trange +from tqdm import tqdm from abc import ABC, abstractmethod from typing import Any, Union, List, Dict, Literal, Optional -import queue import multiprocessing as mp from multiprocessing import Queue @@ -13,6 +12,8 @@ import numpy as np from transformers import is_torch_npu_available +from ._multi_process import collect_results, run_worker + try: import torch_musa except Exception: @@ -362,20 +363,7 @@ def _encode_multi_process_worker( """ Internal working process to encode sentences in multi-process setup """ - while True: - try: - chunk_id, sentences, kwargs = ( - input_queue.get() - ) - embeddings = model.encode_single_device( - sentences, - device=target_device, - **kwargs - ) - - results_queue.put([chunk_id, embeddings]) - except queue.Empty: - break + run_worker(model.encode_single_device, target_device, input_queue, results_queue) # copied from https://github.com/UKPLab/sentence-transformers/blob/1802076d4eae42ff0a5629e1b04e75785d4e193b/sentence_transformers/SentenceTransformer.py#L857 @staticmethod @@ -407,6 +395,7 @@ def encode_multi_process( pool: Dict[Literal["input", "output", "processes"], Any], **kwargs ): + """Encode chunks in order. A failed worker stops the supplied pool; recreate it before reuse.""" chunk_size = math.ceil(len(sentences) / len(pool["processes"])) input_queue = pool["input"] @@ -426,11 +415,7 @@ def encode_multi_process( input_queue.put([last_chunk_id, chunk, kwargs]) last_chunk_id += 1 - output_queue = pool["output"] - results_list = sorted( - [output_queue.get() for _ in trange(last_chunk_id, desc="Chunks")], - key=lambda x: x[0], - ) + results_list = collect_results(self, pool, last_chunk_id) embeddings = self._concatenate_results_from_multi_process([result[1] for result in results_list]) return embeddings diff --git a/FlagEmbedding/abc/inference/AbsReranker.py b/FlagEmbedding/abc/inference/AbsReranker.py index c7c6fec27..827dd204f 100644 --- a/FlagEmbedding/abc/inference/AbsReranker.py +++ b/FlagEmbedding/abc/inference/AbsReranker.py @@ -9,7 +9,7 @@ import gc import torch import numpy as np -from tqdm import tqdm, trange +from tqdm import tqdm from transformers import is_torch_npu_available try: @@ -17,6 +17,8 @@ except Exception: pass +from ._multi_process import collect_results, run_worker + logger = logging.getLogger(__name__) @@ -287,6 +289,7 @@ def encode_multi_process( pool: Dict[Literal["input", "output", "processes"], Any], **kwargs ) -> np.ndarray: + """Score chunks in order. A failed worker stops the supplied pool; recreate it before reuse.""" chunk_size = math.ceil(len(sentence_pairs) / len(pool["processes"])) input_queue = pool["input"] @@ -306,11 +309,7 @@ def encode_multi_process( input_queue.put([last_chunk_id, chunk, kwargs]) last_chunk_id += 1 - output_queue = pool["output"] - results_list = sorted( - [output_queue.get() for _ in trange(last_chunk_id, desc="Chunks")], - key=lambda x: x[0], - ) + results_list = collect_results(self, pool, last_chunk_id) scores = np.concatenate([result[1] for result in results_list]) return scores @@ -322,20 +321,7 @@ def _encode_multi_process_worker( """ Internal working process to encode sentences in multi-process setup """ - while True: - try: - chunk_id, sentences, kwargs = ( - input_queue.get() - ) - embeddings = model.compute_score_single_gpu( - sentences, - device=target_device, - **kwargs - ) - - results_queue.put([chunk_id, embeddings]) - except: - break + run_worker(model.compute_score_single_gpu, target_device, input_queue, results_queue) # copied from https://github.com/UKPLab/sentence-transformers/blob/1802076d4eae42ff0a5629e1b04e75785d4e193b/sentence_transformers/SentenceTransformer.py#L857 @staticmethod diff --git a/FlagEmbedding/abc/inference/_multi_process.py b/FlagEmbedding/abc/inference/_multi_process.py new file mode 100644 index 000000000..89e5a03d4 --- /dev/null +++ b/FlagEmbedding/abc/inference/_multi_process.py @@ -0,0 +1,95 @@ +import queue +import threading +import traceback +from typing import NamedTuple + +from tqdm import trange + + +class _WorkerError(NamedTuple): + traceback: str + + +def run_worker(compute, target_device, input_queue, results_queue): + """Return an error as text: arbitrary exception objects may not be picklable.""" + while True: + try: + chunk_id, sentences, kwargs = input_queue.get() + except queue.Empty: + return + try: + result = compute(sentences, device=target_device, **kwargs) + except Exception: + results_queue.put([chunk_id, _WorkerError(traceback.format_exc())]) + # Keep the sender alive until the parent consumes the failure and stops + # the pool, including its Queue feeder and any outstanding results. + continue + results_queue.put([chunk_id, result]) + + +def _watch_workers(pool, stopped, failures): + while not stopped.wait(0.1): + exited = [ + f"{process.pid} (exit code {process.exitcode})" + for process in pool["processes"] + if not process.is_alive() + ] + if not exited: + continue + failures.extend(exited) + for process in pool["processes"]: + if process.is_alive(): + process.terminate() + for process in pool["processes"]: + process.join(timeout=1) + if process.is_alive(): + process.kill() + # Queue.get(timeout=...) cannot time out a partial frame. Once all child + # writers are gone, close the parent's unused send handle too so recv sees + # EOF. Queue.close() alone does not close it in a receive-only process. + pool["output"]._writer.close() + return + + +def collect_results(model, pool, num_chunks): + """Collect ordered results, discarding a failed pool and any outstanding work.""" + stopped = threading.Event() + failures = [] + watcher = threading.Thread( + target=_watch_workers, args=(pool, stopped, failures), daemon=True, + name="FlagEmbedding worker monitor", + ) + watcher.start() + try: + try: + results = [] + for _ in trange(num_chunks, desc="Chunks"): + try: + chunk_id, result = pool["output"].get() + except (EOFError, OSError) as error: + if failures: + raise RuntimeError( + "Inference worker exited before returning all results: " + ", ".join(failures) + ) from error + raise + if isinstance(result, _WorkerError): + raise RuntimeError( + f"Inference worker failed while processing chunk {chunk_id}:\n{result.traceback}" + ) + results.append([chunk_id, result]) + finally: + stopped.set() + watcher.join() + if failures: + raise RuntimeError( + "Inference worker exited before returning all results: " + ", ".join(failures) + ) + return sorted(results, key=lambda item: item[0]) + except BaseException: + # Pending input may exceed the pipe buffer after its consumers are terminated. + pool["input"].cancel_join_thread() + model.stop_multi_process_pool(pool) + for attribute in ("pool", "query_pool"): + if getattr(model, attribute, None) is pool: + setattr(model, attribute, None) + raise diff --git a/FlagEmbedding/inference/embedder/decoder_only/icl.py b/FlagEmbedding/inference/embedder/decoder_only/icl.py index d4d6e087a..084840927 100644 --- a/FlagEmbedding/inference/embedder/decoder_only/icl.py +++ b/FlagEmbedding/inference/embedder/decoder_only/icl.py @@ -1,7 +1,6 @@ from tqdm import tqdm, trange from typing import cast, Any, List, Union, Optional -import queue from multiprocessing import Queue import gc @@ -10,6 +9,7 @@ from transformers import AutoModel, AutoTokenizer from FlagEmbedding.abc.inference import AbsEmbedder +from FlagEmbedding.abc.inference._multi_process import run_worker from FlagEmbedding.utils.tokenizer_compat import pad_with_compat @@ -305,20 +305,7 @@ def _encode_queries_multi_process_worker( """ Internal working process to encode sentences in multi-process setup """ - while True: - try: - chunk_id, sentences, kwargs = ( - input_queue.get() - ) - embeddings = model.encode_queries_single_device( - sentences, - device=target_device, - **kwargs - ) - - results_queue.put([chunk_id, embeddings]) - except queue.Empty: - break + run_worker(model.encode_queries_single_device, target_device, input_queue, results_queue) @torch.no_grad() def encode_queries_single_device( diff --git a/FlagEmbedding/inference/embedder/encoder_only/m3.py b/FlagEmbedding/inference/embedder/encoder_only/m3.py index dc533f554..c5d0e646f 100644 --- a/FlagEmbedding/inference/embedder/encoder_only/m3.py +++ b/FlagEmbedding/inference/embedder/encoder_only/m3.py @@ -1,6 +1,5 @@ import math import torch -import queue import logging import numpy as np from tqdm import tqdm, trange @@ -10,6 +9,7 @@ from typing import Any, List, Union, Dict, Literal, Tuple, Optional from FlagEmbedding.abc.inference import AbsEmbedder +from FlagEmbedding.abc.inference._multi_process import collect_results, run_worker from FlagEmbedding.utils.tokenizer_compat import pad_with_compat from FlagEmbedding.finetune.embedder.encoder_only.m3 import ( EncoderOnlyEmbedderM3ModelForInference, EncoderOnlyEmbedderM3Runner @@ -547,6 +547,7 @@ def compute_score_multi_process( pool: Dict[Literal["input", "output", "processes"], Any], **kwargs ): + """Score chunks in order. A failed worker stops the supplied pool; recreate it before reuse.""" chunk_size = math.ceil(len(sentence_pairs) / len(pool["processes"])) input_queue = pool["input"] @@ -566,11 +567,7 @@ def compute_score_multi_process( input_queue.put([last_chunk_id, chunk, kwargs]) last_chunk_id += 1 - output_queue = pool["output"] - results_list = sorted( - [output_queue.get() for _ in trange(last_chunk_id, desc="Chunks")], - key=lambda x: x[0], - ) + results_list = collect_results(self, pool, last_chunk_id) scores_dict = self._concatenate_compute_score_results_from_multi_process([result[1] for result in results_list]) return scores_dict @@ -583,20 +580,7 @@ def _compute_score_multi_process_worker( """ Internal working process to encode sentences in multi-process setup """ - while True: - try: - chunk_id, sentences, kwargs = ( - input_queue.get() - ) - embeddings = model.compute_score_single_device( - sentences, - device=target_device, - **kwargs - ) - - results_queue.put([chunk_id, embeddings]) - except queue.Empty: - break + run_worker(model.compute_score_single_device, target_device, input_queue, results_queue) @torch.no_grad() def compute_score_single_device( diff --git a/tests/test_multiprocess_failures.py b/tests/test_multiprocess_failures.py new file mode 100644 index 000000000..c3935c331 --- /dev/null +++ b/tests/test_multiprocess_failures.py @@ -0,0 +1,268 @@ +import multiprocessing as mp +import os +import queue +import threading +import time + +import numpy as np +import pytest +import torch + +from FlagEmbedding import FlagModel +from FlagEmbedding.abc.inference import AbsEmbedder, AbsReranker +from FlagEmbedding.inference.embedder.decoder_only.icl import ICLLLMEmbedder +from FlagEmbedding.inference.embedder.encoder_only.m3 import M3Embedder + + +_STARTED_POOLS = [] + + +class TrackPools: + def start_multi_process_pool(self, *args): + pool = super().start_multi_process_pool(*args) + _STARTED_POOLS.append((pool, [p.pid for p in pool["processes"]])) + return pool + + +class ToyEmbedder(TrackPools, AbsEmbedder): + def __init__(self): + super().__init__("local", devices=["cpu", "cpu"], use_fp16=False) + self.model = torch.nn.Linear(1, 1) + + def encode_single_device(self, sentences, **kwargs): + if "fail" in sentences: + raise ValueError("controlled inference failure") + if "exit" in sentences: + os._exit(7) + # Force the later chunk to finish first. + if sentences[0] == "slow": + time.sleep(0.2) + return np.array([[len(sentence)] for sentence in sentences], dtype=np.float32) + + +class ToyReranker(TrackPools, AbsReranker): + def __init__(self): + super().__init__("local", devices=["cpu", "cpu"], use_fp16=False) + self.model = torch.nn.Linear(1, 1) + + def compute_score_single_gpu(self, pairs, **kwargs): + return ToyEmbedder.encode_single_device(self, [pair[0] for pair in pairs]).reshape(-1) + + +class ToyICL(TrackPools, ICLLLMEmbedder): + def __init__(self): + AbsEmbedder.__init__(self, "local", devices=["cpu", "cpu"], use_fp16=False) + self.model = torch.nn.Linear(1, 1) + self.query_pool = None + + encode_queries_single_device = ToyEmbedder.encode_single_device + + +class ToyM3(TrackPools, M3Embedder): + def __init__(self): + AbsEmbedder.__init__(self, "local", devices=["cpu", "cpu"], use_fp16=False) + self.model = torch.nn.Linear(1, 1) + + def compute_score_single_device(self, pairs, **kwargs): + scores = ToyEmbedder.encode_single_device(self, [pair[0] for pair in pairs]).reshape(-1).tolist() + return {mode: scores for mode in ( + "dense", "sparse", "colbert", "sparse+dense", "colbert+sparse+dense" + )} + + +class LocalFlagModel(TrackPools, FlagModel): + pass + + +@pytest.fixture +def started_pools(): + _STARTED_POOLS.clear() + yield _STARTED_POOLS + for pool, _ in _STARTED_POOLS: + try: + AbsEmbedder.stop_multi_process_pool(pool) + except ValueError: + # A failed request or M3's per-call pool may already have closed it. + pass + + +def invoke(call, started_pools, timeout=60, abort=None): + """Bound a broken queue wait, including on the unfixed source.""" + outcome = queue.Queue() + + def run(): + try: + outcome.put((True, call())) + except BaseException as error: + outcome.put((False, error)) + + thread = threading.Thread(target=run, daemon=True) + thread.start() + thread.join(timeout) + if thread.is_alive(): + if abort is not None: + abort() + # Release the old collector before failing, so the regression itself leaks no + # blocked threads. Production queues and spawned workers are otherwise real. + for pool, _ in started_pools if abort is None else []: + try: + for chunk in range(2): + pool["output"].put([chunk, np.zeros((1, 1))]) + except ValueError: + pass + thread.join(5) + pytest.fail("The caller blocked after an inference worker failed") + success, value = outcome.get_nowait() + if not success: + raise value + return value + + +@pytest.mark.parametrize("failure, message", [ + ("fail", "ValueError: controlled inference failure"), + ("exit", "worker"), +]) +def test_encode_worker_failure_and_recovery(started_pools, failure, message): + model = ToyEmbedder() + try: + with pytest.raises(RuntimeError, match=message): + invoke(lambda: model.encode([failure, "ok"]), started_pools) + assert model.pool is None + failed_pids = started_pools[0][1] + assert not set(failed_pids) & {p.pid for p in mp.active_children()} + + actual = invoke(lambda: model.encode(["slow", "a", "abc"]), started_pools) + np.testing.assert_array_equal(actual, [[4], [1], [3]]) + assert len(started_pools) == 2 + finally: + model.stop_self_pool() + + +def send_large_result(output): + output.put([0, np.zeros((8192, 1024), dtype=np.float32)]) + # Keep the process alive while its Queue feeder transmits the result. + time.sleep(60) + + +def hold_output_queue(output): + time.sleep(60) + + +@pytest.mark.parametrize("terminate_sender", [False, True]) +def test_large_result_transfer(started_pools, terminate_sender): + ctx = mp.get_context("spawn") + output, inputs = ctx.Queue(), ctx.Queue() + worker = ctx.Process(target=send_large_result, args=(output,), daemon=True) + peer = ctx.Process(target=hold_output_queue, args=(output,), daemon=True) + worker.start() + peer.start() + pool = {"input": inputs, "output": output, "processes": [worker, peer]} + pids = [worker.pid, peer.pid] + started_pools.append((pool, pids)) + model = ToyEmbedder() + model.pool = pool + try: + # A large frame cannot fit in the pipe. Kill its sender after the header + # arrives but before the collector starts consuming the remaining bytes. + assert output._reader.poll(30) + if terminate_sender: + worker.kill() + worker.join(5) + assert worker.exitcode is not None + assert peer.is_alive() + + def abort_transfer(): + peer.terminate() + peer.join(5) + output._writer.close() + + with pytest.raises(RuntimeError, match="worker"): + invoke( + lambda: model.encode_multi_process(["a"], pool), started_pools, + timeout=5, abort=abort_transfer, + ) + assert model.pool is None + assert not set(pids) & {p.pid for p in mp.active_children()} + else: + actual = invoke(lambda: model.encode_multi_process(["a"], pool), started_pools) + assert actual.shape == (8192, 1024) + assert not actual.any() + assert not any(t.name == "FlagEmbedding worker monitor" for t in threading.enumerate()) + finally: + model.stop_self_pool() + + +@pytest.mark.parametrize("kind", ["reranker", "icl", "m3"]) +def test_specialized_worker_failure_and_recovery(started_pools, kind): + factories = {"reranker": ToyReranker, "icl": ToyICL, "m3": ToyM3} + model = factories[kind]() + + def call(texts): + if kind == "icl": + return model.encode_queries(texts) + return model.compute_score([(text, "passage") for text in texts]) + + try: + with pytest.raises(RuntimeError, match="ValueError: controlled inference failure"): + invoke(lambda: call(["fail", "ok"]), started_pools) + assert model.pool is None + if kind == "icl": + assert model.query_pool is None + assert not set(started_pools[0][1]) & {p.pid for p in mp.active_children()} + + actual = invoke(lambda: call(["slow", "a", "abc"]), started_pools) + if kind == "m3": + assert all(scores == [4, 1, 3] for scores in actual.values()) + else: + np.testing.assert_array_equal(actual, [[4], [1], [3]] if kind == "icl" else [4, 1, 3]) + assert len(started_pools) == 2 + finally: + model.stop_self_pool() + if kind == "icl": + model.stop_self_query_pool() + + +def test_real_model_forward_failure_and_recovery(tmp_path, started_pools): + from tokenizers import Tokenizer + from tokenizers.models import WordLevel + from tokenizers.pre_tokenizers import Whitespace + from tokenizers.processors import TemplateProcessing + from transformers import PreTrainedTokenizerFast, XLMRobertaConfig, XLMRobertaModel + + vocabulary = {token: i for i, token in enumerate( + ["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]", "a", "b", "c", "d"] + )} + backend = Tokenizer(WordLevel(vocabulary, unk_token="[UNK]")) + backend.pre_tokenizer = Whitespace() + backend.post_processor = TemplateProcessing( + single="[CLS] $A [SEP]", special_tokens=[("[CLS]", 2), ("[SEP]", 3)] + ) + tokenizer = PreTrainedTokenizerFast( + tokenizer_object=backend, pad_token="[PAD]", unk_token="[UNK]", model_max_length=64, + model_input_names=["input_ids", "token_type_ids", "attention_mask"], + ) + tokenizer.save_pretrained(tmp_path) + torch.manual_seed(0) + backbone = XLMRobertaModel(XLMRobertaConfig( + vocab_size=len(vocabulary), hidden_size=8, intermediate_size=16, + num_attention_heads=2, num_hidden_layers=1, max_position_embeddings=8, + pad_token_id=0, + )) + backbone.save_pretrained(tmp_path) + + model = LocalFlagModel(str(tmp_path), devices=["cpu", "cpu"], use_fp16=False) + too_long = "a b c d a b c d a b" + valid = ["a", "b c", "d"] + # A genuine embedding-layer IndexError, not an injected inference exception. + with pytest.raises(IndexError): + model.encode_single_device([too_long], device="cpu", max_length=64) + expected = model.encode_single_device(valid, device="cpu", max_length=8) + try: + with pytest.raises(RuntimeError, match="IndexError"): + invoke(lambda: model.encode([too_long, "a"], max_length=64), started_pools) + assert model.pool is None + actual = invoke(lambda: model.encode(valid, max_length=8), started_pools) + np.testing.assert_allclose(actual, expected, rtol=1e-6, atol=1e-6) + assert not set(started_pools[0][1]) & {p.pid for p in mp.active_children()} + finally: + model.stop_self_pool()