Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 6 additions & 21 deletions FlagEmbedding/abc/inference/AbsEmbedder.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
import logging
from tqdm import tqdm, trange
from tqdm import tqdm
from abc import ABC, abstractmethod
from typing import Any, Union, List, Dict, Literal, Optional

import queue
import multiprocessing as mp
from multiprocessing import Queue

Expand All @@ -13,6 +12,8 @@
import numpy as np
from transformers import is_torch_npu_available

from ._multi_process import collect_results, run_worker

try:
import torch_musa
except Exception:
Expand Down Expand Up @@ -362,20 +363,7 @@ def _encode_multi_process_worker(
"""
Internal working process to encode sentences in multi-process setup
"""
while True:
try:
chunk_id, sentences, kwargs = (
input_queue.get()
)
embeddings = model.encode_single_device(
sentences,
device=target_device,
**kwargs
)

results_queue.put([chunk_id, embeddings])
except queue.Empty:
break
run_worker(model.encode_single_device, target_device, input_queue, results_queue)

# copied from https://github.com/UKPLab/sentence-transformers/blob/1802076d4eae42ff0a5629e1b04e75785d4e193b/sentence_transformers/SentenceTransformer.py#L857
@staticmethod
Expand Down Expand Up @@ -407,6 +395,7 @@ def encode_multi_process(
pool: Dict[Literal["input", "output", "processes"], Any],
**kwargs
):
"""Encode chunks in order. A failed worker stops the supplied pool; recreate it before reuse."""
chunk_size = math.ceil(len(sentences) / len(pool["processes"]))

input_queue = pool["input"]
Expand All @@ -426,11 +415,7 @@ def encode_multi_process(
input_queue.put([last_chunk_id, chunk, kwargs])
last_chunk_id += 1

output_queue = pool["output"]
results_list = sorted(
[output_queue.get() for _ in trange(last_chunk_id, desc="Chunks")],
key=lambda x: x[0],
)
results_list = collect_results(self, pool, last_chunk_id)
embeddings = self._concatenate_results_from_multi_process([result[1] for result in results_list])
return embeddings

Expand Down
26 changes: 6 additions & 20 deletions FlagEmbedding/abc/inference/AbsReranker.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,16 @@
import gc
import torch
import numpy as np
from tqdm import tqdm, trange
from tqdm import tqdm
from transformers import is_torch_npu_available

try:
import torch_musa
except Exception:
pass

from ._multi_process import collect_results, run_worker

logger = logging.getLogger(__name__)


Expand Down Expand Up @@ -287,6 +289,7 @@ def encode_multi_process(
pool: Dict[Literal["input", "output", "processes"], Any],
**kwargs
) -> np.ndarray:
"""Score chunks in order. A failed worker stops the supplied pool; recreate it before reuse."""
chunk_size = math.ceil(len(sentence_pairs) / len(pool["processes"]))

input_queue = pool["input"]
Expand All @@ -306,11 +309,7 @@ def encode_multi_process(
input_queue.put([last_chunk_id, chunk, kwargs])
last_chunk_id += 1

output_queue = pool["output"]
results_list = sorted(
[output_queue.get() for _ in trange(last_chunk_id, desc="Chunks")],
key=lambda x: x[0],
)
results_list = collect_results(self, pool, last_chunk_id)
scores = np.concatenate([result[1] for result in results_list])
return scores

Expand All @@ -322,20 +321,7 @@ def _encode_multi_process_worker(
"""
Internal working process to encode sentences in multi-process setup
"""
while True:
try:
chunk_id, sentences, kwargs = (
input_queue.get()
)
embeddings = model.compute_score_single_gpu(
sentences,
device=target_device,
**kwargs
)

results_queue.put([chunk_id, embeddings])
except:
break
run_worker(model.compute_score_single_gpu, target_device, input_queue, results_queue)

# copied from https://github.com/UKPLab/sentence-transformers/blob/1802076d4eae42ff0a5629e1b04e75785d4e193b/sentence_transformers/SentenceTransformer.py#L857
@staticmethod
Expand Down
95 changes: 95 additions & 0 deletions FlagEmbedding/abc/inference/_multi_process.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
import queue
import threading
import traceback
from typing import NamedTuple

from tqdm import trange


class _WorkerError(NamedTuple):
traceback: str


def run_worker(compute, target_device, input_queue, results_queue):
"""Return an error as text: arbitrary exception objects may not be picklable."""
while True:
try:
chunk_id, sentences, kwargs = input_queue.get()
except queue.Empty:
return
try:
result = compute(sentences, device=target_device, **kwargs)
except Exception:
results_queue.put([chunk_id, _WorkerError(traceback.format_exc())])
# Keep the sender alive until the parent consumes the failure and stops
# the pool, including its Queue feeder and any outstanding results.
continue
results_queue.put([chunk_id, result])


def _watch_workers(pool, stopped, failures):
while not stopped.wait(0.1):
exited = [
f"{process.pid} (exit code {process.exitcode})"
for process in pool["processes"]
if not process.is_alive()
]
if not exited:
continue
failures.extend(exited)
for process in pool["processes"]:
if process.is_alive():
process.terminate()
for process in pool["processes"]:
process.join(timeout=1)
if process.is_alive():
process.kill()
# Queue.get(timeout=...) cannot time out a partial frame. Once all child
# writers are gone, close the parent's unused send handle too so recv sees
# EOF. Queue.close() alone does not close it in a receive-only process.
pool["output"]._writer.close()
return


def collect_results(model, pool, num_chunks):
"""Collect ordered results, discarding a failed pool and any outstanding work."""
stopped = threading.Event()
failures = []
watcher = threading.Thread(
target=_watch_workers, args=(pool, stopped, failures), daemon=True,
name="FlagEmbedding worker monitor",
)
watcher.start()
try:
try:
results = []
for _ in trange(num_chunks, desc="Chunks"):
try:
chunk_id, result = pool["output"].get()
except (EOFError, OSError) as error:
if failures:
raise RuntimeError(
"Inference worker exited before returning all results: " + ", ".join(failures)
) from error
raise
if isinstance(result, _WorkerError):
raise RuntimeError(
f"Inference worker failed while processing chunk {chunk_id}:\n{result.traceback}"
)
results.append([chunk_id, result])
finally:
stopped.set()
watcher.join()
if failures:
raise RuntimeError(
"Inference worker exited before returning all results: " + ", ".join(failures)
)
return sorted(results, key=lambda item: item[0])
except BaseException:
# Pending input may exceed the pipe buffer after its consumers are terminated.
pool["input"].cancel_join_thread()
model.stop_multi_process_pool(pool)
for attribute in ("pool", "query_pool"):
if getattr(model, attribute, None) is pool:
setattr(model, attribute, None)
raise
17 changes: 2 additions & 15 deletions FlagEmbedding/inference/embedder/decoder_only/icl.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
from tqdm import tqdm, trange
from typing import cast, Any, List, Union, Optional

import queue
from multiprocessing import Queue

import gc
Expand All @@ -10,6 +9,7 @@
from transformers import AutoModel, AutoTokenizer

from FlagEmbedding.abc.inference import AbsEmbedder
from FlagEmbedding.abc.inference._multi_process import run_worker
from FlagEmbedding.utils.tokenizer_compat import pad_with_compat


Expand Down Expand Up @@ -305,20 +305,7 @@ def _encode_queries_multi_process_worker(
"""
Internal working process to encode sentences in multi-process setup
"""
while True:
try:
chunk_id, sentences, kwargs = (
input_queue.get()
)
embeddings = model.encode_queries_single_device(
sentences,
device=target_device,
**kwargs
)

results_queue.put([chunk_id, embeddings])
except queue.Empty:
break
run_worker(model.encode_queries_single_device, target_device, input_queue, results_queue)

@torch.no_grad()
def encode_queries_single_device(
Expand Down
24 changes: 4 additions & 20 deletions FlagEmbedding/inference/embedder/encoder_only/m3.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import math
import torch
import queue
import logging
import numpy as np
from tqdm import tqdm, trange
Expand All @@ -10,6 +9,7 @@
from typing import Any, List, Union, Dict, Literal, Tuple, Optional

from FlagEmbedding.abc.inference import AbsEmbedder
from FlagEmbedding.abc.inference._multi_process import collect_results, run_worker
from FlagEmbedding.utils.tokenizer_compat import pad_with_compat
from FlagEmbedding.finetune.embedder.encoder_only.m3 import (
EncoderOnlyEmbedderM3ModelForInference, EncoderOnlyEmbedderM3Runner
Expand Down Expand Up @@ -547,6 +547,7 @@ def compute_score_multi_process(
pool: Dict[Literal["input", "output", "processes"], Any],
**kwargs
):
"""Score chunks in order. A failed worker stops the supplied pool; recreate it before reuse."""
chunk_size = math.ceil(len(sentence_pairs) / len(pool["processes"]))

input_queue = pool["input"]
Expand All @@ -566,11 +567,7 @@ def compute_score_multi_process(
input_queue.put([last_chunk_id, chunk, kwargs])
last_chunk_id += 1

output_queue = pool["output"]
results_list = sorted(
[output_queue.get() for _ in trange(last_chunk_id, desc="Chunks")],
key=lambda x: x[0],
)
results_list = collect_results(self, pool, last_chunk_id)

scores_dict = self._concatenate_compute_score_results_from_multi_process([result[1] for result in results_list])
return scores_dict
Expand All @@ -583,20 +580,7 @@ def _compute_score_multi_process_worker(
"""
Internal working process to encode sentences in multi-process setup
"""
while True:
try:
chunk_id, sentences, kwargs = (
input_queue.get()
)
embeddings = model.compute_score_single_device(
sentences,
device=target_device,
**kwargs
)

results_queue.put([chunk_id, embeddings])
except queue.Empty:
break
run_worker(model.compute_score_single_device, target_device, input_queue, results_queue)

@torch.no_grad()
def compute_score_single_device(
Expand Down
Loading