Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions backends/nxp/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@ The eIQ Neutron NPU Backend should be considered as prototype quality at this mo
improvements. NXP and the ExecuTorch community is actively developing this codebase.

## Neutron Backend implementation and SW architecture
Neutron Backend uses the eIQ Neutron Converter as ML compiler to compile the delegated subgraph to Neutron microcode.
The Neutron Converter accepts the ML model in LiteRT format, for the **eIQ Neutron N3** class therefore the Neutron Backend
uses the LiteRT flatbuffers format as IR between the ExecuTorch and Neutron Converter ML compiler.
Neutron Backend uses the eIQ Neutron Compiler as ML compiler to compile the delegated subgraph to Neutron microcode.
The Neutron Compiler accepts the ML model in LiteRT format, for the **eIQ Neutron N3** class therefore the Neutron Backend
uses the LiteRT flatbuffers format as IR between the ExecuTorch and Neutron Compiler ML compiler.

## Layout
* `backend/ir/` - TFLite/LiteRT based IR to represent the Edge Subgraph, taken from onnx2tflite code base and extended to
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

class AddBatchSizeFor3DInputPool2DOps(PassBase):
"""Adds batch size dimension for aten.adaptive_avg_pool2d.default, aten.avg_pool2d.default
and aten.max_pool2d.default ops with 3D input, as the Neutron Converter is unable to convert these ops with 3D input.
and aten.max_pool2d.default ops with 3D input, as the Neutron Compiler is unable to compile these ops with 3D input.

┌──────▼──────┐
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ def supports_partitioning_result(
is_alone_in_partition = cls.is_node_alone_in_partition(node, partition_list)

if is_alone_in_partition and keepdim and all(input_shape[d] == 1 for d in dim):
# The operator is a no-op, so the Neutron Converter will skip it. If it's the only node in the
# The operator is a no-op, so the Neutron Compiler will skip it. If it's the only node in the
# partition, the graph would end up empty.
return False

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -400,7 +400,7 @@ def supports_partitioning_result(
is_alone_in_partition = cls.is_node_alone_in_partition(node, partition_list)
if has_static_input and is_alone_in_partition:
# Transpose with a static input is a no-op on Neutron. If it was the only operator in the partition,
# Neutron Converter would produce and empty graph, so delegation is prohibited.
# Neutron Compiler would produce and empty graph, so delegation is prohibited.
return False

return True
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ def supports_partitioning_result(
is_alone_in_partition = cls.is_node_alone_in_partition(node, partition_list)

if is_alone_in_partition and input_shape == output_shape:
# The operator is a no-op, so the Neutron Converter will skip it. If it's the only node in the
# The operator is a no-op, so the Neutron Compiler will skip it. If it's the only node in the
# partition, the graph would end up empty.
return False

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ def supports_partitioning_result(
is_alone_in_partition = cls.is_node_alone_in_partition(node, partition_list)

if is_alone_in_partition and h_scale == w_scale == 1:
# The operator is a no-op, so the Neutron Converter will skip it. If it's the only node in the
# The operator is a no-op, so the Neutron Compiler will skip it. If it's the only node in the
# partition, the graph would end up empty.
return False

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,27 @@
import os

try:
from eiq_neutron_sdk import neutron_converter, neutron_library_utils
from eiq_neutron_sdk import neutron_compiler, neutron_library_utils

_USING_NEUTRON_COMPILER = True
except ImportError:
raise RuntimeError(
"eIQ Neutron SDK not found. To install it, run 'examples/nxp/setup.sh'."
)
try:
from eiq_neutron_sdk import (
neutron_converter as neutron_compiler,
neutron_library_utils,
)

_USING_NEUTRON_COMPILER = False
except ImportError:
raise RuntimeError(
"eIQ Neutron SDK not found. To install it, run 'examples/nxp/setup.sh'."
)


def _build_compilation_context(compilation_opts):
"""Build a CompilationContext from a plain dict of options."""
cctx = neutron_converter.CompilationContext()
cctx.targetOpts = neutron_converter.getNeutronTarget(compilation_opts["target"])
cctx = neutron_compiler.CompilationContext()
cctx.targetOpts = neutron_compiler.getNeutronTarget(compilation_opts["target"])
cctx.compilationOpts.minNumOpsPerGraph = compilation_opts["minNumOpsPerGraph"]
cctx.compilationOpts.excludeGraphPasses = compilation_opts["excludeGraphPasses"]
cctx.compilationOpts.fetchConstantsToSRAM = compilation_opts["fetchConstantsToSRAM"]
Expand All @@ -37,19 +47,22 @@ def _build_compilation_context(compilation_opts):
return cctx


def convert_unsafe(tflite_model, compilation_opts, queue):
def compile_unsafe(tflite_model, compilation_opts, queue):
"""
Run neutron_converter on given tflite_model with the provided compilation options.
Run neutron_compiler on given tflite_model with the provided compilation options.
This routine is supposed to run in a separate process.
If properly finished, the output queue contains the converted model,
otherwise the neutron_converter exits and the output queue is empty.
If properly finished, the output queue contains the compiled model,
otherwise the neutron_compiler exits and the output queue is empty.
"""
cctx = _build_compilation_context(compilation_opts)
model_converted = neutron_converter.convertModel(list(tflite_model), cctx)
queue.put(model_converted)
if _USING_NEUTRON_COMPILER:
model_compiled = neutron_compiler.compileModel(list(tflite_model), cctx)
else:
model_compiled = neutron_compiler.convertModel(list(tflite_model), cctx)
queue.put(model_compiled)


class NeutronConverterManager:
class NeutronCompilerManager:
"""
Manager for conversion of TFLite model in flatbuffers format into TFLite model that
contains NeutronGraph nodes.
Expand All @@ -69,8 +82,8 @@ def _rename_partition_kernel_selection_file(delegation_tag):
except OSError:
logging.error("Failed to rename partition kernel selection file.")

def get_converter(self):
return neutron_converter
def get_compiler(self):
return neutron_compiler

def get_library_utils(self):
return neutron_library_utils
Expand All @@ -84,7 +97,7 @@ def verify_target(self, target: str):
f"Target `{target}` is not a valid target. Must be one of `{valid_targets}`."
)

def convert(
def compile(
self,
tflite_model: bytes,
target: str,
Expand All @@ -93,9 +106,9 @@ def convert(
use_profiling: bool = False,
) -> bytes:
"""
Call Neutron Converter.
Call Neutron Compiler.

:param tflite_model: A generic TFLite model to be converted.
:param tflite_model: A generic TFLite model to be compiled.
:param target: The target platform.
:param delegation_tag: The delegation tag of model partition.
:param fetch_constants_to_sram: Add microcode that fetches weights from external memory.
Expand All @@ -104,7 +117,7 @@ def convert(

:return: TFLite model with Neutron microcode as bytes.
"""
# Neutron converter crashes if we provide invalid target -> verify.
# Neutron compiler crashes if we provide invalid target -> verify.
self.verify_target(target)

compilation_opts = {
Expand All @@ -124,28 +137,31 @@ def convert(
queue = multiprocessing.Manager().Queue()

process = multiprocessing.Process(
target=convert_unsafe,
target=compile_unsafe,
args=(tflite_model, compilation_opts, queue),
)
process.start()
process.join() # waits until the subprocess is complete

if queue.empty(): # signals the unsafe task did not run till the end
raise RuntimeError(
f"Neutron converter module terminated unexpectedly with exit code {process.exitcode}"
f"Neutron compiler module terminated unexpectedly with exit code {process.exitcode}"
)

model_converted = queue.get()
model_compiled = queue.get()
process.close()
except (EOFError, OSError, TypeError) as e:
# Multiprocessing failed (likely due to environment restrictions)
# Fall back to direct execution
logging.warning(
f"Multiprocessing not available ({e}), running neutron converter directly"
f"Multiprocessing not available ({e}), running neutron compiler directly"
)
cctx = _build_compilation_context(compilation_opts)
model_converted = neutron_converter.convertModel(list(tflite_model), cctx)
if _USING_NEUTRON_COMPILER:
model_compiled = neutron_compiler.compileModel(list(tflite_model), cctx)
else:
model_compiled = neutron_compiler.convertModel(list(tflite_model), cctx)
if self.dump_kernel_selection_code:
self._rename_partition_kernel_selection_file(delegation_tag)

return bytes(model_converted)
return bytes(model_compiled)
22 changes: 11 additions & 11 deletions backends/nxp/backend/neutron_map.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,15 +91,15 @@ def get_tensors_name(tensors: str) -> list[str]:


class NeutronMap:
"""Mapping between Neutron, TFLite, and Edge operators based on the Neutron converter log.
"""Mapping between Neutron, TFLite, and Edge operators based on the Neutron compiler log.

Parses the Neutron converter log to extract information about TFLite nodes and Neutron subgraphs.
Parses the Neutron compiler log to extract information about TFLite nodes and Neutron subgraphs.
Maps TFLite operators to corresponding Neutron operators.
Maps Edge operators to Neutron operators via the Edge-to-TFLite mapping.

Attributes:
tflite_nodes (list[Node]): TFLite node information extracted from the converter log.
neutron_subgraphs (list[SubgraphInfo]): Neutron subgraph information extracted from the converter log.
tflite_nodes (list[Node]): TFLite node information extracted from the compiler log.
neutron_subgraphs (list[SubgraphInfo]): Neutron subgraph information extracted from the compiler log.
neutron_graphs (list[int]): Indices of final Neutron graphs derived from neutron_subgraphs.
edge_to_tflite_map (dict[int, tuple[int, ...]]): Mapping from Edge operators to TFLite operators.
edge_to_neutron_map (dict[int, tuple[int, ...]]): Mapping from Edge operators to Neutron operators.
Expand All @@ -118,12 +118,12 @@ class NeutronMap:
tflite_to_neutron_map: dict[int, tuple[int, ...]]

def __init__(
self, neutron_converter_log: str, edge_to_tflite_map: dict[int, tuple[int, ...]]
self, neutron_compiler_log: str, edge_to_tflite_map: dict[int, tuple[int, ...]]
) -> None:
"""Initialize neutron map from neutron converter log.
"""Initialize neutron map from neutron compiler log.

:param neutron_converter_log: neutron converter log obtained during model conversion. It should contain
original tflite graph and neutron graph dump. To add these dumps to converter log the dumpAfterImport and
:param neutron_compiler_log: neutron compiler log obtained during model compilation. It should contain
original tflite graph and neutron graph dump. To add these dumps to compiler log the dumpAfterImport and
dumpAfterGenerate flags have to be set to "console".
"""
super().__init__()
Expand All @@ -134,12 +134,12 @@ def __init__(
self.tflite_to_neutron_map = {}
self.edge_to_neutron_map = {}
self.neutron_kernels_num = 0
self._split_profiling_log(neutron_converter_log)
self._split_profiling_log(neutron_compiler_log)

def _split_profiling_log(self, log: str) -> None:
"""Process profiling log to split it into original TFLite and converted Neutron nodes.

:param log: Neutron converter log obtained during model conversion, containing the original
:param log: Neutron compiler log obtained during model compilation, containing the original
TFLite graph and Neutron graph dump.
:return: None. Sets class attributes tflite_nodes and neutron_subgraphs with node information.
"""
Expand Down Expand Up @@ -175,7 +175,7 @@ def _split_profiling_log(self, log: str) -> None:
def _get_neutron_subgraphs(self, graph_dump: str) -> list[SubgraphInfo]:
"""Parse Neutron graph dump and extract subgraph information.

:param graph_dump: String containing the Neutron graph dump from the converter log.
:param graph_dump: String containing the Neutron graph dump from the compiler log.
:return: List of SubgraphInfo objects containing subgraph metadata and operator nodes.
"""

Expand Down
12 changes: 6 additions & 6 deletions backends/nxp/backend/neutron_target_spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
from enum import Enum

import torch
from executorch.backends.nxp.backend.neutron_converter_manager import (
NeutronConverterManager,
from executorch.backends.nxp.backend.neutron_compiler_manager import (
NeutronCompilerManager,
)
from executorch.exir.dialects._ops import ops as exir_ops
from torch.fx import Node
Expand Down Expand Up @@ -98,10 +98,10 @@ class NeutronTargetSpec:

def __init__(self, target: str):

converter_manager = NeutronConverterManager()
converter_manager.verify_target(target)
neutron_converter = converter_manager.get_converter()
self.neutron_target = neutron_converter.getNeutronTarget(target)
compiler_manager = NeutronCompilerManager()
compiler_manager.verify_target(target)
neutron_compiler = compiler_manager.get_compiler()
self.neutron_target = neutron_compiler.getNeutronTarget(target)

if self.is_subsystem():
raise ValueError(
Expand Down
14 changes: 7 additions & 7 deletions backends/nxp/nxp_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@
EdgeProgramToIRConverter,
)
from executorch.backends.nxp.backend.ir.conversion_config import ConversionConfig
from executorch.backends.nxp.backend.neutron_converter_manager import (
NeutronConverterManager,
from executorch.backends.nxp.backend.neutron_compiler_manager import (
NeutronCompilerManager,
)

from executorch.backends.nxp.backend.neutron_map import NeutronMap
Expand Down Expand Up @@ -86,10 +86,10 @@ def neutron_compile_spec(
:param use_neutron_for_format_conversion: If True, the EdgeProgramToIRConverter will insert `Transpose` ops to
ensure that the IO matches the executorch partition, which will be
delegated to Neutron.
:param fetch_constants_to_sram: If True, the Neutron Converter will insert microinstructions to prefetch weights
:param fetch_constants_to_sram: If True, the Neutron Compiler will insert microinstructions to prefetch weights
from FLASH to SRAM. This should be used when the whole model does not fit into SRAM.
:param dump_kernel_selection_code: Whether Neutron converter dumps kernel selection code.
:param use_profiling: If true Neutron Converter will enable profiling for neutron delegated model
:param dump_kernel_selection_code: Whether Neutron Compiler dumps kernel selection code.
:param use_profiling: If true Neutron Compiler will enable profiling for neutron delegated model
:return: self for method chaining
"""

Expand Down Expand Up @@ -282,9 +282,9 @@ def preprocess( # noqa C901
)

with capture_fd_output() as tmp:
neutron_model = NeutronConverterManager(
neutron_model = NeutronCompilerManager(
dump_kernel_selection_code
).convert(
).compile(
tflite_model,
target,
delegation_tag,
Expand Down
6 changes: 3 additions & 3 deletions backends/nxp/runtime/NeutronDriver.h
Original file line number Diff line number Diff line change
Expand Up @@ -42,23 +42,23 @@ typedef void* NeutronModelHandle;

typedef struct {
/// Neutron microcode buffer address.
/// The Neutron microcode is generated by the Neutron converter tool.
/// The Neutron microcode is generated by the Neutron compiler tool.
/// The microcode buffer, 16 bytes aligned, is allocated and initialized by
/// the application or ML framework. The microcode buffer is passed by
/// reference to the Neutron firmware. The microcode buffer is specific for a
/// given ML model.
const void* microcode;

/// Neutron weights buffer address.
/// The Neutron weights is generated by the Neutron converter tool.
/// The Neutron weights is generated by the Neutron compiler tool.
/// The weights buffer, 16 bytes aligned, is allocated and initialized by the
/// application or ML framework. The weights buffer address is passed by
/// reference to the Neutron-firmware. The weights buffer is specific for a
/// given ML model.
const void* weights;

/// Neutron kernels buffer address.
/// The Neutron kernels are generated by the Neutron converter tool.
/// The Neutron kernels are generated by the Neutron compiler tool.
/// The kernels buffer, 16 bytes aligned, is allocated and initialized by the
/// application or ML framework. The kernels buffer address is passed by
/// reference to the Neutron-firmware. The kernels buffer is specific for a
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -122,10 +122,10 @@ def test_noop_partitions__concatenate_one_tensor_and_add_zeros():

@pytest.mark.xfail(
strict=True,
reason="Neutron Converter currently supports these 2 noops in sequence.",
reason="Neutron Compiler currently supports these 2 noops in sequence.",
)
def test_noop_partitions__concatenate_one_tensor_and_add_zeros__forced_delegation():
# When the noop `Concatenate` and noop `Add` are in sequence, Neutron Converter supports them. This edge case is
# When the noop `Concatenate` and noop `Add` are in sequence, Neutron Compiler supports them. This edge case is
# not reflected in our logic. But as this edge case is extremely rare (and even if it ever happened in a real
# model, the consequences would be minimal), fixing it is not a priority.

Expand Down
Loading
Loading