diff --git a/docs/domain-entity-rename.md b/docs/domain-entity-rename.md index 9a5c0e71..10f4131b 100644 --- a/docs/domain-entity-rename.md +++ b/docs/domain-entity-rename.md @@ -45,8 +45,8 @@ stream = AgentStream.get(name="prod-traces", project_name="my-project") streams = AgentStream.list(project_name="my-project") # Enable evaluators on the stream -from splunk_ao.schema.metrics import SplunkAOMetrics -stream.enable_evaluators([SplunkAOMetrics.correctness, SplunkAOMetrics.completeness]) +from splunk_ao.schema.metrics import SplunkAOEvaluators +stream.enable_evaluators([SplunkAOEvaluators.correctness, SplunkAOEvaluators.completeness]) ``` ```python diff --git a/examples/agent/langgraph-fsi-agent/after/test.py b/examples/agent/langgraph-fsi-agent/after/test.py index 3e1035a7..c5a49aed 100644 --- a/examples/agent/langgraph-fsi-agent/after/test.py +++ b/examples/agent/langgraph-fsi-agent/after/test.py @@ -15,7 +15,7 @@ from langchain_core.callbacks import Callbacks from langchain_core.messages import HumanMessage -from splunk_ao import SplunkAOMetrics, splunk_ao_context +from splunk_ao import SplunkAOEvaluators, splunk_ao_context from splunk_ao.datasets import create_dataset, get_dataset, delete_dataset from splunk_ao.experiments import get_experiment, run_experiment from splunk_ao.handlers.langchain import SplunkAOCallback @@ -108,7 +108,7 @@ def test_run_experiment_with_dataset(): experiment_name="langgraph-fsi-experiment", dataset_name=DATASET_NAME, function=send_message_to_supervisor_agent, - metrics=[SplunkAOMetrics.action_advancement, SplunkAOMetrics.action_completion, SplunkAOMetrics.tool_error_rate, SplunkAOMetrics.tool_selection_quality], + metrics=[SplunkAOEvaluators.action_advancement, SplunkAOEvaluators.action_completion, SplunkAOEvaluators.tool_error_rate, SplunkAOEvaluators.tool_selection_quality], project=os.getenv("SPLUNK_AO_PROJECT"), ) diff --git a/examples/chatbot/sample-project-chatbot/anthropic/test.py b/examples/chatbot/sample-project-chatbot/anthropic/test.py index 50412891..8a5fa447 100644 --- a/examples/chatbot/sample-project-chatbot/anthropic/test.py +++ b/examples/chatbot/sample-project-chatbot/anthropic/test.py @@ -12,7 +12,7 @@ from dotenv import load_dotenv -from splunk_ao import SplunkAOMetrics +from splunk_ao import SplunkAOEvaluators from splunk_ao.datasets import create_dataset, get_dataset from splunk_ao.experiments import get_experiment, run_experiment @@ -75,8 +75,8 @@ def test_run_experiment_with_dataset(): dataset_name="simple-chatbot-unit-test-dataset", function=chat_with_llm, metrics=[ - SplunkAOMetrics.correctness, - SplunkAOMetrics.instruction_adherence, + SplunkAOEvaluators.correctness, + SplunkAOEvaluators.instruction_adherence, ], project=os.getenv("SPLUNK_AO_PROJECT"), ) diff --git a/examples/chatbot/sample-project-chatbot/azure-inference/test.py b/examples/chatbot/sample-project-chatbot/azure-inference/test.py index 50412891..8a5fa447 100644 --- a/examples/chatbot/sample-project-chatbot/azure-inference/test.py +++ b/examples/chatbot/sample-project-chatbot/azure-inference/test.py @@ -12,7 +12,7 @@ from dotenv import load_dotenv -from splunk_ao import SplunkAOMetrics +from splunk_ao import SplunkAOEvaluators from splunk_ao.datasets import create_dataset, get_dataset from splunk_ao.experiments import get_experiment, run_experiment @@ -75,8 +75,8 @@ def test_run_experiment_with_dataset(): dataset_name="simple-chatbot-unit-test-dataset", function=chat_with_llm, metrics=[ - SplunkAOMetrics.correctness, - SplunkAOMetrics.instruction_adherence, + SplunkAOEvaluators.correctness, + SplunkAOEvaluators.instruction_adherence, ], project=os.getenv("SPLUNK_AO_PROJECT"), ) diff --git a/examples/chatbot/sample-project-chatbot/openai-ollama/test.py b/examples/chatbot/sample-project-chatbot/openai-ollama/test.py index 50412891..8a5fa447 100644 --- a/examples/chatbot/sample-project-chatbot/openai-ollama/test.py +++ b/examples/chatbot/sample-project-chatbot/openai-ollama/test.py @@ -12,7 +12,7 @@ from dotenv import load_dotenv -from splunk_ao import SplunkAOMetrics +from splunk_ao import SplunkAOEvaluators from splunk_ao.datasets import create_dataset, get_dataset from splunk_ao.experiments import get_experiment, run_experiment @@ -75,8 +75,8 @@ def test_run_experiment_with_dataset(): dataset_name="simple-chatbot-unit-test-dataset", function=chat_with_llm, metrics=[ - SplunkAOMetrics.correctness, - SplunkAOMetrics.instruction_adherence, + SplunkAOEvaluators.correctness, + SplunkAOEvaluators.instruction_adherence, ], project=os.getenv("SPLUNK_AO_PROJECT"), ) diff --git a/examples/experiments/multi-turn/README.md b/examples/experiments/multi-turn/README.md index f81f61dc..ac09920b 100644 --- a/examples/experiments/multi-turn/README.md +++ b/examples/experiments/multi-turn/README.md @@ -61,13 +61,13 @@ The `METRIC_NAME` variable in this script cites a session-level metric. Pre-defined session-level metrics include: -- `SplunkAOMetrics.conversation_quality` -- `SplunkAOMetrics.action_completion` -- `SplunkAOMetrics.action_advancement` -- `SplunkAOMetrics.agent_efficiency` -- `SplunkAOMetrics.context_adherence` -- `SplunkAOMetrics.context_relevance` -- `SplunkAOMetrics.tool_error_rate` +- `SplunkAOEvaluators.conversation_quality` +- `SplunkAOEvaluators.action_completion` +- `SplunkAOEvaluators.action_advancement` +- `SplunkAOEvaluators.agent_efficiency` +- `SplunkAOEvaluators.context_adherence` +- `SplunkAOEvaluators.context_relevance` +- `SplunkAOEvaluators.tool_error_rate` Related documentation: [Metrics Comparison](https://agent-observability-docs.splunk.com/concepts/metrics/metric-comparison) diff --git a/examples/experiments/multi-turn/basic-example.py b/examples/experiments/multi-turn/basic-example.py index 5d6189fc..5e167532 100644 --- a/examples/experiments/multi-turn/basic-example.py +++ b/examples/experiments/multi-turn/basic-example.py @@ -1,7 +1,7 @@ import os import time -from splunk_ao import SplunkAOMetrics, splunk_ao_context +from splunk_ao import SplunkAOEvaluators, splunk_ao_context from splunk_ao.experiments import create_experiment from splunk_ao.projects import create_project, get_project from splunk_ao.resources.models import MetricSuccess @@ -9,7 +9,7 @@ from splunk_ao.utils.metrics import create_metric_configs # Provide the name of a session-level metric -METRIC_NAME = SplunkAOMetrics.conversation_quality +METRIC_NAME = SplunkAOEvaluators.conversation_quality # example custom metric name (must be set up in advance) # METRIC_NAME = "multi-turn-session-test-metric-apples" diff --git a/examples/experiments/rag-and-tools/experiment.py b/examples/experiments/rag-and-tools/experiment.py index aa4c19a8..1cca3d6f 100644 --- a/examples/experiments/rag-and-tools/experiment.py +++ b/examples/experiments/rag-and-tools/experiment.py @@ -1,6 +1,6 @@ import os -from splunk_ao import SplunkAOMetrics +from splunk_ao import SplunkAOEvaluators from splunk_ao.experiments import run_experiment from app import get_users_horoscope @@ -25,10 +25,10 @@ def main(): dataset=dataset, function=get_users_horoscope, metrics=[ - SplunkAOMetrics.tool_error_rate, - SplunkAOMetrics.tool_selection_quality, - SplunkAOMetrics.chunk_attribution_utilization, - SplunkAOMetrics.context_adherence, + SplunkAOEvaluators.tool_error_rate, + SplunkAOEvaluators.tool_selection_quality, + SplunkAOEvaluators.chunk_attribution_utilization, + SplunkAOEvaluators.context_adherence, ], project=os.environ["SPLUNK_AO_PROJECT"], ) diff --git a/examples/experiments/upload_experiment/README.md b/examples/experiments/upload_experiment/README.md index a17cad83..e5420cdb 100644 --- a/examples/experiments/upload_experiment/README.md +++ b/examples/experiments/upload_experiment/README.md @@ -117,12 +117,12 @@ Edit `dataset.json` with your own evaluation data. The example uses space missio In `upload_existing_results.py`, modify the `metrics` parameter in `upload_experiment()`: ```python -from splunk_ao.schema.metrics import SplunkAOMetrics +from splunk_ao.schema.metrics import SplunkAOEvaluators custom_metrics = [ - SplunkAOMetrics.ground_truth_adherence, - SplunkAOMetrics.context_adherence, - SplunkAOMetrics.correctness, + SplunkAOEvaluators.ground_truth_adherence, + SplunkAOEvaluators.context_adherence, + SplunkAOEvaluators.correctness, # Add any other Splunk AO metrics you want ] diff --git a/examples/experiments/upload_experiment/upload_existing_results.py b/examples/experiments/upload_experiment/upload_existing_results.py index 4673a181..a0a13a4f 100644 --- a/examples/experiments/upload_experiment/upload_existing_results.py +++ b/examples/experiments/upload_experiment/upload_existing_results.py @@ -32,7 +32,7 @@ # Splunk AO imports from splunk_ao.datasets import create_dataset, get_dataset from splunk_ao.experiments import run_experiment -from splunk_ao.schema.metrics import SplunkAOMetrics +from splunk_ao.schema.metrics import SplunkAOEvaluators from splunk_ao import splunk_ao_context @@ -239,11 +239,11 @@ def upload_experiment( # Use default metrics if none provided if metrics is None: metrics = [ - SplunkAOMetrics.ground_truth_adherence, - SplunkAOMetrics.context_adherence, - SplunkAOMetrics.chunk_attribution_utilization, - SplunkAOMetrics.completeness, - SplunkAOMetrics.correctness, + SplunkAOEvaluators.ground_truth_adherence, + SplunkAOEvaluators.context_adherence, + SplunkAOEvaluators.chunk_attribution_utilization, + SplunkAOEvaluators.completeness, + SplunkAOEvaluators.correctness, ] # Run experiment with your data diff --git a/splunk-ao-migration-tool/README.md b/splunk-ao-migration-tool/README.md index a4b96273..c972e237 100644 --- a/splunk-ao-migration-tool/README.md +++ b/splunk-ao-migration-tool/README.md @@ -196,12 +196,12 @@ All sub-module paths follow the same rename pattern: | Old | New | |-----|-----| | `GalileoMetric` | `SplunkAOMetric` | -| `GalileoMetrics` | `SplunkAOMetrics` | +| `GalileoMetrics` | `SplunkAOEvaluators` | | `GalileoScorers` | **Removed** (see §5.2) | ```diff - from galileo import GalileoMetric, GalileoMetrics -+ from splunk_ao import SplunkAOMetric, SplunkAOMetrics ++ from splunk_ao import SplunkAOMetric, SplunkAOEvaluators ``` ### 3.4 Handlers & Middleware @@ -330,14 +330,14 @@ The following symbols from `galileo` are **not available** in `splunk-ao`: ### 5.2 `GalileoScorers` Removed -The `GalileoScorers` enum has been removed entirely. Migrate to `SplunkAOMetrics`: +The `GalileoScorers` enum has been removed entirely. Migrate to `SplunkAOEvaluators`: ```diff - from galileo.schema.metrics import GalileoScorers -+ from splunk_ao.schema.metrics import SplunkAOMetrics ++ from splunk_ao.schema.metrics import SplunkAOEvaluators - scorer = GalileoScorers.completeness -+ scorer = SplunkAOMetrics.completeness ++ scorer = SplunkAOEvaluators.completeness ``` --- @@ -441,8 +441,8 @@ The following are **unchanged** between galileo and splunk-ao and require no mig - [ ] Rename `GalileoLoggerException` → `SplunkAOLoggerException` - [ ] Rename `GalileoFutureError` → `SplunkAOFutureError` - [ ] Rename `GalileoMetric` → `SplunkAOMetric` -- [ ] Rename `GalileoMetrics` → `SplunkAOMetrics` -- [ ] Replace `GalileoScorers` with `SplunkAOMetrics` +- [ ] Rename `GalileoMetrics` → `SplunkAOEvaluators` +- [ ] Replace `GalileoScorers` with `SplunkAOEvaluators` - [ ] Rename `GalileoAgentControlBridge` → `SplunkAOAgentControlBridge` - [ ] Rename all `GALILEO_*` environment variables to `SPLUNK_AO_*` - [ ] Update `.env`, `.env.example`, CI/CD secrets, and deployment configs diff --git a/src/splunk_ao/__init__.py b/src/splunk_ao/__init__.py index d435ae8a..54d1f98a 100644 --- a/src/splunk_ao/__init__.py +++ b/src/splunk_ao/__init__.py @@ -67,7 +67,7 @@ from splunk_ao.provider import AnthropicProvider, AzureProvider, BedrockProvider, OpenAIProvider, Provider from splunk_ao.resources.models.document import Document from splunk_ao.schema.message import Message -from splunk_ao.schema.metrics import SplunkAOMetrics +from splunk_ao.schema.metrics import SplunkAOEvaluators from splunk_ao.shared.base import SyncState from splunk_ao.shared.exceptions import ( AmbiguousConfigurationError, @@ -149,7 +149,7 @@ "SplunkAOFutureError", "SplunkAOLogger", "SplunkAOLoggerException", - "SplunkAOMetrics", + "SplunkAOEvaluators", "StepType", "StepWithChildSpans", "SyncState", diff --git a/src/splunk_ao/agent_stream.py b/src/splunk_ao/agent_stream.py index 23e7c905..5ba50dfc 100644 --- a/src/splunk_ao/agent_stream.py +++ b/src/splunk_ao/agent_stream.py @@ -21,7 +21,7 @@ from splunk_ao.resources.models.log_records_available_columns_response import LogRecordsAvailableColumnsResponse from splunk_ao.resources.types import Unset from splunk_ao.schema.filters import FilterType -from splunk_ao.schema.metrics import LocalMetricConfig, Metric, SplunkAOMetrics +from splunk_ao.schema.metrics import LocalMetricConfig, Metric, SplunkAOEvaluators from splunk_ao.search import RecordType, Search from splunk_ao.shared.base import StateManagementMixin, SyncState from splunk_ao.shared.exceptions import ValidationError @@ -78,10 +78,10 @@ class AgentStream(StateManagementMixin): agent_stream = project.create_agent_stream(name="Production Logs") # Enable metrics on the log stream - from splunk_ao.schema.metrics import SplunkAOMetrics + from splunk_ao.schema.metrics import SplunkAOEvaluators local_metrics = log_stream.enable_evaluators([ - SplunkAOMetrics.correctness, - SplunkAOMetrics.completeness, + SplunkAOEvaluators.correctness, + SplunkAOEvaluators.completeness, "context_relevance" ]) @@ -441,7 +441,7 @@ def get_metrics(self) -> builtins.list[str]: return metric_names def set_metrics( - self, metrics: builtins.list[SplunkAOMetrics | Metric | LocalMetricConfig | str] + self, metrics: builtins.list[SplunkAOEvaluators | Metric | LocalMetricConfig | str] ) -> builtins.list[LocalMetricConfig]: """ Set (replace) the metrics on this log stream. @@ -451,7 +451,7 @@ def set_metrics( Args: metrics: List of metrics to set. Supports: - - SplunkAOMetrics enum values (e.g., SplunkAOMetrics.correctness) + - SplunkAOEvaluators enum values (e.g., SplunkAOEvaluators.correctness) - Metric objects (including from Metric.get(id="...")) - LocalMetricConfig objects for custom scoring functions - String names of built-in metrics diff --git a/src/splunk_ao/agent_streams.py b/src/splunk_ao/agent_streams.py index d3d8c0f6..84ad68e6 100644 --- a/src/splunk_ao/agent_streams.py +++ b/src/splunk_ao/agent_streams.py @@ -12,7 +12,7 @@ from splunk_ao.resources.models.log_stream_create_request import LogStreamCreateRequest from splunk_ao.resources.models.log_stream_response import LogStreamResponse from splunk_ao.resources.types import Unset -from splunk_ao.schema.metrics import LocalMetricConfig, Metric, SplunkAOMetrics +from splunk_ao.schema.metrics import LocalMetricConfig, Metric, SplunkAOEvaluators from splunk_ao.utils.env_helpers import _get_agent_stream_from_env, _get_project_from_env from splunk_ao.utils.log_config import get_logger from splunk_ao.utils.metrics import create_metric_configs @@ -78,7 +78,7 @@ class AgentStream(LogStreamResponse): # Enable metrics on a log stream - RECOMMENDED APPROACH from splunk_ao.agent_streams import enable_evaluators - from splunk_ao.schema.metrics import SplunkAOMetrics + from splunk_ao.schema.metrics import SplunkAOEvaluators # Set environment variables first # export SPLUNK_AO_AGENT_STREAM="Production Logs" @@ -86,8 +86,8 @@ class AgentStream(LogStreamResponse): # Clean and simple - just pass the metrics! local_metrics = enable_evaluators([ - SplunkAOMetrics.correctness, - SplunkAOMetrics.completeness, + SplunkAOEvaluators.correctness, + SplunkAOEvaluators.completeness, "context_relevance" ]) @@ -123,7 +123,7 @@ def __init__(self, agent_stream: None | LogStreamResponse = None): return def enable_evaluators( - self, metrics: builtins.list[SplunkAOMetrics | Metric | LocalMetricConfig | str] + self, metrics: builtins.list[SplunkAOEvaluators | Metric | LocalMetricConfig | str] ) -> builtins.list[LocalMetricConfig]: """ Enable metrics directly on this log stream instance. @@ -139,10 +139,10 @@ def enable_evaluators( Parameters ---------- - metrics : builtins.list[Union[SplunkAOMetrics, Metric, LocalMetricConfig, str]] + metrics : builtins.list[Union[SplunkAOEvaluators, Metric, LocalMetricConfig, str]] List of metrics to enable on this log stream. Supports multiple input formats: - - **SplunkAOMetrics enum values**: Built-in metrics like `SplunkAOMetrics.correctness` + - **SplunkAOEvaluators enum values**: Built-in metrics like `SplunkAOEvaluators.correctness` - **Metric objects**: Custom metrics with optional version specifications - **LocalMetricConfig objects**: Client-side metrics with custom scoring functions - **String names**: Built-in metric names like "correctness" or "toxicity" @@ -169,7 +169,7 @@ def enable_evaluators( ```python from splunk_ao.agent_streams import AgentStreams - from splunk_ao.schema.metrics import SplunkAOMetrics + from splunk_ao.schema.metrics import SplunkAOEvaluators # Get a log stream first log_streams = AgentStreams() @@ -177,8 +177,8 @@ def enable_evaluators( # Enable metrics directly - clean and intuitive! local_metrics = log_stream.enable_evaluators([ - SplunkAOMetrics.correctness, - SplunkAOMetrics.completeness, + SplunkAOEvaluators.correctness, + SplunkAOEvaluators.completeness, "context_relevance", "toxicity" ]) @@ -196,7 +196,7 @@ def custom_scorer(trace_or_span): return 0.75 # Your scoring logic local_metrics = log_stream.enable_evaluators([ - SplunkAOMetrics.correctness, + SplunkAOEvaluators.correctness, "completeness", Metric(name="domain_relevance", version=3), LocalMetricConfig(name="custom_metric", scorer_fn=custom_scorer) @@ -481,7 +481,7 @@ def enable_evaluators( *, agent_stream_name: str | None = None, project_name: str | None = None, - metrics: builtins.list[SplunkAOMetrics | Metric | LocalMetricConfig | str], + metrics: builtins.list[SplunkAOEvaluators | Metric | LocalMetricConfig | str], ) -> builtins.list[LocalMetricConfig]: """ Enable metrics for a log stream by configuring scorers. @@ -498,9 +498,9 @@ def enable_evaluators( The name of the log stream. Takes precedence over the SPLUNK_AO_AGENT_STREAM environment variable. Defaults to None. project_name : Optional[str], optional The name of the project. Takes precedence over the SPLUNK_AO_PROJECT environment variable. Defaults to None. - metrics : builtins.list[Union[SplunkAOMetrics, Metric, LocalMetricConfig, str]] + metrics : builtins.list[Union[SplunkAOEvaluators, Metric, LocalMetricConfig, str]] List of metrics to enable. Can include: - - SplunkAOMetrics enum values (e.g., SplunkAOMetrics.correctness) + - SplunkAOEvaluators enum values (e.g., SplunkAOEvaluators.correctness) - Metric objects with name and optional version - LocalMetricConfig objects for custom local metrics - String names of built-in metrics @@ -520,15 +520,15 @@ def enable_evaluators( ```python # Enable built-in metrics with explicit parameters from splunk_ao.agent_streams import AgentStreams - from splunk_ao.schema.metrics import SplunkAOMetrics + from splunk_ao.schema.metrics import SplunkAOEvaluators log_streams = AgentStreams() scorer_configs, local_metrics = log_streams.enable_evaluators( agent_stream_name="Production Logs", project_name="My AI Project", metrics=[ - SplunkAOMetrics.correctness, - SplunkAOMetrics.completeness, + SplunkAOEvaluators.correctness, + SplunkAOEvaluators.completeness, "context_relevance", ], ) @@ -686,7 +686,7 @@ def enable_evaluators( *, agent_stream_name: str | None = None, project_name: str | None = None, - metrics: builtins.list[SplunkAOMetrics | Metric | LocalMetricConfig | str], + metrics: builtins.list[SplunkAOEvaluators | Metric | LocalMetricConfig | str], ) -> builtins.list[LocalMetricConfig]: """ Enable metrics for a log stream with flexible parameter and environment variable support. @@ -714,9 +714,9 @@ def enable_evaluators( project_name : Optional[str], optional The name of the project. Takes precedence over SPLUNK_AO_PROJECT environment variable. If None, will use SPLUNK_AO_PROJECT env var. Defaults to None. - metrics : builtins.list[Union[SplunkAOMetrics, Metric, LocalMetricConfig, str]] + metrics : builtins.list[Union[SplunkAOEvaluators, Metric, LocalMetricConfig, str]] List of metrics to enable on the log stream. Can include: - - SplunkAOMetrics enum values (e.g., SplunkAOMetrics.correctness) + - SplunkAOEvaluators enum values (e.g., SplunkAOEvaluators.correctness) - Metric objects with name and optional version for custom metrics - LocalMetricConfig objects for client-side custom scoring functions - String names of built-in metrics (e.g., "correctness", "toxicity") @@ -742,14 +742,14 @@ def enable_evaluators( ```python # Enable built-in metrics with explicit parameters from splunk_ao.agent_streams import enable_evaluators - from splunk_ao.schema.metrics import SplunkAOMetrics + from splunk_ao.schema.metrics import SplunkAOEvaluators local_metrics = enable_evaluators( agent_stream_name="Production Logs", project_name="My AI Project", metrics=[ - SplunkAOMetrics.correctness, - SplunkAOMetrics.completeness, + SplunkAOEvaluators.correctness, + SplunkAOEvaluators.completeness, "context_relevance", ], ) @@ -772,7 +772,7 @@ def response_length_scorer(trace_or_span): local_metrics = enable_evaluators( agent_stream_name="Development Logs", metrics=[ - SplunkAOMetrics.correctness, + SplunkAOEvaluators.correctness, "toxicity", Metric(name="my_custom_metric", version=2), LocalMetricConfig( diff --git a/src/splunk_ao/evaluator.py b/src/splunk_ao/evaluator.py index ef64164c..310db4a2 100644 --- a/src/splunk_ao/evaluator.py +++ b/src/splunk_ao/evaluator.py @@ -39,7 +39,7 @@ ) from splunk_ao.resources.models.invalid_result import InvalidResult from splunk_ao.resources.types import UNSET, File, Unset -from splunk_ao.schema.metrics import LocalMetricConfig, SplunkAOMetrics +from splunk_ao.schema.metrics import LocalMetricConfig, SplunkAOEvaluators from splunk_ao.schema.metrics import Metric as SchemaMetric from splunk_ao.scorers import Scorers from splunk_ao.shared.base import StateManagementMixin, SyncState @@ -68,17 +68,17 @@ class BuiltInEvaluators: Evaluator.metrics.toxicity """ - def __getattr__(self, name: str) -> SplunkAOMetrics: + def __getattr__(self, name: str) -> SplunkAOEvaluators: """Allow attribute-style access to built-in metrics.""" # Try to find the metric by name (enum names match UI-visible names) - for scorer in SplunkAOMetrics: + for scorer in SplunkAOEvaluators: if scorer.name == name: return scorer - raise AttributeError(f"Built-in metric '{name}' not found. Available: {[s.name for s in SplunkAOMetrics]}") + raise AttributeError(f"Built-in metric '{name}' not found. Available: {[s.name for s in SplunkAOEvaluators]}") def __dir__(self) -> list[str]: """Return list of available metric names for autocomplete.""" - return [scorer.name for scorer in SplunkAOMetrics] + return [scorer.name for scorer in SplunkAOEvaluators] # Backwards-compatible alias diff --git a/src/splunk_ao/experiment.py b/src/splunk_ao/experiment.py index bb432c10..6cd4bc8b 100644 --- a/src/splunk_ao/experiment.py +++ b/src/splunk_ao/experiment.py @@ -45,7 +45,7 @@ # TODO: DatasetRecord needed for function-based experiments # from splunk_ao.schema.datasets import DatasetRecord from splunk_ao.schema.filters import FilterType -from splunk_ao.schema.metrics import LocalMetricConfig, Metric, SplunkAOMetrics +from splunk_ao.schema.metrics import LocalMetricConfig, Metric, SplunkAOEvaluators from splunk_ao.search import RecordType, Search from splunk_ao.shared.base import StateManagementMixin, SyncState from splunk_ao.shared.exceptions import ValidationError @@ -176,7 +176,7 @@ class Experiment(StateManagementMixin): prompt_name: str | None created_at: datetime.datetime | None updated_at: datetime.datetime | None - metrics: builtins.list[SplunkAOMetrics | Metric | LocalMetricConfig | str] | None + metrics: builtins.list[SplunkAOEvaluators | Metric | LocalMetricConfig | str] | None # TODO: Function-based experiments temporarily disabled - need to validate implementation # function: Callable | None model_alias: str | None @@ -209,7 +209,7 @@ def __init__( prompt: Prompt | PromptTemplate | str | None = None, prompt_name: str | None = None, model: Model | str | None = None, - metrics: builtins.list[SplunkAOMetrics | Metric | LocalMetricConfig | str] | None = None, + metrics: builtins.list[SplunkAOEvaluators | Metric | LocalMetricConfig | str] | None = None, project_id: str | None = None, project_name: str | None = None, prompt_settings: PromptRunSettings | None = None, @@ -2031,14 +2031,14 @@ def experiment_columns(self) -> ColumnCollection: columns = [Column(col) for col in response.columns] return ColumnCollection(columns) - def get_metric_aggregate(self, metric: SplunkAOMetrics | str) -> MetricAggregates | None: + def get_metric_aggregate(self, metric: SplunkAOEvaluators | str) -> MetricAggregates | None: """Return aggregate statistics for a specific metric. Looks up a metric by any of the following identifiers, tried in order: - 1. :class:`~splunk_ao.schema.metrics.SplunkAOMetrics` enum value — its + 1. :class:`~splunk_ao.schema.metrics.SplunkAOEvaluators` enum value — its ``value`` IS the human-readable label (e.g. - ``SplunkAOMetrics.correctness`` → ``"Correctness"``). + ``SplunkAOEvaluators.correctness`` → ``"Correctness"``). 2. Scorer UUID string — direct lookup in :attr:`metric_aggregates`, no column resolution needed. 3. Human-readable label string (e.g. ``"Correctness"``) — resolved @@ -2052,7 +2052,7 @@ def get_metric_aggregate(self, metric: SplunkAOMetrics | str) -> MetricAggregate Parameters ---------- metric : - Any of: a :class:`SplunkAOMetrics` enum value, scorer UUID string, + Any of: a :class:`SplunkAOEvaluators` enum value, scorer UUID string, human-readable label, or legacy metric_key_alias. Returns @@ -2066,21 +2066,21 @@ def get_metric_aggregate(self, metric: SplunkAOMetrics | str) -> MetricAggregate -------- Poll until a specific metric is computed, then assert:: - from splunk_ao.schema.metrics import SplunkAOMetrics + from splunk_ao.schema.metrics import SplunkAOEvaluators - while experiment.get_metric_aggregate(SplunkAOMetrics.correctness) is None: + while experiment.get_metric_aggregate(SplunkAOEvaluators.correctness) is None: time.sleep(5) experiment.refresh() - agg = experiment.get_metric_aggregate(SplunkAOMetrics.correctness) + agg = experiment.get_metric_aggregate(SplunkAOEvaluators.correctness) assert agg.avg >= 0.95 """ aggregates = self.metric_aggregates if not aggregates: return None - # SplunkAOMetrics.value IS the human-readable label (e.g. "Correctness") - metric_str = metric.value if isinstance(metric, SplunkAOMetrics) else metric + # SplunkAOEvaluators.value IS the human-readable label (e.g. "Correctness") + metric_str = metric.value if isinstance(metric, SplunkAOEvaluators) else metric # Scorer UUID → direct lookup, no column resolution needed if _UUID_RE.fullmatch(metric_str): diff --git a/src/splunk_ao/experiments.py b/src/splunk_ao/experiments.py index b61b6fbe..52e5d1e5 100644 --- a/src/splunk_ao/experiments.py +++ b/src/splunk_ao/experiments.py @@ -28,7 +28,7 @@ ) from splunk_ao.schema.datasets import DatasetRecord from splunk_ao.schema.experiment_group import ExperimentGroupResponse -from splunk_ao.schema.metrics import LocalMetricConfig, Metric, SplunkAOMetrics +from splunk_ao.schema.metrics import LocalMetricConfig, Metric, SplunkAOEvaluators from splunk_ao.utils.datasets import create_rows_from_records, load_dataset from splunk_ao.utils.exceptions import _format_http_validation_error from splunk_ao.utils.headers_data import get_sdk_header @@ -318,7 +318,7 @@ def run_experiment( dataset: Dataset | list[dict[str, Any] | str] | str | None = None, dataset_id: str | None = None, dataset_name: str | None = None, - metrics: list[SplunkAOMetrics | Metric | LocalMetricConfig | str] | None = None, + metrics: list[SplunkAOEvaluators | Metric | LocalMetricConfig | str] | None = None, function: Callable | None = None, experiment_tags: dict[str, str] | None = None, on_error: Callable[[Exception], None] | None = None, diff --git a/src/splunk_ao/schema/metrics.py b/src/splunk_ao/schema/metrics.py index 04c022dd..c41896ce 100644 --- a/src/splunk_ao/schema/metrics.py +++ b/src/splunk_ao/schema/metrics.py @@ -11,7 +11,7 @@ from galileo_core.schemas.shared.metric import MetricValueType -class SplunkAOMetrics(StrEnum): +class SplunkAOEvaluators(StrEnum): """Built-in Splunk AO metric scorers. Values are human-readable UI labels used for scorer lookup via the API. diff --git a/src/splunk_ao/types.py b/src/splunk_ao/types.py index dda394e2..778189c7 100644 --- a/src/splunk_ao/types.py +++ b/src/splunk_ao/types.py @@ -6,11 +6,11 @@ """ from splunk_ao.evaluator import Evaluator -from splunk_ao.schema.metrics import LocalMetricConfig, SplunkAOMetrics +from splunk_ao.schema.metrics import LocalMetricConfig, SplunkAOEvaluators # Unified metric type that accepts all valid metric specifications MetricSpec = ( - SplunkAOMetrics # Built-in scorer enum (e.g., SplunkAOMetrics.correctness) + SplunkAOEvaluators # Built-in scorer enum (e.g., SplunkAOEvaluators.correctness) | Evaluator # Custom or local evaluator object | LocalMetricConfig # Legacy local metric config | str # String name of built-in metric (e.g., "correctness") diff --git a/src/splunk_ao/utils/metrics.py b/src/splunk_ao/utils/metrics.py index ceb05621..93b33884 100644 --- a/src/splunk_ao/utils/metrics.py +++ b/src/splunk_ao/utils/metrics.py @@ -9,7 +9,7 @@ from galileo_core.schemas.shared.metric import MetricValueType from splunk_ao.resources.models.scorer_config import ScorerConfig from splunk_ao.resources.models.scorer_response import ScorerResponse -from splunk_ao.schema.metrics import LocalMetricConfig, Metric, SplunkAOMetrics +from splunk_ao.schema.metrics import LocalMetricConfig, Metric, SplunkAOEvaluators from splunk_ao.scorers import Scorers, ScorerSettings logger = logging.getLogger(__name__) @@ -95,7 +95,7 @@ def _is_uuid(value: str) -> bool: def create_metric_configs( project_id: str, run_id: str | None, # Can be experiment_id, agent_stream_id, or None (for trigger=True flow) - metrics: builtins.list[SplunkAOMetrics | Metric | LocalMetricConfig | str], + metrics: builtins.list[SplunkAOEvaluators | Metric | LocalMetricConfig | str], ) -> tuple[builtins.list[ScorerConfig], builtins.list[LocalMetricConfig]]: """ Process metrics and create scorer configurations for experiments or log streams. @@ -104,7 +104,7 @@ def create_metric_configs( validates they exist, and registers server-side metrics with Splunk AO. Metrics can be specified as: - - SplunkAOMetrics enum values (human-readable labels like "Correctness") + - SplunkAOEvaluators enum values (human-readable labels like "Correctness") - Metric objects with name and optional version - UUID strings (scorer IDs for direct lookup) - Plain strings (searched by label with name fallback) @@ -138,7 +138,7 @@ def create_metric_configs( # Categorize metrics by type for metric in metrics: - if isinstance(metric, SplunkAOMetrics): + if isinstance(metric, SplunkAOEvaluators): label_searches.append((metric.value, None)) elif isinstance(metric, Metric): label_searches.append((metric.name, metric.version)) diff --git a/tests/schemas/test_metrics.py b/tests/schemas/test_metrics.py index c8b0451d..667f0b87 100644 --- a/tests/schemas/test_metrics.py +++ b/tests/schemas/test_metrics.py @@ -1,4 +1,4 @@ -from splunk_ao.schema.metrics import Metric, SplunkAOMetrics +from splunk_ao.schema.metrics import Metric, SplunkAOEvaluators def test_metric_custom_with_version() -> None: @@ -16,15 +16,15 @@ def test_metric_custom_no_version() -> None: def test_galileo_metrics_values_are_nonempty_strings() -> None: - """All SplunkAOMetrics values are non-empty human-readable strings.""" - for member in SplunkAOMetrics: + """All SplunkAOEvaluators values are non-empty human-readable strings.""" + for member in SplunkAOEvaluators: assert isinstance(member.value, str), f"{member.name} value is not a string" assert len(member.value.strip()) > 0, f"{member.name} has an empty value" def test_galileo_metrics_is_str_compatible() -> None: - """SplunkAOMetrics members are str-compatible (usable as plain strings).""" - member = SplunkAOMetrics.correctness + """SplunkAOEvaluators members are str-compatible (usable as plain strings).""" + member = SplunkAOEvaluators.correctness assert isinstance(member, str) assert member == "Correctness" assert member.value == "Correctness" @@ -33,15 +33,15 @@ def test_galileo_metrics_is_str_compatible() -> None: def test_galileo_metrics_naming_convention() -> None: """Base names map to LLM versions, _luna suffix maps to SLM versions.""" # LLM versions (base names) should NOT have "(SLM)" in the label - assert "(SLM)" not in SplunkAOMetrics.input_pii.value - assert "(SLM)" not in SplunkAOMetrics.input_tone.value - assert "(SLM)" not in SplunkAOMetrics.output_pii.value - assert "(SLM)" not in SplunkAOMetrics.output_tone.value - assert "(SLM)" not in SplunkAOMetrics.correctness.value + assert "(SLM)" not in SplunkAOEvaluators.input_pii.value + assert "(SLM)" not in SplunkAOEvaluators.input_tone.value + assert "(SLM)" not in SplunkAOEvaluators.output_pii.value + assert "(SLM)" not in SplunkAOEvaluators.output_tone.value + assert "(SLM)" not in SplunkAOEvaluators.correctness.value # SLM versions (_luna suffix) should have "(SLM)" in the label - assert "(SLM)" in SplunkAOMetrics.input_pii_luna.value - assert "(SLM)" in SplunkAOMetrics.input_tone_luna.value - assert "(SLM)" in SplunkAOMetrics.output_pii_luna.value - assert "(SLM)" in SplunkAOMetrics.output_tone_luna.value - assert "(SLM)" in SplunkAOMetrics.completeness_luna.value + assert "(SLM)" in SplunkAOEvaluators.input_pii_luna.value + assert "(SLM)" in SplunkAOEvaluators.input_tone_luna.value + assert "(SLM)" in SplunkAOEvaluators.output_pii_luna.value + assert "(SLM)" in SplunkAOEvaluators.output_tone_luna.value + assert "(SLM)" in SplunkAOEvaluators.completeness_luna.value diff --git a/tests/test_agent_streams_evaluators.py b/tests/test_agent_streams_evaluators.py index 6ba420c1..9247338c 100644 --- a/tests/test_agent_streams_evaluators.py +++ b/tests/test_agent_streams_evaluators.py @@ -8,7 +8,7 @@ from splunk_ao.projects import Project from splunk_ao.resources.models import ProjectCreateResponse, ScorerResponse, ScorerTypes from splunk_ao.resources.models.log_stream_response import LogStreamResponse -from splunk_ao.schema.metrics import LocalMetricConfig, SplunkAOMetrics +from splunk_ao.schema.metrics import LocalMetricConfig, SplunkAOEvaluators from splunk_ao.utils.metrics import create_metric_configs @@ -90,7 +90,7 @@ def test_create_metric_configs_with_builtin_metrics( # Test with built-in metrics scorers, local_metrics = create_metric_configs( - "project-123", "logstream-456", [SplunkAOMetrics.correctness, "completeness"] + "project-123", "logstream-456", [SplunkAOEvaluators.correctness, "completeness"] ) # Verify scorers list_by_labels was called @@ -143,7 +143,7 @@ def custom_scorer(trace_or_span) -> float: # Test with mixed metrics (only valid ones to avoid decorator error handling) scorers, local_metrics = create_metric_configs( - "project-123", "logstream-456", [SplunkAOMetrics.correctness, local_metric] + "project-123", "logstream-456", [SplunkAOEvaluators.correctness, local_metric] ) # Verify local metrics @@ -431,7 +431,7 @@ def test_mixed_uuid_and_label(self, mock_settings_class, mock_scorers_class) -> mock_settings_class.return_value.create.return_value = None # When: passing both a UUID and an enum - scorers, _ = create_metric_configs("project-123", "run-456", [scorer_id, SplunkAOMetrics.completeness]) + scorers, _ = create_metric_configs("project-123", "run-456", [scorer_id, SplunkAOEvaluators.completeness]) # Then: both lookup methods are called mock_scorers_class.return_value.list_by_ids.assert_called_once() @@ -465,7 +465,7 @@ def test_run_id_none_skips_registration(self, mock_settings_class, mock_scorers_ ] # When: run_id is None - scorers, _ = create_metric_configs("project-123", None, [SplunkAOMetrics.correctness]) + scorers, _ = create_metric_configs("project-123", None, [SplunkAOEvaluators.correctness]) # Then: scorers are returned but registration is skipped mock_settings_class.return_value.create.assert_not_called() diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 83ff8860..665690cd 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -11,7 +11,7 @@ from splunk_ao.resources.models.column_category import ColumnCategory from splunk_ao.resources.models.column_info import ColumnInfo from splunk_ao.resources.models.data_type import DataType -from splunk_ao.schema.metrics import SplunkAOMetrics +from splunk_ao.schema.metrics import SplunkAOEvaluators from splunk_ao.search import RecordType from splunk_ao.shared.base import SyncState from splunk_ao.shared.column import ColumnCollection @@ -107,7 +107,7 @@ def test_init_with_prompt_name(self, reset_configuration: None) -> None: def test_init_with_metrics(self, reset_configuration: None) -> None: """Test initializing an experiment with metrics.""" - metrics = [SplunkAOMetrics.correctness, "completeness"] + metrics = [SplunkAOEvaluators.correctness, "completeness"] experiment = Experiment( name="Test Experiment", dataset_name="test-dataset", @@ -1492,7 +1492,7 @@ def test_returns_none_when_metric_aggregates_empty( synced_experiment._experiment_response = mock_response # When: getting a metric aggregate - result = synced_experiment.get_metric_aggregate(SplunkAOMetrics.correctness) + result = synced_experiment.get_metric_aggregate(SplunkAOEvaluators.correctness) # Then: None is returned without error assert result is None @@ -1513,8 +1513,8 @@ def test_lookup_by_galileo_metrics_enum( ) mock_config_class.get.return_value = MagicMock() - # When: looking up by SplunkAOMetrics enum (value == label "Correctness") - result = experiment.get_metric_aggregate(SplunkAOMetrics.correctness) + # When: looking up by SplunkAOEvaluators enum (value == label "Correctness") + result = experiment.get_metric_aggregate(SplunkAOEvaluators.correctness) # Then: the aggregate for the scorer UUID is returned assert result is not None diff --git a/tests/test_experiments.py b/tests/test_experiments.py index 50e9b7d2..323957d5 100644 --- a/tests/test_experiments.py +++ b/tests/test_experiments.py @@ -47,7 +47,7 @@ from splunk_ao.resources.types import UNSET from splunk_ao.schema.datasets import DatasetRecord from splunk_ao.schema.experiment_group import ExperimentGroupResponse -from splunk_ao.schema.metrics import LocalMetricConfig, SplunkAOMetrics +from splunk_ao.schema.metrics import LocalMetricConfig, SplunkAOEvaluators from splunk_ao.utils.datasets import load_dataset_and_records from splunk_ao.utils.exceptions import _format_http_validation_error from tests.testutils.setup import setup_mock_logstreams_client, setup_mock_projects_client, setup_mock_traces_client @@ -929,7 +929,7 @@ def test_run_experiment_w_prompt_template_and_metrics( project="awesome-new-project", dataset_id=dataset_id, prompt_template=prompt_template(), - metrics=[SplunkAOMetrics.correctness], + metrics=[SplunkAOEvaluators.correctness], ) mock_get_project.assert_called_once_with(id=None, name="awesome-new-project") diff --git a/tests/test_public_imports.py b/tests/test_public_imports.py index c6a6eb6a..22da7f07 100644 --- a/tests/test_public_imports.py +++ b/tests/test_public_imports.py @@ -18,7 +18,7 @@ def test_scorer_prompt_types_are_exported(): "Session", "Message", "Document", - "SplunkAOMetrics", + "SplunkAOEvaluators", ] missing = [name for name in types if not hasattr(splunk_ao, name)] assert not missing, f"Types missing from `splunk_ao` public API: {missing}"