improvement(mothership): agent model dropdown validations, recommendation system

icecrasher321 · icecrasher321 · commit 8698db33fa38 · 2026-04-17T00:28:11.000-07:00
diff --git a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.test.ts b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.test.ts
@@ -4,8 +4,9 @@
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 import { normalizeConditionRouterIds } from './builders'
 
-const { mockValidateSelectorIds } = vi.hoisted(() => ({
+const { mockValidateSelectorIds, mockGetModelOptions } = vi.hoisted(() => ({
   mockValidateSelectorIds: vi.fn(),
+  mockGetModelOptions: vi.fn(() => []),
 }))
 
 const conditionBlockConfig = {
@@ -26,7 +27,24 @@ const routerBlockConfig = {
   type: 'router_v2',
   name: 'Router',
   outputs: {},
-  subBlocks: [{ id: 'routes', type: 'router-input' }],
+  subBlocks: [
+    { id: 'routes', type: 'router-input' },
+    { id: 'model', type: 'combobox', options: mockGetModelOptions },
+  ],
+}
+
+const agentBlockConfig = {
+  type: 'agent',
+  name: 'Agent',
+  outputs: {},
+  subBlocks: [{ id: 'model', type: 'combobox', options: mockGetModelOptions }],
+}
+
+const huggingfaceBlockConfig = {
+  type: 'huggingface',
+  name: 'HuggingFace',
+  outputs: {},
+  subBlocks: [{ id: 'model', type: 'short-input' }],
 }
 
 vi.mock('@/blocks/registry', () => ({
@@ -37,7 +55,15 @@ vi.mock('@/blocks/registry', () => ({
         ? oauthBlockConfig
         : type === 'router_v2'
           ? routerBlockConfig
-          : undefined,
+          : type === 'agent'
+            ? agentBlockConfig
+            : type === 'huggingface'
+              ? huggingfaceBlockConfig
+              : undefined,
+}))
+
+vi.mock('@/blocks/utils', () => ({
+  getModelOptions: mockGetModelOptions,
 }))
 
 vi.mock('@/lib/copilot/validation/selector-validator', () => ({
@@ -83,6 +109,95 @@ describe('validateInputsForBlock', () => {
     expect(result.errors).toHaveLength(1)
     expect(result.errors[0]?.error).toContain('expected a JSON array')
   })
+
+  it('accepts known agent model ids', () => {
+    const result = validateInputsForBlock('agent', { model: 'claude-sonnet-4-6' }, 'agent-1')
+
+    expect(result.errors).toHaveLength(0)
+    expect(result.validInputs.model).toBe('claude-sonnet-4-6')
+  })
+
+  it('rejects hallucinated agent model ids that match a static provider pattern', () => {
+    const result = validateInputsForBlock('agent', { model: 'claude-sonnet-4.6' }, 'agent-1')
+
+    expect(result.validInputs.model).toBeUndefined()
+    expect(result.errors).toHaveLength(1)
+    expect(result.errors[0]?.field).toBe('model')
+    expect(result.errors[0]?.error).toContain('Unknown model id')
+    expect(result.errors[0]?.error).toContain('claude-sonnet-4-6')
+  })
+
+  it('rejects legacy claude-4.5-haiku style ids', () => {
+    const result = validateInputsForBlock('agent', { model: 'claude-4.5-haiku' }, 'agent-1')
+
+    expect(result.errors).toHaveLength(1)
+    expect(result.errors[0]?.error).toContain('Unknown model id')
+  })
+
+  it('allows empty model values', () => {
+    const result = validateInputsForBlock('agent', { model: '' }, 'agent-1')
+
+    expect(result.errors).toHaveLength(0)
+    expect(result.validInputs.model).toBe('')
+  })
+
+  it('allows custom ollama-prefixed model ids', () => {
+    const result = validateInputsForBlock('agent', { model: 'ollama/my-private-model' }, 'agent-1')
+
+    expect(result.errors).toHaveLength(0)
+    expect(result.validInputs.model).toBe('ollama/my-private-model')
+  })
+
+  it('validates the model field on router_v2 blocks too', () => {
+    const valid = validateInputsForBlock('router_v2', { model: 'claude-sonnet-4-6' }, 'router-1')
+    expect(valid.errors).toHaveLength(0)
+    expect(valid.validInputs.model).toBe('claude-sonnet-4-6')
+
+    const invalid = validateInputsForBlock('router_v2', { model: 'claude-sonnet-4.6' }, 'router-1')
+    expect(invalid.validInputs.model).toBeUndefined()
+    expect(invalid.errors).toHaveLength(1)
+    expect(invalid.errors[0]?.blockType).toBe('router_v2')
+    expect(invalid.errors[0]?.field).toBe('model')
+    expect(invalid.errors[0]?.error).toContain('Unknown model id')
+  })
+
+  it("does not apply model validation to blocks whose model field is not Sim's catalog", () => {
+    const result = validateInputsForBlock(
+      'huggingface',
+      { model: 'mistralai/Mistral-7B-Instruct-v0.3' },
+      'hf-1'
+    )
+
+    expect(result.errors).toHaveLength(0)
+    expect(result.validInputs.model).toBe('mistralai/Mistral-7B-Instruct-v0.3')
+  })
+
+  it('accepts date-suffixed variants of known Anthropic ids', () => {
+    const result = validateInputsForBlock(
+      'agent',
+      { model: 'claude-sonnet-4-5-20250929' },
+      'agent-1'
+    )
+
+    expect(result.errors).toHaveLength(0)
+    expect(result.validInputs.model).toBe('claude-sonnet-4-5-20250929')
+  })
+
+  it('trims whitespace around catalog model ids and stores the trimmed value', () => {
+    const result = validateInputsForBlock('agent', { model: '  gpt-5.4  ' }, 'agent-1')
+
+    expect(result.errors).toHaveLength(0)
+    expect(result.validInputs.model).toBe('gpt-5.4')
+  })
+
+  it('rejects a pattern-matching but uncataloged id even with surrounding whitespace', () => {
+    const result = validateInputsForBlock('agent', { model: '  gpt-100-ultra  ' }, 'agent-1')
+
+    expect(result.validInputs.model).toBeUndefined()
+    expect(result.errors).toHaveLength(1)
+    expect(result.errors[0]?.error).toContain('gpt-100-ultra')
+    expect(result.errors[0]?.error).not.toMatch(/\s{2,}/)
+  })
 })
 
 describe('normalizeConditionRouterIds', () => {
diff --git a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.ts b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.ts
@@ -3,7 +3,9 @@ import { validateSelectorIds } from '@/lib/copilot/validation/selector-validator
 import type { PermissionGroupConfig } from '@/lib/permission-groups/types'
 import { getBlock } from '@/blocks/registry'
 import type { SubBlockConfig } from '@/blocks/types'
+import { getModelOptions } from '@/blocks/utils'
 import { EDGE, normalizeName } from '@/executor/constants'
+import { isKnownModelId, suggestModelIdsForUnknownModel } from '@/providers/models'
 import { TRIGGER_RUNTIME_SUBBLOCK_IDS } from '@/triggers/constants'
 import type {
   EdgeHandleValidationResult,
@@ -350,12 +352,37 @@ export function validateValueForSubBlockType(
     case 'short-input':
     case 'long-input':
     case 'combobox': {
-      // Should be string (combobox allows custom values)
+      let stringValue: string
       if (typeof value !== 'string' && typeof value !== 'number') {
-        // Convert to string but don't error
-        return { valid: true, value: String(value) }
+        stringValue = String(value)
+      } else {
+        stringValue = typeof value === 'number' ? String(value) : value
       }
-      return { valid: true, value }
+
+      const usesProviderCatalog =
+        fieldName === 'model' && subBlockConfig.options === getModelOptions
+
+      if (usesProviderCatalog) {
+        const trimmed = stringValue.trim()
+        if (trimmed !== '' && !isKnownModelId(trimmed)) {
+          const suggestions = suggestModelIdsForUnknownModel(trimmed)
+          const suggestionText =
+            suggestions.length > 0 ? ` Valid options include: ${suggestions.join(', ')}.` : ''
+          return {
+            valid: false,
+            error: {
+              blockId,
+              blockType,
+              field: fieldName,
+              value,
+              error: `Unknown model id "${trimmed}" for block "${blockType}". Read components/blocks/${blockType}.json (the model.options array) for valid ids; prefer entries with recommended: true and avoid deprecated: true.${suggestionText}`,
+            },
+          }
+        }
+        return { valid: true, value: trimmed }
+      }
+
+      return { valid: true, value: typeof value === 'string' ? value : stringValue }
     }
 
     // Selector types - allow strings (IDs) or arrays of strings
diff --git a/apps/sim/lib/copilot/vfs/serializers.ts b/apps/sim/lib/copilot/vfs/serializers.ts
@@ -320,24 +320,75 @@ export function serializeTableMeta(table: {
  * Excludes dynamic providers (ollama, vllm, openrouter) whose models are user-configured.
  * Includes provider ID and whether the model is hosted by Sim (no API key required).
  */
-function getStaticModelOptionsForVFS(): Array<{
+interface StaticModelOption {
   id: string
   provider: string
   hosted: boolean
-}> {
+  recommended?: boolean
+  speedOptimized?: boolean
+  deprecated?: boolean
+}
+
+interface TierFlags {
+  recommended?: boolean
+  speedOptimized?: boolean
+  deprecated?: boolean
+}
+
+const RESELLER_BASE_PREFIX: Record<string, string> = {
+  'azure-openai': 'azure/',
+  'azure-anthropic': 'azure-anthropic/',
+  vertex: 'vertex/',
+}
+
+const DYNAMIC_PROVIDERS_NOTE = {
+  note: 'The options array above lists Sim\'s static provider catalog. These four providers also accept user-configured models that are NOT enumerated here: the user may have additional ids available at runtime. Any model id prefixed with one of the slashes below is accepted by the server, as is any bare id that does not match a static provider pattern (typically a local Ollama tag like "llama3.1:8b"). The UI dropdown shows the user\'s actual installed models; if the user references one by name, use that id verbatim.',
+  prefixes: ['ollama/', 'vllm/', 'openrouter/', 'fireworks/'],
+} as const
+
+function getStaticModelOptionsForVFS(): StaticModelOption[] {
   const hostedProviders = new Set(['openai', 'anthropic', 'google'])
   const dynamicProviders = new Set(['ollama', 'vllm', 'openrouter', 'fireworks'])
 
-  const models: Array<{ id: string; provider: string; hosted: boolean }> = []
+  const baseTierFlags = new Map<string, TierFlags>()
+  for (const providerId of hostedProviders) {
+    const def = PROVIDER_DEFINITIONS[providerId]
+    if (!def) continue
+    for (const model of def.models) {
+      if (model.recommended || model.speedOptimized || model.deprecated) {
+        baseTierFlags.set(model.id, {
+          ...(model.recommended && { recommended: true }),
+          ...(model.speedOptimized && { speedOptimized: true }),
+          ...(model.deprecated && { deprecated: true }),
+        })
+      }
+    }
+  }
+
+  const models: StaticModelOption[] = []
 
   for (const [providerId, def] of Object.entries(PROVIDER_DEFINITIONS)) {
     if (dynamicProviders.has(providerId)) continue
     for (const model of def.models) {
-      models.push({
+      const option: StaticModelOption = {
         id: model.id,
         provider: providerId,
         hosted: hostedProviders.has(providerId),
-      })
+      }
+      if (model.recommended) option.recommended = true
+      if (model.speedOptimized) option.speedOptimized = true
+      if (model.deprecated) option.deprecated = true
+
+      if (!option.recommended && !option.speedOptimized && !option.deprecated) {
+        const prefix = RESELLER_BASE_PREFIX[providerId]
+        if (prefix && model.id.startsWith(prefix)) {
+          const baseId = model.id.slice(prefix.length)
+          const inherited = baseTierFlags.get(baseId)
+          if (inherited) Object.assign(option, inherited)
+        }
+      }
+
+      models.push(option)
     }
   }
 
@@ -378,9 +429,9 @@ export function serializeBlockSchema(block: BlockConfig): string {
     .map((sb) => {
       const serialized = serializeSubBlock(sb)
 
-      // For model comboboxes with function options, inject static model data with hosting info
       if (sb.id === 'model' && sb.type === 'combobox' && typeof sb.options === 'function') {
         serialized.options = getStaticModelOptionsForVFS()
+        serialized.dynamicProviders = DYNAMIC_PROVIDERS_NOTE
       }
 
       return serialized
diff --git a/apps/sim/providers/models.ts b/apps/sim/providers/models.ts
