{ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://promptfoo.dev/config-schema.json", "title": "Promptfoo Configuration Schema", "$ref": "#/definitions/PromptfooConfigSchema", "definitions": { "PromptfooConfigSchema": { "type": "object", "properties": { "tags": { "$ref": "#/definitions/__schema0" }, "description": { "$ref": "#/definitions/__schema1" }, "providers": { "$ref": "#/definitions/__schema2" }, "prompts": { "$ref": "#/definitions/__schema13" }, "tests": { "$ref": "#/definitions/__schema14" }, "scenarios": { "$ref": "#/definitions/__schema29" }, "defaultTest": { "$ref": "#/definitions/__schema30" }, "outputPath": { "$ref": "#/definitions/__schema31" }, "sharing": { "$ref": "#/definitions/__schema32" }, "nunjucksFilters": { "$ref": "#/definitions/__schema33" }, "env": { "$ref": "#/definitions/__schema34" }, "derivedMetrics": { "$ref": "#/definitions/__schema35" }, "extensions": { "$ref": "#/definitions/__schema36" }, "metadata": { "$ref": "#/definitions/__schema37" }, "redteam": { "$ref": "#/definitions/__schema38" }, "writeLatestResults": { "$ref": "#/definitions/__schema39" }, "tracing": { "$ref": "#/definitions/__schema40" }, "evaluateOptions": { "$ref": "#/definitions/__schema41" }, "commandLineOptions": { "$ref": "#/definitions/__schema43" }, "targets": { "$ref": "#/definitions/__schema44" } }, "required": ["prompts"], "additionalProperties": false }, "__schema0": { "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "type": "string" } }, "__schema1": { "type": "string" }, "__schema2": { "allOf": [ { "$ref": "#/definitions/__schema3" } ] }, "__schema3": { "anyOf": [ { "type": "string" }, { "$ref": "#/definitions/__schema4" }, { "type": "array", "items": { "anyOf": [ { "type": "string" }, { "$ref": "#/definitions/__schema4" }, { "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "$ref": "#/definitions/__schema5" } }, { "$ref": "#/definitions/__schema5" } ] } } ] }, "__schema4": {}, "__schema5": { "type": "object", "properties": { "id": {}, "label": {}, "config": {}, "prompts": { "type": "array", "items": { "type": "string" } }, "transform": { "type": "string" }, "delay": { "type": "number" }, "env": { "allOf": [ { "$ref": "#/definitions/__schema7" } ] }, "inputs": { "allOf": [ { "$ref": "#/definitions/__schema8" } ] } }, "additionalProperties": false }, "__schema6": { "type": "string" }, "__schema7": { "type": "object", "properties": { "ABLIT_API_BASE_URL": { "type": "string" }, "ABLIT_KEY": { "type": "string" }, "AI21_API_BASE_URL": { "type": "string" }, "AI21_API_KEY": { "type": "string" }, "AIML_API_KEY": { "type": "string" }, "ANTHROPIC_API_KEY": { "type": "string" }, "ANTHROPIC_BASE_URL": { "type": "string" }, "ATLASCLOUD_API_KEY": { "type": "string" }, "AWS_BEDROCK_REGION": { "type": "string" }, "AZURE_API_BASE_URL": { "type": "string" }, "AZURE_API_HOST": { "type": "string" }, "AZURE_API_KEY": { "type": "string" }, "AZURE_AUTHORITY_HOST": { "type": "string" }, "AZURE_CLIENT_ID": { "type": "string" }, "AZURE_CLIENT_SECRET": { "type": "string" }, "AZURE_DEPLOYMENT_NAME": { "type": "string" }, "AZURE_EMBEDDING_DEPLOYMENT_NAME": { "type": "string" }, "AZURE_OPENAI_API_BASE_URL": { "type": "string" }, "AZURE_OPENAI_API_HOST": { "type": "string" }, "AZURE_OPENAI_API_KEY": { "type": "string" }, "AZURE_OPENAI_BASE_URL": { "type": "string" }, "AZURE_OPENAI_DEPLOYMENT_NAME": { "type": "string" }, "AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME": { "type": "string" }, "AZURE_TENANT_ID": { "type": "string" }, "AZURE_TOKEN_SCOPE": { "type": "string" }, "CLAUDE_CODE_USE_BEDROCK": { "type": "string" }, "CLAUDE_CODE_USE_VERTEX": { "type": "string" }, "CLOUDFLARE_ACCOUNT_ID": { "type": "string" }, "CLOUDFLARE_API_KEY": { "type": "string" }, "CLOUDFLARE_GATEWAY_ID": { "type": "string" }, "CF_AIG_TOKEN": { "type": "string" }, "COMETAPI_KEY": { "type": "string" }, "COHERE_API_KEY": { "type": "string" }, "COHERE_CLIENT_NAME": { "type": "string" }, "DATABRICKS_TOKEN": { "type": "string" }, "DATABRICKS_WORKSPACE_URL": { "type": "string" }, "DOCKER_MODEL_RUNNER_BASE_URL": { "type": "string" }, "DOCKER_MODEL_RUNNER_API_KEY": { "type": "string" }, "ELEVENLABS_API_KEY": { "type": "string" }, "FAL_KEY": { "type": "string" }, "GITHUB_TOKEN": { "type": "string" }, "GOOGLE_API_HOST": { "type": "string" }, "GOOGLE_API_BASE_URL": { "type": "string" }, "GOOGLE_API_KEY": { "type": "string" }, "GOOGLE_PROJECT_ID": { "type": "string" }, "GOOGLE_LOCATION": { "type": "string" }, "GOOGLE_GENERATIVE_AI_API_KEY": { "type": "string" }, "GEMINI_API_KEY": { "type": "string" }, "GROQ_API_KEY": { "type": "string" }, "HELICONE_API_KEY": { "type": "string" }, "HF_API_TOKEN": { "type": "string" }, "HF_TOKEN": { "type": "string" }, "HYPERBOLIC_API_KEY": { "type": "string" }, "HUGGING_FACE_HUB_TOKEN": { "type": "string" }, "JFROG_API_KEY": { "type": "string" }, "LANGFUSE_HOST": { "type": "string" }, "LANGFUSE_PUBLIC_KEY": { "type": "string" }, "LANGFUSE_SECRET_KEY": { "type": "string" }, "LITELLM_API_BASE": { "type": "string" }, "LLAMA_BASE_URL": { "type": "string" }, "LOCALAI_BASE_URL": { "type": "string" }, "MISTRAL_API_BASE_URL": { "type": "string" }, "MISTRAL_API_HOST": { "type": "string" }, "MISTRAL_API_KEY": { "type": "string" }, "MODELSLAB_API_KEY": { "type": "string" }, "NSCALE_SERVICE_TOKEN": { "type": "string" }, "NSCALE_API_KEY": { "type": "string" }, "OLLAMA_API_KEY": { "type": "string" }, "OLLAMA_BASE_URL": { "type": "string" }, "OPENAI_API_BASE_URL": { "type": "string" }, "OPENAI_API_HOST": { "type": "string" }, "OPENAI_API_KEY": { "type": "string" }, "OPENAI_BASE_URL": { "type": "string" }, "OPENAI_ORGANIZATION": { "type": "string" }, "CLAWDBOT_GATEWAY_PASSWORD": { "type": "string" }, "CLAWDBOT_GATEWAY_TOKEN": { "type": "string" }, "CLAWDBOT_GATEWAY_URL": { "type": "string" }, "CODEX_API_KEY": { "type": "string" }, "OPENCLAW_CONFIG_PATH": { "type": "string" }, "OPENCLAW_GATEWAY_PASSWORD": { "type": "string" }, "OPENCLAW_GATEWAY_PORT": { "type": "string" }, "OPENCLAW_GATEWAY_TOKEN": { "type": "string" }, "OPENCLAW_GATEWAY_URL": { "type": "string" }, "PALM_API_HOST": { "type": "string" }, "PALM_API_KEY": { "type": "string" }, "PORTKEY_API_KEY": { "type": "string" }, "PROMPTFOO_CA_CERT_PATH": { "type": "string" }, "PROMPTFOO_PFX_CERT_PATH": { "type": "string" }, "PROMPTFOO_PFX_PASSWORD": { "type": "string" }, "PROMPTFOO_JKS_CERT_PATH": { "type": "string" }, "PROMPTFOO_JKS_PASSWORD": { "type": "string" }, "PROMPTFOO_JKS_ALIAS": { "type": "string" }, "PROMPTFOO_INSECURE_SSL": { "type": "string" }, "QUIVERAI_API_KEY": { "type": "string" }, "REPLICATE_API_KEY": { "type": "string" }, "REPLICATE_API_TOKEN": { "type": "string" }, "SHAREPOINT_BASE_URL": { "type": "string" }, "SHAREPOINT_CERT_PATH": { "type": "string" }, "SHAREPOINT_CLIENT_ID": { "type": "string" }, "SHAREPOINT_TENANT_ID": { "type": "string" }, "VERCEL_AI_GATEWAY_API_KEY": { "type": "string" }, "VERCEL_AI_GATEWAY_BASE_URL": { "type": "string" }, "VERTEX_API_HOST": { "type": "string" }, "VERTEX_API_KEY": { "type": "string" }, "VERTEX_API_VERSION": { "type": "string" }, "VERTEX_PROJECT_ID": { "type": "string" }, "VERTEX_PUBLISHER": { "type": "string" }, "VERTEX_REGION": { "type": "string" }, "VOYAGE_API_BASE_URL": { "type": "string" }, "VOYAGE_API_KEY": { "type": "string" }, "WATSONX_AI_APIKEY": { "type": "string" }, "WATSONX_AI_AUTH_TYPE": { "type": "string" }, "WATSONX_AI_BEARER_TOKEN": { "type": "string" }, "WATSONX_AI_PROJECT_ID": { "type": "string" }, "XAI_API_BASE_URL": { "type": "string" }, "XAI_API_KEY": { "type": "string" }, "AZURE_CONTENT_SAFETY_ENDPOINT": { "type": "string" }, "AZURE_CONTENT_SAFETY_API_KEY": { "type": "string" }, "AZURE_CONTENT_SAFETY_API_VERSION": { "type": "string" }, "AWS_REGION": { "type": "string" }, "AWS_DEFAULT_REGION": { "type": "string" }, "AWS_SAGEMAKER_MAX_TOKENS": { "type": "string" }, "AWS_SAGEMAKER_TEMPERATURE": { "type": "string" }, "AWS_SAGEMAKER_TOP_P": { "type": "string" }, "AWS_SAGEMAKER_MAX_RETRIES": { "type": "string" }, "PROMPTFOO_EVAL_TIMEOUT_MS": { "type": "string" } }, "additionalProperties": false }, "__schema8": { "type": "object", "propertyNames": { "type": "string", "pattern": "^[a-zA-Z_][a-zA-Z0-9_]*$" }, "additionalProperties": { "anyOf": [ { "type": "string", "minLength": 1 }, { "type": "object", "properties": { "config": { "$ref": "#/definitions/__schema9" }, "description": { "$ref": "#/definitions/__schema11" }, "type": { "$ref": "#/definitions/__schema12" } }, "required": ["description"], "additionalProperties": false } ] } }, "__schema9": { "type": "object", "properties": { "benign": { "type": "boolean" }, "inputPurpose": { "type": "string", "minLength": 1 }, "injectionPlacements": { "minItems": 1, "type": "array", "items": { "$ref": "#/definitions/__schema10" } } }, "additionalProperties": false }, "__schema10": { "type": "string", "minLength": 1 }, "__schema11": { "type": "string", "minLength": 1 }, "__schema12": { "type": "string", "enum": ["text", "pdf", "docx", "image"] }, "__schema13": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "anyOf": [ { "type": "string" }, { "type": "object", "properties": { "id": { "type": "string" }, "label": { "type": "string" }, "raw": { "type": "string" } }, "required": ["id"], "additionalProperties": false }, { "type": "object", "properties": { "id": { "type": "string" }, "raw": { "type": "string" }, "template": { "type": "string" }, "display": { "type": "string" }, "label": { "type": "string" }, "function": {}, "config": {} }, "required": ["raw", "label"], "additionalProperties": false } ] } }, { "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "type": "string" } } ] }, "__schema14": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "anyOf": [ { "type": "string" }, { "$ref": "#/definitions/__schema15" }, { "$ref": "#/definitions/__schema28" } ] } }, { "$ref": "#/definitions/__schema28" } ] }, "__schema15": { "type": "object", "properties": { "description": { "$ref": "#/definitions/__schema16" }, "vars": { "$ref": "#/definitions/__schema17" }, "provider": { "$ref": "#/definitions/__schema18" }, "providers": { "$ref": "#/definitions/__schema19" }, "prompts": { "$ref": "#/definitions/__schema20" }, "providerOutput": { "$ref": "#/definitions/__schema21" }, "assert": { "$ref": "#/definitions/__schema22" }, "assertScoringFunction": { "$ref": "#/definitions/__schema24" }, "options": { "$ref": "#/definitions/__schema25" }, "threshold": { "$ref": "#/definitions/__schema26" }, "metadata": { "$ref": "#/definitions/__schema27" } }, "additionalProperties": false }, "__schema16": { "type": "string" }, "__schema17": {}, "__schema18": { "anyOf": [ { "type": "string" }, { "$ref": "#/definitions/__schema5" }, { "type": "object", "properties": { "id": {}, "callApi": {}, "callEmbeddingApi": {}, "callClassificationApi": {}, "label": {}, "transform": { "type": "string" }, "delay": { "type": "number" }, "config": {}, "inputs": { "allOf": [ { "$ref": "#/definitions/__schema8" } ] } }, "required": ["id", "callApi"], "additionalProperties": false } ] }, "__schema19": { "type": "array", "items": { "type": "string" } }, "__schema20": { "type": "array", "items": { "type": "string" } }, "__schema21": { "anyOf": [ { "type": "string" }, { "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": {} } ] }, "__schema22": { "type": "array", "items": { "anyOf": [ { "type": "object", "properties": { "type": { "type": "string", "const": "assert-set" }, "assert": { "type": "array", "items": { "allOf": [ { "$ref": "#/definitions/__schema23" } ] } }, "weight": { "type": "number" }, "metric": { "type": "string" }, "threshold": { "type": "number" }, "config": { "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": {} } }, "required": ["type", "assert"], "additionalProperties": false }, { "$ref": "#/definitions/__schema23" } ] } }, "__schema23": { "type": "object", "properties": { "type": { "anyOf": [ { "type": "string", "enum": [ "answer-relevance", "bleu", "classifier", "contains", "contains-all", "contains-any", "contains-html", "contains-json", "contains-sql", "contains-xml", "context-faithfulness", "context-recall", "context-relevance", "conversation-relevance", "cost", "equals", "factuality", "finish-reason", "g-eval", "gleu", "guardrails", "icontains", "icontains-all", "icontains-any", "is-html", "is-json", "is-refusal", "is-sql", "is-valid-function-call", "is-valid-openai-function-call", "is-valid-openai-tools-call", "is-xml", "javascript", "latency", "levenshtein", "llm-rubric", "pi", "meteor", "model-graded-closedqa", "model-graded-factuality", "moderation", "perplexity", "perplexity-score", "python", "regex", "rouge-n", "ruby", "similar", "similar:cosine", "similar:dot", "similar:euclidean", "starts-with", "tool-call-f1", "skill-used", "trajectory:goal-success", "trajectory:tool-args-match", "trajectory:step-count", "trajectory:tool-sequence", "trajectory:tool-used", "trace-error-spans", "trace-span-count", "trace-span-duration", "search-rubric", "webhook", "word-count" ] }, { "type": "string", "enum": [ "answer-relevance", "bleu", "classifier", "contains", "contains-all", "contains-any", "contains-html", "contains-json", "contains-sql", "contains-xml", "context-faithfulness", "context-recall", "context-relevance", "conversation-relevance", "cost", "equals", "factuality", "finish-reason", "g-eval", "gleu", "guardrails", "icontains", "icontains-all", "icontains-any", "is-html", "is-json", "is-refusal", "is-sql", "is-valid-function-call", "is-valid-openai-function-call", "is-valid-openai-tools-call", "is-xml", "javascript", "latency", "levenshtein", "llm-rubric", "pi", "meteor", "model-graded-closedqa", "model-graded-factuality", "moderation", "perplexity", "perplexity-score", "python", "regex", "rouge-n", "ruby", "similar", "similar:cosine", "similar:dot", "similar:euclidean", "starts-with", "tool-call-f1", "skill-used", "trajectory:goal-success", "trajectory:tool-args-match", "trajectory:step-count", "trajectory:tool-sequence", "trajectory:tool-used", "trace-error-spans", "trace-span-count", "trace-span-duration", "search-rubric", "webhook", "word-count" ] }, { "type": "string", "enum": ["select-best", "human", "max-score"] }, {} ] }, "value": {}, "config": { "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": {} }, "threshold": { "type": "number" }, "weight": { "type": "number" }, "provider": {}, "rubricPrompt": {}, "metric": { "type": "string" }, "transform": { "type": "string" }, "contextTransform": { "type": "string" } }, "required": ["type"], "additionalProperties": false }, "__schema24": { "anyOf": [ { "type": "string", "pattern": "^file:\\/\\/.*\\.(js|cjs|mjs|ts|cts|mts|py)(?::[\\w.]+)?$" }, {} ] }, "__schema25": { "type": "object", "properties": { "prefix": { "type": "string" }, "suffix": { "type": "string" }, "postprocess": { "type": "string" }, "transform": { "type": "string" }, "transformVars": { "type": "string" }, "storeOutputAs": { "type": "string" }, "rubricPrompt": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "array", "items": { "type": "object", "properties": { "role": { "type": "string" }, "content": { "type": "string" } }, "required": ["role", "content"], "additionalProperties": false } } ] }, "provider": { "anyOf": [ { "type": "string" }, {}, { "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "anyOf": [ { "type": "string" }, {} ] } } ] }, "factuality": { "type": "object", "properties": { "subset": { "type": "number" }, "superset": { "type": "number" }, "agree": { "type": "number" }, "disagree": { "type": "number" }, "differButFactual": { "type": "number" } }, "additionalProperties": false }, "disableVarExpansion": { "type": "boolean" }, "disableConversationVar": { "type": "boolean" }, "disableDefaultAsserts": { "type": "boolean" }, "runSerially": { "type": "boolean" } }, "additionalProperties": {} }, "__schema26": { "type": "number" }, "__schema27": { "type": "object", "properties": { "pluginConfig": {}, "strategyConfig": {} }, "additionalProperties": {} }, "__schema28": { "type": "object", "properties": { "path": { "type": "string" }, "config": { "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "boolean" }, { "type": "array", "items": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "boolean" } ] } }, { "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": {} }, {} ] } } }, "required": ["path"], "additionalProperties": false }, "__schema29": { "type": "array", "items": { "anyOf": [ { "type": "string" }, { "type": "object", "properties": { "description": { "type": "string" }, "config": { "type": "array", "items": { "type": "object", "properties": { "description": { "allOf": [ { "$ref": "#/definitions/__schema16" } ] }, "vars": { "allOf": [ { "$ref": "#/definitions/__schema17" } ] }, "provider": { "allOf": [ { "$ref": "#/definitions/__schema18" } ] }, "providers": { "allOf": [ { "$ref": "#/definitions/__schema19" } ] }, "prompts": { "allOf": [ { "$ref": "#/definitions/__schema20" } ] }, "providerOutput": { "allOf": [ { "$ref": "#/definitions/__schema21" } ] }, "assert": { "allOf": [ { "$ref": "#/definitions/__schema22" } ] }, "assertScoringFunction": { "allOf": [ { "$ref": "#/definitions/__schema24" } ] }, "options": { "allOf": [ { "$ref": "#/definitions/__schema25" } ] }, "threshold": { "allOf": [ { "$ref": "#/definitions/__schema26" } ] }, "metadata": { "allOf": [ { "$ref": "#/definitions/__schema27" } ] } }, "additionalProperties": false } }, "tests": { "type": "array", "items": { "$ref": "#/definitions/__schema15" } } }, "required": ["config", "tests"], "additionalProperties": false } ] } }, "__schema30": { "anyOf": [ { "type": "string" }, { "type": "object", "properties": { "vars": { "$ref": "#/definitions/__schema17" }, "provider": { "$ref": "#/definitions/__schema18" }, "providers": { "$ref": "#/definitions/__schema19" }, "prompts": { "$ref": "#/definitions/__schema20" }, "providerOutput": { "$ref": "#/definitions/__schema21" }, "assert": { "$ref": "#/definitions/__schema22" }, "assertScoringFunction": { "$ref": "#/definitions/__schema24" }, "options": { "$ref": "#/definitions/__schema25" }, "threshold": { "$ref": "#/definitions/__schema26" }, "metadata": { "$ref": "#/definitions/__schema27" } }, "additionalProperties": false } ] }, "__schema31": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ] }, "__schema32": { "anyOf": [ { "type": "boolean" }, { "type": "object", "properties": { "apiBaseUrl": { "type": "string" }, "appBaseUrl": { "type": "string" } }, "additionalProperties": false } ] }, "__schema33": { "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "type": "string" } }, "__schema34": { "anyOf": [ { "$ref": "#/definitions/__schema7" }, { "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "boolean" } ] } } ] }, "__schema35": { "type": "array", "items": { "type": "object", "properties": { "name": { "type": "string" }, "value": { "anyOf": [ { "type": "string" }, {} ] } }, "required": ["name", "value"], "additionalProperties": false } }, "__schema36": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "null" } ] }, "__schema37": { "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": {} }, "__schema38": { "type": "object", "properties": { "injectVar": { "description": "Variable to inject. Can be a string or array of strings. If string, it's transformed to an array. Inferred from the prompts by default.", "type": "string" }, "purpose": { "description": "Purpose override string - describes the prompt templates", "type": "string" }, "testGenerationInstructions": { "description": "Additional instructions for test generation applied to each plugin", "type": "string" }, "provider": { "description": "Provider used for generating adversarial inputs", "anyOf": [ { "type": "string" }, { "type": "object", "properties": { "id": {}, "callApi": {}, "callEmbeddingApi": {}, "callClassificationApi": {}, "label": {}, "transform": { "type": "string" }, "delay": { "type": "number" }, "config": {}, "inputs": { "type": "object", "propertyNames": { "type": "string", "pattern": "^[a-zA-Z_][a-zA-Z0-9_]*$" }, "additionalProperties": { "anyOf": [ { "type": "string", "minLength": 1 }, { "type": "object", "properties": { "config": { "type": "object", "properties": { "benign": { "type": "boolean" }, "inputPurpose": { "type": "string", "minLength": 1 }, "injectionPlacements": { "minItems": 1, "type": "array", "items": { "type": "string", "minLength": 1 } } }, "additionalProperties": false }, "description": { "type": "string", "minLength": 1 }, "type": { "type": "string", "enum": ["text", "pdf", "docx", "image"] } }, "required": ["description"], "additionalProperties": false } ] } } }, "required": ["id", "callApi"], "additionalProperties": false }, { "type": "object", "properties": { "id": {}, "label": {}, "config": {}, "prompts": { "type": "array", "items": { "type": "string" } }, "transform": { "type": "string" }, "delay": { "type": "number" }, "env": { "type": "object", "properties": { "ABLIT_API_BASE_URL": { "type": "string" }, "ABLIT_KEY": { "type": "string" }, "AI21_API_BASE_URL": { "type": "string" }, "AI21_API_KEY": { "type": "string" }, "AIML_API_KEY": { "type": "string" }, "ANTHROPIC_API_KEY": { "type": "string" }, "ANTHROPIC_BASE_URL": { "type": "string" }, "ATLASCLOUD_API_KEY": { "type": "string" }, "AWS_BEDROCK_REGION": { "type": "string" }, "AZURE_API_BASE_URL": { "type": "string" }, "AZURE_API_HOST": { "type": "string" }, "AZURE_API_KEY": { "type": "string" }, "AZURE_AUTHORITY_HOST": { "type": "string" }, "AZURE_CLIENT_ID": { "type": "string" }, "AZURE_CLIENT_SECRET": { "type": "string" }, "AZURE_DEPLOYMENT_NAME": { "type": "string" }, "AZURE_EMBEDDING_DEPLOYMENT_NAME": { "type": "string" }, "AZURE_OPENAI_API_BASE_URL": { "type": "string" }, "AZURE_OPENAI_API_HOST": { "type": "string" }, "AZURE_OPENAI_API_KEY": { "type": "string" }, "AZURE_OPENAI_BASE_URL": { "type": "string" }, "AZURE_OPENAI_DEPLOYMENT_NAME": { "type": "string" }, "AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME": { "type": "string" }, "AZURE_TENANT_ID": { "type": "string" }, "AZURE_TOKEN_SCOPE": { "type": "string" }, "CLAUDE_CODE_USE_BEDROCK": { "type": "string" }, "CLAUDE_CODE_USE_VERTEX": { "type": "string" }, "CLOUDFLARE_ACCOUNT_ID": { "type": "string" }, "CLOUDFLARE_API_KEY": { "type": "string" }, "CLOUDFLARE_GATEWAY_ID": { "type": "string" }, "CF_AIG_TOKEN": { "type": "string" }, "COMETAPI_KEY": { "type": "string" }, "COHERE_API_KEY": { "type": "string" }, "COHERE_CLIENT_NAME": { "type": "string" }, "DATABRICKS_TOKEN": { "type": "string" }, "DATABRICKS_WORKSPACE_URL": { "type": "string" }, "DOCKER_MODEL_RUNNER_BASE_URL": { "type": "string" }, "DOCKER_MODEL_RUNNER_API_KEY": { "type": "string" }, "ELEVENLABS_API_KEY": { "type": "string" }, "FAL_KEY": { "type": "string" }, "GITHUB_TOKEN": { "type": "string" }, "GOOGLE_API_HOST": { "type": "string" }, "GOOGLE_API_BASE_URL": { "type": "string" }, "GOOGLE_API_KEY": { "type": "string" }, "GOOGLE_PROJECT_ID": { "type": "string" }, "GOOGLE_LOCATION": { "type": "string" }, "GOOGLE_GENERATIVE_AI_API_KEY": { "type": "string" }, "GEMINI_API_KEY": { "type": "string" }, "GROQ_API_KEY": { "type": "string" }, "HELICONE_API_KEY": { "type": "string" }, "HF_API_TOKEN": { "type": "string" }, "HF_TOKEN": { "type": "string" }, "HYPERBOLIC_API_KEY": { "type": "string" }, "HUGGING_FACE_HUB_TOKEN": { "type": "string" }, "JFROG_API_KEY": { "type": "string" }, "LANGFUSE_HOST": { "type": "string" }, "LANGFUSE_PUBLIC_KEY": { "type": "string" }, "LANGFUSE_SECRET_KEY": { "type": "string" }, "LITELLM_API_BASE": { "type": "string" }, "LLAMA_BASE_URL": { "type": "string" }, "LOCALAI_BASE_URL": { "type": "string" }, "MISTRAL_API_BASE_URL": { "type": "string" }, "MISTRAL_API_HOST": { "type": "string" }, "MISTRAL_API_KEY": { "type": "string" }, "MODELSLAB_API_KEY": { "type": "string" }, "NSCALE_SERVICE_TOKEN": { "type": "string" }, "NSCALE_API_KEY": { "type": "string" }, "OLLAMA_API_KEY": { "type": "string" }, "OLLAMA_BASE_URL": { "type": "string" }, "OPENAI_API_BASE_URL": { "type": "string" }, "OPENAI_API_HOST": { "type": "string" }, "OPENAI_API_KEY": { "type": "string" }, "OPENAI_BASE_URL": { "type": "string" }, "OPENAI_ORGANIZATION": { "type": "string" }, "CLAWDBOT_GATEWAY_PASSWORD": { "type": "string" }, "CLAWDBOT_GATEWAY_TOKEN": { "type": "string" }, "CLAWDBOT_GATEWAY_URL": { "type": "string" }, "CODEX_API_KEY": { "type": "string" }, "OPENCLAW_CONFIG_PATH": { "type": "string" }, "OPENCLAW_GATEWAY_PASSWORD": { "type": "string" }, "OPENCLAW_GATEWAY_PORT": { "type": "string" }, "OPENCLAW_GATEWAY_TOKEN": { "type": "string" }, "OPENCLAW_GATEWAY_URL": { "type": "string" }, "PALM_API_HOST": { "type": "string" }, "PALM_API_KEY": { "type": "string" }, "PORTKEY_API_KEY": { "type": "string" }, "PROMPTFOO_CA_CERT_PATH": { "type": "string" }, "PROMPTFOO_PFX_CERT_PATH": { "type": "string" }, "PROMPTFOO_PFX_PASSWORD": { "type": "string" }, "PROMPTFOO_JKS_CERT_PATH": { "type": "string" }, "PROMPTFOO_JKS_PASSWORD": { "type": "string" }, "PROMPTFOO_JKS_ALIAS": { "type": "string" }, "PROMPTFOO_INSECURE_SSL": { "type": "string" }, "QUIVERAI_API_KEY": { "type": "string" }, "REPLICATE_API_KEY": { "type": "string" }, "REPLICATE_API_TOKEN": { "type": "string" }, "SHAREPOINT_BASE_URL": { "type": "string" }, "SHAREPOINT_CERT_PATH": { "type": "string" }, "SHAREPOINT_CLIENT_ID": { "type": "string" }, "SHAREPOINT_TENANT_ID": { "type": "string" }, "VERCEL_AI_GATEWAY_API_KEY": { "type": "string" }, "VERCEL_AI_GATEWAY_BASE_URL": { "type": "string" }, "VERTEX_API_HOST": { "type": "string" }, "VERTEX_API_KEY": { "type": "string" }, "VERTEX_API_VERSION": { "type": "string" }, "VERTEX_PROJECT_ID": { "type": "string" }, "VERTEX_PUBLISHER": { "type": "string" }, "VERTEX_REGION": { "type": "string" }, "VOYAGE_API_BASE_URL": { "type": "string" }, "VOYAGE_API_KEY": { "type": "string" }, "WATSONX_AI_APIKEY": { "type": "string" }, "WATSONX_AI_AUTH_TYPE": { "type": "string" }, "WATSONX_AI_BEARER_TOKEN": { "type": "string" }, "WATSONX_AI_PROJECT_ID": { "type": "string" }, "XAI_API_BASE_URL": { "type": "string" }, "XAI_API_KEY": { "type": "string" }, "AZURE_CONTENT_SAFETY_ENDPOINT": { "type": "string" }, "AZURE_CONTENT_SAFETY_API_KEY": { "type": "string" }, "AZURE_CONTENT_SAFETY_API_VERSION": { "type": "string" }, "AWS_REGION": { "type": "string" }, "AWS_DEFAULT_REGION": { "type": "string" }, "AWS_SAGEMAKER_MAX_TOKENS": { "type": "string" }, "AWS_SAGEMAKER_TEMPERATURE": { "type": "string" }, "AWS_SAGEMAKER_TOP_P": { "type": "string" }, "AWS_SAGEMAKER_MAX_RETRIES": { "type": "string" }, "PROMPTFOO_EVAL_TIMEOUT_MS": { "type": "string" } }, "additionalProperties": false }, "inputs": { "type": "object", "propertyNames": { "type": "string", "pattern": "^[a-zA-Z_][a-zA-Z0-9_]*$" }, "additionalProperties": { "anyOf": [ { "type": "string", "minLength": 1 }, { "type": "object", "properties": { "config": { "type": "object", "properties": { "benign": { "type": "boolean" }, "inputPurpose": { "type": "string", "minLength": 1 }, "injectionPlacements": { "minItems": 1, "type": "array", "items": { "type": "string", "minLength": 1 } } }, "additionalProperties": false }, "description": { "type": "string", "minLength": 1 }, "type": { "type": "string", "enum": ["text", "pdf", "docx", "image"] } }, "required": ["description"], "additionalProperties": false } ] } } }, "additionalProperties": false } ] }, "numTests": { "description": "Number of tests to generate", "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 }, "language": { "description": "Language(s) of tests to generate for this plugin", "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ] }, "frameworks": { "description": "Compliance frameworks to include across reports and commands", "minItems": 1, "type": "array", "items": { "type": "string", "enum": [ "mitre:atlas", "nist:ai:measure", "owasp:api", "owasp:llm", "owasp:agentic", "eu:ai-act", "iso:42001", "gdpr", "dod:ai:ethics" ] } }, "entities": { "description": "Names of people, brands, or organizations related to your LLM application", "type": "array", "items": { "type": "string" } }, "contexts": { "description": "Security contexts for testing multiple states - each context has its own purpose", "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "description": "Unique identifier for the context" }, "purpose": { "type": "string", "description": "Purpose/context for this context - used for generation and grading" }, "vars": { "description": "Variables passed to provider (e.g., context_file, user_role)", "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "type": "string" } } }, "required": ["id", "purpose"], "additionalProperties": false } }, "plugins": { "type": "array", "items": { "anyOf": [ { "anyOf": [ { "type": "string", "enum": [ "aegis", "agentic:memory-poisoning", "ascii-smuggling", "beavertails", "bfla", "bias", "bias:age", "bias:disability", "bias:gender", "bias:race", "bola", "cca", "coding-agent:all", "coding-agent:automation-poisoning", "coding-agent:core", "coding-agent:delayed-ci-exfil", "coding-agent:generated-vulnerability", "coding-agent:network-egress-bypass", "coding-agent:procfs-credential-read", "coding-agent:repo-prompt-injection", "coding-agent:sandbox-read-escape", "coding-agent:sandbox-write-escape", "coding-agent:secret-env-read", "coding-agent:secret-file-read", "coding-agent:steganographic-exfil", "coding-agent:terminal-output-injection", "coding-agent:verifier-sabotage", "competitors", "contracts", "coppa", "cross-session-leak", "cyberseceval", "data-exfil", "debug-access", "default", "divergent-repetition", "dod:ai:ethics", "dod:ai:ethics:01", "dod:ai:ethics:02", "dod:ai:ethics:03", "dod:ai:ethics:04", "dod:ai:ethics:05", "donotanswer", "ecommerce", "ecommerce:compliance-bypass", "ecommerce:order-fraud", "ecommerce:pci-dss", "ecommerce:price-manipulation", "eu:ai-act", "eu:ai-act:annex3:biometric-id", "eu:ai-act:annex3:critical-infrastructure", "eu:ai-act:annex3:education", "eu:ai-act:annex3:employment", "eu:ai-act:annex3:essential-services", "eu:ai-act:annex3:justice-democracy", "eu:ai-act:annex3:law-enforcement", "eu:ai-act:annex3:migration-border", "eu:ai-act:art5:biometric-categorisation", "eu:ai-act:art5:exploitation-of-vulnerabilities", "eu:ai-act:art5:remote-biometric-id-live", "eu:ai-act:art5:remote-biometric-id-post", "eu:ai-act:art5:social-scoring", "eu:ai-act:art5:subliminal-manipulation", "excessive-agency", "ferpa", "financial", "financial:calculation-error", "financial:compliance-violation", "financial:confidential-disclosure", "financial:counterfactual", "financial:data-leakage", "financial:defamation", "financial:hallucination", "financial:impartiality", "financial:japan-fiea-suitability", "financial:misconduct", "financial:sox-compliance", "financial:sycophancy", "foundation", "gdpr", "gdpr:art15", "gdpr:art17", "gdpr:art22", "gdpr:art25", "gdpr:art32", "gdpr:art5", "gdpr:art9", "goal-misalignment", "guardrails-eval", "hallucination", "harmbench", "harmful", "harmful:chemical-biological-weapons", "harmful:child-exploitation", "harmful:copyright-violations", "harmful:cybercrime", "harmful:cybercrime:malicious-code", "harmful:graphic-content", "harmful:harassment-bullying", "harmful:hate", "harmful:illegal-activities", "harmful:illegal-drugs", "harmful:illegal-drugs:meth", "harmful:indiscriminate-weapons", "harmful:insults", "harmful:intellectual-property", "harmful:misinformation-disinformation", "harmful:non-violent-crime", "harmful:privacy", "harmful:profanity", "harmful:radicalization", "harmful:self-harm", "harmful:sex-crime", "harmful:sexual-content", "harmful:specialized-advice", "harmful:unsafe-practices", "harmful:violent-crime", "harmful:weapons:ied", "hijacking", "illegal-activity", "imitation", "indirect-prompt-injection", "insurance", "insurance:coverage-discrimination", "insurance:data-disclosure", "insurance:network-misinformation", "insurance:phi-disclosure", "intent", "iso:42001", "iso:42001:accountability", "iso:42001:fairness", "iso:42001:privacy", "iso:42001:robustness", "iso:42001:safety", "iso:42001:security", "iso:42001:transparency", "mcp", "medical", "medical:anchoring-bias", "medical:fda:ai-disclosure", "medical:fda:cyber-access-control", "medical:fda:cyber-audit-tampering", "medical:hallucination", "medical:incorrect-knowledge", "medical:off-label-use", "medical:prioritization-error", "medical:sycophancy", "misinformation", "mitre:atlas", "mitre:atlas:ai-attack-staging", "mitre:atlas:ai-model-access", "mitre:atlas:collection", "mitre:atlas:command-and-control", "mitre:atlas:credential-access", "mitre:atlas:defense-evasion", "mitre:atlas:discovery", "mitre:atlas:execution", "mitre:atlas:exfiltration", "mitre:atlas:impact", "mitre:atlas:initial-access", "mitre:atlas:lateral-movement", "mitre:atlas:ml-attack-staging", "mitre:atlas:persistence", "mitre:atlas:privilege-escalation", "mitre:atlas:reconnaissance", "mitre:atlas:resource-development", "model-identification", "nist:ai", "nist:ai:measure", "nist:ai:measure:1.1", "nist:ai:measure:1.2", "nist:ai:measure:2.1", "nist:ai:measure:2.10", "nist:ai:measure:2.11", "nist:ai:measure:2.12", "nist:ai:measure:2.13", "nist:ai:measure:2.2", "nist:ai:measure:2.3", "nist:ai:measure:2.4", "nist:ai:measure:2.5", "nist:ai:measure:2.6", "nist:ai:measure:2.7", "nist:ai:measure:2.8", "nist:ai:measure:2.9", "nist:ai:measure:3.1", "nist:ai:measure:3.2", "nist:ai:measure:3.3", "nist:ai:measure:4.1", "nist:ai:measure:4.2", "nist:ai:measure:4.3", "off-topic", "overreliance", "owasp:agentic", "owasp:agentic:asi01", "owasp:agentic:asi02", "owasp:agentic:asi03", "owasp:agentic:asi04", "owasp:agentic:asi05", "owasp:agentic:asi06", "owasp:agentic:asi07", "owasp:agentic:asi08", "owasp:agentic:asi09", "owasp:agentic:asi10", "owasp:api", "owasp:api:01", "owasp:api:02", "owasp:api:03", "owasp:api:04", "owasp:api:05", "owasp:api:06", "owasp:api:07", "owasp:api:08", "owasp:api:09", "owasp:api:10", "owasp:llm", "owasp:llm:01", "owasp:llm:02", "owasp:llm:03", "owasp:llm:04", "owasp:llm:05", "owasp:llm:06", "owasp:llm:07", "owasp:llm:08", "owasp:llm:09", "owasp:llm:10", "owasp:llm:redteam:implementation", "owasp:llm:redteam:model", "owasp:llm:redteam:runtime", "owasp:llm:redteam:system", "personal-safety", "pharmacy", "pharmacy:controlled-substance-compliance", "pharmacy:dosage-calculation", "pharmacy:drug-interaction", "pii", "pii:api-db", "pii:direct", "pii:session", "pii:social", "pliny", "policy", "politics", "prompt-extraction", "rag-document-exfiltration", "rag-poisoning", "rag-source-attribution", "rbac", "realestate", "realestate:accessibility-discrimination", "realestate:advertising-discrimination", "realestate:discriminatory-listings", "realestate:fair-housing-discrimination", "realestate:lending-discrimination", "realestate:source-of-income", "realestate:steering", "realestate:valuation-bias", "reasoning-dos", "religion", "shell-injection", "special-token-injection", "sql-injection", "ssrf", "system-prompt-override", "teen-safety", "teen-safety:age-restricted-goods-and-services", "teen-safety:dangerous-content", "teen-safety:dangerous-roleplay", "teen-safety:harmful-body-ideals", "telecom", "telecom:accessibility-violation", "telecom:account-takeover", "telecom:billing-misinformation", "telecom:coverage-misinformation", "telecom:cpni-disclosure", "telecom:e911-misinformation", "telecom:fraud-enablement", "telecom:law-enforcement-request-handling", "telecom:location-disclosure", "telecom:porting-misinformation", "telecom:tcpa-violation", "telecom:unauthorized-changes", "tool-discovery", "tool-discovery:multi-turn", "toxic-chat", "toxicity", "unsafebench", "unverifiable-claims", "vlguard", "vlsu", "wordplay", "xstest" ] }, { "type": "string" } ], "description": "Name of the plugin or path to custom plugin" }, { "type": "object", "properties": { "id": { "anyOf": [ { "type": "string", "enum": [ "aegis", "agentic:memory-poisoning", "ascii-smuggling", "beavertails", "bfla", "bias", "bias:age", "bias:disability", "bias:gender", "bias:race", "bola", "cca", "coding-agent:all", "coding-agent:automation-poisoning", "coding-agent:core", "coding-agent:delayed-ci-exfil", "coding-agent:generated-vulnerability", "coding-agent:network-egress-bypass", "coding-agent:procfs-credential-read", "coding-agent:repo-prompt-injection", "coding-agent:sandbox-read-escape", "coding-agent:sandbox-write-escape", "coding-agent:secret-env-read", "coding-agent:secret-file-read", "coding-agent:steganographic-exfil", "coding-agent:terminal-output-injection", "coding-agent:verifier-sabotage", "competitors", "contracts", "coppa", "cross-session-leak", "cyberseceval", "data-exfil", "debug-access", "default", "divergent-repetition", "dod:ai:ethics", "dod:ai:ethics:01", "dod:ai:ethics:02", "dod:ai:ethics:03", "dod:ai:ethics:04", "dod:ai:ethics:05", "donotanswer", "ecommerce", "ecommerce:compliance-bypass", "ecommerce:order-fraud", "ecommerce:pci-dss", "ecommerce:price-manipulation", "eu:ai-act", "eu:ai-act:annex3:biometric-id", "eu:ai-act:annex3:critical-infrastructure", "eu:ai-act:annex3:education", "eu:ai-act:annex3:employment", "eu:ai-act:annex3:essential-services", "eu:ai-act:annex3:justice-democracy", "eu:ai-act:annex3:law-enforcement", "eu:ai-act:annex3:migration-border", "eu:ai-act:art5:biometric-categorisation", "eu:ai-act:art5:exploitation-of-vulnerabilities", "eu:ai-act:art5:remote-biometric-id-live", "eu:ai-act:art5:remote-biometric-id-post", "eu:ai-act:art5:social-scoring", "eu:ai-act:art5:subliminal-manipulation", "excessive-agency", "ferpa", "financial", "financial:calculation-error", "financial:compliance-violation", "financial:confidential-disclosure", "financial:counterfactual", "financial:data-leakage", "financial:defamation", "financial:hallucination", "financial:impartiality", "financial:japan-fiea-suitability", "financial:misconduct", "financial:sox-compliance", "financial:sycophancy", "foundation", "gdpr", "gdpr:art15", "gdpr:art17", "gdpr:art22", "gdpr:art25", "gdpr:art32", "gdpr:art5", "gdpr:art9", "goal-misalignment", "guardrails-eval", "hallucination", "harmbench", "harmful", "harmful:chemical-biological-weapons", "harmful:child-exploitation", "harmful:copyright-violations", "harmful:cybercrime", "harmful:cybercrime:malicious-code", "harmful:graphic-content", "harmful:harassment-bullying", "harmful:hate", "harmful:illegal-activities", "harmful:illegal-drugs", "harmful:illegal-drugs:meth", "harmful:indiscriminate-weapons", "harmful:insults", "harmful:intellectual-property", "harmful:misinformation-disinformation", "harmful:non-violent-crime", "harmful:privacy", "harmful:profanity", "harmful:radicalization", "harmful:self-harm", "harmful:sex-crime", "harmful:sexual-content", "harmful:specialized-advice", "harmful:unsafe-practices", "harmful:violent-crime", "harmful:weapons:ied", "hijacking", "illegal-activity", "imitation", "indirect-prompt-injection", "insurance", "insurance:coverage-discrimination", "insurance:data-disclosure", "insurance:network-misinformation", "insurance:phi-disclosure", "intent", "iso:42001", "iso:42001:accountability", "iso:42001:fairness", "iso:42001:privacy", "iso:42001:robustness", "iso:42001:safety", "iso:42001:security", "iso:42001:transparency", "mcp", "medical", "medical:anchoring-bias", "medical:fda:ai-disclosure", "medical:fda:cyber-access-control", "medical:fda:cyber-audit-tampering", "medical:hallucination", "medical:incorrect-knowledge", "medical:off-label-use", "medical:prioritization-error", "medical:sycophancy", "misinformation", "mitre:atlas", "mitre:atlas:ai-attack-staging", "mitre:atlas:ai-model-access", "mitre:atlas:collection", "mitre:atlas:command-and-control", "mitre:atlas:credential-access", "mitre:atlas:defense-evasion", "mitre:atlas:discovery", "mitre:atlas:execution", "mitre:atlas:exfiltration", "mitre:atlas:impact", "mitre:atlas:initial-access", "mitre:atlas:lateral-movement", "mitre:atlas:ml-attack-staging", "mitre:atlas:persistence", "mitre:atlas:privilege-escalation", "mitre:atlas:reconnaissance", "mitre:atlas:resource-development", "model-identification", "nist:ai", "nist:ai:measure", "nist:ai:measure:1.1", "nist:ai:measure:1.2", "nist:ai:measure:2.1", "nist:ai:measure:2.10", "nist:ai:measure:2.11", "nist:ai:measure:2.12", "nist:ai:measure:2.13", "nist:ai:measure:2.2", "nist:ai:measure:2.3", "nist:ai:measure:2.4", "nist:ai:measure:2.5", "nist:ai:measure:2.6", "nist:ai:measure:2.7", "nist:ai:measure:2.8", "nist:ai:measure:2.9", "nist:ai:measure:3.1", "nist:ai:measure:3.2", "nist:ai:measure:3.3", "nist:ai:measure:4.1", "nist:ai:measure:4.2", "nist:ai:measure:4.3", "off-topic", "overreliance", "owasp:agentic", "owasp:agentic:asi01", "owasp:agentic:asi02", "owasp:agentic:asi03", "owasp:agentic:asi04", "owasp:agentic:asi05", "owasp:agentic:asi06", "owasp:agentic:asi07", "owasp:agentic:asi08", "owasp:agentic:asi09", "owasp:agentic:asi10", "owasp:api", "owasp:api:01", "owasp:api:02", "owasp:api:03", "owasp:api:04", "owasp:api:05", "owasp:api:06", "owasp:api:07", "owasp:api:08", "owasp:api:09", "owasp:api:10", "owasp:llm", "owasp:llm:01", "owasp:llm:02", "owasp:llm:03", "owasp:llm:04", "owasp:llm:05", "owasp:llm:06", "owasp:llm:07", "owasp:llm:08", "owasp:llm:09", "owasp:llm:10", "owasp:llm:redteam:implementation", "owasp:llm:redteam:model", "owasp:llm:redteam:runtime", "owasp:llm:redteam:system", "personal-safety", "pharmacy", "pharmacy:controlled-substance-compliance", "pharmacy:dosage-calculation", "pharmacy:drug-interaction", "pii", "pii:api-db", "pii:direct", "pii:session", "pii:social", "pliny", "policy", "politics", "prompt-extraction", "rag-document-exfiltration", "rag-poisoning", "rag-source-attribution", "rbac", "realestate", "realestate:accessibility-discrimination", "realestate:advertising-discrimination", "realestate:discriminatory-listings", "realestate:fair-housing-discrimination", "realestate:lending-discrimination", "realestate:source-of-income", "realestate:steering", "realestate:valuation-bias", "reasoning-dos", "religion", "shell-injection", "special-token-injection", "sql-injection", "ssrf", "system-prompt-override", "teen-safety", "teen-safety:age-restricted-goods-and-services", "teen-safety:dangerous-content", "teen-safety:dangerous-roleplay", "teen-safety:harmful-body-ideals", "telecom", "telecom:accessibility-violation", "telecom:account-takeover", "telecom:billing-misinformation", "telecom:coverage-misinformation", "telecom:cpni-disclosure", "telecom:e911-misinformation", "telecom:fraud-enablement", "telecom:law-enforcement-request-handling", "telecom:location-disclosure", "telecom:porting-misinformation", "telecom:tcpa-violation", "telecom:unauthorized-changes", "tool-discovery", "tool-discovery:multi-turn", "toxic-chat", "toxicity", "unsafebench", "unverifiable-claims", "vlguard", "vlsu", "wordplay", "xstest" ] }, { "type": "string" } ], "description": "Name of the plugin" }, "numTests": { "description": "Number of tests to generate for this plugin", "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 }, "config": { "description": "Plugin-specific configuration", "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": {} }, "severity": { "description": "Severity level for this plugin", "type": "string", "enum": ["critical", "high", "medium", "low", "informational"] } }, "required": ["id", "numTests"], "additionalProperties": false } ] }, "description": "Plugins to use for redteam generation" }, "strategies": { "type": "array", "items": { "anyOf": [ { "anyOf": [ { "type": "string", "enum": [ "audio", "authoritative-markup-injection", "base64", "basic", "best-of-n", "camelcase", "citation", "crescendo", "custom", "default", "emoji", "gcg", "goat", "hex", "homoglyph", "image", "indirect-web-pwn", "jailbreak", "jailbreak-templates", "jailbreak:composite", "jailbreak:hydra", "jailbreak:likert", "jailbreak:meta", "jailbreak:tree", "layer", "leetspeak", "math-prompt", "mischievous-user", "morse", "multilingual", "other-encodings", "piglatin", "prompt-injection", "retry", "rot13", "video" ] }, { "type": "string" }, { "type": "string" } ] }, { "type": "object", "properties": { "id": { "anyOf": [ { "type": "string", "enum": [ "audio", "authoritative-markup-injection", "base64", "basic", "best-of-n", "camelcase", "citation", "crescendo", "custom", "default", "emoji", "gcg", "goat", "hex", "homoglyph", "image", "indirect-web-pwn", "jailbreak", "jailbreak-templates", "jailbreak:composite", "jailbreak:hydra", "jailbreak:likert", "jailbreak:meta", "jailbreak:tree", "layer", "leetspeak", "math-prompt", "mischievous-user", "morse", "multilingual", "other-encodings", "piglatin", "prompt-injection", "retry", "rot13", "video" ] }, { "type": "string" }, { "type": "string" } ] }, "config": { "description": "Strategy-specific configuration", "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": {} } }, "required": ["id"], "additionalProperties": false } ] }, "description": "Strategies to use for redteam generation.\n\nDefaults to basic, jailbreak:meta, jailbreak:composite\nSupports audio, authoritative-markup-injection, base64, basic, best-of-n, camelcase, citation, crescendo, custom, default, emoji, gcg, goat, hex, homoglyph, image, indirect-web-pwn, jailbreak, jailbreak-templates, jailbreak:composite, jailbreak:hydra, jailbreak:likert, jailbreak:meta, jailbreak:tree, layer, leetspeak, math-prompt, mischievous-user, morse, multilingual, other-encodings, piglatin, prompt-injection, retry, rot13, video" }, "maxConcurrency": { "description": "Maximum number of concurrent API calls", "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 }, "maxCharsPerMessage": { "description": "Maximum number of characters allowed per generated user message", "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 }, "delay": { "description": "Delay in milliseconds between plugin API calls", "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "excludeTargetOutputFromAgenticAttackGeneration": { "description": "Whether to exclude target output from the agentific attack generation process", "type": "boolean" }, "tracing": { "description": "Tracing defaults applied to all strategies unless overridden", "allOf": [ { "$ref": "#/definitions/__schema0" } ] }, "graderExamples": { "description": "Global grading examples that apply to all plugins", "type": "array", "items": { "type": "object", "properties": { "output": { "type": "string" }, "pass": { "type": "boolean" }, "score": { "type": "number" }, "reason": { "type": "string" } }, "required": ["output", "pass", "score", "reason"], "additionalProperties": false } } }, "required": ["plugins", "strategies"], "additionalProperties": false, "definitions": { "__schema0": { "type": "object", "properties": { "enabled": { "type": "boolean" }, "includeInAttack": { "type": "boolean" }, "includeInGrading": { "type": "boolean" }, "includeInternalSpans": { "type": "boolean" }, "maxSpans": { "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 }, "maxDepth": { "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 }, "maxRetries": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "retryDelayMs": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "spanFilter": { "type": "array", "items": { "type": "string" } }, "sanitizeAttributes": { "type": "boolean" }, "strategies": { "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "allOf": [ { "$ref": "#/definitions/__schema0" } ] } } }, "additionalProperties": false } } }, "__schema39": { "type": "boolean" }, "__schema40": { "type": "object", "properties": { "enabled": { "type": "boolean" }, "otlp": { "type": "object", "properties": { "http": { "type": "object", "properties": { "enabled": { "type": "boolean" }, "port": { "type": "number" }, "host": { "type": "string" }, "acceptFormats": { "type": "array", "items": { "type": "string", "enum": ["protobuf", "json"] } } }, "required": ["enabled", "port", "host", "acceptFormats"], "additionalProperties": false }, "grpc": { "type": "object", "properties": { "enabled": { "type": "boolean" }, "port": { "type": "number" } }, "required": ["enabled", "port"], "additionalProperties": false } }, "additionalProperties": false }, "storage": { "type": "object", "properties": { "type": { "type": "string", "enum": ["sqlite"] }, "retentionDays": { "type": "number" } }, "required": ["type", "retentionDays"], "additionalProperties": false }, "forwarding": { "type": "object", "properties": { "enabled": { "type": "boolean" }, "endpoint": { "type": "string" }, "headers": { "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "type": "string" } } }, "required": ["enabled", "endpoint"], "additionalProperties": false } }, "required": ["enabled"], "additionalProperties": false }, "__schema41": { "type": "object", "properties": { "cache": { "type": "boolean" }, "delay": { "type": "number" }, "eventSource": { "type": "string" }, "generateSuggestions": { "type": "boolean" }, "suggestionsCount": { "type": "integer", "exclusiveMinimum": 0, "maximum": 50 }, "interactiveProviders": { "type": "boolean" }, "maxConcurrency": { "type": "number" }, "progressCallback": {}, "repeat": { "type": "number" }, "showProgressBar": { "type": "boolean" }, "timeoutMs": { "type": "number" }, "maxEvalTimeMs": { "type": "number" }, "isRedteam": { "type": "boolean" }, "silent": { "type": "boolean" }, "filterRange": { "$ref": "#/definitions/__schema42" } }, "additionalProperties": false }, "__schema42": { "type": "string" }, "__schema43": { "type": "object", "properties": { "description": { "type": "string" }, "prompts": { "type": "array", "items": { "type": "string" } }, "providers": { "type": "array", "items": { "type": "string" } }, "output": { "type": "array", "items": { "type": "string" } }, "maxConcurrency": { "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 }, "repeat": { "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 }, "delay": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "vars": { "type": "string" }, "tests": { "type": "string" }, "config": { "type": "array", "items": { "type": "string" } }, "assertions": { "type": "string" }, "modelOutputs": { "type": "string" }, "verbose": { "type": "boolean" }, "grader": { "type": "string" }, "tableCellMaxLength": { "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 }, "write": { "type": "boolean" }, "cache": { "type": "boolean" }, "table": { "type": "boolean" }, "share": { "type": "boolean" }, "noShare": { "type": "boolean" }, "progressBar": { "type": "boolean" }, "watch": { "type": "boolean" }, "filterErrorsOnly": { "type": "string" }, "filterFailing": { "type": "string" }, "filterFailingOnly": { "type": "string" }, "filterFirstN": { "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 }, "filterMetadata": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ] }, "filterPattern": { "type": "string" }, "filterPrompts": { "type": "string" }, "filterProviders": { "type": "string" }, "filterRange": { "allOf": [ { "$ref": "#/definitions/__schema42" } ] }, "filterSample": { "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 }, "filterTargets": { "type": "string" }, "var": { "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "type": "string" } }, "generateSuggestions": { "type": "boolean" }, "suggestionsCount": { "type": "integer", "exclusiveMinimum": 0, "maximum": 50 }, "promptPrefix": { "type": "string" }, "promptSuffix": { "type": "string" }, "retryErrors": { "type": "boolean" }, "envPath": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ] }, "extension": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "__schema44": { "allOf": [ { "$ref": "#/definitions/__schema3" } ] } } }