{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [], "authorship_tag": "ABX9TyNwIDQIlsu+ZzpWul+ZsKbM", "include_colab_link": true }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" } }, "cells": [ { "cell_type": "markdown", "metadata": { "id": "view-in-github", "colab_type": "text" }, "source": [ "\"Open" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "RmW-833YVU2R", "outputId": "7e7b9ad3-39d8-4e28-fd0b-4f626958ce6d" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Collecting deepteam\n", " Downloading deepteam-0.2.7-py3-none-any.whl.metadata (15 kB)\n", "Collecting deepeval\n", " Downloading deepeval-3.6.7-py3-none-any.whl.metadata (18 kB)\n", "Requirement already satisfied: aiohttp>=3.11.18 in /usr/local/lib/python3.12/dist-packages (from deepteam) (3.13.0)\n", " Downloading deepeval-3.6.2-py3-none-any.whl.metadata (18 kB)\n", "Requirement already satisfied: grpcio>=1.67.1 in /usr/local/lib/python3.12/dist-packages (from deepteam) (1.75.1)\n", "Requirement already satisfied: openai>=1.76.2 in /usr/local/lib/python3.12/dist-packages (from deepteam) (1.109.1)\n", "Requirement already satisfied: pyyaml>=6.0.2 in /usr/local/lib/python3.12/dist-packages (from deepteam) (6.0.3)\n", "Requirement already satisfied: requests>=2.31.0 in /usr/local/lib/python3.12/dist-packages (from deepteam) (2.32.4)\n", "Requirement already satisfied: tabulate>=0.9.0 in /usr/local/lib/python3.12/dist-packages (from deepteam) (0.9.0)\n", "Requirement already satisfied: tqdm>=4.66.1 in /usr/local/lib/python3.12/dist-packages (from deepteam) (4.67.1)\n", "Collecting anthropic (from deepeval)\n", " Downloading anthropic-0.71.0-py3-none-any.whl.metadata (28 kB)\n", "Collecting click<8.3.0,>=8.0.0 (from deepeval)\n", " Downloading click-8.2.1-py3-none-any.whl.metadata (2.5 kB)\n", "Requirement already satisfied: google-genai<2.0.0,>=1.9.0 in /usr/local/lib/python3.12/dist-packages (from deepeval) (1.41.0)\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.12/dist-packages (from deepeval) (3.1.6)\n", "Requirement already satisfied: nest_asyncio in /usr/local/lib/python3.12/dist-packages (from deepeval) (1.6.0)\n", "Collecting ollama (from deepeval)\n", " Downloading ollama-0.6.0-py3-none-any.whl.metadata (4.3 kB)\n", "Requirement already satisfied: opentelemetry-api<2.0.0,>=1.24.0 in /usr/local/lib/python3.12/dist-packages (from deepeval) (1.37.0)\n", "Collecting opentelemetry-exporter-otlp-proto-grpc<2.0.0,>=1.24.0 (from deepeval)\n", " Downloading opentelemetry_exporter_otlp_proto_grpc-1.38.0-py3-none-any.whl.metadata (2.4 kB)\n", "Requirement already satisfied: opentelemetry-sdk<2.0.0,>=1.24.0 in /usr/local/lib/python3.12/dist-packages (from deepeval) (1.37.0)\n", "Collecting portalocker (from deepeval)\n", " Downloading portalocker-3.2.0-py3-none-any.whl.metadata (8.7 kB)\n", "Collecting posthog<7.0.0,>=6.3.0 (from deepeval)\n", " Downloading posthog-6.7.8-py3-none-any.whl.metadata (6.0 kB)\n", "Requirement already satisfied: pydantic<3.0.0,>=2.11.7 in /usr/local/lib/python3.12/dist-packages (from deepeval) (2.11.10)\n", "Requirement already satisfied: pydantic-settings<3.0.0,>=2.10.1 in /usr/local/lib/python3.12/dist-packages (from deepeval) (2.11.0)\n", "Collecting pyfiglet (from deepeval)\n", " Downloading pyfiglet-1.0.4-py3-none-any.whl.metadata (7.4 kB)\n", "Requirement already satisfied: pytest in /usr/local/lib/python3.12/dist-packages (from deepeval) (8.4.2)\n", "Collecting pytest-asyncio (from deepeval)\n", " Downloading pytest_asyncio-1.2.0-py3-none-any.whl.metadata (4.1 kB)\n", "Collecting pytest-repeat (from deepeval)\n", " Downloading pytest_repeat-0.9.4-py3-none-any.whl.metadata (4.9 kB)\n", "Collecting pytest-rerunfailures<13.0,>=12.0 (from deepeval)\n", " Downloading pytest_rerunfailures-12.0-py3-none-any.whl.metadata (18 kB)\n", "Collecting pytest-xdist (from deepeval)\n", " Downloading pytest_xdist-3.8.0-py3-none-any.whl.metadata (3.0 kB)\n", "Requirement already satisfied: python-dotenv<2.0.0,>=1.1.1 in /usr/local/lib/python3.12/dist-packages (from deepeval) (1.1.1)\n", "Requirement already satisfied: rich<15.0.0,>=13.6.0 in /usr/local/lib/python3.12/dist-packages (from deepeval) (13.9.4)\n", "Requirement already satisfied: sentry-sdk in /usr/local/lib/python3.12/dist-packages (from deepeval) (2.40.0)\n", "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (from deepeval) (75.2.0)\n", "Requirement already satisfied: tenacity<=10.0.0,>=8.0.0 in /usr/local/lib/python3.12/dist-packages (from deepeval) (8.5.0)\n", "Requirement already satisfied: typer<1.0.0,>=0.9 in /usr/local/lib/python3.12/dist-packages (from deepeval) (0.19.2)\n", "Requirement already satisfied: wheel in /usr/local/lib/python3.12/dist-packages (from deepeval) (0.45.1)\n", "Requirement already satisfied: aiohappyeyeballs>=2.5.0 in /usr/local/lib/python3.12/dist-packages (from aiohttp>=3.11.18->deepteam) (2.6.1)\n", "Requirement already satisfied: aiosignal>=1.4.0 in /usr/local/lib/python3.12/dist-packages (from aiohttp>=3.11.18->deepteam) (1.4.0)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.12/dist-packages (from aiohttp>=3.11.18->deepteam) (25.4.0)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.12/dist-packages (from aiohttp>=3.11.18->deepteam) (1.8.0)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.12/dist-packages (from aiohttp>=3.11.18->deepteam) (6.7.0)\n", "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.12/dist-packages (from aiohttp>=3.11.18->deepteam) (0.3.2)\n", "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.12/dist-packages (from aiohttp>=3.11.18->deepteam) (1.22.0)\n", "Requirement already satisfied: anyio<5.0.0,>=4.8.0 in /usr/local/lib/python3.12/dist-packages (from google-genai<2.0.0,>=1.9.0->deepeval) (4.11.0)\n", "Requirement already satisfied: google-auth<3.0.0,>=2.14.1 in /usr/local/lib/python3.12/dist-packages (from google-genai<2.0.0,>=1.9.0->deepeval) (2.38.0)\n", "Requirement already satisfied: httpx<1.0.0,>=0.28.1 in /usr/local/lib/python3.12/dist-packages (from google-genai<2.0.0,>=1.9.0->deepeval) (0.28.1)\n", "Requirement already satisfied: websockets<15.1.0,>=13.0.0 in /usr/local/lib/python3.12/dist-packages (from google-genai<2.0.0,>=1.9.0->deepeval) (15.0.1)\n", "Requirement already satisfied: typing-extensions<5.0.0,>=4.11.0 in /usr/local/lib/python3.12/dist-packages (from google-genai<2.0.0,>=1.9.0->deepeval) (4.15.0)\n", "Requirement already satisfied: distro<2,>=1.7.0 in /usr/local/lib/python3.12/dist-packages (from openai>=1.76.2->deepteam) (1.9.0)\n", "Requirement already satisfied: jiter<1,>=0.4.0 in /usr/local/lib/python3.12/dist-packages (from openai>=1.76.2->deepteam) (0.11.0)\n", "Requirement already satisfied: sniffio in /usr/local/lib/python3.12/dist-packages (from openai>=1.76.2->deepteam) (1.3.1)\n", "Requirement already satisfied: importlib-metadata<8.8.0,>=6.0 in /usr/local/lib/python3.12/dist-packages (from opentelemetry-api<2.0.0,>=1.24.0->deepeval) (8.7.0)\n", "Requirement already satisfied: googleapis-common-protos~=1.57 in /usr/local/lib/python3.12/dist-packages (from opentelemetry-exporter-otlp-proto-grpc<2.0.0,>=1.24.0->deepeval) (1.70.0)\n", "Collecting opentelemetry-exporter-otlp-proto-common==1.38.0 (from opentelemetry-exporter-otlp-proto-grpc<2.0.0,>=1.24.0->deepeval)\n", " Downloading opentelemetry_exporter_otlp_proto_common-1.38.0-py3-none-any.whl.metadata (1.8 kB)\n", "Collecting opentelemetry-proto==1.38.0 (from opentelemetry-exporter-otlp-proto-grpc<2.0.0,>=1.24.0->deepeval)\n", " Downloading opentelemetry_proto-1.38.0-py3-none-any.whl.metadata (2.3 kB)\n", "Collecting opentelemetry-sdk<2.0.0,>=1.24.0 (from deepeval)\n", " Downloading opentelemetry_sdk-1.38.0-py3-none-any.whl.metadata (1.5 kB)\n", "Requirement already satisfied: protobuf<7.0,>=5.0 in /usr/local/lib/python3.12/dist-packages (from opentelemetry-proto==1.38.0->opentelemetry-exporter-otlp-proto-grpc<2.0.0,>=1.24.0->deepeval) (5.29.5)\n", "Collecting opentelemetry-api<2.0.0,>=1.24.0 (from deepeval)\n", " Downloading opentelemetry_api-1.38.0-py3-none-any.whl.metadata (1.5 kB)\n", "Collecting opentelemetry-semantic-conventions==0.59b0 (from opentelemetry-sdk<2.0.0,>=1.24.0->deepeval)\n", " Downloading opentelemetry_semantic_conventions-0.59b0-py3-none-any.whl.metadata (2.4 kB)\n", "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.12/dist-packages (from posthog<7.0.0,>=6.3.0->deepeval) (1.17.0)\n", "Requirement already satisfied: python-dateutil>=2.2 in /usr/local/lib/python3.12/dist-packages (from posthog<7.0.0,>=6.3.0->deepeval) (2.9.0.post0)\n", "Collecting backoff>=1.10.0 (from posthog<7.0.0,>=6.3.0->deepeval)\n", " Downloading backoff-2.2.1-py3-none-any.whl.metadata (14 kB)\n", "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.12/dist-packages (from pydantic<3.0.0,>=2.11.7->deepeval) (0.7.0)\n", "Requirement already satisfied: pydantic-core==2.33.2 in /usr/local/lib/python3.12/dist-packages (from pydantic<3.0.0,>=2.11.7->deepeval) (2.33.2)\n", "Requirement already satisfied: typing-inspection>=0.4.0 in /usr/local/lib/python3.12/dist-packages (from pydantic<3.0.0,>=2.11.7->deepeval) (0.4.2)\n", "Requirement already satisfied: packaging>=17.1 in /usr/local/lib/python3.12/dist-packages (from pytest-rerunfailures<13.0,>=12.0->deepeval) (25.0)\n", "Requirement already satisfied: iniconfig>=1 in /usr/local/lib/python3.12/dist-packages (from pytest->deepeval) (2.1.0)\n", "Requirement already satisfied: pluggy<2,>=1.5 in /usr/local/lib/python3.12/dist-packages (from pytest->deepeval) (1.6.0)\n", "Requirement already satisfied: pygments>=2.7.2 in /usr/local/lib/python3.12/dist-packages (from pytest->deepeval) (2.19.2)\n", "Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.12/dist-packages (from requests>=2.31.0->deepteam) (3.4.3)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.12/dist-packages (from requests>=2.31.0->deepteam) (3.10)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.12/dist-packages (from requests>=2.31.0->deepteam) (2.5.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.12/dist-packages (from requests>=2.31.0->deepteam) (2025.10.5)\n", "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.12/dist-packages (from rich<15.0.0,>=13.6.0->deepeval) (4.0.0)\n", "Requirement already satisfied: shellingham>=1.3.0 in /usr/local/lib/python3.12/dist-packages (from typer<1.0.0,>=0.9->deepeval) (1.5.4)\n", "Requirement already satisfied: docstring-parser<1,>=0.15 in /usr/local/lib/python3.12/dist-packages (from anthropic->deepeval) (0.17.0)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.12/dist-packages (from jinja2->deepeval) (3.0.3)\n", "Collecting execnet>=2.1 (from pytest-xdist->deepeval)\n", " Downloading execnet-2.1.1-py3-none-any.whl.metadata (2.9 kB)\n", "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.12/dist-packages (from google-auth<3.0.0,>=2.14.1->google-genai<2.0.0,>=1.9.0->deepeval) (5.5.2)\n", "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.12/dist-packages (from google-auth<3.0.0,>=2.14.1->google-genai<2.0.0,>=1.9.0->deepeval) (0.4.2)\n", "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.12/dist-packages (from google-auth<3.0.0,>=2.14.1->google-genai<2.0.0,>=1.9.0->deepeval) (4.9.1)\n", "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.12/dist-packages (from httpx<1.0.0,>=0.28.1->google-genai<2.0.0,>=1.9.0->deepeval) (1.0.9)\n", "Requirement already satisfied: h11>=0.16 in /usr/local/lib/python3.12/dist-packages (from httpcore==1.*->httpx<1.0.0,>=0.28.1->google-genai<2.0.0,>=1.9.0->deepeval) (0.16.0)\n", "Requirement already satisfied: zipp>=3.20 in /usr/local/lib/python3.12/dist-packages (from importlib-metadata<8.8.0,>=6.0->opentelemetry-api<2.0.0,>=1.24.0->deepeval) (3.23.0)\n", "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py>=2.2.0->rich<15.0.0,>=13.6.0->deepeval) (0.1.2)\n", "Requirement already satisfied: pyasn1<0.7.0,>=0.6.1 in /usr/local/lib/python3.12/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.0,>=2.14.1->google-genai<2.0.0,>=1.9.0->deepeval) (0.6.1)\n", "Downloading deepteam-0.2.7-py3-none-any.whl (459 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m459.1/459.1 kB\u001b[0m \u001b[31m9.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading deepeval-3.6.2-py3-none-any.whl (617 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m617.7/617.7 kB\u001b[0m \u001b[31m24.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading click-8.2.1-py3-none-any.whl (102 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m102.2/102.2 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading opentelemetry_exporter_otlp_proto_grpc-1.38.0-py3-none-any.whl (19 kB)\n", "Downloading opentelemetry_exporter_otlp_proto_common-1.38.0-py3-none-any.whl (18 kB)\n", "Downloading opentelemetry_proto-1.38.0-py3-none-any.whl (72 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.5/72.5 kB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading opentelemetry_sdk-1.38.0-py3-none-any.whl (132 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.3/132.3 kB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading opentelemetry_api-1.38.0-py3-none-any.whl (65 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m65.9/65.9 kB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading opentelemetry_semantic_conventions-0.59b0-py3-none-any.whl (207 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m208.0/208.0 kB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading posthog-6.7.8-py3-none-any.whl (137 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m137.3/137.3 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading pytest_rerunfailures-12.0-py3-none-any.whl (12 kB)\n", "Downloading anthropic-0.71.0-py3-none-any.whl (355 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m355.0/355.0 kB\u001b[0m \u001b[31m22.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading ollama-0.6.0-py3-none-any.whl (14 kB)\n", "Downloading portalocker-3.2.0-py3-none-any.whl (22 kB)\n", "Downloading pyfiglet-1.0.4-py3-none-any.whl (1.8 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m50.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading pytest_asyncio-1.2.0-py3-none-any.whl (15 kB)\n", "Downloading pytest_repeat-0.9.4-py3-none-any.whl (4.2 kB)\n", "Downloading pytest_xdist-3.8.0-py3-none-any.whl (46 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.4/46.4 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading backoff-2.2.1-py3-none-any.whl (15 kB)\n", "Downloading execnet-2.1.1-py3-none-any.whl (40 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.6/40.6 kB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hInstalling collected packages: pyfiglet, portalocker, opentelemetry-proto, execnet, click, backoff, pytest-xdist, pytest-rerunfailures, pytest-repeat, pytest-asyncio, posthog, opentelemetry-exporter-otlp-proto-common, opentelemetry-api, opentelemetry-semantic-conventions, ollama, anthropic, opentelemetry-sdk, opentelemetry-exporter-otlp-proto-grpc, deepeval, deepteam\n", " Attempting uninstall: click\n", " Found existing installation: click 8.3.0\n", " Uninstalling click-8.3.0:\n", " Successfully uninstalled click-8.3.0\n", " Attempting uninstall: opentelemetry-api\n", " Found existing installation: opentelemetry-api 1.37.0\n", " Uninstalling opentelemetry-api-1.37.0:\n", " Successfully uninstalled opentelemetry-api-1.37.0\n", " Attempting uninstall: opentelemetry-semantic-conventions\n", " Found existing installation: opentelemetry-semantic-conventions 0.58b0\n", " Uninstalling opentelemetry-semantic-conventions-0.58b0:\n", " Successfully uninstalled opentelemetry-semantic-conventions-0.58b0\n", " Attempting uninstall: opentelemetry-sdk\n", " Found existing installation: opentelemetry-sdk 1.37.0\n", " Uninstalling opentelemetry-sdk-1.37.0:\n", " Successfully uninstalled opentelemetry-sdk-1.37.0\n", "Successfully installed anthropic-0.71.0 backoff-2.2.1 click-8.2.1 deepeval-3.6.2 deepteam-0.2.7 execnet-2.1.1 ollama-0.6.0 opentelemetry-api-1.38.0 opentelemetry-exporter-otlp-proto-common-1.38.0 opentelemetry-exporter-otlp-proto-grpc-1.38.0 opentelemetry-proto-1.38.0 opentelemetry-sdk-1.38.0 opentelemetry-semantic-conventions-0.59b0 portalocker-3.2.0 posthog-6.7.8 pyfiglet-1.0.4 pytest-asyncio-1.2.0 pytest-repeat-0.9.4 pytest-rerunfailures-12.0 pytest-xdist-3.8.0\n" ] } ], "source": [ "!pip install deepteam deepeval" ] }, { "cell_type": "code", "source": [ "import os\n", "\n", "os.environ[\"OPENAI_API_KEY\"] = \"YOUR_API_KEY\"" ], "metadata": { "id": "AKBci10zVfTK" }, "execution_count": 2, "outputs": [] }, { "cell_type": "markdown", "source": [ "Want to use other evaluation models? [Click here](https://deepeval.com/integrations/models/openai) to see all supported models and their usage instructions." ], "metadata": { "id": "By1mvBjjVv_k" } }, { "cell_type": "code", "source": [ "from openai import OpenAI\n", "from deepteam.vulnerabilities import Competition\n", "from deepteam.attacks.single_turn import PermissionEscalation, PromptInjection\n", "from deepteam.attacks.multi_turn import LinearJailbreaking, SequentialJailbreak\n", "from deepteam.test_case import RTTurn\n", "from deepeval.models import GeminiModel\n", "\n", "def model_callback(attack, turn_history=None):\n", " model = GeminiModel(\n", " model_name=\"gemini-2.5-flash\",\n", " api_key=\"YOUR_API_KEY_HERE\",\n", " temperature=0\n", " )\n", " return model.generate(attack)[0]\n", "\n", "\n", "competetion = Competition()\n", "\n", "test_cases = competetion.simulate_attacks()\n", "\n", "permission_escalation = PermissionEscalation()\n", "prompt_injection = PromptInjection()\n", "linear_jailbreaking = LinearJailbreaking()\n", "sequential_jailbreak = SequentialJailbreak()\n", "\n", "safe_model = True\n", "\n", "for test_case in test_cases:\n", " enhanced_attack_1 = permission_escalation.enhance(test_case.input)\n", " enhanced_attack_2 = prompt_injection.enhance(enhanced_attack_1)\n", " turns = [RTTurn(role=\"user\", content=enhanced_attack_2)]\n", " new_turns_1 = linear_jailbreaking._get_turns(\n", " model_callback, turns, test_case.vulnerability, test_case.vulnerability_type\n", " )\n", " new_turns_2 = sequential_jailbreak._get_turns(\n", " model_callback, new_turns_1, test_case.vulnerability, test_case.vulnerability_type\n", " )\n", " test_case.turns = new_turns_2\n", " metric = competetion._get_metric(test_case.vulnerability_type)\n", " metric.measure(test_case)\n", " if metric.score < 1:\n", " safe_model = False\n", " print(\"LLM application failed. 😞\")\n", " print(metric.score)\n", " print(metric.reason)\n", " break\n", "\n", "if safe_model:\n", " print(\"LLM application passed! 🥳\")" ], "metadata": { "id": "JLGdy0KMVc9v" }, "execution_count": null, "outputs": [] } ] }