{ "cells": [ { "cell_type": "code", "execution_count": 5, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "executionInfo": { "elapsed": 2008, "status": "ok", "timestamp": 1710126277679, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "UnSA6Jvg4Nhk", "outputId": "d3ae32e6-c9ab-4abd-adde-2b2a6fe81edf" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n" ] } ], "source": [ "# by nitta\n", "import os\n", "is_colab = 'google.colab' in str(get_ipython()) # for Google Colab\n", "\n", "if is_colab:\n", " from google.colab import drive\n", " drive.mount('/content/drive')\n", " SAVE_PATH='/content/drive/MyDrive/DeepLearning/llm_book2'\n", "else:\n", " SAVE_PATH='.'\n", "\n", "# # variable definition in 'account.py' to access HuggingFace Hub\n", "# MyAccount = {\n", "# 'HuggingFace': {\n", "# 'user': 'YourUserName',\n", "# 'token': 'YourTokenWithWritePermission'\n", "# },\n", "# 'OpenAI': {\n", "# 'api-key': 'YourOpenAI_API_Key'\n", "# }\n", "# }\n", "ACCOUNT_FILE = os.path.join(SAVE_PATH, 'account.py')\n", "%run {ACCOUNT_FILE} # set 'MyAccount' variable" ] }, { "cell_type": "markdown", "metadata": { "id": "FN_ayuoyQ8wp" }, "source": [ "# LangChain\n", "\n", "\n", "\n", "LangChain は、さまざまなモジュールを組み合わせることで、複雑なタスクを実行するアプリケーションを作成する。\n", "このモジュールを Chain と呼ぶ。\n", "\n", "Agent モジュールを使うことで、ユーザの入力から回答として求められているものを推論し、必要なツールを選択して実行する(= 行動)ことで、最適な回答を導き出す。\n", "\n", "Agent は、回答が不十分であると判断した場合は、推論と行動を繰り返して、高い精度で複雑なタスクを実行する。" ] }, { "cell_type": "markdown", "metadata": { "id": "MyOn2EawR9GA" }, "source": [ "# Quickstart\n", "\n", "\n", "\n", "次の方法を説明する。\n", "\n", "\n", "\n" ] }, { "cell_type": "markdown", "metadata": { "id": "tAz5VQEyBO9C" }, "source": [ "# セットアップ\n", "\n" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "executionInfo": { "elapsed": 13269, "status": "ok", "timestamp": 1710126290936, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "ORMcWgkQBZ0M", "outputId": "02f92091-08d0-4ba0-8a24-d80749eb6e99" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting langchain\n", " Downloading langchain-0.1.11-py3-none-any.whl (807 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m807.5/807.5 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: PyYAML>=5.3 in /usr/local/lib/python3.10/dist-packages (from langchain) (6.0.1)\n", "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.0.28)\n", "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /usr/local/lib/python3.10/dist-packages (from langchain) (3.9.3)\n", "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (4.0.3)\n", "Collecting dataclasses-json<0.7,>=0.5.7 (from langchain)\n", " Downloading dataclasses_json-0.6.4-py3-none-any.whl (28 kB)\n", "Collecting jsonpatch<2.0,>=1.33 (from langchain)\n", " Downloading jsonpatch-1.33-py2.py3-none-any.whl (12 kB)\n", "Collecting langchain-community<0.1,>=0.0.25 (from langchain)\n", " Downloading langchain_community-0.0.27-py3-none-any.whl (1.8 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m26.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting langchain-core<0.2,>=0.1.29 (from langchain)\n", " Downloading langchain_core-0.1.30-py3-none-any.whl (256 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m256.9/256.9 kB\u001b[0m \u001b[31m34.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting langchain-text-splitters<0.1,>=0.0.1 (from langchain)\n", " Downloading langchain_text_splitters-0.0.1-py3-none-any.whl (21 kB)\n", "Collecting langsmith<0.2.0,>=0.1.17 (from langchain)\n", " Downloading langsmith-0.1.23-py3-none-any.whl (66 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m66.6/66.6 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: numpy<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.25.2)\n", "Requirement already satisfied: pydantic<3,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.6.3)\n", "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.31.0)\n", "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (8.2.3)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.2.0)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.4.1)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.5)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.9.4)\n", "Collecting marshmallow<4.0.0,>=3.18.0 (from dataclasses-json<0.7,>=0.5.7->langchain)\n", " Downloading marshmallow-3.21.1-py3-none-any.whl (49 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.4/49.4 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting typing-inspect<1,>=0.4.0 (from dataclasses-json<0.7,>=0.5.7->langchain)\n", " Downloading typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n", "Collecting jsonpointer>=1.9 (from jsonpatch<2.0,>=1.33->langchain)\n", " Downloading jsonpointer-2.4-py2.py3-none-any.whl (7.8 kB)\n", "Requirement already satisfied: anyio<5,>=3 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2,>=0.1.29->langchain) (3.7.1)\n", "Requirement already satisfied: packaging<24.0,>=23.2 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2,>=0.1.29->langchain) (23.2)\n", "Collecting orjson<4.0.0,>=3.9.14 (from langsmith<0.2.0,>=0.1.17->langchain)\n", " Downloading orjson-3.9.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (138 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m138.5/138.5 kB\u001b[0m \u001b[31m19.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (0.6.0)\n", "Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (2.16.3)\n", "Requirement already satisfied: typing-extensions>=4.6.1 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (4.10.0)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (3.3.2)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (3.6)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (2.0.7)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (2024.2.2)\n", "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy<3,>=1.4->langchain) (3.0.3)\n", "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3->langchain-core<0.2,>=0.1.29->langchain) (1.3.1)\n", "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3->langchain-core<0.2,>=0.1.29->langchain) (1.2.0)\n", "Collecting mypy-extensions>=0.3.0 (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain)\n", " Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n", "Installing collected packages: orjson, mypy-extensions, marshmallow, jsonpointer, typing-inspect, jsonpatch, langsmith, dataclasses-json, langchain-core, langchain-text-splitters, langchain-community, langchain\n", "Successfully installed dataclasses-json-0.6.4 jsonpatch-1.33 jsonpointer-2.4 langchain-0.1.11 langchain-community-0.0.27 langchain-core-0.1.30 langchain-text-splitters-0.0.1 langsmith-0.1.23 marshmallow-3.21.1 mypy-extensions-1.0.0 orjson-3.9.15 typing-inspect-0.9.0\n" ] } ], "source": [ "! pip install langchain" ] }, { "cell_type": "markdown", "metadata": { "id": "yrA7d2b8Bipk" }, "source": [ "LangChain を用いて構築されるアプリケーションでは、LLM呼び出しを複数回呼び出す手順を含んでいる。chain や agent 内で何が起きているかを調べる最良の方法は LangSmith を使うことである。\n", "\n", "LangSmith を使う場合は、次の環境変数を設定すること。\n", "
\n",
    "export LANGCHAIN_TRACING_V2=\"true\"\n",
    "export LANGCHAIN_API_KEY=\"...\"\n",
    "
" ] }, { "cell_type": "markdown", "metadata": { "id": "H_QV_U0QljpQ" }, "source": [ "# LangChain を用いた構築\n", "\n", "LangChain を使うと、LLMを外部のデータソースや計算に接続するアプリケーションを構築できる。\n", "このクィックスタートでは、そのためのいくつかの方法を示す。\n", "返答のためのプロンプト・テンプレートのみに依存する単純な LLM chain から始める。\n", "次に、別のデータベースからデータを取得し、それをプロンプト・テンプレートに渡す chain を構築する。\n", "それから、チャット履歴を追加して、会話検索 chain を構築する。\n", "これにより 前の質問を記憶されているので、LLM とチャット形式で対話できる。\n", "最後に、質問に答えるためにデータを取得する必要があるかどうかを決定するために LLM を使う agent を構築する。\n", "大まかに説明するが、それぞれには詳細がある。\n", "関連するリンクを貼っておくので必要ならば参照してほしい。" ] }, { "cell_type": "markdown", "metadata": { "id": "-KmFir53n604" }, "source": [ "# LLM Chain: OpenAI" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "executionInfo": { "elapsed": 13607, "status": "ok", "timestamp": 1710126304539, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "JOpWamKDoBn0", "outputId": "f56e75bc-ccb7-4c37-e167-c6370cd69a32" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting langchain-openai\n", " Downloading langchain_openai-0.0.8-py3-none-any.whl (32 kB)\n", "Requirement already satisfied: langchain-core<0.2.0,>=0.1.27 in /usr/local/lib/python3.10/dist-packages (from langchain-openai) (0.1.30)\n", "Collecting openai<2.0.0,>=1.10.0 (from langchain-openai)\n", " Downloading openai-1.13.3-py3-none-any.whl (227 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m227.4/227.4 kB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting tiktoken<1,>=0.5.2 (from langchain-openai)\n", " Downloading tiktoken-0.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.8 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: PyYAML>=5.3 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2.0,>=0.1.27->langchain-openai) (6.0.1)\n", "Requirement already satisfied: anyio<5,>=3 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2.0,>=0.1.27->langchain-openai) (3.7.1)\n", "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2.0,>=0.1.27->langchain-openai) (1.33)\n", "Requirement already satisfied: langsmith<0.2.0,>=0.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2.0,>=0.1.27->langchain-openai) (0.1.23)\n", "Requirement already satisfied: packaging<24.0,>=23.2 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2.0,>=0.1.27->langchain-openai) (23.2)\n", "Requirement already satisfied: pydantic<3,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2.0,>=0.1.27->langchain-openai) (2.6.3)\n", "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2.0,>=0.1.27->langchain-openai) (2.31.0)\n", "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2.0,>=0.1.27->langchain-openai) (8.2.3)\n", "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai<2.0.0,>=1.10.0->langchain-openai) (1.7.0)\n", "Collecting httpx<1,>=0.23.0 (from openai<2.0.0,>=1.10.0->langchain-openai)\n", " Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.10.0->langchain-openai) (1.3.1)\n", "Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.10.0->langchain-openai) (4.66.2)\n", "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.10.0->langchain-openai) (4.10.0)\n", "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken<1,>=0.5.2->langchain-openai) (2023.12.25)\n", "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3->langchain-core<0.2.0,>=0.1.27->langchain-openai) (3.6)\n", "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3->langchain-core<0.2.0,>=0.1.27->langchain-openai) (1.2.0)\n", "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai<2.0.0,>=1.10.0->langchain-openai) (2024.2.2)\n", "Collecting httpcore==1.* (from httpx<1,>=0.23.0->openai<2.0.0,>=1.10.0->langchain-openai)\n", " Downloading httpcore-1.0.4-py3-none-any.whl (77 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.8/77.8 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting h11<0.15,>=0.13 (from httpcore==1.*->httpx<1,>=0.23.0->openai<2.0.0,>=1.10.0->langchain-openai)\n", " Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: jsonpointer>=1.9 in /usr/local/lib/python3.10/dist-packages (from jsonpatch<2.0,>=1.33->langchain-core<0.2.0,>=0.1.27->langchain-openai) (2.4)\n", "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in /usr/local/lib/python3.10/dist-packages (from langsmith<0.2.0,>=0.1.0->langchain-core<0.2.0,>=0.1.27->langchain-openai) (3.9.15)\n", "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain-core<0.2.0,>=0.1.27->langchain-openai) (0.6.0)\n", "Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain-core<0.2.0,>=0.1.27->langchain-openai) (2.16.3)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain-core<0.2.0,>=0.1.27->langchain-openai) (3.3.2)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain-core<0.2.0,>=0.1.27->langchain-openai) (2.0.7)\n", "Installing collected packages: h11, tiktoken, httpcore, httpx, openai, langchain-openai\n", "Successfully installed h11-0.14.0 httpcore-1.0.4 httpx-0.27.0 langchain-openai-0.0.8 openai-1.13.3 tiktoken-0.6.0\n" ] } ], "source": [ "#\n", "! pip install langchain-openai" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "executionInfo": { "elapsed": 13, "status": "ok", "timestamp": 1710126304539, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "831ZJQA24JCv" }, "outputs": [], "source": [ "import os\n", "os.environ[\"OPENAI_API_KEY\"] = MyAccount[\"OpenAI\"][\"api-key\"]" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "executionInfo": { "elapsed": 1751, "status": "ok", "timestamp": 1710126306277, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "bzkQ8A4p8dnT" }, "outputs": [], "source": [ "from langchain_openai import ChatOpenAI\n", "\n", "llm = ChatOpenAI()\n", "# llm = ChatOpenAI(openai_api_key=MyAccount[\"OpenAI\"][\"api-key\"])" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "executionInfo": { "elapsed": 1552, "status": "ok", "timestamp": 1710126307827, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "0H_zesRh8zA-", "outputId": "b8efa767-150b-49f8-a6aa-ab728135e0c0" }, "outputs": [ { "data": { "text/plain": [ "AIMessage(content='Langsmith can help with testing by providing automated testing capabilities, such as unit testing and integration testing. It can also help by generating test cases based on grammar and syntax rules, and by providing tools for analyzing test coverage and identifying potential issues in the code. Additionally, Langsmith can be used to simulate various scenarios and test edge cases, helping to ensure the reliability and quality of the software being developed.')" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "llm.invoke(\"how can langsmith help with testing?\")" ] }, { "cell_type": "markdown", "metadata": { "id": "K8k6FTXurNKU" }, "source": [ "

上記回答の機械翻訳

\n", "\n", "Langsmith は、次のようなさまざまな方法でテストを支援できます。\n", "\n", "1. さまざまなプログラミング言語で記述されたコードのテストを迅速かつ効率的に実行できる自動テスト ツールを提供します。\n", "\n", "2. テストを実行する前にコード内の潜在的なバグやエラーを特定するコード分析機能を提供します。\n", "\n", "3. テスト プロセスを合理化し、テストの作成と実行を容易にするテスト フレームワークとライブラリを提供します。\n", "\n", "4. 単体テスト、統合テスト、エンドツーエンド テストなど、さまざまなテスト手法のサポートを提供します。\n", "\n", "5. 継続的な統合および展開ツールを提供して、テスト プロセスを自動化し、コードの変更が本番環境に展開される前に徹底的にテストされるようにします。\n", "\n", "全体として、Langsmith は包括的なテスト機能を通じてソフトウェアの品質と信頼性の向上に役立ちます。" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "executionInfo": { "elapsed": 2, "status": "ok", "timestamp": 1710126307827, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "E7dZIo54o6HU" }, "outputs": [], "source": [ "# prompt template を使って回答を誘導することができる\n", "from langchain_core.prompts import ChatPromptTemplate\n", "prompt = ChatPromptTemplate.from_messages([\n", " (\"system\", \"You are world class technical documentation writer.\"),\n", " (\"user\", \"{input}\")\n", "])" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "executionInfo": { "elapsed": 2, "status": "ok", "timestamp": 1710126307827, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "JLDGM75jpXzb" }, "outputs": [], "source": [ "# combine these into a simple LLM chain\n", "# In python, '|' is short hand for calling the object's __or__ method.\n", "\n", "chain = prompt | llm" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "executionInfo": { "elapsed": 3310, "status": "ok", "timestamp": 1710126311135, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "3b0B5pimqorB", "outputId": "5f3e8ba5-85d5-4eb0-abb8-a364b3e4f15f" }, "outputs": [ { "data": { "text/plain": [ "AIMessage(content='Langsmith can help with testing in several ways:\\n\\n1. Automated Testing: Langsmith can be used to generate test data for automated testing scripts. By creating realistic and diverse test data sets, Langsmith can help improve the coverage and effectiveness of automated tests.\\n\\n2. Performance Testing: Langsmith can generate large volumes of data to simulate real-world conditions and stress test applications. This can help identify performance bottlenecks and potential issues under heavy loads.\\n\\n3. Security Testing: Langsmith can generate various types of data to test the security of applications, such as SQL injection attacks, cross-site scripting, and other vulnerabilities. By providing realistic test data, Langsmith can help uncover security weaknesses in applications.\\n\\n4. User Acceptance Testing: Langsmith can generate test data that closely resembles real user behavior, allowing testers to validate the functionality and usability of applications from an end-user perspective.\\n\\n5. Data Validation: Langsmith can be used to validate the accuracy and integrity of data in databases or applications by generating test data sets and comparing the results with expected outcomes.\\n\\nOverall, Langsmith can be a valuable tool in the testing process by providing high-quality test data, enabling comprehensive testing coverage, and improving the overall quality of software applications.')" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# 上の単純な llm への質問と同じ内容を送ってみる。\"techincal writer\" として回答してくれるはず。\n", "chain.invoke({\"input\": \"how can langsmith help with testing?\"})" ] }, { "cell_type": "markdown", "metadata": { "id": "fWewRRu3rzwE" }, "source": [ "

上記回答の機械翻訳

\n", "\n", "Langsmith は、次のようなさまざまな方法でテストを支援できます。\n", "\n", "1. テストの自動化: ラングスミスは、言語仕様に基づいてテスト スクリプトまたはテスト ケースを生成することにより、テスト プロセスを自動化できます。 これは、手作業の労力を軽減し、より一貫性のある包括的なテスト範囲を確保するのに役立ちます。\n", "\n", "2. テスト データの生成: Langsmith は、言語のルールと制約に基づいて、さまざまなシナリオのテスト データの生成を支援します。 これは、さまざまなエッジケースをテストし、システムの堅牢性を確保するのに役立ちます。\n", "\n", "3. テスト カバレッジ分析: ラングスミスは、言語仕様に基づいてテスト カバレッジを分析し、さらにテストが必要な領域についての洞察を提供します。 これは、テスト プロセス全体の品質の向上に役立ちます。\n", "\n", "4. 回帰テスト: Langsmith は、コード変更後にテスト ケースを自動的に再実行し、回帰を特定することで、回帰テストの実行を支援します。 これは、新しい変更によって既存の機能が損なわれないようにするのに役立ちます。\n", "\n", "5. パフォーマンス テスト: Langsmith は、言語仕様に基づいて負荷テストを生成することにより、パフォーマンス テストを支援することもできます。 これは、さまざまな負荷条件下でのシステムのパフォーマンスを評価するのに役立ちます。\n", "\n", "全体として、Langsmith は言語分析機能を活用することで、テスト プロセスを合理化し、効率を向上させ、テストの品質を向上させることができます。" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "executionInfo": { "elapsed": 3, "status": "ok", "timestamp": 1710126311136, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "fR4cG4trsRwT" }, "outputs": [], "source": [ "# ChatModel の出力は message であるが、文字列として扱うと便利な場合が多い。\n", "# 単純な output parser を使って、message を string に変換できる。\n", "\n", "from langchain_core.output_parsers import StrOutputParser\n", "\n", "chain = prompt | llm | StrOutputParser()" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 163 }, "executionInfo": { "elapsed": 3490, "status": "ok", "timestamp": 1710126314624, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "kgzakIBprHNX", "outputId": "ae26bb3b-44b0-4e37-8325-046902ac7030" }, "outputs": [ { "data": { "application/vnd.google.colaboratory.intrinsic+json": { "type": "string" }, "text/plain": [ "'Langsmith is a powerful tool that can greatly assist in testing by automating various aspects of the testing process. Here are some ways Langsmith can help with testing:\\n\\n1. Automated Test Generation: Langsmith can automatically generate test cases based on the specifications provided, saving time and effort in creating test cases manually.\\n\\n2. Code Analysis: Langsmith can analyze the code under test to identify potential issues such as code smells, bugs, or vulnerabilities, helping testers focus on areas that require attention.\\n\\n3. Test Execution: Langsmith can execute test cases automatically, providing quick feedback on the quality of the code and identifying any failures or regressions.\\n\\n4. Integration with Testing Frameworks: Langsmith can integrate with popular testing frameworks like JUnit, TestNG, or Selenium, making it easier to incorporate automated testing into the development process.\\n\\n5. Continuous Integration: Langsmith can be integrated into continuous integration pipelines to automate testing as part of the build process, ensuring that new code changes are thoroughly tested before deployment.\\n\\nOverall, Langsmith can streamline the testing process, improve test coverage, and help identify issues early in the development cycle, ultimately leading to a more robust and reliable software product.'" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "chain.invoke({\"input\": \"how can langsmith help with testing?\"})" ] }, { "cell_type": "markdown", "metadata": { "id": "MQaoZF9TtB4q" }, "source": [ "基本的な LLM chain のせってアップ方法を解説してきた。\n", "プロンプトやモデル、出力パーサの基礎に触れたに過ぎない。\n", "詳しく知りたい場合は\n", "\n", "Model I/O\n", " を参照すること。" ] }, { "cell_type": "markdown", "metadata": { "id": "6oEewi2ItuJb" }, "source": [ "# Retrieval Chain (検索チェイン)\n", "\n", "元の質問 \"how can langsmith help with testing?\" に適切に回答するために、LLMに追加の文脈を提供する必要がある。\n", "retrieval を用いてこれを行うことができる。\n", "retreaval は、LLMに直接渡すには多すぎるデータがあるときに、役に立つ。\n", "最も関連した部分だけを取得して渡すには retreaver を用いる。\n", "\n", "この手順では、関連文書を Retriever から検索して、プロンプトに渡す。\n", "retriever は\n", "SQL テーブル、インターネットなど、なんでも裏で使えるが、\n", "今回はvector storeを設定してretriever として使ってみる。\n", "詳しい説明は\n", "\n", "Vector stores\n", "を参照のこと。\n", "\n", "最初に、インデックスしたいデータをロードする必要がある。\n", "そのために WebBaseLoader を使う。\n", "これには BeautifulSoup をインストールする必要がある。" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "executionInfo": { "elapsed": 7227, "status": "ok", "timestamp": 1710126321848, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "Chq4C4G-s4Y4", "outputId": "23280d08-99a4-4601-e5da-2ca272421f34" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (4.12.3)\n", "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4) (2.5)\n" ] } ], "source": [ "! pip install beautifulsoup4" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "executionInfo": { "elapsed": 1442, "status": "ok", "timestamp": 1710126323278, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "Fg1VJllvZheN" }, "outputs": [], "source": [ "from langchain_community.document_loaders import WebBaseLoader\n", "\n", "loader = WebBaseLoader(\"https://docs.smith.langchain.com/user_guide\")\n", "\n", "docs = loader.load()" ] }, { "cell_type": "markdown", "metadata": { "id": "NBc6lKeOXDh7" }, "source": [ "次に、これを vector store へインデックスする必要がある。\n", "そのためには、いくつかのコンポーネント、すなわち embedding model と vectorstore が必要となる。\n", "\n", "### OpenAI の embedding model の場合\n" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "executionInfo": { "elapsed": 3, "status": "ok", "timestamp": 1710126323278, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "-BYZu8rGZuAz" }, "outputs": [], "source": [ "# lancchain_openai パッケージをインストールしたうえで、環境変数が設定されている必要がある。\n", "from langchain_openai import OpenAIEmbeddings\n", "\n", "embeddings = OpenAIEmbeddings()" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "executionInfo": { "elapsed": 16983, "status": "ok", "timestamp": 1710126340259, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "i1HJ4-f8YLm8", "outputId": "5345a62a-1f59-44c9-ee1f-f1695fd207a3" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting faiss-gpu\n", " Downloading faiss_gpu-1.7.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (85.5 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m85.5/85.5 MB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hInstalling collected packages: faiss-gpu\n", "Successfully installed faiss-gpu-1.7.2\n" ] } ], "source": [ "# 簡単にするためローカルな vector store である FAISS を使う。\n", "# まず、パッケージをインストールする\n", "! pip install faiss-gpu # faiss-cpu" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "executionInfo": { "elapsed": 1785, "status": "ok", "timestamp": 1710126342031, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "fBnBx0_MYoGP" }, "outputs": [], "source": [ "# 次に、自分の index を構築する\n", "from langchain_community.vectorstores import FAISS\n", "from langchain_text_splitters import RecursiveCharacterTextSplitter\n", "\n", "text_splitter = RecursiveCharacterTextSplitter()\n", "documents = text_splitter.split_documents(docs)\n", "vector = FAISS.from_documents(documents, embeddings)" ] }, { "cell_type": "markdown", "metadata": { "id": "QAqrpfeuZKOI" }, "source": [ "このデータは vectorstore 内で index 付けされたので、retrieval chain を作成できる。\n", "この chain は入力される質問を受け取り、関連文書を検索し、元の質問と共にこれらの文書をLLM に渡し、元の質問に回答するように依頼する。\n", "\n" ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "executionInfo": { "elapsed": 1098, "status": "ok", "timestamp": 1710126343128, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "9C_4-7zmZCtG" }, "outputs": [], "source": [ "# 最初に、質問を受け取り、文書を検索し、回答を生成する chain を設定する。\n", "from langchain.chains.combine_documents import create_stuff_documents_chain\n", "\n", "prompt = ChatPromptTemplate.from_template(\n", " \"\"\"\n", " Answer the following question based only on the provided context:\n", "\n", " \n", " {context}\n", " \n", "\n", " Question: {input}\n", " \"\"\" )\n", "\n", "document_chain = create_stuff_documents_chain(llm, prompt)" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 36 }, "executionInfo": { "elapsed": 381, "status": "ok", "timestamp": 1710126343508, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "_PCCKxCPai8d", "outputId": "84ea9e3e-6e64-4993-bbab-ae685b4bc36d" }, "outputs": [ { "data": { "application/vnd.google.colaboratory.intrinsic+json": { "type": "string" }, "text/plain": [ "'Langsmith can help with testing by allowing you to visualize test results.'" ] }, "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# 必要ならば、文書に直接渡すことでこれを自分で実行することもできる。\n", "from langchain_core.documents import Document\n", "\n", "document_chain.invoke({\n", " \"input\": \"how can langsmith help with testing?\",\n", " \"context\": [Document(page_content=\"langsmith can let you visualize test results\")]\n", "})" ] }, { "cell_type": "markdown", "metadata": { "id": "ttk86c_WbtC-" }, "source": [ "＃しかし、最初に設定した retriever から文書を得ようと思う。\n", "このやり方では、与えられた質問に対して、retriever を用いて最も関連が深い文書を動的に選択して、それを渡すことができる。" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "executionInfo": { "elapsed": 3, "status": "ok", "timestamp": 1710126343508, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "bcJA2xIKbQoh" }, "outputs": [], "source": [ "from langchain.chains import create_retrieval_chain\n", "\n", "retriever = vector.as_retriever()\n", "retrieval_chain = create_retrieval_chain(retriever, document_chain)" ] }, { "cell_type": "code", "execution_count": 24, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "executionInfo": { "elapsed": 1640, "status": "ok", "timestamp": 1710126345145, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "xPa3h0omiavY", "outputId": "240737e2-6735-4fb8-aa0c-6d6f1b89b7d9" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "LangSmith can help with testing by allowing developers to create datasets, run tests on LLM applications, compare different versions of applications, use a playground environment for rapid iteration and experimentation, collect feedback from users, annotate traces for evaluation, add runs to datasets for refinement, and monitor key metrics for performance analysis.\n" ] } ], "source": [ "# これで、chain を呼び出すことができるようになった。辞書を返すが、LLMからの返答では \"answer\" key が回答である。\n", "response = retrieval_chain.invoke({\n", " \"input\": \"how can langsmith :help with testing?\"\n", " })\n", "\n", "print(response[\"answer\"])" ] }, { "cell_type": "markdown", "metadata": { "id": "3s3GSXFolRiR" }, "source": [ "これで 基本的な retrieval chain を設定できたことになる。\n", "より詳しい情報は\n", "\n", "Retrieval\n", " を参照すること。" ] }, { "cell_type": "markdown", "metadata": { "id": "bqDv5F4hjgDa" }, "source": [ "# Conversation Retrieval Chain (会話検索チェイン)\n", "\n", "これまで作成してきた chain は、単一の質問に答えるだけであった。\n", "LLM アプリケーションの主要なタイプの一つは、chat bot である。\n", "chain を続く質問に答えられるように調整する方法について述べる。\n", "\n", "\n", "やはり create_retrieval_chain 関数を使うことができるが、\n", "2種類の変更点がある。\n", "\n", "\n", "\n", "## Updating Retrieval (検索を更新する)\n", "\n", "検索を更新するために、新しい chain を作成する。\n", "この chain は、最新の質問を input に、\n", "会話履歴を chat_history に受け取り、LLM を使用して\n", "質問クエリを生成する。" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "executionInfo": { "elapsed": 2, "status": "ok", "timestamp": 1710126345145, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "qilJRVJXjH_G" }, "outputs": [], "source": [ "from langchain.chains import create_history_aware_retriever\n", "from langchain_core.prompts import MessagesPlaceholder\n", "\n", "# 最初に、LLM に渡して検索クエリを生成させるための、プロンプトが必要となる。\n", "\n", "prompt = ChatPromptTemplate.from_messages([\n", " MessagesPlaceholder(variable_name=\"chat_history\"),\n", " (\"user\", \"{input}\"),\n", " (\"user\", \"Given the above conversation, generate a search query to look up in order to get information relevant to the conversation\")\n", "])\n", "\n", "retriever_chain = create_history_aware_retriever(llm, retriever, prompt)" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "executionInfo": { "elapsed": 964, "status": "ok", "timestamp": 1710126346108, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "QLSAElgipNfc", "outputId": "325e46ba-0033-421f-c8c8-590fad3388f8" }, "outputs": [ { "data": { "text/plain": [ "[Document(page_content='Skip to main content\\uf8ffü¶úÔ∏è\\uf8ffüõ†Ô∏è LangSmith DocsLangChain Python DocsLangChain JS/TS DocsLangSmith API DocsSearchGo to AppLangSmithUser GuideSetupPricing (Coming Soon)Self-HostingTracingEvaluationMonitoringPrompt HubProxyUser GuideOn this pageLangSmith User GuideLangSmith is a platform for LLM application development, monitoring, and testing. In this guide, we‚Äôll highlight the breadth of workflows LangSmith supports and how they fit into each stage of the application development lifecycle. We hope this will inform users how to best utilize this powerful platform or give them something to consider if they‚Äôre just starting their journey.Prototyping‚ÄãPrototyping LLM applications often involves quick experimentation between prompts, model types, retrieval strategy and other parameters.\\nThe ability to rapidly understand how the model is performing ‚Äî and debug where it is failing ‚Äî is incredibly important for this phase.Debugging‚ÄãWhen developing new LLM applications, we suggest having LangSmith tracing enabled by default.\\nOftentimes, it isn‚Äôt necessary to look at every single trace. However, when things go wrong (an unexpected end result, infinite agent loop, slower than expected execution, higher than expected token usage), it‚Äôs extremely helpful to debug by looking through the application traces. LangSmith gives clear visibility and debugging information at each step of an LLM sequence, making it much easier to identify and root-cause issues.\\nWe provide native rendering of chat messages, functions, and retrieve documents.Initial Test Set‚ÄãWhile many developers still ship an initial version of their application based on ‚Äúvibe checks‚Äù, we‚Äôve seen an increasing number of engineering teams start to adopt a more test driven approach. LangSmith allows developers to create datasets, which are collections of inputs and reference outputs, and use these to run tests on their LLM applications.\\nThese test cases can be uploaded in bulk, created on the fly, or exported from application traces. LangSmith also makes it easy to run custom evaluations (both LLM and heuristic based) to score test results.Comparison View‚ÄãWhen prototyping different versions of your applications and making changes, it‚Äôs important to see whether or not you‚Äôve regressed with respect to your initial test cases.\\nOftentimes, changes in the prompt, retrieval strategy, or model choice can have huge implications in responses produced by your application.\\nIn order to get a sense for which variant is performing better, it‚Äôs useful to be able to view results for different configurations on the same datapoints side-by-side. We‚Äôve invested heavily in a user-friendly comparison view for test runs to track and diagnose regressions in test scores across multiple revisions of your application.Playground‚ÄãLangSmith provides a playground environment for rapid iteration and experimentation.\\nThis allows you to quickly test out different prompts and models. You can open the playground from any prompt or model run in your trace.', metadata={'source': 'https://docs.smith.langchain.com/user_guide', 'title': 'LangSmith User Guide | \\uf8ffü¶úÔ∏è\\uf8ffüõ†Ô∏è LangSmith', 'description': 'LangSmith is a platform for LLM application development, monitoring, and testing. In this guide, we‚Äôll highlight the breadth of workflows LangSmith supports and how they fit into each stage of the application development lifecycle. We hope this will inform users how to best utilize this powerful platform or give them something to consider if they‚Äôre just starting their journey.', 'language': 'en'}),\n", " Document(page_content='LangSmith User Guide | \\uf8ffü¶úÔ∏è\\uf8ffüõ†Ô∏è LangSmith', metadata={'source': 'https://docs.smith.langchain.com/user_guide', 'title': 'LangSmith User Guide | \\uf8ffü¶úÔ∏è\\uf8ffüõ†Ô∏è LangSmith', 'description': 'LangSmith is a platform for LLM application development, monitoring, and testing. In this guide, we‚Äôll highlight the breadth of workflows LangSmith supports and how they fit into each stage of the application development lifecycle. We hope this will inform users how to best utilize this powerful platform or give them something to consider if they‚Äôre just starting their journey.', 'language': 'en'}),\n", " Document(page_content=\"This allows you to quickly test out different prompts and models. You can open the playground from any prompt or model run in your trace.\\nEvery playground run is logged in the system and can be used to create test cases or compare with other runs.Beta Testing‚ÄãBeta testing allows developers to collect more data on how their LLM applications are performing in real-world scenarios. In this phase, it‚Äôs important to develop an understanding for the types of inputs the app is performing well or poorly on and how exactly it‚Äôs breaking down in those cases. Both feedback collection and run annotation are critical for this workflow. This will help in curation of test cases that can help track regressions/improvements and development of automatic evaluations.Collecting Feedback‚ÄãWhen launching your application to an initial set of users, it‚Äôs important to gather human feedback on the responses it‚Äôs producing. This helps draw attention to the most interesting runs and highlight edge cases that are causing problematic responses. LangSmith allows you to attach feedback scores to logged traces (oftentimes, this is hooked up to a feedback button in your app), then filter on traces that have a specific feedback tag and score. A common workflow is to filter on traces that receive a poor user feedback score, then drill down into problematic points using the detailed trace view.Annotating Traces‚ÄãLangSmith also supports sending runs to annotation queues, which allow annotators to closely inspect interesting traces and annotate them with respect to different criteria. Annotators can be PMs, engineers, or even subject matter experts. This allows users to catch regressions across important evaluation criteria.Adding Runs to a Dataset‚ÄãAs your application progresses through the beta testing phase, it's essential to continue collecting data to refine and improve its performance. LangSmith enables you to add runs as examples to datasets (from both the project page and within an annotation queue), expanding your test coverage on real-world scenarios. This is a key benefit in having your logging system and your evaluation/testing system in the same platform.Production‚ÄãClosely inspecting key data points, growing benchmarking datasets, annotating traces, and drilling down into important data in trace view are workflows you‚Äôll also want to do once your app hits production. However, especially at the production stage, it‚Äôs crucial to get a high-level overview of application performance with respect to latency, cost, and feedback scores. This ensures that it's delivering desirable results at scale.Monitoring and A/B Testing‚ÄãLangSmith provides monitoring charts that allow you to track key metrics over time. You can expand to view metrics for a given period and drill down into a specific data point to get a trace table for that time period ‚Äî this is especially handy for debugging production issues.LangSmith also allows for tag and metadata grouping, which allows users to mark different versions of their applications with different identifiers and view how they are performing side-by-side within each chart. This is helpful for A/B testing changes in prompt, model, or retrieval strategy.Was this page helpful?PreviousLangSmithNextSetupPrototypingBeta TestingProductionCommunityDiscordTwitterGitHubDocs CodeLangSmith SDKPythonJS/TSMoreHomepageBlogCopyright ¬© 2024 LangChain, Inc.\", metadata={'source': 'https://docs.smith.langchain.com/user_guide', 'title': 'LangSmith User Guide | \\uf8ffü¶úÔ∏è\\uf8ffüõ†Ô∏è LangSmith', 'description': 'LangSmith is a platform for LLM application development, monitoring, and testing. In this guide, we‚Äôll highlight the breadth of workflows LangSmith supports and how they fit into each stage of the application development lifecycle. We hope this will inform users how to best utilize this powerful platform or give them something to consider if they‚Äôre just starting their journey.', 'language': 'en'})]" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# ユーザーが引き続く質問をしたインスタンスを渡して、テストする。\n", "from langchain_core.messages import HumanMessage, AIMessage\n", "\n", "chat_history = [\n", " HumanMessage(content=\"Can LangSmith help test my LLM applications?\"),\n", " AIMessage(content=\"Yes!\")\n", "]\n", "retriever_chain.invoke({\n", " \"chat_history\": chat_history,\n", " \"input\": \"Tell me how\"\n", "})" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "executionInfo": { "elapsed": 2, "status": "ok", "timestamp": 1710126346108, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "RTKJ7Cn5pNuT" }, "outputs": [], "source": [ "# 新しい retriever を得て、検索済みの文書を記憶しながら、会話を続けるための新しい chain を生成することができる。\n", "prompt = ChatPromptTemplate.from_messages([\n", " (\"system\", \"\"\"\n", " Answer the user's questions based on the below context:\n", "\n", " {context}\n", " \"\"\"),\n", " MessagesPlaceholder(variable_name=\"chat_history\"),\n", " (\"user\", \"{input}\"),\n", "])\n", "document_chain = create_stuff_documents_chain(llm, prompt)\n", "\n", "retrieval_chain = create_retrieval_chain(retriever_chain, document_chain)" ] }, { "cell_type": "code", "execution_count": 28, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "executionInfo": { "elapsed": 3908, "status": "ok", "timestamp": 1710126350015, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "0AQN38-duQHu", "outputId": "cf8c0164-dbd6-4f5e-a2a9-08a97d2dac10" }, "outputs": [ { "data": { "text/plain": [ "{'chat_history': [HumanMessage(content='Can LangSmith help test my LLM applications?'),\n", " AIMessage(content='Yes!')],\n", " 'input': 'Tell me how',\n", " 'context': [Document(page_content='Skip to main content\\uf8ffü¶úÔ∏è\\uf8ffüõ†Ô∏è LangSmith DocsLangChain Python DocsLangChain JS/TS DocsLangSmith API DocsSearchGo to AppLangSmithUser GuideSetupPricing (Coming Soon)Self-HostingTracingEvaluationMonitoringPrompt HubProxyUser GuideOn this pageLangSmith User GuideLangSmith is a platform for LLM application development, monitoring, and testing. In this guide, we‚Äôll highlight the breadth of workflows LangSmith supports and how they fit into each stage of the application development lifecycle. We hope this will inform users how to best utilize this powerful platform or give them something to consider if they‚Äôre just starting their journey.Prototyping‚ÄãPrototyping LLM applications often involves quick experimentation between prompts, model types, retrieval strategy and other parameters.\\nThe ability to rapidly understand how the model is performing ‚Äî and debug where it is failing ‚Äî is incredibly important for this phase.Debugging‚ÄãWhen developing new LLM applications, we suggest having LangSmith tracing enabled by default.\\nOftentimes, it isn‚Äôt necessary to look at every single trace. However, when things go wrong (an unexpected end result, infinite agent loop, slower than expected execution, higher than expected token usage), it‚Äôs extremely helpful to debug by looking through the application traces. LangSmith gives clear visibility and debugging information at each step of an LLM sequence, making it much easier to identify and root-cause issues.\\nWe provide native rendering of chat messages, functions, and retrieve documents.Initial Test Set‚ÄãWhile many developers still ship an initial version of their application based on ‚Äúvibe checks‚Äù, we‚Äôve seen an increasing number of engineering teams start to adopt a more test driven approach. LangSmith allows developers to create datasets, which are collections of inputs and reference outputs, and use these to run tests on their LLM applications.\\nThese test cases can be uploaded in bulk, created on the fly, or exported from application traces. LangSmith also makes it easy to run custom evaluations (both LLM and heuristic based) to score test results.Comparison View‚ÄãWhen prototyping different versions of your applications and making changes, it‚Äôs important to see whether or not you‚Äôve regressed with respect to your initial test cases.\\nOftentimes, changes in the prompt, retrieval strategy, or model choice can have huge implications in responses produced by your application.\\nIn order to get a sense for which variant is performing better, it‚Äôs useful to be able to view results for different configurations on the same datapoints side-by-side. We‚Äôve invested heavily in a user-friendly comparison view for test runs to track and diagnose regressions in test scores across multiple revisions of your application.Playground‚ÄãLangSmith provides a playground environment for rapid iteration and experimentation.\\nThis allows you to quickly test out different prompts and models. You can open the playground from any prompt or model run in your trace.', metadata={'source': 'https://docs.smith.langchain.com/user_guide', 'title': 'LangSmith User Guide | \\uf8ffü¶úÔ∏è\\uf8ffüõ†Ô∏è LangSmith', 'description': 'LangSmith is a platform for LLM application development, monitoring, and testing. In this guide, we‚Äôll highlight the breadth of workflows LangSmith supports and how they fit into each stage of the application development lifecycle. We hope this will inform users how to best utilize this powerful platform or give them something to consider if they‚Äôre just starting their journey.', 'language': 'en'}),\n", " Document(page_content='LangSmith User Guide | \\uf8ffü¶úÔ∏è\\uf8ffüõ†Ô∏è LangSmith', metadata={'source': 'https://docs.smith.langchain.com/user_guide', 'title': 'LangSmith User Guide | \\uf8ffü¶úÔ∏è\\uf8ffüõ†Ô∏è LangSmith', 'description': 'LangSmith is a platform for LLM application development, monitoring, and testing. In this guide, we‚Äôll highlight the breadth of workflows LangSmith supports and how they fit into each stage of the application development lifecycle. We hope this will inform users how to best utilize this powerful platform or give them something to consider if they‚Äôre just starting their journey.', 'language': 'en'}),\n", " Document(page_content=\"This allows you to quickly test out different prompts and models. You can open the playground from any prompt or model run in your trace.\\nEvery playground run is logged in the system and can be used to create test cases or compare with other runs.Beta Testing‚ÄãBeta testing allows developers to collect more data on how their LLM applications are performing in real-world scenarios. In this phase, it‚Äôs important to develop an understanding for the types of inputs the app is performing well or poorly on and how exactly it‚Äôs breaking down in those cases. Both feedback collection and run annotation are critical for this workflow. This will help in curation of test cases that can help track regressions/improvements and development of automatic evaluations.Collecting Feedback‚ÄãWhen launching your application to an initial set of users, it‚Äôs important to gather human feedback on the responses it‚Äôs producing. This helps draw attention to the most interesting runs and highlight edge cases that are causing problematic responses. LangSmith allows you to attach feedback scores to logged traces (oftentimes, this is hooked up to a feedback button in your app), then filter on traces that have a specific feedback tag and score. A common workflow is to filter on traces that receive a poor user feedback score, then drill down into problematic points using the detailed trace view.Annotating Traces‚ÄãLangSmith also supports sending runs to annotation queues, which allow annotators to closely inspect interesting traces and annotate them with respect to different criteria. Annotators can be PMs, engineers, or even subject matter experts. This allows users to catch regressions across important evaluation criteria.Adding Runs to a Dataset‚ÄãAs your application progresses through the beta testing phase, it's essential to continue collecting data to refine and improve its performance. LangSmith enables you to add runs as examples to datasets (from both the project page and within an annotation queue), expanding your test coverage on real-world scenarios. This is a key benefit in having your logging system and your evaluation/testing system in the same platform.Production‚ÄãClosely inspecting key data points, growing benchmarking datasets, annotating traces, and drilling down into important data in trace view are workflows you‚Äôll also want to do once your app hits production. However, especially at the production stage, it‚Äôs crucial to get a high-level overview of application performance with respect to latency, cost, and feedback scores. This ensures that it's delivering desirable results at scale.Monitoring and A/B Testing‚ÄãLangSmith provides monitoring charts that allow you to track key metrics over time. You can expand to view metrics for a given period and drill down into a specific data point to get a trace table for that time period ‚Äî this is especially handy for debugging production issues.LangSmith also allows for tag and metadata grouping, which allows users to mark different versions of their applications with different identifiers and view how they are performing side-by-side within each chart. This is helpful for A/B testing changes in prompt, model, or retrieval strategy.Was this page helpful?PreviousLangSmithNextSetupPrototypingBeta TestingProductionCommunityDiscordTwitterGitHubDocs CodeLangSmith SDKPythonJS/TSMoreHomepageBlogCopyright ¬© 2024 LangChain, Inc.\", metadata={'source': 'https://docs.smith.langchain.com/user_guide', 'title': 'LangSmith User Guide | \\uf8ffü¶úÔ∏è\\uf8ffüõ†Ô∏è LangSmith', 'description': 'LangSmith is a platform for LLM application development, monitoring, and testing. In this guide, we‚Äôll highlight the breadth of workflows LangSmith supports and how they fit into each stage of the application development lifecycle. We hope this will inform users how to best utilize this powerful platform or give them something to consider if they‚Äôre just starting their journey.', 'language': 'en'})],\n", " 'answer': 'LangSmith can help test your LLM applications by providing features such as:\\n\\n1. Prototyping: Allows quick experimentation between prompts, model types, and retrieval strategies.\\n2. Debugging: Offers clear visibility and debugging information at each step of an LLM sequence.\\n3. Initial Test Set: Enables developers to create datasets for running tests on their applications.\\n4. Comparison View: Helps track and diagnose regressions in test scores across multiple revisions of your application.\\n5. Playground: Provides a playground environment for rapid iteration and experimentation with different prompts and models.\\n6. Beta Testing: Allows developers to collect more data on how their applications are performing in real-world scenarios.\\n7. Monitoring and A/B Testing: Provides monitoring charts to track key metrics over time and compare different versions of applications.\\n\\nThese features in LangSmith facilitate testing and evaluation of LLM applications at various stages of development and deployment.'}" ] }, "execution_count": 28, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# end-to-end でテストできるようになった。\n", "\n", "chat_history = [\n", " HumanMessage(content=\"Can LangSmith help test my LLM applications?\"),\n", " AIMessage(content=\"Yes!\")\n", "]\n", "retrieval_chain.invoke({\n", " \"chat_history\": chat_history,\n", " \"input\": \"Tell me how\"\n", "})" ] }, { "cell_type": "markdown", "metadata": { "id": "ry7cHDHzvsDg" }, "source": [ "LangSimth でのテストに関する文書を、これが返すことがわかる。\n", "これは、続く質問を会話履歴に結合して、LLM が新しい質問を生成したおかげである。\n", "\n", "この新しい retriver を得たので、検索した文書を心にとめつつ、会話を続ける新しい chain を生成できる。" ] }, { "cell_type": "code", "execution_count": 29, "metadata": { "executionInfo": { "elapsed": 12, "status": "ok", "timestamp": 1710126350015, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "KqZpKzTyunEL" }, "outputs": [], "source": [ "prompt = ChatPromptTemplate.from_messages([\n", " (\"system\", \"\"\"\n", " Answer the user's questions based on the below context:\n", "\n", " {context}\n", " \"\"\"),\n", " MessagesPlaceholder(variable_name=\"chat_history\"),\n", " (\"user\", \"{input}\"),\n", "])\n", "\n", "document_chain = create_stuff_documents_chain(llm, prompt)\n", "\n", "retrieval_chain = create_retrieval_chain(retriever_chain, document_chain)" ] }, { "cell_type": "code", "execution_count": 30, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "executionInfo": { "elapsed": 3330, "status": "ok", "timestamp": 1710126353334, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "AhjdG-HExZ6E", "outputId": "85c91d40-362b-4331-b2b5-464e3e4027c8" }, "outputs": [ { "data": { "text/plain": [ "{'chat_history': [HumanMessage(content='Can LangSmith help test my LLM applications?'),\n", " AIMessage(content='Yes!')],\n", " 'input': 'Tell me how',\n", " 'context': [Document(page_content='Skip to main content\\uf8ffü¶úÔ∏è\\uf8ffüõ†Ô∏è LangSmith DocsLangChain Python DocsLangChain JS/TS DocsLangSmith API DocsSearchGo to AppLangSmithUser GuideSetupPricing (Coming Soon)Self-HostingTracingEvaluationMonitoringPrompt HubProxyUser GuideOn this pageLangSmith User GuideLangSmith is a platform for LLM application development, monitoring, and testing. In this guide, we‚Äôll highlight the breadth of workflows LangSmith supports and how they fit into each stage of the application development lifecycle. We hope this will inform users how to best utilize this powerful platform or give them something to consider if they‚Äôre just starting their journey.Prototyping‚ÄãPrototyping LLM applications often involves quick experimentation between prompts, model types, retrieval strategy and other parameters.\\nThe ability to rapidly understand how the model is performing ‚Äî and debug where it is failing ‚Äî is incredibly important for this phase.Debugging‚ÄãWhen developing new LLM applications, we suggest having LangSmith tracing enabled by default.\\nOftentimes, it isn‚Äôt necessary to look at every single trace. However, when things go wrong (an unexpected end result, infinite agent loop, slower than expected execution, higher than expected token usage), it‚Äôs extremely helpful to debug by looking through the application traces. LangSmith gives clear visibility and debugging information at each step of an LLM sequence, making it much easier to identify and root-cause issues.\\nWe provide native rendering of chat messages, functions, and retrieve documents.Initial Test Set‚ÄãWhile many developers still ship an initial version of their application based on ‚Äúvibe checks‚Äù, we‚Äôve seen an increasing number of engineering teams start to adopt a more test driven approach. LangSmith allows developers to create datasets, which are collections of inputs and reference outputs, and use these to run tests on their LLM applications.\\nThese test cases can be uploaded in bulk, created on the fly, or exported from application traces. LangSmith also makes it easy to run custom evaluations (both LLM and heuristic based) to score test results.Comparison View‚ÄãWhen prototyping different versions of your applications and making changes, it‚Äôs important to see whether or not you‚Äôve regressed with respect to your initial test cases.\\nOftentimes, changes in the prompt, retrieval strategy, or model choice can have huge implications in responses produced by your application.\\nIn order to get a sense for which variant is performing better, it‚Äôs useful to be able to view results for different configurations on the same datapoints side-by-side. We‚Äôve invested heavily in a user-friendly comparison view for test runs to track and diagnose regressions in test scores across multiple revisions of your application.Playground‚ÄãLangSmith provides a playground environment for rapid iteration and experimentation.\\nThis allows you to quickly test out different prompts and models. You can open the playground from any prompt or model run in your trace.', metadata={'source': 'https://docs.smith.langchain.com/user_guide', 'title': 'LangSmith User Guide | \\uf8ffü¶úÔ∏è\\uf8ffüõ†Ô∏è LangSmith', 'description': 'LangSmith is a platform for LLM application development, monitoring, and testing. In this guide, we‚Äôll highlight the breadth of workflows LangSmith supports and how they fit into each stage of the application development lifecycle. We hope this will inform users how to best utilize this powerful platform or give them something to consider if they‚Äôre just starting their journey.', 'language': 'en'}),\n", " Document(page_content='LangSmith User Guide | \\uf8ffü¶úÔ∏è\\uf8ffüõ†Ô∏è LangSmith', metadata={'source': 'https://docs.smith.langchain.com/user_guide', 'title': 'LangSmith User Guide | \\uf8ffü¶úÔ∏è\\uf8ffüõ†Ô∏è LangSmith', 'description': 'LangSmith is a platform for LLM application development, monitoring, and testing. In this guide, we‚Äôll highlight the breadth of workflows LangSmith supports and how they fit into each stage of the application development lifecycle. We hope this will inform users how to best utilize this powerful platform or give them something to consider if they‚Äôre just starting their journey.', 'language': 'en'}),\n", " Document(page_content=\"This allows you to quickly test out different prompts and models. You can open the playground from any prompt or model run in your trace.\\nEvery playground run is logged in the system and can be used to create test cases or compare with other runs.Beta Testing‚ÄãBeta testing allows developers to collect more data on how their LLM applications are performing in real-world scenarios. In this phase, it‚Äôs important to develop an understanding for the types of inputs the app is performing well or poorly on and how exactly it‚Äôs breaking down in those cases. Both feedback collection and run annotation are critical for this workflow. This will help in curation of test cases that can help track regressions/improvements and development of automatic evaluations.Collecting Feedback‚ÄãWhen launching your application to an initial set of users, it‚Äôs important to gather human feedback on the responses it‚Äôs producing. This helps draw attention to the most interesting runs and highlight edge cases that are causing problematic responses. LangSmith allows you to attach feedback scores to logged traces (oftentimes, this is hooked up to a feedback button in your app), then filter on traces that have a specific feedback tag and score. A common workflow is to filter on traces that receive a poor user feedback score, then drill down into problematic points using the detailed trace view.Annotating Traces‚ÄãLangSmith also supports sending runs to annotation queues, which allow annotators to closely inspect interesting traces and annotate them with respect to different criteria. Annotators can be PMs, engineers, or even subject matter experts. This allows users to catch regressions across important evaluation criteria.Adding Runs to a Dataset‚ÄãAs your application progresses through the beta testing phase, it's essential to continue collecting data to refine and improve its performance. LangSmith enables you to add runs as examples to datasets (from both the project page and within an annotation queue), expanding your test coverage on real-world scenarios. This is a key benefit in having your logging system and your evaluation/testing system in the same platform.Production‚ÄãClosely inspecting key data points, growing benchmarking datasets, annotating traces, and drilling down into important data in trace view are workflows you‚Äôll also want to do once your app hits production. However, especially at the production stage, it‚Äôs crucial to get a high-level overview of application performance with respect to latency, cost, and feedback scores. This ensures that it's delivering desirable results at scale.Monitoring and A/B Testing‚ÄãLangSmith provides monitoring charts that allow you to track key metrics over time. You can expand to view metrics for a given period and drill down into a specific data point to get a trace table for that time period ‚Äî this is especially handy for debugging production issues.LangSmith also allows for tag and metadata grouping, which allows users to mark different versions of their applications with different identifiers and view how they are performing side-by-side within each chart. This is helpful for A/B testing changes in prompt, model, or retrieval strategy.Was this page helpful?PreviousLangSmithNextSetupPrototypingBeta TestingProductionCommunityDiscordTwitterGitHubDocs CodeLangSmith SDKPythonJS/TSMoreHomepageBlogCopyright ¬© 2024 LangChain, Inc.\", metadata={'source': 'https://docs.smith.langchain.com/user_guide', 'title': 'LangSmith User Guide | \\uf8ffü¶úÔ∏è\\uf8ffüõ†Ô∏è LangSmith', 'description': 'LangSmith is a platform for LLM application development, monitoring, and testing. In this guide, we‚Äôll highlight the breadth of workflows LangSmith supports and how they fit into each stage of the application development lifecycle. We hope this will inform users how to best utilize this powerful platform or give them something to consider if they‚Äôre just starting their journey.', 'language': 'en'})],\n", " 'answer': 'LangSmith can help test your LLM applications by providing features such as tracing, test case creation, custom evaluations, comparison view, playground environment for rapid iteration and experimentation, beta testing support, feedback collection, annotation queues, adding runs to datasets, monitoring charts for tracking key metrics, and A/B testing capabilities. These features allow you to assess and improve the performance of your LLM applications at various stages of development and production.'}" ] }, "execution_count": 30, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# end-to-end でテストする\n", "chat_history = [\n", " HumanMessage(content=\"Can LangSmith help test my LLM applications?\"),\n", " AIMessage(content=\"Yes!\")\n", "]\n", "\n", "retrieval_chain.invoke({\n", " \"chat_history\": chat_history,\n", " \"input\": \"Tell me how\"\n", "})" ] }, { "cell_type": "markdown", "metadata": { "id": "P26qPgYPyJhk" }, "source": [ "一貫した回答が得られたことがわかる。retrieval chain (質問チェイン)を chat bot へと調整することができた。" ] }, { "cell_type": "markdown", "metadata": { "id": "OEoE-BQwyYwb" }, "source": [ "# Agent\n", "\n", "各ステップが事前にわかっている chain 例を作成してきた。\n", "最後に作成するのは agent で、agent では LLM がどのステップを取るかを決定する。\n", "\n", "[注意] ローカルなモデルはまだ信頼性がないので、OpenAI モデルを使った agent の作り方のみを説明する。\n", "\n", "agent を構築するときにさいしょにすべきなのは、どのツールにアクセスできるかを決定することである。\n", "この例では、agent が2つのツールにアクセスできるものとする。\n", "\n", "
    \n", "
  1. たった今作成した retriever。これにより、LangSmith に関する質問に簡単に答えることができる。
    2. \n", "
  2. 検索ツール。これにより、最新のデータ情報が必要な質問に簡単に答えることができる。
    3. \n", "
\n", "\n", "最初に、たった今作成した retriever 用のツールをセットアップする。" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "executionInfo": { "elapsed": 14, "status": "ok", "timestamp": 1710126353335, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "0_hJUxJEx768" }, "outputs": [], "source": [ "from langchain.tools.retriever import create_retriever_tool\n", "\n", "retriever_tool = create_retriever_tool(\n", " retriever,\n", " \"langsmith_search\",\n", " \"\"\"\\\n", " Search for information about LangSmith. \\\n", " For any questions about LangSmith, \\\n", " you must use this tool!\\\n", " \"\"\",\n", ")" ] }, { "cell_type": "markdown", "metadata": { "id": "_K3SNAxb9b3N" }, "source": [ "使用する検索ツールは Tavily である。\n", "https://tavily.com/\n", "\n", "Tavily API の使用は、限定的ならば無料である。\n", "クレジットカードの入力無しでサインアップして API キーを取得できる。" ] }, { "cell_type": "code", "execution_count": 32, "metadata": { "executionInfo": { "elapsed": 13, "status": "ok", "timestamp": 1710126353335, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "lWcGd1pzF17c" }, "outputs": [], "source": [ "# 検察ツールは Tavily である。\n", "import os\n", "os.environ[\"TAVILY_API_KEY\"] = MyAccount[\"Tavily\"][\"api-key\"]" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "executionInfo": { "elapsed": 13, "status": "ok", "timestamp": 1710126353335, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "a3xZgIbE0gIe" }, "outputs": [], "source": [ "from langchain_community.tools.tavily_search import TavilySearchResults\n", "\n", "search = TavilySearchResults()" ] }, { "cell_type": "code", "execution_count": 34, "metadata": { "executionInfo": { "elapsed": 13, "status": "ok", "timestamp": 1710126353335, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "fTNQlHF_-HCq" }, "outputs": [], "source": [ "tools = [retriever_tool, search]" ] }, { "cell_type": "markdown", "metadata": { "id": "9NydhaOV-62v" }, "source": [ "ツールを準備したので、これらを使う agent を生成する。\n" ] }, { "cell_type": "code", "execution_count": 35, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "executionInfo": { "elapsed": 6472, "status": "ok", "timestamp": 1710126359795, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "GoOfIlUJ-0bn", "outputId": "45df48f1-361a-44c0-c1c6-5ae853f9192f" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting langchainhub\n", " Downloading langchainhub-0.1.15-py3-none-any.whl (4.6 kB)\n", "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchainhub) (2.31.0)\n", "Collecting types-requests<3.0.0.0,>=2.31.0.2 (from langchainhub)\n", " Downloading types_requests-2.31.0.20240311-py3-none-any.whl (14 kB)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchainhub) (3.3.2)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchainhub) (3.6)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchainhub) (2.0.7)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchainhub) (2024.2.2)\n", "Installing collected packages: types-requests, langchainhub\n", "Successfully installed langchainhub-0.1.15 types-requests-2.31.0.20240311\n" ] } ], "source": [ "! pip install langchainhub" ] }, { "cell_type": "code", "execution_count": 36, "metadata": { "executionInfo": { "elapsed": 2279, "status": "ok", "timestamp": 1710126362063, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "HLN3P3fP-4Q7" }, "outputs": [], "source": [ "from langchain_openai import ChatOpenAI\n", "from langchain import hub\n", "from langchain.agents import create_openai_functions_agent\n", "from langchain.agents import AgentExecutor\n", "\n", "prompt = hub.pull(\"hwchase17/openai-functions-agent\")\n", "llm = ChatOpenAI(model=\"gpt-3.5-turbo\", temperature=0)\n", "agent = create_openai_functions_agent(llm, tools, prompt)\n", "agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=True)" ] }, { "cell_type": "code", "execution_count": 37, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "executionInfo": { "elapsed": 8193, "status": "ok", "timestamp": 1710126370254, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "fwRm2Pc4_9z2", "outputId": "49e439ff-6f05-4540-cfb1-4e1b1c9b788d" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "\n", "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n", "\u001b[32;1m\u001b[1;3m\n", "Invoking: `langsmith_search` with `{'query': 'how can LangSmith help with testing'}`\n", "\n", "\n", "\u001b[0m\u001b[36;1m\u001b[1;3mSkip to main contentü¶úÔ∏èüõ†Ô∏è LangSmith DocsLangChain Python DocsLangChain JS/TS DocsLangSmith API DocsSearchGo to AppLangSmithUser GuideSetupPricing (Coming Soon)Self-HostingTracingEvaluationMonitoringPrompt HubProxyUser GuideOn this pageLangSmith User GuideLangSmith is a platform for LLM application development, monitoring, and testing. In this guide, we‚Äôll highlight the breadth of workflows LangSmith supports and how they fit into each stage of the application development lifecycle. We hope this will inform users how to best utilize this powerful platform or give them something to consider if they‚Äôre just starting their journey.Prototyping‚ÄãPrototyping LLM applications often involves quick experimentation between prompts, model types, retrieval strategy and other parameters.\n", "The ability to rapidly understand how the model is performing ‚Äî and debug where it is failing ‚Äî is incredibly important for this phase.Debugging‚ÄãWhen developing new LLM applications, we suggest having LangSmith tracing enabled by default.\n", "Oftentimes, it isn‚Äôt necessary to look at every single trace. However, when things go wrong (an unexpected end result, infinite agent loop, slower than expected execution, higher than expected token usage), it‚Äôs extremely helpful to debug by looking through the application traces. LangSmith gives clear visibility and debugging information at each step of an LLM sequence, making it much easier to identify and root-cause issues.\n", "We provide native rendering of chat messages, functions, and retrieve documents.Initial Test Set‚ÄãWhile many developers still ship an initial version of their application based on ‚Äúvibe checks‚Äù, we‚Äôve seen an increasing number of engineering teams start to adopt a more test driven approach. LangSmith allows developers to create datasets, which are collections of inputs and reference outputs, and use these to run tests on their LLM applications.\n", "These test cases can be uploaded in bulk, created on the fly, or exported from application traces. LangSmith also makes it easy to run custom evaluations (both LLM and heuristic based) to score test results.Comparison View‚ÄãWhen prototyping different versions of your applications and making changes, it‚Äôs important to see whether or not you‚Äôve regressed with respect to your initial test cases.\n", "Oftentimes, changes in the prompt, retrieval strategy, or model choice can have huge implications in responses produced by your application.\n", "In order to get a sense for which variant is performing better, it‚Äôs useful to be able to view results for different configurations on the same datapoints side-by-side. We‚Äôve invested heavily in a user-friendly comparison view for test runs to track and diagnose regressions in test scores across multiple revisions of your application.Playground‚ÄãLangSmith provides a playground environment for rapid iteration and experimentation.\n", "This allows you to quickly test out different prompts and models. You can open the playground from any prompt or model run in your trace.\n", "\n", "This allows you to quickly test out different prompts and models. You can open the playground from any prompt or model run in your trace.\n", "Every playground run is logged in the system and can be used to create test cases or compare with other runs.Beta Testing‚ÄãBeta testing allows developers to collect more data on how their LLM applications are performing in real-world scenarios. In this phase, it‚Äôs important to develop an understanding for the types of inputs the app is performing well or poorly on and how exactly it‚Äôs breaking down in those cases. Both feedback collection and run annotation are critical for this workflow. This will help in curation of test cases that can help track regressions/improvements and development of automatic evaluations.Collecting Feedback‚ÄãWhen launching your application to an initial set of users, it‚Äôs important to gather human feedback on the responses it‚Äôs producing. This helps draw attention to the most interesting runs and highlight edge cases that are causing problematic responses. LangSmith allows you to attach feedback scores to logged traces (oftentimes, this is hooked up to a feedback button in your app), then filter on traces that have a specific feedback tag and score. A common workflow is to filter on traces that receive a poor user feedback score, then drill down into problematic points using the detailed trace view.Annotating Traces‚ÄãLangSmith also supports sending runs to annotation queues, which allow annotators to closely inspect interesting traces and annotate them with respect to different criteria. Annotators can be PMs, engineers, or even subject matter experts. This allows users to catch regressions across important evaluation criteria.Adding Runs to a Dataset‚ÄãAs your application progresses through the beta testing phase, it's essential to continue collecting data to refine and improve its performance. LangSmith enables you to add runs as examples to datasets (from both the project page and within an annotation queue), expanding your test coverage on real-world scenarios. This is a key benefit in having your logging system and your evaluation/testing system in the same platform.Production‚ÄãClosely inspecting key data points, growing benchmarking datasets, annotating traces, and drilling down into important data in trace view are workflows you‚Äôll also want to do once your app hits production. However, especially at the production stage, it‚Äôs crucial to get a high-level overview of application performance with respect to latency, cost, and feedback scores. This ensures that it's delivering desirable results at scale.Monitoring and A/B Testing‚ÄãLangSmith provides monitoring charts that allow you to track key metrics over time. You can expand to view metrics for a given period and drill down into a specific data point to get a trace table for that time period ‚Äî this is especially handy for debugging production issues.LangSmith also allows for tag and metadata grouping, which allows users to mark different versions of their applications with different identifiers and view how they are performing side-by-side within each chart. This is helpful for A/B testing changes in prompt, model, or retrieval strategy.Was this page helpful?PreviousLangSmithNextSetupPrototypingBeta TestingProductionCommunityDiscordTwitterGitHubDocs CodeLangSmith SDKPythonJS/TSMoreHomepageBlogCopyright ¬© 2024 LangChain, Inc.\n", "\n", "LangSmith User Guide | ü¶úÔ∏èüõ†Ô∏è LangSmith\u001b[0m\u001b[32;1m\u001b[1;3mLangSmith can help with testing in various ways throughout the application development lifecycle. Here are some key ways LangSmith supports testing:\n", "\n", "1. Prototyping: LangSmith allows for quick experimentation between prompts, model types, retrieval strategies, and other parameters to understand how the model is performing and debug any issues.\n", "\n", "2. Debugging: LangSmith provides clear visibility and debugging information at each step of an LLM sequence, making it easier to identify and root-cause issues when things go wrong.\n", "\n", "3. Initial Test Set: Developers can create datasets of inputs and reference outputs to run tests on their LLM applications. LangSmith makes it easy to run custom evaluations to score test results.\n", "\n", "4. Comparison View: LangSmith offers a user-friendly comparison view for test runs to track and diagnose regressions in test scores across multiple revisions of an application.\n", "\n", "5. Playground: LangSmith provides a playground environment for rapid iteration and experimentation with different prompts and models, allowing users to create test cases and compare different runs.\n", "\n", "6. Beta Testing: Developers can collect more data on how their LLM applications perform in real-world scenarios during beta testing. LangSmith supports feedback collection and run annotation to track regressions and improvements.\n", "\n", "7. Collecting Feedback: LangSmith allows developers to gather human feedback on the responses produced by their applications, helping to highlight edge cases and problematic responses.\n", "\n", "8. Annotating Traces: Users can send runs to annotation queues in LangSmith to closely inspect interesting traces and annotate them with different criteria, catching regressions across important evaluation criteria.\n", "\n", "9. Production: In the production stage, LangSmith enables users to closely inspect key data points, grow benchmarking datasets, and monitor application performance with respect to latency, cost, and feedback scores.\n", "\n", "10. Monitoring and A/B Testing: LangSmith provides monitoring charts to track key metrics over time and allows for A/B testing changes in prompt, model, or retrieval strategy.\n", "\n", "Overall, LangSmith offers a comprehensive platform for LLM application development, monitoring, and testing, supporting developers at every stage of the application lifecycle.\u001b[0m\n", "\n", "\u001b[1m> Finished chain.\u001b[0m\n" ] }, { "data": { "text/plain": [ "{'input': 'how can langsmith help with testing?',\n", " 'output': 'LangSmith can help with testing in various ways throughout the application development lifecycle. Here are some key ways LangSmith supports testing:\\n\\n1. Prototyping: LangSmith allows for quick experimentation between prompts, model types, retrieval strategies, and other parameters to understand how the model is performing and debug any issues.\\n\\n2. Debugging: LangSmith provides clear visibility and debugging information at each step of an LLM sequence, making it easier to identify and root-cause issues when things go wrong.\\n\\n3. Initial Test Set: Developers can create datasets of inputs and reference outputs to run tests on their LLM applications. LangSmith makes it easy to run custom evaluations to score test results.\\n\\n4. Comparison View: LangSmith offers a user-friendly comparison view for test runs to track and diagnose regressions in test scores across multiple revisions of an application.\\n\\n5. Playground: LangSmith provides a playground environment for rapid iteration and experimentation with different prompts and models, allowing users to create test cases and compare different runs.\\n\\n6. Beta Testing: Developers can collect more data on how their LLM applications perform in real-world scenarios during beta testing. LangSmith supports feedback collection and run annotation to track regressions and improvements.\\n\\n7. Collecting Feedback: LangSmith allows developers to gather human feedback on the responses produced by their applications, helping to highlight edge cases and problematic responses.\\n\\n8. Annotating Traces: Users can send runs to annotation queues in LangSmith to closely inspect interesting traces and annotate them with different criteria, catching regressions across important evaluation criteria.\\n\\n9. Production: In the production stage, LangSmith enables users to closely inspect key data points, grow benchmarking datasets, and monitor application performance with respect to latency, cost, and feedback scores.\\n\\n10. Monitoring and A/B Testing: LangSmith provides monitoring charts to track key metrics over time and allows for A/B testing changes in prompt, model, or retrieval strategy.\\n\\nOverall, LangSmith offers a comprehensive platform for LLM application development, monitoring, and testing, supporting developers at every stage of the application lifecycle.'}" ] }, "execution_count": 37, "metadata": {}, "output_type": "execute_result" } ], "source": [ "agent_executor.invoke({\"input\":\"how can langsmith help with testing?\"})" ] }, { "cell_type": "code", "execution_count": 38, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "executionInfo": { "elapsed": 7603, "status": "ok", "timestamp": 1710126377844, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "gEqgTtW8EA1P", "outputId": "fad18b66-90f4-43e2-a2af-680317983fd2" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "\n", "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n", "\u001b[32;1m\u001b[1;3m\n", "Invoking: `tavily_search_results_json` with `{'query': 'weather in San Francisco'}`\n", "\n", "\n", "\u001b[0m\u001b[33;1m\u001b[1;3m[{'url': 'https://www.accuweather.com/en/us/san-francisco/94103/hourly-weather-forecast/347629', 'content': 'Get the latest hourly weather conditions and outlook for San Francisco, CA, including temperature, precipitation, wind, air quality and more. See the cloud cover, visibility and cloud ceiling for each hour until 11 PM.'}, {'url': 'https://weather.com/weather/tenday/l/San Francisco CA USCA0987:1:US', 'content': \"Comfy & Cozy\\nThat's Not What Was Expected\\nOutside\\n'No-Name Storms' In Florida\\nGifts From On High\\nWhat To Do For Wheezing\\nSurviving The Season\\nStay Safe\\nAir Quality Index\\nAir quality is considered satisfactory, and air pollution poses little or no risk.\\n Health & Activities\\nSeasonal Allergies and Pollen Count Forecast\\nNo pollen detected in your area\\nCold & Flu Forecast\\nFlu risk is low in your area\\nWe recognize our responsibility to use data and technology for good. recents\\nSpecialty Forecasts\\n10 Day Weather-San Francisco, CA\\nToday\\nMon 18 | Day\\nConsiderable cloudiness. Tue 19\\nTue 19 | Day\\nLight rain early...then remaining cloudy with showers in the afternoon. Wed 27\\nWed 27 | Day\\nOvercast with rain showers at times.\"}, {'url': 'https://www.accuweather.com/en/us/san-francisco/94103/current-weather/347629', 'content': 'Get the latest weather conditions and outlook for San Francisco, CA, including temperature, humidity, wind, pressure, and cloud cover. See alerts, sunrise and sunset times, and historical data for the city.'}, {'url': 'https://www.accuweather.com/en/us/san-francisco/94103/weather-forecast/347629', 'content': 'Get the current and future weather conditions for San Francisco, CA, including temperature, precipitation, wind, air quality and more. See the hourly and 10-day outlook, radar maps, alerts and allergy information.'}, {'url': 'https://www.wunderground.com/forecast/us/ca/san-francisco', 'content': 'Get the latest weather information for San Francisco, CA, including temperature, precipitation, wind speed, and humidity. See the hourly and 10-day forecast for the South of Market station and other nearby weather stations.'}]\u001b[0m\u001b[32;1m\u001b[1;3mI found some sources where you can check the weather in San Francisco:\n", "\n", "1. [AccuWeather - Hourly Weather Forecast](https://www.accuweather.com/en/us/san-francisco/94103/hourly-weather-forecast/347629): Get the latest hourly weather conditions and outlook for San Francisco, CA, including temperature, precipitation, wind, air quality, and more.\n", "\n", "2. [The Weather Channel - 10 Day Weather Forecast](https://weather.com/weather/tenday/l/San Francisco CA USCA0987:1:US): Check the 10-day weather forecast for San Francisco, CA, including details on temperature, precipitation, and more.\n", "\n", "3. [AccuWeather - Current Weather Conditions](https://www.accuweather.com/en/us/san-francisco/94103/current-weather/347629): Get the latest weather conditions and outlook for San Francisco, CA, including temperature, humidity, wind, pressure, and cloud cover.\n", "\n", "4. [AccuWeather - Weather Forecast](https://www.accuweather.com/en/us/san-francisco/94103/weather-forecast/347629): Check the current and future weather conditions for San Francisco, CA, including temperature, precipitation, wind, air quality, and more.\n", "\n", "5. [Weather Underground - Forecast](https://www.wunderground.com/forecast/us/ca/san-francisco): Get the latest weather information for San Francisco, CA, including temperature, precipitation, wind speed, and humidity. See the hourly and 10-day forecast for the South of Market station and other nearby weather stations.\u001b[0m\n", "\n", "\u001b[1m> Finished chain.\u001b[0m\n" ] }, { "data": { "text/plain": [ "{'input': 'what is the weather in SF?',\n", " 'output': 'I found some sources where you can check the weather in San Francisco:\\n\\n1. [AccuWeather - Hourly Weather Forecast](https://www.accuweather.com/en/us/san-francisco/94103/hourly-weather-forecast/347629): Get the latest hourly weather conditions and outlook for San Francisco, CA, including temperature, precipitation, wind, air quality, and more.\\n\\n2. [The Weather Channel - 10 Day Weather Forecast](https://weather.com/weather/tenday/l/San Francisco CA USCA0987:1:US): Check the 10-day weather forecast for San Francisco, CA, including details on temperature, precipitation, and more.\\n\\n3. [AccuWeather - Current Weather Conditions](https://www.accuweather.com/en/us/san-francisco/94103/current-weather/347629): Get the latest weather conditions and outlook for San Francisco, CA, including temperature, humidity, wind, pressure, and cloud cover.\\n\\n4. [AccuWeather - Weather Forecast](https://www.accuweather.com/en/us/san-francisco/94103/weather-forecast/347629): Check the current and future weather conditions for San Francisco, CA, including temperature, precipitation, wind, air quality, and more.\\n\\n5. [Weather Underground - Forecast](https://www.wunderground.com/forecast/us/ca/san-francisco): Get the latest weather information for San Francisco, CA, including temperature, precipitation, wind speed, and humidity. See the hourly and 10-day forecast for the South of Market station and other nearby weather stations.'}" ] }, "execution_count": 38, "metadata": {}, "output_type": "execute_result" } ], "source": [ "agent_executor.invoke({\"input\":\"what is the weather in SF?\"})" ] }, { "cell_type": "code", "execution_count": 39, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "executionInfo": { "elapsed": 5408, "status": "ok", "timestamp": 1710126383238, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "3woupmIDEf-x", "outputId": "f8bee928-9fa3-44eb-f696-b31202000913" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "\n", "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n", "\u001b[32;1m\u001b[1;3m\n", "Invoking: `langsmith_search` with `{'query': 'LangSmith LLM application testing services'}`\n", "\n", "\n", "\u001b[0m\u001b[36;1m\u001b[1;3mSkip to main contentü¶úÔ∏èüõ†Ô∏è LangSmith DocsLangChain Python DocsLangChain JS/TS DocsLangSmith API DocsSearchGo to AppLangSmithUser GuideSetupPricing (Coming Soon)Self-HostingTracingEvaluationMonitoringPrompt HubProxyUser GuideOn this pageLangSmith User GuideLangSmith is a platform for LLM application development, monitoring, and testing. In this guide, we‚Äôll highlight the breadth of workflows LangSmith supports and how they fit into each stage of the application development lifecycle. We hope this will inform users how to best utilize this powerful platform or give them something to consider if they‚Äôre just starting their journey.Prototyping‚ÄãPrototyping LLM applications often involves quick experimentation between prompts, model types, retrieval strategy and other parameters.\n", "The ability to rapidly understand how the model is performing ‚Äî and debug where it is failing ‚Äî is incredibly important for this phase.Debugging‚ÄãWhen developing new LLM applications, we suggest having LangSmith tracing enabled by default.\n", "Oftentimes, it isn‚Äôt necessary to look at every single trace. However, when things go wrong (an unexpected end result, infinite agent loop, slower than expected execution, higher than expected token usage), it‚Äôs extremely helpful to debug by looking through the application traces. LangSmith gives clear visibility and debugging information at each step of an LLM sequence, making it much easier to identify and root-cause issues.\n", "We provide native rendering of chat messages, functions, and retrieve documents.Initial Test Set‚ÄãWhile many developers still ship an initial version of their application based on ‚Äúvibe checks‚Äù, we‚Äôve seen an increasing number of engineering teams start to adopt a more test driven approach. LangSmith allows developers to create datasets, which are collections of inputs and reference outputs, and use these to run tests on their LLM applications.\n", "These test cases can be uploaded in bulk, created on the fly, or exported from application traces. LangSmith also makes it easy to run custom evaluations (both LLM and heuristic based) to score test results.Comparison View‚ÄãWhen prototyping different versions of your applications and making changes, it‚Äôs important to see whether or not you‚Äôve regressed with respect to your initial test cases.\n", "Oftentimes, changes in the prompt, retrieval strategy, or model choice can have huge implications in responses produced by your application.\n", "In order to get a sense for which variant is performing better, it‚Äôs useful to be able to view results for different configurations on the same datapoints side-by-side. We‚Äôve invested heavily in a user-friendly comparison view for test runs to track and diagnose regressions in test scores across multiple revisions of your application.Playground‚ÄãLangSmith provides a playground environment for rapid iteration and experimentation.\n", "This allows you to quickly test out different prompts and models. You can open the playground from any prompt or model run in your trace.\n", "\n", "LangSmith User Guide | ü¶úÔ∏èüõ†Ô∏è LangSmith\n", "\n", "This allows you to quickly test out different prompts and models. You can open the playground from any prompt or model run in your trace.\n", "Every playground run is logged in the system and can be used to create test cases or compare with other runs.Beta Testing‚ÄãBeta testing allows developers to collect more data on how their LLM applications are performing in real-world scenarios. In this phase, it‚Äôs important to develop an understanding for the types of inputs the app is performing well or poorly on and how exactly it‚Äôs breaking down in those cases. Both feedback collection and run annotation are critical for this workflow. This will help in curation of test cases that can help track regressions/improvements and development of automatic evaluations.Collecting Feedback‚ÄãWhen launching your application to an initial set of users, it‚Äôs important to gather human feedback on the responses it‚Äôs producing. This helps draw attention to the most interesting runs and highlight edge cases that are causing problematic responses. LangSmith allows you to attach feedback scores to logged traces (oftentimes, this is hooked up to a feedback button in your app), then filter on traces that have a specific feedback tag and score. A common workflow is to filter on traces that receive a poor user feedback score, then drill down into problematic points using the detailed trace view.Annotating Traces‚ÄãLangSmith also supports sending runs to annotation queues, which allow annotators to closely inspect interesting traces and annotate them with respect to different criteria. Annotators can be PMs, engineers, or even subject matter experts. This allows users to catch regressions across important evaluation criteria.Adding Runs to a Dataset‚ÄãAs your application progresses through the beta testing phase, it's essential to continue collecting data to refine and improve its performance. LangSmith enables you to add runs as examples to datasets (from both the project page and within an annotation queue), expanding your test coverage on real-world scenarios. This is a key benefit in having your logging system and your evaluation/testing system in the same platform.Production‚ÄãClosely inspecting key data points, growing benchmarking datasets, annotating traces, and drilling down into important data in trace view are workflows you‚Äôll also want to do once your app hits production. However, especially at the production stage, it‚Äôs crucial to get a high-level overview of application performance with respect to latency, cost, and feedback scores. This ensures that it's delivering desirable results at scale.Monitoring and A/B Testing‚ÄãLangSmith provides monitoring charts that allow you to track key metrics over time. You can expand to view metrics for a given period and drill down into a specific data point to get a trace table for that time period ‚Äî this is especially handy for debugging production issues.LangSmith also allows for tag and metadata grouping, which allows users to mark different versions of their applications with different identifiers and view how they are performing side-by-side within each chart. This is helpful for A/B testing changes in prompt, model, or retrieval strategy.Was this page helpful?PreviousLangSmithNextSetupPrototypingBeta TestingProductionCommunityDiscordTwitterGitHubDocs CodeLangSmith SDKPythonJS/TSMoreHomepageBlogCopyright ¬© 2024 LangChain, Inc.\u001b[0m\u001b[32;1m\u001b[1;3mLangSmith is a platform for LLM application development, monitoring, and testing. Here are some ways LangSmith can help test your LLM applications:\n", "\n", "1. Prototyping: LangSmith supports quick experimentation between prompts, model types, retrieval strategies, and other parameters for prototyping LLM applications.\n", "\n", "2. Debugging: LangSmith provides tracing capabilities to help debug issues in LLM applications by giving clear visibility and debugging information at each step of an LLM sequence.\n", "\n", "3. Initial Test Set: Developers can create datasets of inputs and reference outputs to run tests on their LLM applications. LangSmith makes it easy to run custom evaluations to score test results.\n", "\n", "4. Comparison View: LangSmith offers a user-friendly comparison view for test runs to track and diagnose regressions in test scores across multiple revisions of your application.\n", "\n", "5. Playground: LangSmith provides a playground environment for rapid iteration and experimentation with different prompts and models.\n", "\n", "6. Beta Testing: Developers can collect more data on how their LLM applications are performing in real-world scenarios during beta testing phase.\n", "\n", "7. Collecting Feedback: LangSmith allows developers to gather human feedback on the responses produced by their applications to improve performance.\n", "\n", "8. Annotating Traces: LangSmith supports sending runs to annotation queues for annotators to inspect and annotate interesting traces with respect to different criteria.\n", "\n", "9. Monitoring and A/B Testing: LangSmith provides monitoring charts to track key metrics over time and allows for A/B testing changes in prompt, model, or retrieval strategy.\n", "\n", "These are some of the ways LangSmith can assist in testing your LLM applications.\u001b[0m\n", "\n", "\u001b[1m> Finished chain.\u001b[0m\n" ] }, { "data": { "text/plain": [ "{'chat_history': [HumanMessage(content='Can LangSmith help test my LLM applications?'),\n", " AIMessage(content='Yes!')],\n", " 'input': 'Tell me how',\n", " 'output': 'LangSmith is a platform for LLM application development, monitoring, and testing. Here are some ways LangSmith can help test your LLM applications:\\n\\n1. Prototyping: LangSmith supports quick experimentation between prompts, model types, retrieval strategies, and other parameters for prototyping LLM applications.\\n\\n2. Debugging: LangSmith provides tracing capabilities to help debug issues in LLM applications by giving clear visibility and debugging information at each step of an LLM sequence.\\n\\n3. Initial Test Set: Developers can create datasets of inputs and reference outputs to run tests on their LLM applications. LangSmith makes it easy to run custom evaluations to score test results.\\n\\n4. Comparison View: LangSmith offers a user-friendly comparison view for test runs to track and diagnose regressions in test scores across multiple revisions of your application.\\n\\n5. Playground: LangSmith provides a playground environment for rapid iteration and experimentation with different prompts and models.\\n\\n6. Beta Testing: Developers can collect more data on how their LLM applications are performing in real-world scenarios during beta testing phase.\\n\\n7. Collecting Feedback: LangSmith allows developers to gather human feedback on the responses produced by their applications to improve performance.\\n\\n8. Annotating Traces: LangSmith supports sending runs to annotation queues for annotators to inspect and annotate interesting traces with respect to different criteria.\\n\\n9. Monitoring and A/B Testing: LangSmith provides monitoring charts to track key metrics over time and allows for A/B testing changes in prompt, model, or retrieval strategy.\\n\\nThese are some of the ways LangSmith can assist in testing your LLM applications.'}" ] }, "execution_count": 39, "metadata": {}, "output_type": "execute_result" } ], "source": [ "chat_history = [\n", " HumanMessage(\n", " content=\"Can LangSmith help test my LLM applications?\"\n", " ),\n", " AIMessage(content=\"Yes!\")\n", "]\n", "agent_executor.invoke({\n", " \"chat_history\": chat_history,\n", " \"input\": \"Tell me how\"\n", "})" ] }, { "cell_type": "markdown", "metadata": { "id": "4CqnyNJFFRAk" }, "source": [ "基本的な agent を設定することができた。\n", "もっと詳しい情報は\n", "\n", "Agents\n", " を参照してほしい。" ] }, { "cell_type": "markdown", "metadata": { "id": "t200Z7NnFvo8" }, "source": [ "# Serving with LangServe (LangServe でサービスを提供する)\n", "\n", "アプリケーションを構築したので、それを外部に提供する。\n", "ここで LangServe が必要になる。\n", "LangServe は LangChain chain を REST API として配備するのに役に立つ。\n", "\n", "この文書はこれまで Jupyter で実行することを意図していたが、これ以降は Python ファイルを作成してコマンドラインから対話実行する。" ] }, { "cell_type": "code", "execution_count": 40, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "executionInfo": { "elapsed": 6880, "status": "ok", "timestamp": 1710126390106, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "ylv8BbCzFFbG", "outputId": "504986e5-7457-48c3-9b20-0a22ef6c7fe2" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting langserve[all]\n", " Downloading langserve-0.0.47-py3-none-any.whl (937 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m938.0/938.0 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: httpx>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from langserve[all]) (0.27.0)\n", "Requirement already satisfied: langchain>=0.0.333 in /usr/local/lib/python3.10/dist-packages (from langserve[all]) (0.1.11)\n", "Requirement already satisfied: orjson>=2 in /usr/local/lib/python3.10/dist-packages (from langserve[all]) (3.9.15)\n", "Requirement already satisfied: pydantic>=1 in /usr/local/lib/python3.10/dist-packages (from langserve[all]) (2.6.3)\n", "Collecting fastapi<1,>=0.90.1 (from langserve[all])\n", " Downloading fastapi-0.110.0-py3-none-any.whl (92 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.1/92.1 kB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting httpx-sse>=0.3.1 (from langserve[all])\n", " Downloading httpx_sse-0.4.0-py3-none-any.whl (7.8 kB)\n", "Collecting sse-starlette<2.0.0,>=1.3.0 (from langserve[all])\n", " Downloading sse_starlette-1.8.2-py3-none-any.whl (8.9 kB)\n", "Collecting starlette<0.37.0,>=0.36.3 (from fastapi<1,>=0.90.1->langserve[all])\n", " Downloading starlette-0.36.3-py3-none-any.whl (71 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.5/71.5 kB\u001b[0m \u001b[31m11.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from fastapi<1,>=0.90.1->langserve[all]) (4.10.0)\n", "Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.23.0->langserve[all]) (3.7.1)\n", "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx>=0.23.0->langserve[all]) (2024.2.2)\n", "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx>=0.23.0->langserve[all]) (1.0.4)\n", "Requirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx>=0.23.0->langserve[all]) (3.6)\n", "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.23.0->langserve[all]) (1.3.1)\n", "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx>=0.23.0->langserve[all]) (0.14.0)\n", "Requirement already satisfied: PyYAML>=5.3 in /usr/local/lib/python3.10/dist-packages (from langchain>=0.0.333->langserve[all]) (6.0.1)\n", "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.10/dist-packages (from langchain>=0.0.333->langserve[all]) (2.0.28)\n", "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /usr/local/lib/python3.10/dist-packages (from langchain>=0.0.333->langserve[all]) (3.9.3)\n", "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from langchain>=0.0.333->langserve[all]) (4.0.3)\n", "Requirement already satisfied: dataclasses-json<0.7,>=0.5.7 in /usr/local/lib/python3.10/dist-packages (from langchain>=0.0.333->langserve[all]) (0.6.4)\n", "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /usr/local/lib/python3.10/dist-packages (from langchain>=0.0.333->langserve[all]) (1.33)\n", "Requirement already satisfied: langchain-community<0.1,>=0.0.25 in /usr/local/lib/python3.10/dist-packages (from langchain>=0.0.333->langserve[all]) (0.0.27)\n", "Requirement already satisfied: langchain-core<0.2,>=0.1.29 in /usr/local/lib/python3.10/dist-packages (from langchain>=0.0.333->langserve[all]) (0.1.30)\n", "Requirement already satisfied: langchain-text-splitters<0.1,>=0.0.1 in /usr/local/lib/python3.10/dist-packages (from langchain>=0.0.333->langserve[all]) (0.0.1)\n", "Requirement already satisfied: langsmith<0.2.0,>=0.1.17 in /usr/local/lib/python3.10/dist-packages (from langchain>=0.0.333->langserve[all]) (0.1.23)\n", "Requirement already satisfied: numpy<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain>=0.0.333->langserve[all]) (1.25.2)\n", "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchain>=0.0.333->langserve[all]) (2.31.0)\n", "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain>=0.0.333->langserve[all]) (8.2.3)\n", "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1->langserve[all]) (0.6.0)\n", "Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1->langserve[all]) (2.16.3)\n", "Collecting uvicorn (from sse-starlette<2.0.0,>=1.3.0->langserve[all])\n", " Downloading uvicorn-0.28.0-py3-none-any.whl (60 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.6/60.6 kB\u001b[0m \u001b[31m8.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain>=0.0.333->langserve[all]) (1.3.1)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain>=0.0.333->langserve[all]) (23.2.0)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain>=0.0.333->langserve[all]) (1.4.1)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain>=0.0.333->langserve[all]) (6.0.5)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain>=0.0.333->langserve[all]) (1.9.4)\n", "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json<0.7,>=0.5.7->langchain>=0.0.333->langserve[all]) (3.21.1)\n", "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json<0.7,>=0.5.7->langchain>=0.0.333->langserve[all]) (0.9.0)\n", "Requirement already satisfied: jsonpointer>=1.9 in /usr/local/lib/python3.10/dist-packages (from jsonpatch<2.0,>=1.33->langchain>=0.0.333->langserve[all]) (2.4)\n", "Requirement already satisfied: packaging<24.0,>=23.2 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2,>=0.1.29->langchain>=0.0.333->langserve[all]) (23.2)\n", "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx>=0.23.0->langserve[all]) (1.2.0)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain>=0.0.333->langserve[all]) (3.3.2)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain>=0.0.333->langserve[all]) (2.0.7)\n", "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy<3,>=1.4->langchain>=0.0.333->langserve[all]) (3.0.3)\n", "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.10/dist-packages (from uvicorn->sse-starlette<2.0.0,>=1.3.0->langserve[all]) (8.1.7)\n", "Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain>=0.0.333->langserve[all]) (1.0.0)\n", "Installing collected packages: uvicorn, httpx-sse, starlette, fastapi, sse-starlette, langserve\n", "Successfully installed fastapi-0.110.0 httpx-sse-0.4.0 langserve-0.0.47 sse-starlette-1.8.2 starlette-0.36.3 uvicorn-0.28.0\n" ] } ], "source": [ "! pip install \"langserve[all]\"" ] }, { "cell_type": "markdown", "metadata": { "id": "xCvr_iJTHV_E" }, "source": [ "## Server\n", "略\n", "\n" ] }, { "cell_type": "code", "execution_count": 41, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "executionInfo": { "elapsed": 5, "status": "ok", "timestamp": 1710126390106, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "CXlwgAb7cejr", "outputId": "77092876-41fd-43dc-c2c4-5ba556ed9526" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "python3: can't open file '/content/drive/MyDrive/DeepLearning/llm_book2/langserve_serv.py': [Errno 2] No such file or directory\n" ] } ], "source": [ "!python {SAVE_PATH}/langserve_serv.py" ] }, { "cell_type": "markdown", "metadata": { "id": "5-Gmj18ncZUF" }, "source": [ "\n", "## CLient\n", "\n", "略" ] }, { "cell_type": "code", "execution_count": 41, "metadata": { "executionInfo": { "elapsed": 2, "status": "ok", "timestamp": 1710126390106, "user": { "displayName": "Yoshihisa Nitta", "userId": "15888006800030996813" }, "user_tz": -540 }, "id": "UMvHJwTRHGgb" }, "outputs": [], "source": [] } ], "metadata": { "accelerator": "GPU", "colab": { "authorship_tag": "ABX9TyP/FOBlYVilmgknKP8+O0gj", "gpuType": "T4", "provenance": [] }, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.13" } }, "nbformat": 4, "nbformat_minor": 1 }