diff --git a/toolkits/arcade_expert/.pre-commit-config.yaml b/toolkits/arcade_expert/.pre-commit-config.yaml new file mode 100644 index 000000000..3953e996e --- /dev/null +++ b/toolkits/arcade_expert/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +files: ^./ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: "v4.4.0" + hooks: + - id: check-case-conflict + - id: check-merge-conflict + - id: check-toml + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.6.7 + hooks: + - id: ruff + args: [--fix] + - id: ruff-format diff --git a/toolkits/arcade_expert/.ruff.toml b/toolkits/arcade_expert/.ruff.toml new file mode 100644 index 000000000..9519fe6c3 --- /dev/null +++ b/toolkits/arcade_expert/.ruff.toml @@ -0,0 +1,44 @@ +target-version = "py310" +line-length = 100 +fix = true + +[lint] +select = [ + # flake8-2020 + "YTT", + # flake8-bandit + "S", + # flake8-bugbear + "B", + # flake8-builtins + "A", + # flake8-comprehensions + "C4", + # flake8-debugger + "T10", + # flake8-simplify + "SIM", + # isort + "I", + # mccabe + "C90", + # pycodestyle + "E", "W", + # pyflakes + "F", + # pygrep-hooks + "PGH", + # pyupgrade + "UP", + # ruff + "RUF", + # tryceratops + "TRY", +] + +[lint.per-file-ignores] +"**/tests/*" = ["S101"] + +[format] +preview = true +skip-magic-trailing-comma = false diff --git a/toolkits/arcade_expert/LICENSE b/toolkits/arcade_expert/LICENSE new file mode 100644 index 000000000..8c2d4f375 --- /dev/null +++ b/toolkits/arcade_expert/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025, Arcade + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/toolkits/arcade_expert/Makefile b/toolkits/arcade_expert/Makefile new file mode 100644 index 000000000..c8e999003 --- /dev/null +++ b/toolkits/arcade_expert/Makefile @@ -0,0 +1,58 @@ +.PHONY: help + +help: + @echo "🛠️ arcade_expert Commands:\n" + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +.PHONY: install +install: ## Install the poetry environment and install the pre-commit hooks + @echo "📦 Checking if Poetry is installed" + @if ! command -v poetry >/dev/null 2>&1; then \ + echo "📦 Poetry not found. Checking if pip is available"; \ + if ! command -v pip >/dev/null 2>&1; then \ + echo "❌ pip is not installed. Please install pip first."; \ + exit 1; \ + fi; \ + echo "📦 Installing Poetry with pip"; \ + pip install poetry==1.8.5; \ + else \ + echo "📦 Poetry is already installed"; \ + fi + @echo "🚀 Installing package in development mode with all extras" + poetry install --all-extras + +.PHONY: build +build: clean-build ## Build wheel file using poetry + @echo "🚀 Creating wheel file" + poetry build + +.PHONY: clean-build +clean-build: ## clean build artifacts + @echo "🗑️ Cleaning dist directory" + rm -rf dist + +.PHONY: test +test: ## Test the code with pytest + @echo "🚀 Testing code: Running pytest" + @poetry run pytest -W ignore -v --cov --cov-config=pyproject.toml --cov-report=xml + +.PHONY: coverage +coverage: ## Generate coverage report + @echo "coverage report" + coverage report + @echo "Generating coverage report" + coverage html + +.PHONY: bump-version +bump-version: ## Bump the version in the pyproject.toml file + @echo "🚀 Bumping version in pyproject.toml" + poetry version patch + +.PHONY: check +check: ## Run code quality tools. + @echo "🚀 Checking Poetry lock file consistency with 'pyproject.toml': Running poetry check" + @poetry check + @echo "🚀 Linting code: Running pre-commit" + @poetry run pre-commit run -a + @echo "🚀 Static type checking: Running mypy" + @poetry run mypy --config-file=pyproject.toml diff --git a/toolkits/arcade_expert/arcade_arcade_expert/__init__.py b/toolkits/arcade_expert/arcade_arcade_expert/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/toolkits/arcade_expert/arcade_arcade_expert/models.py b/toolkits/arcade_expert/arcade_arcade_expert/models.py new file mode 100644 index 000000000..9eb73bf44 --- /dev/null +++ b/toolkits/arcade_expert/arcade_arcade_expert/models.py @@ -0,0 +1,16 @@ +import validators +from pydantic import BaseModel + + +class Links(BaseModel): + links: list[str] + + def validate_links(self) -> None: + """Validate links, removing any invalid ones""" + valid_links = [] + + for link in self.links: + if validators.url(link): + valid_links.append(link) + + self.links = valid_links diff --git a/toolkits/arcade_expert/arcade_arcade_expert/tools/__init__.py b/toolkits/arcade_expert/arcade_arcade_expert/tools/__init__.py new file mode 100644 index 000000000..68cf121da --- /dev/null +++ b/toolkits/arcade_expert/arcade_arcade_expert/tools/__init__.py @@ -0,0 +1,3 @@ +from arcade_arcade_expert.tools.search_documentation import search_documentation + +__all__ = ["search_documentation"] diff --git a/toolkits/arcade_expert/arcade_arcade_expert/tools/search_documentation.py b/toolkits/arcade_expert/arcade_arcade_expert/tools/search_documentation.py new file mode 100644 index 000000000..db086f754 --- /dev/null +++ b/toolkits/arcade_expert/arcade_arcade_expert/tools/search_documentation.py @@ -0,0 +1,67 @@ +from typing import Annotated + +import httpx +from arcade.sdk import ToolContext, tool +from markdownify import markdownify +from openai import OpenAI +from openai.types.chat import ParsedChatCompletion + +from arcade_arcade_expert.models import Links + + +@tool(requires_secrets=["OPENAI_API_KEY"]) +async def search_documentation( + context: ToolContext, + query: Annotated[str, "The query to use to search for relevant Arcade.dev documentation"], +) -> Annotated[str, "The answer to the query"]: + """Search Arcade.dev's documentation for the content of pages that are relevant to the query. + + Arcade.dev securely connects your AI to APIs, data, code, and other systems. + + Arcade is an AI Tool-calling Platform. For the first time, AI can securely act on behalf + of users through Arcade's authenticated integrations, or "tools" in AI lingo. Connect AI + to email, files, calendars, and APIs to build assistants that don't just chat - they get + work done. Start building in minutes with our pre-built connectors or custom SDK. + """ + openai_api_key = context.get_secret("OPENAI_API_KEY") + openai_client = OpenAI(api_key=openai_api_key) + + # Get Arcade.dev documentation's llms.txt file + url = "https://docs.arcade.dev/llms.txt" + async with httpx.AsyncClient() as client: + response = await client.get(url) + response.raise_for_status() + data = markdownify(response.text) + + # Get relevant links from the llms.txt file + chat_response: ParsedChatCompletion = openai_client.beta.chat.completions.parse( + model="gpt-4o", + messages=[ + { + "role": "system", + "content": ( + "Provided a query, you are an expert at selecting the most relevant URLs " + "from a list of URLs. You return at most 5 URLs." + ), + }, + {"role": "user", "content": f"Question: {query}\n\nCandidate Links: {data}"}, + ], + response_format=Links, + ) + links = chat_response.choices[0].message.parsed + + if not links: + return "No relevant documentation found." + + # Get the content of the relevant links + documentation_content: list[str] = [] + async with httpx.AsyncClient() as client: + sources = [] + for link in links.links: + response = await client.get(link) + if 200 <= response.status_code < 300: + documentation_content.append(markdownify(response.text)) + sources.append(link) + + sources_str = "\n".join(sources) + return "\n\n".join(documentation_content) + f"\n\nSources: {sources_str}" diff --git a/toolkits/arcade_expert/evals/eval_arcade_expert.py b/toolkits/arcade_expert/evals/eval_arcade_expert.py new file mode 100644 index 000000000..36c93b0f7 --- /dev/null +++ b/toolkits/arcade_expert/evals/eval_arcade_expert.py @@ -0,0 +1,62 @@ +from arcade.sdk import ToolCatalog +from arcade.sdk.eval import ( + EvalRubric, + EvalSuite, + ExpectedToolCall, + SimilarityCritic, + tool_eval, +) + +import arcade_arcade_expert +from arcade_arcade_expert.tools import search_documentation + +# Evaluation rubric +rubric = EvalRubric( + fail_threshold=0.85, + warn_threshold=0.95, +) + + +catalog = ToolCatalog() +catalog.add_module(arcade_arcade_expert) + + +@tool_eval() +def arcade_expert_eval_suite() -> EvalSuite: + suite = EvalSuite( + name="Search Documentation Tool Evaluation", + system_message="Help the user with their queries", + catalog=catalog, + rubric=rubric, + ) + + suite.add_case( + name="Finding engine.yaml location", + user_message="where is my engine.yaml file that Arcade is telling me that I need", + expected_tool_calls=[ + ExpectedToolCall( + func=search_documentation, + args={"query": "engine.yaml file location"}, + ) + ], + rubric=rubric, + critics=[ + SimilarityCritic(critic_field="query", weight=0.3, similarity_threshold=0.4), + ], + ) + + suite.extend_case( + name="Add a custom Reddit OAuth Provider", + user_message="I want to create a new Reddit OAuth Provider. How do I do this?", + expected_tool_calls=[ + ExpectedToolCall( + func=search_documentation, args={"query": "create a new Reddit OAuth Provider"} + ), + ], + rubric=rubric, + critics=[ + SimilarityCritic(critic_field="query", weight=0.3, similarity_threshold=0.4), + ], + ) + + return suite diff --git a/toolkits/arcade_expert/pyproject.toml b/toolkits/arcade_expert/pyproject.toml new file mode 100644 index 000000000..4cf48116c --- /dev/null +++ b/toolkits/arcade_expert/pyproject.toml @@ -0,0 +1,43 @@ +[tool.poetry] +name = "arcade_arcade_expert" +version = "0.0.1" +description = "Arcade.dev LLM tools for everything Arcade.dev related. Super meta, we know!" +authors = ["Arcade "] + +[tool.poetry.dependencies] +python = "^3.10" +arcade-ai = "^1.0.5" +httpx = "^0.28.1" +markdownify = "^1.1.0" +openai = "^1.55.3" +validators = "^0.35.0" + +[tool.poetry.dev-dependencies] +pytest = "^8.3.0" +pytest-cov = "^4.0.0" +mypy = "^1.5.1" +pre-commit = "^3.4.0" +tox = "^4.11.1" +ruff = "^0.7.4" + +[build-system] +requires = ["poetry-core>=1.0.0,<2.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.mypy] +files = ["arcade_arcade_expert/**/*.py"] +python_version = "3.10" +disallow_untyped_defs = "True" +disallow_any_unimported = "True" +no_implicit_optional = "True" +check_untyped_defs = "True" +warn_return_any = "True" +warn_unused_ignores = "True" +show_error_codes = "True" +ignore_missing_imports = "True" + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.coverage.report] +skip_empty = true diff --git a/toolkits/arcade_expert/tests/__init__.py b/toolkits/arcade_expert/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/toolkits/arcade_expert/tests/test_models.py b/toolkits/arcade_expert/tests/test_models.py new file mode 100644 index 000000000..3b5b332f2 --- /dev/null +++ b/toolkits/arcade_expert/tests/test_models.py @@ -0,0 +1,28 @@ +from arcade_arcade_expert.models import Links + + +def test_links_initialization(): + # Test that Links can be initialized with a list of links + links = Links(links=["https://example.com", "https://test.com"]) + assert links.links == ["https://example.com", "https://test.com"] + + +def test_validate_links(): + # Test that validate_links removes invalid URLs + links = Links(links=["https://example.com", "not-a-valid-url", "https://test.com"]) + links.validate_links() + assert links.links == ["https://example.com", "https://test.com"] + + +def test_validate_links_empty(): + # Test with empty list + links = Links(links=[]) + links.validate_links() + assert links.links == [] + + +def test_validate_links_all_invalid(): + # Test with all invalid links + links = Links(links=["not-valid-1", "not-valid-2"]) + links.validate_links() + assert links.links == []