test_create_agent_template_reasoning.py 6.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146
  1. """Tests for create_agent_template reasoning and validation behavior."""
  2. from pathlib import Path
  3. from unittest.mock import patch
  4. from agency_swarm.utils.create_agent_template import create_agent_template
  5. class TestCreateAgentTemplateReasoning:
  6. def test_reasoning_model_with_reasoning_parameter(self, tmp_path: Path) -> None:
  7. """Test creating agent with reasoning model and reasoning parameter."""
  8. assert (
  9. create_agent_template(
  10. agent_name="Smart Agent",
  11. agent_description="Uses reasoning capabilities",
  12. model="gpt-5.4-mini",
  13. reasoning="high",
  14. path=str(tmp_path),
  15. )
  16. is True
  17. )
  18. agent_content = (tmp_path / "smart_agent" / "smart_agent.py").read_text(encoding="utf-8")
  19. assert "from openai.types.shared import Reasoning" in agent_content
  20. assert 'model="gpt-5.4-mini"' in agent_content
  21. assert 'reasoning=Reasoning(effort="high", summary="auto")' in agent_content
  22. assert "temperature=" not in agent_content
  23. def test_reasoning_model_with_temperature_shows_error(self, tmp_path: Path, capsys) -> None:
  24. """Test that reasoning model with temperature shows error but continues."""
  25. assert (
  26. create_agent_template(
  27. agent_name="Bad Agent",
  28. agent_description="Test error handling",
  29. model="gpt-5.4-mini",
  30. temperature=0.7,
  31. reasoning="medium",
  32. path=str(tmp_path),
  33. )
  34. is True
  35. )
  36. captured = capsys.readouterr()
  37. assert "ERROR: Reasoning models (like gpt-5.4-mini) do not support the temperature parameter" in captured.out
  38. assert "Temperature parameter will be ignored" in captured.out
  39. agent_content = (tmp_path / "bad_agent" / "bad_agent.py").read_text(encoding="utf-8")
  40. assert 'reasoning=Reasoning(effort="medium", summary="auto")' in agent_content
  41. assert "temperature=" not in agent_content
  42. def test_non_reasoning_model_with_reasoning_shows_error(self, tmp_path: Path, capsys) -> None:
  43. """Test that non-reasoning model with reasoning shows error and ignores reasoning."""
  44. assert (
  45. create_agent_template(
  46. agent_name="Regular Agent",
  47. agent_description="Test error handling",
  48. model="gpt-4.1",
  49. reasoning="high",
  50. temperature=0.5,
  51. path=str(tmp_path),
  52. )
  53. is True
  54. )
  55. captured = capsys.readouterr()
  56. assert "ERROR: Non-reasoning models (like gpt-4.1) do not support the reasoning parameter" in captured.out
  57. assert "Reasoning parameter will be ignored" in captured.out
  58. agent_content = (tmp_path / "regular_agent" / "regular_agent.py").read_text(encoding="utf-8")
  59. assert "temperature=0.5" in agent_content
  60. assert "reasoning=" not in agent_content
  61. assert "from openai.types.shared import Reasoning" not in agent_content
  62. @patch("builtins.input")
  63. def test_invalid_agent_name_validation(self, mock_input, tmp_path: Path, capsys) -> None:
  64. """Test that invalid agent names are rejected."""
  65. mock_input.return_value = ""
  66. result = create_agent_template(agent_name=None, path=str(tmp_path))
  67. captured = capsys.readouterr()
  68. assert "ERROR: Agent name cannot be empty" in captured.out
  69. assert result is False
  70. def test_invalid_characters_in_name(self, tmp_path: Path, capsys) -> None:
  71. """Test that agent names with invalid characters show error."""
  72. result = create_agent_template(agent_name="Bad<Name>", path=str(tmp_path))
  73. captured = capsys.readouterr()
  74. assert "ERROR: Agent name contains invalid characters" in captured.out
  75. assert result is False
  76. def test_temperature_validation(self, tmp_path: Path, capsys) -> None:
  77. """Test temperature validation."""
  78. result = create_agent_template(agent_name="Test Agent", temperature=3.0, path=str(tmp_path))
  79. captured = capsys.readouterr()
  80. assert "ERROR: Temperature must be between 0.0 and 2.0" in captured.out
  81. assert result is False
  82. def test_description_with_quotes_escaped(self, tmp_path: Path) -> None:
  83. """Test that quotes in description are properly escaped in instructions."""
  84. assert (
  85. create_agent_template(
  86. agent_name="Quote Agent",
  87. agent_description='An agent that uses "quotes" in description',
  88. path=str(tmp_path),
  89. )
  90. is True
  91. )
  92. instructions = (tmp_path / "quote_agent" / "instructions.md").read_text(encoding="utf-8")
  93. assert "An agent that uses 'quotes' in description" in instructions
  94. assert 'An agent that uses "quotes" in description' not in instructions
  95. def test_gpt5_reasoning_includes_summary(self, tmp_path: Path) -> None:
  96. """Test that GPT-5 models include summary parameter in Reasoning."""
  97. assert (
  98. create_agent_template(
  99. agent_name="GPT5 Agent",
  100. model="gpt-5.4-mini",
  101. reasoning="high",
  102. path=str(tmp_path),
  103. )
  104. is True
  105. )
  106. agent_content = (tmp_path / "gpt5_agent" / "gpt5_agent.py").read_text(encoding="utf-8")
  107. assert 'Reasoning(effort="high", summary="auto")' in agent_content
  108. assert "temperature=" not in agent_content
  109. def test_non_gpt5_reasoning_no_summary(self, tmp_path: Path) -> None:
  110. """Test that non-GPT-5 reasoning models don't include summary in Reasoning."""
  111. assert (
  112. create_agent_template(
  113. agent_name="O1 Agent",
  114. model="o1-preview",
  115. reasoning="high",
  116. path=str(tmp_path),
  117. )
  118. is True
  119. )
  120. agent_content = (tmp_path / "o1_agent" / "o1_agent.py").read_text(encoding="utf-8")
  121. assert 'Reasoning(effort="high")' in agent_content
  122. assert 'summary="auto"' not in agent_content
  123. assert "temperature=" not in agent_content