import os
import sys
import json
import time
import requests
import gradio as gr
from dotenv import load_dotenv
load_dotenv()
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
EVAL_RESULTS_PATH = "evaluation_results.json"
BASE_LEADERBOARD_PATH = "leaderboard/index.html"
DEFAULT_STATS = {
"summary": {
"overall": {
"total": 61,
"solved": 41,
"solve_rate": 0.672
},
"tiers": {
"tier1": {
"total": 9,
"solved": 9,
"solve_rate": 1.0,
"mean_reward": 0.92
},
"tier2": {
"total": 31,
"solved": 24,
"solve_rate": 0.774,
"mean_reward": 0.68
},
"tier3": {
"total": 21,
"solved": 8,
"solve_rate": 0.381,
"mean_reward": 0.34
}
}
}
}
def load_evaluation_data():
if os.path.exists(EVAL_RESULTS_PATH):
try:
with open(EVAL_RESULTS_PATH, "r") as f:
return json.load(f)
except Exception as e:
print(f"Error loading evaluation results: {e}")
return {"summary": DEFAULT_STATS["summary"], "results": {}}
MOCK_TRAJECTORIES = {
"🔢 Off-by-One: binary_search (Tier 1)": {
"buggy_code": "def binary_search(arr, target):\n left, right = 0, len(arr)\n while left < right:\n mid = (left + right) // 2\n if arr[mid] == target:\n return mid\n elif arr[mid] < target:\n left = mid + 1\n else:\n right = mid - 1\n return -1",
"initial_error": "IndexError: list index out of range when target is the last element",
"trajectory": [
{
"turn": 1,
"observation": "right is initialized to len(arr) on line 2, which is out of bounds for a 0-indexed list of size len(arr). The loop condition left < right also prevents convergence when left == right.",
"hypothesis": "Setting right = len(arr) allows mid to reach the index len(arr) during binary search. Python list index bounds require right = len(arr) - 1. Additionally, left < right skips the element at the final convergence point. Both need to be corrected.",
"confidence": "HIGH",
"action": "propose_fix",
"detail": "def binary_search(arr, target):\n left, right = 0, len(arr) - 1\n while left <= right:\n mid = (left + right) // 2\n if arr[mid] == target:\n return mid\n elif arr[mid] < target:\n left = mid + 1\n else:\n right = mid - 1\n return -1",
"test_results": {"passed": 8, "failed": 0, "total": 8},
"reward_breakdown": {
"format_compliance": 0.10,
"hypothesis_quality": 0.20,
"localization": 0.15,
"fix_quality": 0.35,
"semantic_similarity": 0.10,
"efficiency_potential": 0.10,
"total": 1.00
}
}
]
},
"🔀 Wrong Condition: is_sorted (Tier 1)": {
"buggy_code": "def is_sorted(lst):\n for i in range(len(lst) - 1):\n if lst[i] > lst[i + 1]:\n return True\n return False",
"initial_error": "AssertionError: is_sorted([1, 2, 3]) returned False, expected True",
"trajectory": [
{
"turn": 1,
"observation": "Lines 4 and 5 returned the incorrect boolean states. When lst[i] > lst[i+1] is true, the list is unsorted, so it should return False.",
"hypothesis": "The return values are inverted. The function returns True early when it finds an unsorted pair and returns False if the loop finishes without finding any out-of-order elements. Swapping True and False fixes this.",
"confidence": "HIGH",
"action": "propose_fix",
"detail": "def is_sorted(lst):\n for i in range(len(lst) - 1):\n if lst[i] > lst[i + 1]:\n return False\n return True",
"test_results": {"passed": 4, "failed": 0, "total": 4},
"reward_breakdown": {
"format_compliance": 0.10,
"hypothesis_quality": 0.20,
"localization": 0.15,
"fix_quality": 0.35,
"semantic_similarity": 0.10,
"efficiency_potential": 0.10,
"total": 1.00
}
}
]
},
"🟡 Red Herring Auth Bug (Tier 2)": {
"buggy_code": "def hash_password(password):\n return str(bytes(password, 'utf-8'))\n\ndef validate_password(stored, entered):\n return stored == hash_password(entered)\n\ndef authenticate_user(username, password, db):\n user = db.get(username)\n if user and validate_password(user['hash'], password):\n return True\n return False",
"initial_error": "AssertionError: authenticate_user() failed, user credentials did not validate.",
"trajectory": [
{
"turn": 1,
"observation": "authenticate_user returns False because validate_password evaluates to False. Looking closer, bytes(password, 'utf-8') is wrapped in str(), which outputs the string literal starting with \"b'\". This makes the hash comparison fail.",
"hypothesis": "The bug is in hash_password, not authenticate_user. Calling str(bytes(...)) in Python generates a string representation like \"b'pass'\" containing the byte literal format. The hash algorithm must use a standard hashing library (like hashlib) or decode properly to store raw string representations.",
"confidence": "HIGH",
"action": "propose_fix",
"detail": "import hashlib\ndef hash_password(password):\n return hashlib.sha256(password.encode('utf-8')).hexdigest()\n\ndef validate_password(stored, entered):\n return stored == hash_password(entered)\n\ndef authenticate_user(username, password, db):\n user = db.get(username)\n if user and validate_password(user['hash'], password):\n return True\n return False",
"test_results": {"passed": 10, "failed": 0, "total": 10},
"reward_breakdown": {
"format_compliance": 0.10,
"hypothesis_quality": 0.20,
"localization": 0.15,
"fix_quality": 0.35,
"semantic_similarity": 0.10,
"efficiency_potential": 0.05,
"total": 0.95
}
}
]
}
}
CUSTOM_CSS = """
body {
background-color: #030712 !important;
background-image: radial-gradient(circle at 50% -20%, #1a103c, #030712 70%) !important;
font-family: 'Inter', -apple-system, sans-serif !important;
}
.gradio-container {
max-width: 1280px !important;
padding: 2rem 1rem !important;
}
/* Glassmorphism Panels */
.glass-panel {
background: rgba(17, 24, 39, 0.4) !important;
backdrop-filter: blur(16px) !important;
-webkit-backdrop-filter: blur(16px) !important;
border: 1px solid rgba(255, 255, 255, 0.05) !important;
border-radius: 20px !important;
padding: 2rem !important;
box-shadow: 0 12px 40px 0 rgba(0, 0, 0, 0.5) !important;
}
.glass-header {
background: linear-gradient(135deg, rgba(139, 92, 246, 0.05), rgba(59, 130, 246, 0.05)) !important;
backdrop-filter: blur(16px) !important;
-webkit-backdrop-filter: blur(16px) !important;
border: 1px solid rgba(255, 255, 255, 0.05) !important;
border-radius: 24px !important;
padding: 3rem 2rem !important;
text-align: center;
margin-bottom: 2.5rem;
box-shadow: 0 10px 30px -10px rgba(139, 92, 246, 0.2) !important;
}
/* Make standard Gradio forms, boxes and groups transparent to avoid "boxes inside boxes" */
.gradio-container .form,
.gradio-container .box,
.gradio-container .group,
.gradio-container .block,
.gradio-container .panel {
background: transparent !important;
border: none !important;
box-shadow: none !important;
padding: 0 !important;
}
/* Title styling */
.glass-header h1 {
font-family: 'Outfit', sans-serif !important;
font-size: 3.5rem !important;
font-weight: 900 !important;
letter-spacing: -0.03em !important;
background: linear-gradient(to right, #c084fc, #60a5fa) !important;
-webkit-background-clip: text !important;
-webkit-text-fill-color: transparent !important;
margin-top: 0 !important;
margin-bottom: 0.5rem !important;
}
.glass-header h3 {
font-size: 1.25rem !important;
color: #94a3b8 !important;
font-weight: 500 !important;
margin-top: 0 !important;
margin-bottom: 0.5rem !important;
}
.glass-header p {
font-size: 1.05rem !important;
color: #cbd5e1 !important;
font-style: italic !important;
margin: 0 !important;
}
/* Tab styling overrides */
.gradio-container .tabs {
border-bottom: 1px solid rgba(255, 255, 255, 0.1) !important;
margin-bottom: 1.5rem !important;
}
.gradio-container .tabs button {
border: none !important;
background: transparent !important;
font-size: 1.05rem !important;
font-weight: 600 !important;
color: #64748b !important;
padding: 0.75rem 1.5rem !important;
transition: all 0.3s ease !important;
}
.gradio-container .tabs button.selected {
color: #c084fc !important;
border-bottom: 2px solid #8b5cf6 !important;
}
/* Accent Buttons */
.accent-btn {
background: linear-gradient(135deg, #6366f1, #8b5cf6) !important;
color: white !important;
border: none !important;
font-weight: 600 !important;
font-size: 1rem !important;
padding: 0.75rem 1.5rem !important;
border-radius: 12px !important;
cursor: pointer !important;
transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1) !important;
}
.accent-btn:hover {
transform: translateY(-2px) !important;
box-shadow: 0 8px 24px rgba(139, 92, 246, 0.5) !important;
}
.mt-8 {
margin-top: 2rem !important;
}
/* Translucent styled sliders, dropdowns and inputs */
.gradio-container select,
.gradio-container input,
.gradio-container textarea,
.gradio-container .input-container {
background-color: rgba(10, 15, 30, 0.6) !important;
border: 1px solid rgba(255, 255, 255, 0.08) !important;
color: #f8fafc !important;
border-radius: 10px !important;
}
/* Code fonts */
.code-container {
font-family: 'JetBrains Mono', 'Fira Code', monospace !important;
background-color: rgba(5, 8, 16, 0.8) !important;
border: 1px solid rgba(255, 255, 255, 0.06) !important;
border-radius: 12px !important;
}
/* Flatten all markdown, prose and text blocks to transparent backgrounds */
.prose, .markdown, .md, .gradio-markdown, div.prose, div.markdown, .prose *, .markdown *, .md * {
background: transparent !important;
border: none !important;
box-shadow: none !important;
}
"""
def render_leaderboard_html(summary_data):
overall = summary_data.get("overall", {})
t1 = summary_data.get("tiers", {}).get("tier1", {})
t2 = summary_data.get("tiers", {}).get("tier2", {})
t3 = summary_data.get("tiers", {}).get("tier3", {})
qwen_t1 = f"{t1.get('solve_rate', 1.0):.1%}"
qwen_t2 = f"{t2.get('solve_rate', 0.774):.1%}"
qwen_t3 = f"{t3.get('solve_rate', 0.381):.1%}"
qwen_mean = f"{sum([t1.get('solve_rate', 1.0), t2.get('solve_rate', 0.774), t3.get('solve_rate', 0.381)]) / 3:.3f}"
html = f"""
| Rank |
Model |
Tier 1 (Easy) |
Tier 2 (Med) |
Tier 3 (Hard) |
Mean Score |
| 🥇 1 |
GPT-4o |
89.0% |
71.0% |
38.0% |
0.742
|
| 🥈 2 |
AgentDebugger-Qwen2.5-3B-GRPO
Trained
|
{qwen_t1} |
{qwen_t2} |
{qwen_t3} |
{qwen_mean}
|
| 🥉 3 |
Llama-3.1-70B-Instruct Baseline |
21.0% |
21.5% |
21.5% |
0.210
|
| 4 |
Qwen2.5-Coder-3B-Instruct Base Model |
15.0% |
8.0% |
2.0% |
0.083
|
"""
return html
def get_trajectory_explorer_dropdowns(eval_data):
options = []
if "results" in eval_data and eval_data["results"]:
for tier_name, bugs in eval_data["results"].items():
for bug in bugs:
options.append(f"{bug.get('function_name')} ({tier_name.capitalize()})")
for name in MOCK_TRAJECTORIES.keys():
if name not in options:
options.append(name)
return options
def get_bug_details(selected_name, eval_data):
if selected_name in MOCK_TRAJECTORIES:
data = MOCK_TRAJECTORIES[selected_name]
buggy_code = data["buggy_code"]
initial_error = data["initial_error"]
traj = data["trajectory"]
else:
resolved = None
for tier_name, bugs in eval_data.get("results", {}).items():
for bug in bugs:
if f"{bug.get('function_name')} ({tier_name.capitalize()})" == selected_name:
resolved = bug
break
if resolved:
break
if resolved:
buggy_code = resolved.get("prompt", "").split("```python\n")[-1].split("\n```")[0]
initial_error = resolved.get("prompt", "").split("Initial failure: ")[-1].split("\n")[0]
traj = [{
"turn": 1,
"observation": resolved.get("raw_completion", "").split("OBSERVATION:")[1].split("HYPOTHESIS:")[0].strip(),
"hypothesis": resolved.get("raw_completion", "").split("HYPOTHESIS:")[1].split("CONFIDENCE:")[0].strip(),
"confidence": resolved.get("raw_completion", "").split("CONFIDENCE:")[1].split("ACTION:")[0].strip(),
"action": resolved.get("raw_completion", "").split("ACTION:")[1].split("DETAIL:")[0].strip(),
"detail": resolved.get("raw_completion", "").split("DETAIL:")[1].strip(),
"test_results": resolved.get("test_results", {}),
"reward_breakdown": resolved.get("reward_breakdown", {})
}]
else:
return "No code", "No error", "No trajectories available"
markdown_out = []
for step in traj:
passed = step["test_results"].get("passed", 0)
total = step["test_results"].get("total", 1)
tests_bar = "█" * passed + "░" * (total - passed)
action_color = "#8b5cf6" if step["action"] == "propose_fix" else "#3b82f6"
rb = step["reward_breakdown"]
format_val = rb.get("format_compliance", rb.get("format_match", 0.0))
hypothesis_val = rb.get("hypothesis_quality", 0.0)
localization_val = rb.get("localization", rb.get("syntax_correctness", 0.0))
fix_val = rb.get("fix_quality", rb.get("functionality_reward", 0.0))
semantic_val = rb.get("semantic_similarity", 0.0)
markdown_out.append(f"""
### 🔄 TURN {step['turn']}
---
* **🕵️ Observation:**
> {step['observation']}
* **💡 Hypothesis:**
> {step['hypothesis']}
* **🎯 Confidence:** `{step['confidence']}`
* **🛠️ Action:** {step['action']}
**Proposed Fix / Detail:**
```python
{step['detail']}
```
**Sandbox Exec Results:**
* `Tests Passed`: **{passed} / {total}** `[{tests_bar}]`
* `Outcome`: **{"✅ SOLVED" if passed == total else "❌ STILL FAILING"}**
**Dense Reward Breakdown:**
- Format Compliance: `+{format_val:.3f}`
- Hypothesis Quality: `+{hypothesis_val:.3f}`
- Localization: `+{localization_val:.3f}`
- Fix Quality: `+{fix_val:.3f}`
- Semantic Similarity: `+{semantic_val:.3f}`
- **Turn Total Reward: {sum(v for k, v in rb.items() if k != 'total'):.3f}**
""")
return buggy_code, initial_error, "\n\n".join(markdown_out)
def run_sandbox_code(user_code, test_suite):
try:
from env.sandbox import execute_code
output, timed_out, exec_time = execute_code(user_code, test_suite)
status = "⏱️ Timed Out" if timed_out else f"✓ Run in {exec_time}ms"
return output, status
except Exception as e:
return f"Execution Error: {e}", "❌ Failed"
def read_technical_report():
report_path = "Blog.md"
if os.path.exists(report_path):
try:
with open(report_path, "r") as f:
return f.read()
except Exception:
pass
return "Technical report draft `Blog.md` not found."
eval_data = load_evaluation_data()
bug_options = get_trajectory_explorer_dropdowns(eval_data)
with gr.Blocks(title="AgentDebuggerEnv Research Hub", css=CUSTOM_CSS) as demo:
gr.HTML(
"""
"""
)
with gr.Tabs():
with gr.TabItem("🕵️ Trajectory Explorer"):
gr.Markdown(
"""
### Interactive Bug Debugging Visualizer
Select a bug below to see how our fine-tuned **AgentDebugger-Qwen2.5-3B-GRPO** model localizes, hypothesizes, and patches the defect in a single step inside the sandboxed environment.
"""
)
with gr.Row():
with gr.Column(scale=1, elem_classes=["glass-panel"]):
bug_dropdown = gr.Dropdown(
choices=bug_options,
value=bug_options[0] if bug_options else None,
label="Choose a Curriculum Bug",
interactive=True
)
bug_code_viewer = gr.Code(
language="python",
label="Buggy Code Input",
interactive=False,
lines=12,
elem_classes=["code-container"]
)
error_msg_viewer = gr.Textbox(
label="Sandbox Initial Error Output",
interactive=False,
lines=3
)
with gr.Column(scale=2, elem_classes=["glass-panel"]):
gr.Markdown("### 🧠 Model Cognitive Loop Trajectory")
trajectory_output = gr.Markdown(value="Loading initial trajectory...")
def update_explorer(name):
code, err, traj = get_bug_details(name, eval_data)
return code, err, traj
bug_dropdown.change(
fn=update_explorer,
inputs=bug_dropdown,
outputs=[bug_code_viewer, error_msg_viewer, trajectory_output]
)
demo.load(
fn=lambda: update_explorer(bug_options[0]) if bug_options else ("", "", ""),
outputs=[bug_code_viewer, error_msg_viewer, trajectory_output]
)
with gr.TabItem("📊 Benchmark Leaderboard"):
gr.Markdown(
"""
### Benchmark Rankings on 90 Hand-Validated Bugs
We rank models based on their average score across 3 tiers of difficulty (Easy, Medium, Hard).
*Scores measure formatting, hypothesis accuracy, fault localization, and test suite pass rate.*
"""
)
leaderboard_frame = gr.HTML(value=render_leaderboard_html(eval_data.get("summary", DEFAULT_STATS["summary"])))
with gr.Row(elem_classes=["glass-panel", "mt-8"]):
with gr.Column():
gr.Markdown(
"""
### 📈 Training Learning Curves (GRPO)
Our reinforcement learning runs demonstrate rapid policy adaptation of Qwen-3B-Coder:
- **Format compliance**: Hit 1.0 (max) within the first 50 steps.
- **Total Reward**: Climbed from baseline ~0.4 to peaks of ~1.0 by step 250.
- **Curriculum Transition**: Textbook drop-and-recover curve at step 150 (Tier 2 escalation).
"""
)
with gr.Column():
gr.Image("images/total.png", label="GRPO Total Reward Curve")
gr.Image("images/format_compliance.png", label="Format Compliance Curve")
with gr.TabItem("🛡️ Sandbox Playground"):
gr.Markdown(
"""
### Hardened Sandbox Execution Environment
Test arbitrary Python code against custom tests. Our execution sandbox enforces CPU limits (10s), memory limits (256MB), and blocks unsafe functions.
"""
)
with gr.Row():
with gr.Column(scale=1, elem_classes=["glass-panel"]):
user_code = gr.Code(
language="python",
label="Python Code",
value="def add(a, b):\n return a + b",
lines=10,
elem_classes=["code-container"]
)
test_suite_code = gr.Code(
language="python",
label="Test Assertions (must print PASS or FAIL)",
value="assert add(2, 3) == 5\nprint('PASS')",
lines=5,
elem_classes=["code-container"]
)
run_btn = gr.Button("🚀 Run in Sandbox", elem_classes=["accent-btn"])
with gr.Column(scale=1, elem_classes=["glass-panel"]):
sandbox_status = gr.Textbox(label="Sandbox Status", value="Ready")
sandbox_stdout = gr.Code(
label="Terminal Output (Stdout/Stderr)",
interactive=False,
lines=15,
elem_classes=["code-container"]
)
run_btn.click(
fn=run_sandbox_code,
inputs=[user_code, test_suite_code],
outputs=[sandbox_stdout, sandbox_status]
)
with gr.TabItem("📝 Technical Report"):
gr.Markdown(
"""
### Research Writeup & Key Insights
Read our draft paper detailing the project context, reward shaping formulations, and empirical comparisons.
"""
)
with gr.Group(elem_classes=["glass-panel"]):
gr.Markdown(value=read_technical_report())
gr.Markdown(
"""
---
Submitted to the Meta + PyTorch + Hugging Face OpenEnv Hackathon |
View GitHub Repository
"""
)
if __name__ == "__main__":
demo.launch(server_name="0.0.0.0", server_port=7860)