Pull human AI evaluation data into Python — for notebooks, scripts, and ML pipelines

pip install grandjury

from grandjury import GJClient

gj = GJClient(token="gj_pat_...")

# Leaderboard — pass rates, flags, rankings as pandas
lb = gj.benchmarks.leaderboard("7a97ef08-3d89-42c5-a4db-f6b43b4700ca")
df_lb = lb.to_pandas()
print(df_lb)

models = gj.arena("7a97ef08-...").models()
df_models = models.to_pandas()
print(models)  # ModelList(15 models)

v = gj.arena("7a97ef08-...").votes(
    model=["xai-grok-4-1-fast", "xai-grok-4"]
)

print(v)  # ResultSet(not loaded)

df = v.to_pandas()  # Loads all pages
print(df.shape)     # (3166, 8)

# Or iterate lazily
for vote in v:
    if not vote["verdict"]:
        print(vote["feedback"])

results = gj.arena("7a97ef08-...").votes()

# Nothing loaded yet
print(results)  # ResultSet(not loaded)

# Just the first 10 rows
df_preview = results.head(10).to_pandas()

# Now eagerly load all pages
results.fetch_all()
df_all = results.to_pandas()

from grandjury import GJClient
import pandas as pd

gj = GJClient(token="gj_pat_...")
arena_id = "7a97ef08-3d89-42c5-a4db-f6b43b4700ca"

# 1. Leaderboard overview
lb = gj.benchmarks.leaderboard(arena_id)
df_lb = lb.to_pandas()
print(df_lb[["model_name", "pass_rate", "total_votes", "flag_count"]])

# 2. Which models are enrolled?
models = gj.arena(arena_id).models()
print(models)

# 3. Deep dive — Grok votes
grok_votes = gj.arena(arena_id).votes(
    model=["xai-grok-4-1-fast", "xai-grok-4"]
)
df = grok_votes.to_pandas()

# 4. Flag pattern analysis
flags = df[df["verdict"] == False]
print(f"Flag rate: {len(flags)/len(df)*100:.1f}%")
print(flags["feedback"].value_counts().head(10))