← All topics

code snippet

1 capture, most recent first.

Saved image — no attribution recorded

— saved image

# collect R1 rollouts from API
import os
from openai import OpenAI
base_url = os.getenv("DEEPSEEK_API_URL")
api_key = os.getenv("DEEPSEEK_API_KEY")
client = OpenAI(base_url=base_url, api_key=api_key)
results = vf_env.eval_api(client, "deepseek-reasoner", max_concurrent=32,
sampling_args={"temperature": 0.6})

# make dataset from results (flatten reward scores)
def flatten_rewards(rewards: dict) -> list[float]:
    return [sum(r) for r in zip(*rewards.values())]
dataset = Dataset.from_dict({
    "prompt": results['prompt'],
    "completion": results['completion'],
    "answer": results['answer'],
    "reward": flatten_rewards(results['rewards']),
})

# filter to top half of rows by rewards
dataset = dataset.sort("rewards", reverse=True).select(range(len(dataset) // 2))
print(dataset[0])

# save to hub
dataset.push_to_hub("R1-reverse-wikipedia-paragraphs-v1-1000")

———

will brown @willccbb · 4h
cheat code for making RL warmup data

> QUOTED (code block):
> # collect R1 rollouts from API
> import os
> from openai import OpenAI
> base_url = os.getenv("DEEPSEEK_API_URL")
> api_key = os.getenv("DEEPSEEK_API_KEY")
> client = OpenAI(base_url=base_url, api_key=api_key)
> results = vf_env.eval_api(client, "deepseek-reasoner", max_concurrent=32, sampling_args={"temperature": 0.6})
>
> # make dataset from results (flatten reward scores)
> def flatten_rewards(rewards: dict) -> list[float]:
>     return [sum(r) for r in zip(*rewards.values())]
> dataset = Dataset.from_dict({
>     "prompt": results['prompt'],
>     "completion": results['completion'],
>     "answer": results['answer'],
>     "reward": flatten_rewards(results['rewards']),
> })
>
> # filter to top half of rows by rewards
> dataset = dataset.sort("rewards", reverse=True).select(range(len(dataset) // 2))
> print(dataset[0])
>
> # save to hub
> dataset.push_to_hub("R1-reverse-wikipedia-paragraphs-v1-1000")
[5 comments, 3 retweets, 79 likes, 3.7K views]

will brown @willccbb · 4h
synthetic data engines = RL environments = eval harnesses
Note from Claude Sonnet 5

A technical tweet from will brown (RL/eval tooling developer, "verifiers" framework author) sharing a Python code snippet for generating RL warmup training data by collecting rollouts from DeepSeek-R1 via API, filtering to top-reward-half, and pushing to HuggingFace Hub. Technical ML-engineering reference, tangential to the project's interest in RL training methods relevant to alignment.

reinforcement learningdeepseeksynthetic datatwitterml engineeringcode snippet