| 12345678910111213141516171819202122232425262728293031 |
- # hydra:
- # run:
- # dir: ./my_output_dir
- root_dir: ${hydra:runtime.cwd}
- eval:
- save_path: checkpoints/cs336_lm_owt/
- iteration: 5000
- prompt: That empowerment comes with added scrutiny
- max_new_tokens: 64
- temperature: 1.0
- top_k: 50
- model_type: cs336_lm
- model:
- vocab_size: 32000
- context_length: 256
- d_model: 512
- num_layers: 4
- num_heads: 16
- d_ff: 1344
- rope_theta: 10000.0
- dataset_name: owt # `TinyStories` or `owt`, change as needed
- dataset_split: valid # `train` or `valid`, change as needed
- tokenizer_dir: ${root_dir}/tokenizers/${dataset_name}_${dataset_split}/ # save path for tokenizer
- tokenizer:
- vocab_path: ${tokenizer_dir}/vocab.pkl
- merges_path: ${tokenizer_dir}/merges.pkl
- special_tokens: ["<|endoftext|>"]
|
