You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
34 lines
552 B
34 lines
552 B
context_window: 256
|
|
patch_len: 16
|
|
stride: 8
|
|
padding_patch: None
|
|
time_layers: 6
|
|
d_model: 512
|
|
n_heads: 8
|
|
d_ff: 2048
|
|
norm: BatchNorm
|
|
attn_dropout: 0.1
|
|
dropout: 0.1
|
|
act: gelu
|
|
bias_ff: True
|
|
qkv_bias: True
|
|
pre_norm: False
|
|
llm_name: "DistilBert"
|
|
llm_layers: 6
|
|
hidden_size: 768
|
|
freeze_layers: 3
|
|
|
|
time_project_bias: False
|
|
symbolic_project_bias: False
|
|
embed_dim: 128
|
|
|
|
# Update on momentum encoders
|
|
momentum: 0.995
|
|
# Temperature coefficient
|
|
temp: 0.07
|
|
# The size of the queue
|
|
queue_size: 60000
|
|
|
|
# The proportion of momentum distillation pseudo-targets
|
|
alpha: 0.4
|