prevent gradient explosion

rl_game/get_up/config/ppo_cfg.yaml

@@ -17,7 +17,7 @@ params:
           name: default
         sigma_init:
           name: const_initializer
-          val: 1.2
+          val: 0.5
         fixed_sigma: False
     mlp:
       units: [512, 256, 128]
@@ -39,21 +39,21 @@ params:
     reward_shaper:
       scale_value: 1.0
     normalize_advantage: True
-    gamma: 0.98
+    gamma: 0.96
     tau: 0.95
-    learning_rate: 5e-4
+    learning_rate: 3e-4
     lr_schedule: adaptive
-    kl_threshold: 0.008
+    kl_threshold: 0.015
     score_to_win: 20000
     max_epochs: 500
     save_best_after: 50
     save_frequency: 100
-    grad_norm: 0.5
-    entropy_coef: 0.008
+    grad_norm: 1.0
+    entropy_coef: 0.005
     truncate_grads: True
     bounds_loss_coef: 0.001
     e_clip: 0.2
-    horizon_length: 128
+    horizon_length: 65536
     minibatch_size: 8192
     mini_epochs: 4
     critic_coef: 1