2025-06-23 13:49:32 Config: {'all_joints': [[0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12], [13], [14], [15], [16], [17], [18], [19], [20], [21], [22], [23], [24], [25], [26], [27]], 'all_joints_names': ['nose(bottom)', 'nose(tip)', 'nose(top)', 'pad(top)(left)', 'pad(side)(left)', 'pad(center)', 'pad(top)(right)', 'pad(side)(right)', 'lowerlip', 'upperlip(left)', 'upperlip(right)', 'eye(front)(left)', 'eye(top)(left)', 'eye(back)(left)', 'eye(bottom)(left)', 'eye(front)(right)', 'eye(top)(right)', 'eye(back)(right)', 'eye(bottom)(right)', 'ear(base)(left)', 'ear(top)(left)', 'ear(tip)(left)', 'ear(bottom)(left)', 'ear(base)(right)', 'ear(top)(right)', 'ear(tip)(right)', 'ear(bottom)(right)', 'ref(head-post)'], 'alpha_r': 0.02, 'apply_prob': 0.5, 'batch_size': 1, 'contrast': {'clahe': True, 'claheratio': 0.1, 'histeq': True, 'histeqratio': 0.1}, 'convolution': {'edge': False, 'emboss': {'alpha': [0.0, 1.0], 'strength': [0.5, 1.5]}, 'embossratio': 0.1, 'sharpen': False, 'sharpenratio': 0.3}, 'crop_pad': 0, 'cropratio': 0.4, 'dataset': 'training-datasets/iteration-2/UnaugmentedDataSet_rig1_aug2May28/rig1_aug2_houlab95shuffle1.mat', 'dataset_type': 'imgaug', 'decay_steps': 30000, 'deterministic': False, 'display_iters': 1000, 'fg_fraction': 0.25, 'global_scale': 0.8, 'init_weights': '/data/disk1/daruwal/repos/mouse-fe-analysis/dlc-training/projects/rig1_aug2-houlab-2025-05-28/dlc-models/iteration-1/rig1_aug2May28-trainset95shuffle1/train/snapshot-3100000', 'intermediate_supervision': False, 'intermediate_supervision_layer': 12, 'location_refinement': True, 'locref_huber_loss': True, 'locref_loss_weight': 0.05, 'locref_stdev': 7.2801, 'log_dir': 'log', 'lr_init': 0.0005, 'max_input_size': 1500, 'mean_pixel': [123.68, 116.779, 103.939], 'metadataset': 'training-datasets/iteration-2/UnaugmentedDataSet_rig1_aug2May28/Documentation_data-rig1_aug2_95shuffle1.pickle', 'min_input_size': 64, 'mirror': False, 'multi_stage': False, 'multi_step': [[0.001, 100000]], 'net_type': 'resnet_50', 'num_joints': 28, 'optimizer': 'sgd', 'pairwise_huber_loss': False, 'pairwise_predict': False, 'partaffinityfield_predict': False, 'pos_dist_thresh': 17, 'project_path': '/data/disk1/daruwal/repos/mouse-fe-analysis/dlc-training/projects/rig1_aug2-houlab-2025-05-28', 'regularize': False, 'rotation': 25, 'rotratio': 0.4, 'save_iters': 50000, 'scale_jitter_lo': 0.5, 'scale_jitter_up': 1.25, 'scoremap_dir': 'test', 'shuffle': True, 'snapshot_prefix': '/data/disk1/daruwal/repos/mouse-fe-analysis/dlc-training/projects/rig1_aug2-houlab-2025-05-28/dlc-models/iteration-2/rig1_aug2May28-trainset95shuffle1/train/snapshot', 'stride': 8.0, 'weigh_negatives': False, 'weigh_only_present_joints': False, 'weigh_part_predictions': False, 'weight_decay': 0.0001} 2025-06-23 13:50:02 iteration: 3101000 loss: 0.0031 lr: 0.001 2025-06-23 13:50:22 iteration: 3102000 loss: 0.0027 lr: 0.001 2025-06-23 13:50:41 iteration: 3103000 loss: 0.0025 lr: 0.001 2025-06-23 13:51:00 iteration: 3104000 loss: 0.0023 lr: 0.001 2025-06-23 13:51:19 iteration: 3105000 loss: 0.0020 lr: 0.001 2025-06-23 13:51:39 iteration: 3106000 loss: 0.0020 lr: 0.001 2025-06-23 13:51:58 iteration: 3107000 loss: 0.0020 lr: 0.001 2025-06-23 13:52:17 iteration: 3108000 loss: 0.0019 lr: 0.001 2025-06-23 13:52:36 iteration: 3109000 loss: 0.0019 lr: 0.001 2025-06-23 13:52:55 iteration: 3110000 loss: 0.0019 lr: 0.001 2025-06-23 13:53:15 iteration: 3111000 loss: 0.0018 lr: 0.001 2025-06-23 13:53:35 iteration: 3112000 loss: 0.0018 lr: 0.001 2025-06-23 13:53:54 iteration: 3113000 loss: 0.0018 lr: 0.001 2025-06-23 13:54:14 iteration: 3114000 loss: 0.0018 lr: 0.001 2025-06-23 13:54:32 iteration: 3115000 loss: 0.0018 lr: 0.001 2025-06-23 13:54:51 iteration: 3116000 loss: 0.0017 lr: 0.001 2025-06-23 13:55:11 iteration: 3117000 loss: 0.0016 lr: 0.001 2025-06-23 13:55:30 iteration: 3118000 loss: 0.0017 lr: 0.001 2025-06-23 13:55:49 iteration: 3119000 loss: 0.0017 lr: 0.001 2025-06-23 13:56:07 iteration: 3120000 loss: 0.0016 lr: 0.001 2025-06-23 13:56:27 iteration: 3121000 loss: 0.0016 lr: 0.001 2025-06-23 13:56:45 iteration: 3122000 loss: 0.0016 lr: 0.001 2025-06-23 13:57:05 iteration: 3123000 loss: 0.0017 lr: 0.001 2025-06-23 13:57:24 iteration: 3124000 loss: 0.0016 lr: 0.001 2025-06-23 13:57:43 iteration: 3125000 loss: 0.0015 lr: 0.001 2025-06-23 13:58:02 iteration: 3126000 loss: 0.0015 lr: 0.001 2025-06-23 13:58:21 iteration: 3127000 loss: 0.0015 lr: 0.001 2025-06-23 13:58:41 iteration: 3128000 loss: 0.0016 lr: 0.001 2025-06-23 13:58:59 iteration: 3129000 loss: 0.0015 lr: 0.001 2025-06-23 13:59:18 iteration: 3130000 loss: 0.0015 lr: 0.001 2025-06-23 13:59:37 iteration: 3131000 loss: 0.0016 lr: 0.001 2025-06-23 13:59:56 iteration: 3132000 loss: 0.0015 lr: 0.001 2025-06-23 14:00:16 iteration: 3133000 loss: 0.0015 lr: 0.001 2025-06-23 14:00:37 iteration: 3134000 loss: 0.0015 lr: 0.001 2025-06-23 14:00:55 iteration: 3135000 loss: 0.0016 lr: 0.001 2025-06-23 14:01:14 iteration: 3136000 loss: 0.0015 lr: 0.001 2025-06-23 14:01:33 iteration: 3137000 loss: 0.0015 lr: 0.001 2025-06-23 14:01:52 iteration: 3138000 loss: 0.0015 lr: 0.001 2025-06-23 14:02:11 iteration: 3139000 loss: 0.0015 lr: 0.001 2025-06-23 14:02:30 iteration: 3140000 loss: 0.0014 lr: 0.001 2025-06-23 14:02:49 iteration: 3141000 loss: 0.0015 lr: 0.001 2025-06-23 14:03:08 iteration: 3142000 loss: 0.0015 lr: 0.001 2025-06-23 14:03:27 iteration: 3143000 loss: 0.0014 lr: 0.001 2025-06-23 14:03:47 iteration: 3144000 loss: 0.0015 lr: 0.001 2025-06-23 14:04:06 iteration: 3145000 loss: 0.0014 lr: 0.001 2025-06-23 14:04:25 iteration: 3146000 loss: 0.0015 lr: 0.001 2025-06-23 14:04:44 iteration: 3147000 loss: 0.0015 lr: 0.001 2025-06-23 14:05:03 iteration: 3148000 loss: 0.0014 lr: 0.001 2025-06-23 14:05:23 iteration: 3149000 loss: 0.0015 lr: 0.001 2025-06-23 14:05:42 iteration: 3150000 loss: 0.0014 lr: 0.001 2025-06-23 14:06:02 iteration: 3151000 loss: 0.0014 lr: 0.001 2025-06-23 14:06:22 iteration: 3152000 loss: 0.0014 lr: 0.001 2025-06-23 14:06:41 iteration: 3153000 loss: 0.0014 lr: 0.001 2025-06-23 14:07:00 iteration: 3154000 loss: 0.0014 lr: 0.001 2025-06-23 14:07:20 iteration: 3155000 loss: 0.0014 lr: 0.001 2025-06-23 14:07:40 iteration: 3156000 loss: 0.0014 lr: 0.001 2025-06-23 14:08:00 iteration: 3157000 loss: 0.0014 lr: 0.001 2025-06-23 14:08:18 iteration: 3158000 loss: 0.0014 lr: 0.001 2025-06-23 14:08:37 iteration: 3159000 loss: 0.0014 lr: 0.001 2025-06-23 14:08:56 iteration: 3160000 loss: 0.0014 lr: 0.001 2025-06-23 14:09:16 iteration: 3161000 loss: 0.0013 lr: 0.001 2025-06-23 14:09:36 iteration: 3162000 loss: 0.0014 lr: 0.001 2025-06-23 14:09:56 iteration: 3163000 loss: 0.0014 lr: 0.001 2025-06-23 14:10:16 iteration: 3164000 loss: 0.0013 lr: 0.001 2025-06-23 14:10:35 iteration: 3165000 loss: 0.0013 lr: 0.001 2025-06-23 14:10:55 iteration: 3166000 loss: 0.0013 lr: 0.001 2025-06-23 14:11:14 iteration: 3167000 loss: 0.0014 lr: 0.001 2025-06-23 14:11:33 iteration: 3168000 loss: 0.0014 lr: 0.001 2025-06-23 14:11:51 iteration: 3169000 loss: 0.0013 lr: 0.001 2025-06-23 14:12:10 iteration: 3170000 loss: 0.0013 lr: 0.001 2025-06-23 14:12:30 iteration: 3171000 loss: 0.0014 lr: 0.001 2025-06-23 14:12:49 iteration: 3172000 loss: 0.0013 lr: 0.001 2025-06-23 14:13:08 iteration: 3173000 loss: 0.0013 lr: 0.001 2025-06-23 14:13:28 iteration: 3174000 loss: 0.0013 lr: 0.001 2025-06-23 14:13:48 iteration: 3175000 loss: 0.0013 lr: 0.001 2025-06-23 14:14:09 iteration: 3176000 loss: 0.0013 lr: 0.001 2025-06-23 14:14:29 iteration: 3177000 loss: 0.0013 lr: 0.001 2025-06-23 14:14:48 iteration: 3178000 loss: 0.0013 lr: 0.001 2025-06-23 14:15:07 iteration: 3179000 loss: 0.0013 lr: 0.001 2025-06-23 14:15:26 iteration: 3180000 loss: 0.0013 lr: 0.001 2025-06-23 14:15:47 iteration: 3181000 loss: 0.0013 lr: 0.001 2025-06-23 14:16:06 iteration: 3182000 loss: 0.0013 lr: 0.001 2025-06-23 14:16:26 iteration: 3183000 loss: 0.0013 lr: 0.001 2025-06-23 14:16:45 iteration: 3184000 loss: 0.0013 lr: 0.001 2025-06-23 14:17:04 iteration: 3185000 loss: 0.0013 lr: 0.001 2025-06-23 14:17:24 iteration: 3186000 loss: 0.0013 lr: 0.001 2025-06-23 14:17:43 iteration: 3187000 loss: 0.0013 lr: 0.001 2025-06-23 14:18:03 iteration: 3188000 loss: 0.0013 lr: 0.001 2025-06-23 14:18:23 iteration: 3189000 loss: 0.0013 lr: 0.001 2025-06-23 14:18:42 iteration: 3190000 loss: 0.0013 lr: 0.001 2025-06-23 14:19:01 iteration: 3191000 loss: 0.0013 lr: 0.001 2025-06-23 14:19:20 iteration: 3192000 loss: 0.0013 lr: 0.001 2025-06-23 14:19:39 iteration: 3193000 loss: 0.0013 lr: 0.001 2025-06-23 14:19:59 iteration: 3194000 loss: 0.0013 lr: 0.001 2025-06-23 14:20:19 iteration: 3195000 loss: 0.0013 lr: 0.001 2025-06-23 14:20:38 iteration: 3196000 loss: 0.0013 lr: 0.001 2025-06-23 14:20:57 iteration: 3197000 loss: 0.0013 lr: 0.001 2025-06-23 14:21:16 iteration: 3198000 loss: 0.0013 lr: 0.001 2025-06-23 14:21:35 iteration: 3199000 loss: 0.0013 lr: 0.001 2025-06-23 14:21:55 iteration: 3200000 loss: 0.0013 lr: 0.001 2025-06-23 14:21:56 Config: {'all_joints': [[0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12], [13], [14], [15], [16], [17], [18], [19], [20], [21], [22], [23], [24], [25], [26], [27]], 'all_joints_names': ['nose(bottom)', 'nose(tip)', 'nose(top)', 'pad(top)(left)', 'pad(side)(left)', 'pad(center)', 'pad(top)(right)', 'pad(side)(right)', 'lowerlip', 'upperlip(left)', 'upperlip(right)', 'eye(front)(left)', 'eye(top)(left)', 'eye(back)(left)', 'eye(bottom)(left)', 'eye(front)(right)', 'eye(top)(right)', 'eye(back)(right)', 'eye(bottom)(right)', 'ear(base)(left)', 'ear(top)(left)', 'ear(tip)(left)', 'ear(bottom)(left)', 'ear(base)(right)', 'ear(top)(right)', 'ear(tip)(right)', 'ear(bottom)(right)', 'ref(head-post)'], 'batch_size': 1, 'crop_pad': 0, 'dataset': 'training-datasets/iteration-2/UnaugmentedDataSet_rig1_aug2May28/rig1_aug2_houlab95shuffle1.mat', 'dataset_type': 'imgaug', 'deterministic': False, 'fg_fraction': 0.25, 'global_scale': 0.8, 'init_weights': '/home/daruwal/micromamba/envs/mouse-fe-analysis/lib/python3.10/site-packages/deeplabcut/pose_estimation_tensorflow/models/pretrained/resnet_v1_50.ckpt', 'intermediate_supervision': False, 'intermediate_supervision_layer': 12, 'location_refinement': True, 'locref_huber_loss': True, 'locref_loss_weight': 1.0, 'locref_stdev': 7.2801, 'log_dir': 'log', 'mean_pixel': [123.68, 116.779, 103.939], 'mirror': False, 'net_type': 'resnet_50', 'num_joints': 28, 'optimizer': 'sgd', 'pairwise_huber_loss': True, 'pairwise_predict': False, 'partaffinityfield_predict': False, 'regularize': False, 'scoremap_dir': 'test', 'shuffle': True, 'snapshot_prefix': '/data/disk1/daruwal/repos/mouse-fe-analysis/dlc-training/projects/rig1_aug2-houlab-2025-05-28/dlc-models/iteration-2/rig1_aug2May28-trainset95shuffle1/test/snapshot', 'stride': 8.0, 'weigh_negatives': False, 'weigh_only_present_joints': False, 'weigh_part_predictions': False, 'weight_decay': 0.0001} 2025-06-23 14:25:26 Config: {'all_joints': [[0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12], [13], [14], [15], [16], [17], [18], [19], [20], [21], [22], [23], [24], [25], [26], [27]], 'all_joints_names': ['nose(bottom)', 'nose(tip)', 'nose(top)', 'pad(top)(left)', 'pad(side)(left)', 'pad(center)', 'pad(top)(right)', 'pad(side)(right)', 'lowerlip', 'upperlip(left)', 'upperlip(right)', 'eye(front)(left)', 'eye(top)(left)', 'eye(back)(left)', 'eye(bottom)(left)', 'eye(front)(right)', 'eye(top)(right)', 'eye(back)(right)', 'eye(bottom)(right)', 'ear(base)(left)', 'ear(top)(left)', 'ear(tip)(left)', 'ear(bottom)(left)', 'ear(base)(right)', 'ear(top)(right)', 'ear(tip)(right)', 'ear(bottom)(right)', 'ref(head-post)'], 'alpha_r': 0.02, 'apply_prob': 0.5, 'batch_size': 1, 'contrast': {'clahe': True, 'claheratio': 0.1, 'histeq': True, 'histeqratio': 0.1}, 'convolution': {'edge': False, 'emboss': {'alpha': [0.0, 1.0], 'strength': [0.5, 1.5]}, 'embossratio': 0.1, 'sharpen': False, 'sharpenratio': 0.3}, 'crop_pad': 0, 'cropratio': 0.4, 'dataset': 'training-datasets/iteration-2/UnaugmentedDataSet_rig1_aug2May28/rig1_aug2_houlab95shuffle1.mat', 'dataset_type': 'imgaug', 'decay_steps': 30000, 'deterministic': False, 'display_iters': 1000, 'fg_fraction': 0.25, 'global_scale': 0.8, 'init_weights': '/data/disk1/daruwal/repos/mouse-fe-analysis/dlc-training/projects/rig1_aug2-houlab-2025-05-28/dlc-models/iteration-2/rig1_aug2May28-trainset95shuffle1/train/snapshot-3200000', 'intermediate_supervision': False, 'intermediate_supervision_layer': 12, 'location_refinement': True, 'locref_huber_loss': True, 'locref_loss_weight': 0.05, 'locref_stdev': 7.2801, 'log_dir': 'log', 'lr_init': 0.0005, 'max_input_size': 1500, 'mean_pixel': [123.68, 116.779, 103.939], 'metadataset': 'training-datasets/iteration-2/UnaugmentedDataSet_rig1_aug2May28/Documentation_data-rig1_aug2_95shuffle1.pickle', 'min_input_size': 64, 'mirror': False, 'multi_stage': False, 'multi_step': [[0.001, 500000]], 'net_type': 'resnet_50', 'num_joints': 28, 'optimizer': 'sgd', 'pairwise_huber_loss': False, 'pairwise_predict': False, 'partaffinityfield_predict': False, 'pos_dist_thresh': 17, 'project_path': '/data/disk1/daruwal/repos/mouse-fe-analysis/dlc-training/projects/rig1_aug2-houlab-2025-05-28', 'regularize': False, 'rotation': 25, 'rotratio': 0.4, 'save_iters': 50000, 'scale_jitter_lo': 0.5, 'scale_jitter_up': 1.25, 'scoremap_dir': 'test', 'shuffle': True, 'snapshot_prefix': '/data/disk1/daruwal/repos/mouse-fe-analysis/dlc-training/projects/rig1_aug2-houlab-2025-05-28/dlc-models/iteration-2/rig1_aug2May28-trainset95shuffle1/train/snapshot', 'stride': 8.0, 'weigh_negatives': False, 'weigh_only_present_joints': False, 'weigh_part_predictions': False, 'weight_decay': 0.0001} 2025-06-23 14:25:56 iteration: 3201000 loss: 0.0012 lr: 0.001 2025-06-23 14:26:15 iteration: 3202000 loss: 0.0013 lr: 0.001 2025-06-23 14:26:35 iteration: 3203000 loss: 0.0013 lr: 0.001 2025-06-23 14:26:53 iteration: 3204000 loss: 0.0013 lr: 0.001 2025-06-23 14:27:12 iteration: 3205000 loss: 0.0012 lr: 0.001 2025-06-23 14:27:32 iteration: 3206000 loss: 0.0012 lr: 0.001 2025-06-23 14:27:51 iteration: 3207000 loss: 0.0013 lr: 0.001 2025-06-23 14:28:10 iteration: 3208000 loss: 0.0012 lr: 0.001 2025-06-23 14:28:29 iteration: 3209000 loss: 0.0013 lr: 0.001 2025-06-23 14:28:49 iteration: 3210000 loss: 0.0013 lr: 0.001 2025-06-23 14:29:08 iteration: 3211000 loss: 0.0012 lr: 0.001 2025-06-23 14:29:27 iteration: 3212000 loss: 0.0012 lr: 0.001 2025-06-23 14:29:46 iteration: 3213000 loss: 0.0012 lr: 0.001 2025-06-23 14:30:06 iteration: 3214000 loss: 0.0012 lr: 0.001 2025-06-23 14:30:26 iteration: 3215000 loss: 0.0012 lr: 0.001 2025-06-23 14:30:46 iteration: 3216000 loss: 0.0012 lr: 0.001 2025-06-23 14:31:05 iteration: 3217000 loss: 0.0012 lr: 0.001 2025-06-23 14:31:24 iteration: 3218000 loss: 0.0012 lr: 0.001 2025-06-23 14:31:43 iteration: 3219000 loss: 0.0012 lr: 0.001 2025-06-23 14:32:02 iteration: 3220000 loss: 0.0012 lr: 0.001 2025-06-23 14:32:21 iteration: 3221000 loss: 0.0012 lr: 0.001 2025-06-23 14:32:40 iteration: 3222000 loss: 0.0012 lr: 0.001 2025-06-23 14:32:59 iteration: 3223000 loss: 0.0012 lr: 0.001 2025-06-23 14:33:18 iteration: 3224000 loss: 0.0012 lr: 0.001 2025-06-23 14:33:38 iteration: 3225000 loss: 0.0012 lr: 0.001 2025-06-23 14:33:57 iteration: 3226000 loss: 0.0012 lr: 0.001 2025-06-23 14:34:16 iteration: 3227000 loss: 0.0012 lr: 0.001 2025-06-23 14:34:35 iteration: 3228000 loss: 0.0012 lr: 0.001 2025-06-23 14:34:54 iteration: 3229000 loss: 0.0012 lr: 0.001 2025-06-23 14:35:14 iteration: 3230000 loss: 0.0012 lr: 0.001 2025-06-23 14:35:33 iteration: 3231000 loss: 0.0013 lr: 0.001 2025-06-23 14:35:52 iteration: 3232000 loss: 0.0012 lr: 0.001 2025-06-23 14:36:11 iteration: 3233000 loss: 0.0012 lr: 0.001 2025-06-23 14:36:32 iteration: 3234000 loss: 0.0012 lr: 0.001 2025-06-23 14:36:52 iteration: 3235000 loss: 0.0012 lr: 0.001 2025-06-23 14:37:11 iteration: 3236000 loss: 0.0012 lr: 0.001 2025-06-23 14:37:30 iteration: 3237000 loss: 0.0012 lr: 0.001 2025-06-23 14:37:48 iteration: 3238000 loss: 0.0012 lr: 0.001 2025-06-23 14:38:07 iteration: 3239000 loss: 0.0012 lr: 0.001 2025-06-23 14:38:26 iteration: 3240000 loss: 0.0012 lr: 0.001 2025-06-23 14:38:45 iteration: 3241000 loss: 0.0012 lr: 0.001 2025-06-23 14:39:04 iteration: 3242000 loss: 0.0012 lr: 0.001 2025-06-23 14:39:24 iteration: 3243000 loss: 0.0012 lr: 0.001 2025-06-23 14:39:43 iteration: 3244000 loss: 0.0012 lr: 0.001 2025-06-23 14:40:03 iteration: 3245000 loss: 0.0012 lr: 0.001 2025-06-23 14:40:23 iteration: 3246000 loss: 0.0012 lr: 0.001 2025-06-23 14:40:42 iteration: 3247000 loss: 0.0012 lr: 0.001 2025-06-23 14:41:02 iteration: 3248000 loss: 0.0012 lr: 0.001 2025-06-23 14:41:21 iteration: 3249000 loss: 0.0012 lr: 0.001 2025-06-23 14:41:41 iteration: 3250000 loss: 0.0012 lr: 0.001 2025-06-23 14:42:00 iteration: 3251000 loss: 0.0012 lr: 0.001 2025-06-23 14:42:20 iteration: 3252000 loss: 0.0012 lr: 0.001 2025-06-23 14:42:39 iteration: 3253000 loss: 0.0012 lr: 0.001 2025-06-23 14:42:58 iteration: 3254000 loss: 0.0012 lr: 0.001 2025-06-23 14:43:18 iteration: 3255000 loss: 0.0012 lr: 0.001 2025-06-23 14:43:37 iteration: 3256000 loss: 0.0012 lr: 0.001 2025-06-23 14:43:56 iteration: 3257000 loss: 0.0012 lr: 0.001 2025-06-23 14:44:15 iteration: 3258000 loss: 0.0012 lr: 0.001 2025-06-23 14:44:34 iteration: 3259000 loss: 0.0012 lr: 0.001 2025-06-23 14:44:52 iteration: 3260000 loss: 0.0012 lr: 0.001 2025-06-23 14:45:12 iteration: 3261000 loss: 0.0011 lr: 0.001 2025-06-23 14:45:30 iteration: 3262000 loss: 0.0012 lr: 0.001 2025-06-23 14:45:49 iteration: 3263000 loss: 0.0012 lr: 0.001 2025-06-23 14:46:09 iteration: 3264000 loss: 0.0012 lr: 0.001 2025-06-23 14:46:30 iteration: 3265000 loss: 0.0012 lr: 0.001 2025-06-23 14:46:50 iteration: 3266000 loss: 0.0012 lr: 0.001 2025-06-23 14:47:11 iteration: 3267000 loss: 0.0012 lr: 0.001 2025-06-23 14:47:32 iteration: 3268000 loss: 0.0012 lr: 0.001 2025-06-23 14:47:51 iteration: 3269000 loss: 0.0011 lr: 0.001 2025-06-23 14:48:11 iteration: 3270000 loss: 0.0012 lr: 0.001 2025-06-23 14:48:42 iteration: 3271000 loss: 0.0012 lr: 0.001 2025-06-23 14:49:02 iteration: 3272000 loss: 0.0012 lr: 0.001 2025-06-23 14:49:21 iteration: 3273000 loss: 0.0012 lr: 0.001 2025-06-23 14:49:40 iteration: 3274000 loss: 0.0012 lr: 0.001 2025-06-23 14:50:00 iteration: 3275000 loss: 0.0012 lr: 0.001 2025-06-23 14:50:18 iteration: 3276000 loss: 0.0012 lr: 0.001 2025-06-23 14:50:37 iteration: 3277000 loss: 0.0012 lr: 0.001 2025-06-23 14:50:56 iteration: 3278000 loss: 0.0011 lr: 0.001 2025-06-23 14:51:15 iteration: 3279000 loss: 0.0012 lr: 0.001 2025-06-23 14:51:34 iteration: 3280000 loss: 0.0012 lr: 0.001 2025-06-23 14:51:53 iteration: 3281000 loss: 0.0012 lr: 0.001 2025-06-23 14:52:13 iteration: 3282000 loss: 0.0012 lr: 0.001 2025-06-23 14:52:32 iteration: 3283000 loss: 0.0012 lr: 0.001 2025-06-23 14:52:51 iteration: 3284000 loss: 0.0012 lr: 0.001 2025-06-23 14:53:12 iteration: 3285000 loss: 0.0012 lr: 0.001 2025-06-23 14:53:31 iteration: 3286000 loss: 0.0011 lr: 0.001 2025-06-23 14:53:49 iteration: 3287000 loss: 0.0012 lr: 0.001 2025-06-23 14:54:08 iteration: 3288000 loss: 0.0012 lr: 0.001 2025-06-23 14:54:27 iteration: 3289000 loss: 0.0012 lr: 0.001 2025-06-23 14:54:47 iteration: 3290000 loss: 0.0012 lr: 0.001 2025-06-23 14:55:07 iteration: 3291000 loss: 0.0012 lr: 0.001 2025-06-23 14:55:26 iteration: 3292000 loss: 0.0012 lr: 0.001 2025-06-23 14:55:46 iteration: 3293000 loss: 0.0012 lr: 0.001 2025-06-23 14:56:05 iteration: 3294000 loss: 0.0012 lr: 0.001 2025-06-23 14:56:24 iteration: 3295000 loss: 0.0012 lr: 0.001 2025-06-23 14:56:45 iteration: 3296000 loss: 0.0012 lr: 0.001 2025-06-23 14:57:04 iteration: 3297000 loss: 0.0012 lr: 0.001 2025-06-23 14:57:23 iteration: 3298000 loss: 0.0011 lr: 0.001 2025-06-23 14:57:41 iteration: 3299000 loss: 0.0011 lr: 0.001 2025-06-23 14:58:00 iteration: 3300000 loss: 0.0011 lr: 0.001 2025-06-23 14:58:19 iteration: 3301000 loss: 0.0012 lr: 0.001 2025-06-23 14:58:38 iteration: 3302000 loss: 0.0012 lr: 0.001 2025-06-23 14:58:57 iteration: 3303000 loss: 0.0012 lr: 0.001 2025-06-23 14:59:16 iteration: 3304000 loss: 0.0011 lr: 0.001 2025-06-23 14:59:35 iteration: 3305000 loss: 0.0011 lr: 0.001 2025-06-23 14:59:54 iteration: 3306000 loss: 0.0012 lr: 0.001 2025-06-23 15:00:14 iteration: 3307000 loss: 0.0012 lr: 0.001 2025-06-23 15:00:33 iteration: 3308000 loss: 0.0011 lr: 0.001 2025-06-23 15:00:52 iteration: 3309000 loss: 0.0012 lr: 0.001 2025-06-23 15:01:12 iteration: 3310000 loss: 0.0012 lr: 0.001 2025-06-23 15:01:31 iteration: 3311000 loss: 0.0012 lr: 0.001 2025-06-23 15:01:50 iteration: 3312000 loss: 0.0012 lr: 0.001 2025-06-23 15:02:09 iteration: 3313000 loss: 0.0011 lr: 0.001 2025-06-23 15:02:29 iteration: 3314000 loss: 0.0012 lr: 0.001 2025-06-23 15:02:48 iteration: 3315000 loss: 0.0012 lr: 0.001 2025-06-23 15:03:06 iteration: 3316000 loss: 0.0011 lr: 0.001 2025-06-23 15:03:25 iteration: 3317000 loss: 0.0012 lr: 0.001 2025-06-23 15:03:44 iteration: 3318000 loss: 0.0012 lr: 0.001 2025-06-23 15:04:03 iteration: 3319000 loss: 0.0012 lr: 0.001 2025-06-23 15:04:23 iteration: 3320000 loss: 0.0011 lr: 0.001 2025-06-23 15:04:43 iteration: 3321000 loss: 0.0012 lr: 0.001 2025-06-23 15:05:02 iteration: 3322000 loss: 0.0012 lr: 0.001 2025-06-23 15:05:22 iteration: 3323000 loss: 0.0012 lr: 0.001 2025-06-23 15:05:41 iteration: 3324000 loss: 0.0012 lr: 0.001 2025-06-23 15:05:59 iteration: 3325000 loss: 0.0012 lr: 0.001 2025-06-23 15:06:18 iteration: 3326000 loss: 0.0011 lr: 0.001 2025-06-23 15:06:37 iteration: 3327000 loss: 0.0012 lr: 0.001 2025-06-23 15:06:55 iteration: 3328000 loss: 0.0012 lr: 0.001 2025-06-23 15:07:14 iteration: 3329000 loss: 0.0011 lr: 0.001 2025-06-23 15:07:33 iteration: 3330000 loss: 0.0011 lr: 0.001 2025-06-23 15:07:52 iteration: 3331000 loss: 0.0011 lr: 0.001 2025-06-23 15:08:11 iteration: 3332000 loss: 0.0012 lr: 0.001 2025-06-23 15:08:31 iteration: 3333000 loss: 0.0011 lr: 0.001 2025-06-23 15:08:50 iteration: 3334000 loss: 0.0011 lr: 0.001 2025-06-23 15:09:10 iteration: 3335000 loss: 0.0011 lr: 0.001 2025-06-23 15:09:31 iteration: 3336000 loss: 0.0011 lr: 0.001 2025-06-23 15:09:50 iteration: 3337000 loss: 0.0011 lr: 0.001 2025-06-23 15:10:09 iteration: 3338000 loss: 0.0011 lr: 0.001 2025-06-23 15:10:29 iteration: 3339000 loss: 0.0011 lr: 0.001 2025-06-23 15:10:48 iteration: 3340000 loss: 0.0012 lr: 0.001 2025-06-23 15:11:07 iteration: 3341000 loss: 0.0011 lr: 0.001 2025-06-23 15:11:27 iteration: 3342000 loss: 0.0011 lr: 0.001 2025-06-23 15:11:47 iteration: 3343000 loss: 0.0011 lr: 0.001 2025-06-23 15:12:06 iteration: 3344000 loss: 0.0011 lr: 0.001 2025-06-23 15:12:25 iteration: 3345000 loss: 0.0011 lr: 0.001 2025-06-23 15:12:45 iteration: 3346000 loss: 0.0012 lr: 0.001 2025-06-23 15:13:04 iteration: 3347000 loss: 0.0011 lr: 0.001 2025-06-23 15:13:24 iteration: 3348000 loss: 0.0012 lr: 0.001 2025-06-23 15:13:43 iteration: 3349000 loss: 0.0012 lr: 0.001 2025-06-23 15:14:02 iteration: 3350000 loss: 0.0012 lr: 0.001 2025-06-23 15:14:21 iteration: 3351000 loss: 0.0011 lr: 0.001 2025-06-23 15:14:40 iteration: 3352000 loss: 0.0012 lr: 0.001 2025-06-23 15:14:59 iteration: 3353000 loss: 0.0011 lr: 0.001 2025-06-23 15:15:18 iteration: 3354000 loss: 0.0012 lr: 0.001 2025-06-23 15:15:37 iteration: 3355000 loss: 0.0011 lr: 0.001 2025-06-23 15:15:57 iteration: 3356000 loss: 0.0011 lr: 0.001 2025-06-23 15:16:16 iteration: 3357000 loss: 0.0011 lr: 0.001 2025-06-23 15:16:36 iteration: 3358000 loss: 0.0012 lr: 0.001 2025-06-23 15:16:54 iteration: 3359000 loss: 0.0011 lr: 0.001 2025-06-23 15:17:13 iteration: 3360000 loss: 0.0012 lr: 0.001 2025-06-23 15:17:32 iteration: 3361000 loss: 0.0011 lr: 0.001 2025-06-23 15:17:52 iteration: 3362000 loss: 0.0011 lr: 0.001 2025-06-23 15:18:10 iteration: 3363000 loss: 0.0011 lr: 0.001 2025-06-23 15:18:30 iteration: 3364000 loss: 0.0011 lr: 0.001 2025-06-23 15:18:49 iteration: 3365000 loss: 0.0011 lr: 0.001 2025-06-23 15:19:08 iteration: 3366000 loss: 0.0012 lr: 0.001 2025-06-23 15:19:29 iteration: 3367000 loss: 0.0011 lr: 0.001 2025-06-23 15:19:48 iteration: 3368000 loss: 0.0011 lr: 0.001 2025-06-23 15:20:07 iteration: 3369000 loss: 0.0011 lr: 0.001 2025-06-23 15:20:26 iteration: 3370000 loss: 0.0011 lr: 0.001 2025-06-23 15:20:46 iteration: 3371000 loss: 0.0011 lr: 0.001 2025-06-23 15:21:05 iteration: 3372000 loss: 0.0011 lr: 0.001 2025-06-23 15:21:24 iteration: 3373000 loss: 0.0011 lr: 0.001 2025-06-23 15:21:43 iteration: 3374000 loss: 0.0011 lr: 0.001 2025-06-23 15:22:03 iteration: 3375000 loss: 0.0011 lr: 0.001 2025-06-23 15:22:22 iteration: 3376000 loss: 0.0012 lr: 0.001 2025-06-23 15:22:41 iteration: 3377000 loss: 0.0011 lr: 0.001 2025-06-23 15:23:00 iteration: 3378000 loss: 0.0011 lr: 0.001 2025-06-23 15:23:20 iteration: 3379000 loss: 0.0011 lr: 0.001 2025-06-23 15:23:41 iteration: 3380000 loss: 0.0011 lr: 0.001 2025-06-23 15:24:00 iteration: 3381000 loss: 0.0011 lr: 0.001 2025-06-23 15:24:19 iteration: 3382000 loss: 0.0011 lr: 0.001 2025-06-23 15:24:38 iteration: 3383000 loss: 0.0011 lr: 0.001 2025-06-23 15:24:57 iteration: 3384000 loss: 0.0011 lr: 0.001 2025-06-23 15:25:17 iteration: 3385000 loss: 0.0011 lr: 0.001 2025-06-23 15:25:36 iteration: 3386000 loss: 0.0011 lr: 0.001 2025-06-23 15:25:56 iteration: 3387000 loss: 0.0012 lr: 0.001 2025-06-23 15:26:14 iteration: 3388000 loss: 0.0011 lr: 0.001 2025-06-23 15:26:33 iteration: 3389000 loss: 0.0011 lr: 0.001 2025-06-23 15:26:52 iteration: 3390000 loss: 0.0011 lr: 0.001 2025-06-23 15:27:11 iteration: 3391000 loss: 0.0011 lr: 0.001 2025-06-23 15:27:30 iteration: 3392000 loss: 0.0011 lr: 0.001 2025-06-23 15:27:49 iteration: 3393000 loss: 0.0011 lr: 0.001 2025-06-23 15:28:09 iteration: 3394000 loss: 0.0011 lr: 0.001 2025-06-23 15:28:28 iteration: 3395000 loss: 0.0011 lr: 0.001 2025-06-23 15:28:48 iteration: 3396000 loss: 0.0011 lr: 0.001 2025-06-23 15:29:07 iteration: 3397000 loss: 0.0011 lr: 0.001 2025-06-23 15:29:26 iteration: 3398000 loss: 0.0011 lr: 0.001 2025-06-23 15:29:46 iteration: 3399000 loss: 0.0011 lr: 0.001 2025-06-23 15:30:05 iteration: 3400000 loss: 0.0011 lr: 0.001 2025-06-23 15:30:24 iteration: 3401000 loss: 0.0011 lr: 0.001 2025-06-23 15:30:43 iteration: 3402000 loss: 0.0011 lr: 0.001 2025-06-23 15:31:03 iteration: 3403000 loss: 0.0011 lr: 0.001 2025-06-23 15:31:22 iteration: 3404000 loss: 0.0011 lr: 0.001 2025-06-23 15:31:42 iteration: 3405000 loss: 0.0011 lr: 0.001 2025-06-23 15:32:01 iteration: 3406000 loss: 0.0011 lr: 0.001 2025-06-23 15:32:20 iteration: 3407000 loss: 0.0011 lr: 0.001 2025-06-23 15:32:39 iteration: 3408000 loss: 0.0011 lr: 0.001 2025-06-23 15:32:58 iteration: 3409000 loss: 0.0011 lr: 0.001 2025-06-23 15:33:17 iteration: 3410000 loss: 0.0011 lr: 0.001 2025-06-23 15:33:35 iteration: 3411000 loss: 0.0011 lr: 0.001 2025-06-23 15:33:53 iteration: 3412000 loss: 0.0011 lr: 0.001 2025-06-23 15:34:12 iteration: 3413000 loss: 0.0011 lr: 0.001 2025-06-23 15:34:31 iteration: 3414000 loss: 0.0011 lr: 0.001 2025-06-23 15:34:49 iteration: 3415000 loss: 0.0011 lr: 0.001 2025-06-23 15:35:09 iteration: 3416000 loss: 0.0011 lr: 0.001 2025-06-23 15:35:28 iteration: 3417000 loss: 0.0011 lr: 0.001 2025-06-23 15:35:48 iteration: 3418000 loss: 0.0011 lr: 0.001 2025-06-23 15:36:07 iteration: 3419000 loss: 0.0011 lr: 0.001 2025-06-23 15:36:27 iteration: 3420000 loss: 0.0011 lr: 0.001 2025-06-23 15:36:47 iteration: 3421000 loss: 0.0011 lr: 0.001 2025-06-23 15:37:07 iteration: 3422000 loss: 0.0011 lr: 0.001 2025-06-23 15:37:26 iteration: 3423000 loss: 0.0011 lr: 0.001 2025-06-23 15:37:45 iteration: 3424000 loss: 0.0011 lr: 0.001 2025-06-23 15:38:05 iteration: 3425000 loss: 0.0011 lr: 0.001 2025-06-23 15:38:24 iteration: 3426000 loss: 0.0011 lr: 0.001 2025-06-23 15:38:44 iteration: 3427000 loss: 0.0011 lr: 0.001 2025-06-23 15:39:03 iteration: 3428000 loss: 0.0011 lr: 0.001 2025-06-23 15:39:22 iteration: 3429000 loss: 0.0011 lr: 0.001 2025-06-23 15:39:41 iteration: 3430000 loss: 0.0011 lr: 0.001 2025-06-23 15:40:02 iteration: 3431000 loss: 0.0011 lr: 0.001 2025-06-23 15:40:21 iteration: 3432000 loss: 0.0011 lr: 0.001 2025-06-23 15:40:40 iteration: 3433000 loss: 0.0011 lr: 0.001 2025-06-23 15:41:00 iteration: 3434000 loss: 0.0011 lr: 0.001 2025-06-23 15:41:19 iteration: 3435000 loss: 0.0011 lr: 0.001 2025-06-23 15:41:39 iteration: 3436000 loss: 0.0011 lr: 0.001 2025-06-23 15:41:59 iteration: 3437000 loss: 0.0011 lr: 0.001 2025-06-23 15:42:19 iteration: 3438000 loss: 0.0011 lr: 0.001 2025-06-23 15:42:38 iteration: 3439000 loss: 0.0011 lr: 0.001 2025-06-23 15:42:57 iteration: 3440000 loss: 0.0011 lr: 0.001 2025-06-23 15:43:17 iteration: 3441000 loss: 0.0011 lr: 0.001 2025-06-23 15:43:35 iteration: 3442000 loss: 0.0011 lr: 0.001 2025-06-23 15:43:54 iteration: 3443000 loss: 0.0011 lr: 0.001 2025-06-23 15:44:13 iteration: 3444000 loss: 0.0011 lr: 0.001 2025-06-23 15:44:32 iteration: 3445000 loss: 0.0011 lr: 0.001 2025-06-23 15:44:51 iteration: 3446000 loss: 0.0011 lr: 0.001 2025-06-23 15:45:11 iteration: 3447000 loss: 0.0011 lr: 0.001 2025-06-23 15:45:31 iteration: 3448000 loss: 0.0011 lr: 0.001 2025-06-23 15:45:50 iteration: 3449000 loss: 0.0011 lr: 0.001 2025-06-23 15:46:09 iteration: 3450000 loss: 0.0011 lr: 0.001 2025-06-23 15:46:29 iteration: 3451000 loss: 0.0011 lr: 0.001 2025-06-23 15:46:48 iteration: 3452000 loss: 0.0011 lr: 0.001 2025-06-23 15:47:07 iteration: 3453000 loss: 0.0011 lr: 0.001 2025-06-23 15:47:26 iteration: 3454000 loss: 0.0011 lr: 0.001 2025-06-23 15:47:46 iteration: 3455000 loss: 0.0011 lr: 0.001 2025-06-23 15:48:07 iteration: 3456000 loss: 0.0011 lr: 0.001 2025-06-23 15:48:27 iteration: 3457000 loss: 0.0011 lr: 0.001 2025-06-23 15:48:48 iteration: 3458000 loss: 0.0011 lr: 0.001 2025-06-23 15:49:08 iteration: 3459000 loss: 0.0010 lr: 0.001 2025-06-23 15:49:28 iteration: 3460000 loss: 0.0011 lr: 0.001 2025-06-23 15:49:49 iteration: 3461000 loss: 0.0011 lr: 0.001 2025-06-23 15:50:09 iteration: 3462000 loss: 0.0011 lr: 0.001 2025-06-23 15:50:30 iteration: 3463000 loss: 0.0011 lr: 0.001 2025-06-23 15:50:50 iteration: 3464000 loss: 0.0011 lr: 0.001 2025-06-23 15:51:11 iteration: 3465000 loss: 0.0011 lr: 0.001 2025-06-23 15:51:32 iteration: 3466000 loss: 0.0011 lr: 0.001 2025-06-23 15:51:53 iteration: 3467000 loss: 0.0011 lr: 0.001 2025-06-23 15:52:13 iteration: 3468000 loss: 0.0011 lr: 0.001 2025-06-23 15:52:34 iteration: 3469000 loss: 0.0011 lr: 0.001 2025-06-23 15:52:54 iteration: 3470000 loss: 0.0011 lr: 0.001 2025-06-23 15:53:15 iteration: 3471000 loss: 0.0011 lr: 0.001 2025-06-23 15:53:36 iteration: 3472000 loss: 0.0011 lr: 0.001 2025-06-23 15:53:56 iteration: 3473000 loss: 0.0011 lr: 0.001 2025-06-23 15:54:17 iteration: 3474000 loss: 0.0011 lr: 0.001 2025-06-23 15:54:36 iteration: 3475000 loss: 0.0011 lr: 0.001 2025-06-23 15:54:57 iteration: 3476000 loss: 0.0011 lr: 0.001 2025-06-23 15:55:17 iteration: 3477000 loss: 0.0011 lr: 0.001 2025-06-23 15:55:38 iteration: 3478000 loss: 0.0011 lr: 0.001 2025-06-23 15:55:58 iteration: 3479000 loss: 0.0011 lr: 0.001 2025-06-23 15:56:19 iteration: 3480000 loss: 0.0011 lr: 0.001 2025-06-23 15:56:39 iteration: 3481000 loss: 0.0011 lr: 0.001 2025-06-23 15:56:59 iteration: 3482000 loss: 0.0011 lr: 0.001 2025-06-23 15:57:19 iteration: 3483000 loss: 0.0011 lr: 0.001 2025-06-23 15:57:39 iteration: 3484000 loss: 0.0011 lr: 0.001 2025-06-23 15:57:59 iteration: 3485000 loss: 0.0011 lr: 0.001 2025-06-23 15:58:20 iteration: 3486000 loss: 0.0011 lr: 0.001 2025-06-23 15:58:40 iteration: 3487000 loss: 0.0010 lr: 0.001 2025-06-23 15:59:01 iteration: 3488000 loss: 0.0011 lr: 0.001 2025-06-23 15:59:21 iteration: 3489000 loss: 0.0011 lr: 0.001 2025-06-23 15:59:42 iteration: 3490000 loss: 0.0011 lr: 0.001 2025-06-23 16:00:02 iteration: 3491000 loss: 0.0011 lr: 0.001 2025-06-23 16:00:23 iteration: 3492000 loss: 0.0011 lr: 0.001 2025-06-23 16:00:43 iteration: 3493000 loss: 0.0011 lr: 0.001 2025-06-23 16:01:03 iteration: 3494000 loss: 0.0011 lr: 0.001 2025-06-23 16:01:24 iteration: 3495000 loss: 0.0011 lr: 0.001 2025-06-23 16:01:45 iteration: 3496000 loss: 0.0011 lr: 0.001 2025-06-23 16:02:05 iteration: 3497000 loss: 0.0011 lr: 0.001 2025-06-23 16:02:25 iteration: 3498000 loss: 0.0011 lr: 0.001 2025-06-23 16:02:45 iteration: 3499000 loss: 0.0011 lr: 0.001 2025-06-23 16:03:06 iteration: 3500000 loss: 0.0011 lr: 0.001 2025-06-23 16:03:26 iteration: 3501000 loss: 0.0010 lr: 0.001 2025-06-23 16:03:47 iteration: 3502000 loss: 0.0011 lr: 0.001 2025-06-23 16:04:07 iteration: 3503000 loss: 0.0011 lr: 0.001 2025-06-23 16:04:27 iteration: 3504000 loss: 0.0011 lr: 0.001 2025-06-23 16:04:46 iteration: 3505000 loss: 0.0011 lr: 0.001 2025-06-23 16:05:06 iteration: 3506000 loss: 0.0011 lr: 0.001 2025-06-23 16:05:27 iteration: 3507000 loss: 0.0011 lr: 0.001 2025-06-23 16:05:47 iteration: 3508000 loss: 0.0011 lr: 0.001 2025-06-23 16:06:07 iteration: 3509000 loss: 0.0011 lr: 0.001 2025-06-23 16:06:27 iteration: 3510000 loss: 0.0011 lr: 0.001 2025-06-23 16:06:47 iteration: 3511000 loss: 0.0011 lr: 0.001 2025-06-23 16:07:07 iteration: 3512000 loss: 0.0011 lr: 0.001 2025-06-23 16:07:28 iteration: 3513000 loss: 0.0010 lr: 0.001 2025-06-23 16:07:48 iteration: 3514000 loss: 0.0011 lr: 0.001 2025-06-23 16:08:09 iteration: 3515000 loss: 0.0011 lr: 0.001 2025-06-23 16:08:29 iteration: 3516000 loss: 0.0011 lr: 0.001 2025-06-23 16:08:49 iteration: 3517000 loss: 0.0011 lr: 0.001 2025-06-23 16:09:10 iteration: 3518000 loss: 0.0011 lr: 0.001 2025-06-23 16:09:30 iteration: 3519000 loss: 0.0010 lr: 0.001 2025-06-23 16:09:50 iteration: 3520000 loss: 0.0011 lr: 0.001 2025-06-23 16:10:11 iteration: 3521000 loss: 0.0011 lr: 0.001 2025-06-23 16:10:31 iteration: 3522000 loss: 0.0010 lr: 0.001 2025-06-23 16:10:51 iteration: 3523000 loss: 0.0011 lr: 0.001 2025-06-23 16:11:11 iteration: 3524000 loss: 0.0011 lr: 0.001 2025-06-23 16:11:32 iteration: 3525000 loss: 0.0011 lr: 0.001 2025-06-23 16:11:52 iteration: 3526000 loss: 0.0011 lr: 0.001 2025-06-23 16:12:12 iteration: 3527000 loss: 0.0010 lr: 0.001 2025-06-23 16:12:32 iteration: 3528000 loss: 0.0011 lr: 0.001 2025-06-23 16:12:53 iteration: 3529000 loss: 0.0011 lr: 0.001 2025-06-23 16:13:13 iteration: 3530000 loss: 0.0011 lr: 0.001 2025-06-23 16:13:33 iteration: 3531000 loss: 0.0011 lr: 0.001 2025-06-23 16:13:53 iteration: 3532000 loss: 0.0011 lr: 0.001 2025-06-23 16:14:14 iteration: 3533000 loss: 0.0011 lr: 0.001 2025-06-23 16:14:34 iteration: 3534000 loss: 0.0011 lr: 0.001 2025-06-23 16:14:55 iteration: 3535000 loss: 0.0011 lr: 0.001 2025-06-23 16:15:15 iteration: 3536000 loss: 0.0010 lr: 0.001 2025-06-23 16:15:35 iteration: 3537000 loss: 0.0011 lr: 0.001 2025-06-23 16:15:55 iteration: 3538000 loss: 0.0010 lr: 0.001 2025-06-23 16:16:15 iteration: 3539000 loss: 0.0011 lr: 0.001 2025-06-23 16:16:34 iteration: 3540000 loss: 0.0011 lr: 0.001 2025-06-23 16:16:55 iteration: 3541000 loss: 0.0011 lr: 0.001 2025-06-23 16:17:15 iteration: 3542000 loss: 0.0010 lr: 0.001 2025-06-23 16:17:35 iteration: 3543000 loss: 0.0010 lr: 0.001 2025-06-23 16:17:55 iteration: 3544000 loss: 0.0011 lr: 0.001 2025-06-23 16:18:16 iteration: 3545000 loss: 0.0011 lr: 0.001 2025-06-23 16:18:36 iteration: 3546000 loss: 0.0011 lr: 0.001 2025-06-23 16:18:56 iteration: 3547000 loss: 0.0010 lr: 0.001 2025-06-23 16:19:16 iteration: 3548000 loss: 0.0011 lr: 0.001 2025-06-23 16:19:36 iteration: 3549000 loss: 0.0011 lr: 0.001 2025-06-23 16:19:57 iteration: 3550000 loss: 0.0011 lr: 0.001 2025-06-23 16:20:17 iteration: 3551000 loss: 0.0011 lr: 0.001 2025-06-23 16:20:38 iteration: 3552000 loss: 0.0011 lr: 0.001 2025-06-23 16:20:58 iteration: 3553000 loss: 0.0011 lr: 0.001 2025-06-23 16:21:18 iteration: 3554000 loss: 0.0010 lr: 0.001 2025-06-23 16:21:38 iteration: 3555000 loss: 0.0010 lr: 0.001 2025-06-23 16:21:58 iteration: 3556000 loss: 0.0011 lr: 0.001 2025-06-23 16:22:18 iteration: 3557000 loss: 0.0011 lr: 0.001 2025-06-23 16:22:39 iteration: 3558000 loss: 0.0011 lr: 0.001 2025-06-23 16:22:59 iteration: 3559000 loss: 0.0011 lr: 0.001 2025-06-23 16:23:20 iteration: 3560000 loss: 0.0011 lr: 0.001 2025-06-23 16:23:40 iteration: 3561000 loss: 0.0010 lr: 0.001 2025-06-23 16:24:00 iteration: 3562000 loss: 0.0011 lr: 0.001 2025-06-23 16:24:21 iteration: 3563000 loss: 0.0011 lr: 0.001 2025-06-23 16:24:41 iteration: 3564000 loss: 0.0011 lr: 0.001 2025-06-23 16:25:02 iteration: 3565000 loss: 0.0011 lr: 0.001 2025-06-23 16:25:22 iteration: 3566000 loss: 0.0011 lr: 0.001 2025-06-23 16:25:43 iteration: 3567000 loss: 0.0010 lr: 0.001 2025-06-23 16:26:04 iteration: 3568000 loss: 0.0011 lr: 0.001 2025-06-23 16:26:24 iteration: 3569000 loss: 0.0011 lr: 0.001 2025-06-23 16:26:44 iteration: 3570000 loss: 0.0011 lr: 0.001 2025-06-23 16:27:03 iteration: 3571000 loss: 0.0010 lr: 0.001 2025-06-23 16:27:23 iteration: 3572000 loss: 0.0010 lr: 0.001 2025-06-23 16:27:44 iteration: 3573000 loss: 0.0010 lr: 0.001 2025-06-23 16:28:04 iteration: 3574000 loss: 0.0010 lr: 0.001 2025-06-23 16:28:24 iteration: 3575000 loss: 0.0010 lr: 0.001 2025-06-23 16:28:45 iteration: 3576000 loss: 0.0010 lr: 0.001 2025-06-23 16:29:05 iteration: 3577000 loss: 0.0010 lr: 0.001 2025-06-23 16:29:25 iteration: 3578000 loss: 0.0010 lr: 0.001 2025-06-23 16:29:45 iteration: 3579000 loss: 0.0011 lr: 0.001 2025-06-23 16:30:05 iteration: 3580000 loss: 0.0010 lr: 0.001 2025-06-23 16:30:26 iteration: 3581000 loss: 0.0010 lr: 0.001 2025-06-23 16:30:46 iteration: 3582000 loss: 0.0011 lr: 0.001 2025-06-23 16:31:06 iteration: 3583000 loss: 0.0010 lr: 0.001 2025-06-23 16:31:27 iteration: 3584000 loss: 0.0011 lr: 0.001 2025-06-23 16:31:47 iteration: 3585000 loss: 0.0011 lr: 0.001 2025-06-23 16:32:08 iteration: 3586000 loss: 0.0011 lr: 0.001 2025-06-23 16:32:28 iteration: 3587000 loss: 0.0011 lr: 0.001 2025-06-23 16:32:48 iteration: 3588000 loss: 0.0011 lr: 0.001 2025-06-23 16:33:08 iteration: 3589000 loss: 0.0010 lr: 0.001 2025-06-23 16:33:28 iteration: 3590000 loss: 0.0011 lr: 0.001 2025-06-23 16:33:48 iteration: 3591000 loss: 0.0011 lr: 0.001 2025-06-23 16:34:09 iteration: 3592000 loss: 0.0010 lr: 0.001 2025-06-23 16:34:29 iteration: 3593000 loss: 0.0010 lr: 0.001 2025-06-23 16:34:49 iteration: 3594000 loss: 0.0010 lr: 0.001 2025-06-23 16:35:09 iteration: 3595000 loss: 0.0010 lr: 0.001 2025-06-23 16:35:29 iteration: 3596000 loss: 0.0010 lr: 0.001 2025-06-23 16:35:50 iteration: 3597000 loss: 0.0010 lr: 0.001 2025-06-23 16:36:10 iteration: 3598000 loss: 0.0010 lr: 0.001 2025-06-23 16:36:30 iteration: 3599000 loss: 0.0010 lr: 0.001 2025-06-23 16:36:50 iteration: 3600000 loss: 0.0011 lr: 0.001 2025-06-23 16:37:11 iteration: 3601000 loss: 0.0011 lr: 0.001 2025-06-23 16:37:31 iteration: 3602000 loss: 0.0011 lr: 0.001 2025-06-23 16:37:52 iteration: 3603000 loss: 0.0010 lr: 0.001 2025-06-23 16:38:12 iteration: 3604000 loss: 0.0011 lr: 0.001 2025-06-23 16:38:33 iteration: 3605000 loss: 0.0010 lr: 0.001 2025-06-23 16:38:53 iteration: 3606000 loss: 0.0011 lr: 0.001 2025-06-23 16:39:13 iteration: 3607000 loss: 0.0011 lr: 0.001 2025-06-23 16:39:33 iteration: 3608000 loss: 0.0010 lr: 0.001 2025-06-23 16:39:53 iteration: 3609000 loss: 0.0010 lr: 0.001 2025-06-23 16:40:14 iteration: 3610000 loss: 0.0011 lr: 0.001 2025-06-23 16:40:35 iteration: 3611000 loss: 0.0010 lr: 0.001 2025-06-23 16:40:55 iteration: 3612000 loss: 0.0010 lr: 0.001 2025-06-23 16:41:16 iteration: 3613000 loss: 0.0010 lr: 0.001 2025-06-23 16:41:36 iteration: 3614000 loss: 0.0010 lr: 0.001 2025-06-23 16:41:57 iteration: 3615000 loss: 0.0011 lr: 0.001 2025-06-23 16:42:17 iteration: 3616000 loss: 0.0010 lr: 0.001 2025-06-23 16:42:36 iteration: 3617000 loss: 0.0010 lr: 0.001 2025-06-23 16:42:56 iteration: 3618000 loss: 0.0010 lr: 0.001 2025-06-23 16:43:17 iteration: 3619000 loss: 0.0010 lr: 0.001 2025-06-23 16:43:36 iteration: 3620000 loss: 0.0010 lr: 0.001 2025-06-23 16:43:56 iteration: 3621000 loss: 0.0010 lr: 0.001 2025-06-23 16:44:16 iteration: 3622000 loss: 0.0010 lr: 0.001 2025-06-23 16:44:37 iteration: 3623000 loss: 0.0011 lr: 0.001 2025-06-23 16:44:58 iteration: 3624000 loss: 0.0010 lr: 0.001 2025-06-23 16:45:18 iteration: 3625000 loss: 0.0010 lr: 0.001 2025-06-23 16:45:38 iteration: 3626000 loss: 0.0010 lr: 0.001 2025-06-23 16:45:58 iteration: 3627000 loss: 0.0011 lr: 0.001 2025-06-23 16:46:18 iteration: 3628000 loss: 0.0010 lr: 0.001 2025-06-23 16:46:38 iteration: 3629000 loss: 0.0010 lr: 0.001 2025-06-23 16:46:58 iteration: 3630000 loss: 0.0010 lr: 0.001 2025-06-23 16:47:18 iteration: 3631000 loss: 0.0010 lr: 0.001 2025-06-23 16:47:38 iteration: 3632000 loss: 0.0010 lr: 0.001 2025-06-23 16:47:58 iteration: 3633000 loss: 0.0011 lr: 0.001 2025-06-23 16:48:18 iteration: 3634000 loss: 0.0010 lr: 0.001 2025-06-23 16:48:37 iteration: 3635000 loss: 0.0010 lr: 0.001 2025-06-23 16:48:57 iteration: 3636000 loss: 0.0011 lr: 0.001 2025-06-23 16:49:17 iteration: 3637000 loss: 0.0011 lr: 0.001 2025-06-23 16:49:38 iteration: 3638000 loss: 0.0010 lr: 0.001 2025-06-23 16:49:58 iteration: 3639000 loss: 0.0010 lr: 0.001 2025-06-23 16:50:18 iteration: 3640000 loss: 0.0011 lr: 0.001 2025-06-23 16:50:38 iteration: 3641000 loss: 0.0010 lr: 0.001 2025-06-23 16:50:58 iteration: 3642000 loss: 0.0010 lr: 0.001 2025-06-23 16:51:18 iteration: 3643000 loss: 0.0011 lr: 0.001 2025-06-23 16:51:39 iteration: 3644000 loss: 0.0010 lr: 0.001 2025-06-23 16:51:59 iteration: 3645000 loss: 0.0011 lr: 0.001 2025-06-23 16:52:19 iteration: 3646000 loss: 0.0010 lr: 0.001 2025-06-23 16:52:39 iteration: 3647000 loss: 0.0010 lr: 0.001 2025-06-23 16:52:59 iteration: 3648000 loss: 0.0010 lr: 0.001 2025-06-23 16:53:20 iteration: 3649000 loss: 0.0011 lr: 0.001 2025-06-23 16:53:40 iteration: 3650000 loss: 0.0010 lr: 0.001 2025-06-23 16:54:01 iteration: 3651000 loss: 0.0010 lr: 0.001 2025-06-23 16:54:21 iteration: 3652000 loss: 0.0010 lr: 0.001 2025-06-23 16:54:41 iteration: 3653000 loss: 0.0010 lr: 0.001 2025-06-23 16:55:02 iteration: 3654000 loss: 0.0011 lr: 0.001 2025-06-23 16:55:22 iteration: 3655000 loss: 0.0010 lr: 0.001 2025-06-23 16:55:43 iteration: 3656000 loss: 0.0010 lr: 0.001 2025-06-23 16:56:02 iteration: 3657000 loss: 0.0011 lr: 0.001 2025-06-23 16:56:23 iteration: 3658000 loss: 0.0010 lr: 0.001 2025-06-23 16:56:42 iteration: 3659000 loss: 0.0010 lr: 0.001 2025-06-23 16:57:02 iteration: 3660000 loss: 0.0010 lr: 0.001 2025-06-23 16:57:23 iteration: 3661000 loss: 0.0011 lr: 0.001 2025-06-23 16:57:43 iteration: 3662000 loss: 0.0010 lr: 0.001 2025-06-23 16:58:03 iteration: 3663000 loss: 0.0010 lr: 0.001 2025-06-23 16:58:23 iteration: 3664000 loss: 0.0010 lr: 0.001 2025-06-23 16:58:44 iteration: 3665000 loss: 0.0010 lr: 0.001 2025-06-23 16:59:04 iteration: 3666000 loss: 0.0010 lr: 0.001 2025-06-23 16:59:24 iteration: 3667000 loss: 0.0010 lr: 0.001 2025-06-23 16:59:44 iteration: 3668000 loss: 0.0010 lr: 0.001 2025-06-23 17:00:04 iteration: 3669000 loss: 0.0011 lr: 0.001 2025-06-23 17:00:24 iteration: 3670000 loss: 0.0010 lr: 0.001 2025-06-23 17:00:44 iteration: 3671000 loss: 0.0010 lr: 0.001 2025-06-23 17:01:04 iteration: 3672000 loss: 0.0011 lr: 0.001 2025-06-23 17:01:24 iteration: 3673000 loss: 0.0010 lr: 0.001 2025-06-23 17:01:44 iteration: 3674000 loss: 0.0010 lr: 0.001 2025-06-23 17:02:05 iteration: 3675000 loss: 0.0010 lr: 0.001 2025-06-23 17:02:25 iteration: 3676000 loss: 0.0011 lr: 0.001 2025-06-23 17:02:45 iteration: 3677000 loss: 0.0010 lr: 0.001 2025-06-23 17:03:06 iteration: 3678000 loss: 0.0010 lr: 0.001 2025-06-23 17:03:26 iteration: 3679000 loss: 0.0010 lr: 0.001 2025-06-23 17:03:46 iteration: 3680000 loss: 0.0010 lr: 0.001 2025-06-23 17:04:06 iteration: 3681000 loss: 0.0010 lr: 0.001 2025-06-23 17:04:26 iteration: 3682000 loss: 0.0011 lr: 0.001 2025-06-23 17:04:46 iteration: 3683000 loss: 0.0010 lr: 0.001 2025-06-23 17:05:07 iteration: 3684000 loss: 0.0010 lr: 0.001 2025-06-23 17:05:27 iteration: 3685000 loss: 0.0010 lr: 0.001 2025-06-23 17:05:47 iteration: 3686000 loss: 0.0011 lr: 0.001 2025-06-23 17:06:07 iteration: 3687000 loss: 0.0010 lr: 0.001 2025-06-23 17:06:28 iteration: 3688000 loss: 0.0010 lr: 0.001 2025-06-23 17:06:48 iteration: 3689000 loss: 0.0010 lr: 0.001 2025-06-23 17:07:08 iteration: 3690000 loss: 0.0010 lr: 0.001 2025-06-23 17:07:28 iteration: 3691000 loss: 0.0011 lr: 0.001 2025-06-23 17:07:48 iteration: 3692000 loss: 0.0010 lr: 0.001 2025-06-23 17:08:08 iteration: 3693000 loss: 0.0010 lr: 0.001 2025-06-23 17:08:28 iteration: 3694000 loss: 0.0010 lr: 0.001 2025-06-23 17:08:48 iteration: 3695000 loss: 0.0010 lr: 0.001 2025-06-23 17:09:08 iteration: 3696000 loss: 0.0010 lr: 0.001 2025-06-23 17:09:29 iteration: 3697000 loss: 0.0010 lr: 0.001 2025-06-23 17:09:48 iteration: 3698000 loss: 0.0010 lr: 0.001 2025-06-23 17:10:09 iteration: 3699000 loss: 0.0010 lr: 0.001 2025-06-23 17:10:29 iteration: 3700000 loss: 0.0010 lr: 0.001 2025-06-23 17:10:30 Config: {'all_joints': [[0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12], [13], [14], [15], [16], [17], [18], [19], [20], [21], [22], [23], [24], [25], [26], [27]], 'all_joints_names': ['nose(bottom)', 'nose(tip)', 'nose(top)', 'pad(top)(left)', 'pad(side)(left)', 'pad(center)', 'pad(top)(right)', 'pad(side)(right)', 'lowerlip', 'upperlip(left)', 'upperlip(right)', 'eye(front)(left)', 'eye(top)(left)', 'eye(back)(left)', 'eye(bottom)(left)', 'eye(front)(right)', 'eye(top)(right)', 'eye(back)(right)', 'eye(bottom)(right)', 'ear(base)(left)', 'ear(top)(left)', 'ear(tip)(left)', 'ear(bottom)(left)', 'ear(base)(right)', 'ear(top)(right)', 'ear(tip)(right)', 'ear(bottom)(right)', 'ref(head-post)'], 'batch_size': 1, 'crop_pad': 0, 'dataset': 'training-datasets/iteration-2/UnaugmentedDataSet_rig1_aug2May28/rig1_aug2_houlab95shuffle1.mat', 'dataset_type': 'imgaug', 'deterministic': False, 'fg_fraction': 0.25, 'global_scale': 0.8, 'init_weights': '/home/daruwal/micromamba/envs/mouse-fe-analysis/lib/python3.10/site-packages/deeplabcut/pose_estimation_tensorflow/models/pretrained/resnet_v1_50.ckpt', 'intermediate_supervision': False, 'intermediate_supervision_layer': 12, 'location_refinement': True, 'locref_huber_loss': True, 'locref_loss_weight': 1.0, 'locref_stdev': 7.2801, 'log_dir': 'log', 'mean_pixel': [123.68, 116.779, 103.939], 'mirror': False, 'net_type': 'resnet_50', 'num_joints': 28, 'optimizer': 'sgd', 'pairwise_huber_loss': True, 'pairwise_predict': False, 'partaffinityfield_predict': False, 'regularize': False, 'scoremap_dir': 'test', 'shuffle': True, 'snapshot_prefix': '/data/disk1/daruwal/repos/mouse-fe-analysis/dlc-training/projects/rig1_aug2-houlab-2025-05-28/dlc-models/iteration-2/rig1_aug2May28-trainset95shuffle1/test/snapshot', 'stride': 8.0, 'weigh_negatives': False, 'weigh_only_present_joints': False, 'weigh_part_predictions': False, 'weight_decay': 0.0001}