2025-06-12 07:14:14 Config: {'all_joints': [[0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12], [13], [14], [15], [16], [17], [18], [19], [20], [21], [22], [23], [24], [25], [26], [27]], 'all_joints_names': ['nose(bottom)', 'nose(tip)', 'nose(top)', 'pad(top)(left)', 'pad(side)(left)', 'pad(center)', 'pad(top)(right)', 'pad(side)(right)', 'lowerlip', 'upperlip(left)', 'upperlip(right)', 'eye(front)(left)', 'eye(top)(left)', 'eye(back)(left)', 'eye(bottom)(left)', 'eye(front)(right)', 'eye(top)(right)', 'eye(back)(right)', 'eye(bottom)(right)', 'ear(base)(left)', 'ear(top)(left)', 'ear(tip)(left)', 'ear(bottom)(left)', 'ear(base)(right)', 'ear(top)(right)', 'ear(tip)(right)', 'ear(bottom)(right)', 'ref(head-post)'], 'alpha_r': 0.02, 'apply_prob': 0.5, 'batch_size': 1, 'contrast': {'clahe': True, 'claheratio': 0.1, 'histeq': True, 'histeqratio': 0.1}, 'convolution': {'edge': False, 'emboss': {'alpha': [0.0, 1.0], 'strength': [0.5, 1.5]}, 'embossratio': 0.1, 'sharpen': False, 'sharpenratio': 0.3}, 'crop_pad': 0, 'cropratio': 0.4, 'dataset': 'training-datasets/iteration-1/UnaugmentedDataSet_rig1_aug2May28/rig1_aug2_houlab95shuffle1.mat', 'dataset_type': 'imgaug', 'decay_steps': 30000, 'deterministic': False, 'display_iters': 1000, 'fg_fraction': 0.25, 'global_scale': 0.8, 'init_weights': '/data/disk1/daruwal/repos/mouse-fe-analysis/dlc-training/projects/rig1_aug2-houlab-2025-05-28/dlc-models/iteration-0/rig1_aug2May28-trainset95shuffle1/train/snapshot-3000000', 'intermediate_supervision': False, 'intermediate_supervision_layer': 12, 'location_refinement': True, 'locref_huber_loss': True, 'locref_loss_weight': 0.05, 'locref_stdev': 7.2801, 'log_dir': 'log', 'lr_init': 0.0005, 'max_input_size': 1500, 'mean_pixel': [123.68, 116.779, 103.939], 'metadataset': 'training-datasets/iteration-1/UnaugmentedDataSet_rig1_aug2May28/Documentation_data-rig1_aug2_95shuffle1.pickle', 'min_input_size': 64, 'mirror': False, 'multi_stage': False, 'multi_step': [[0.001, 100000]], 'net_type': 'resnet_50', 'num_joints': 28, 'optimizer': 'sgd', 'pairwise_huber_loss': False, 'pairwise_predict': False, 'partaffinityfield_predict': False, 'pos_dist_thresh': 17, 'project_path': '/data/disk1/daruwal/repos/mouse-fe-analysis/dlc-training/projects/rig1_aug2-houlab-2025-05-28', 'regularize': False, 'rotation': 25, 'rotratio': 0.4, 'save_iters': 50000, 'scale_jitter_lo': 0.5, 'scale_jitter_up': 1.25, 'scoremap_dir': 'test', 'shuffle': True, 'snapshot_prefix': '/data/disk1/daruwal/repos/mouse-fe-analysis/dlc-training/projects/rig1_aug2-houlab-2025-05-28/dlc-models/iteration-1/rig1_aug2May28-trainset95shuffle1/train/snapshot', 'stride': 8.0, 'weigh_negatives': False, 'weigh_only_present_joints': False, 'weigh_part_predictions': False, 'weight_decay': 0.0001} 2025-06-12 07:14:45 iteration: 3001000 loss: 0.0019 lr: 0.001 2025-06-12 07:15:04 iteration: 3002000 loss: 0.0015 lr: 0.001 2025-06-12 07:15:24 iteration: 3003000 loss: 0.0014 lr: 0.001 2025-06-12 07:15:44 iteration: 3004000 loss: 0.0014 lr: 0.001 2025-06-12 07:16:04 iteration: 3005000 loss: 0.0013 lr: 0.001 2025-06-12 07:16:23 iteration: 3006000 loss: 0.0013 lr: 0.001 2025-06-12 07:16:43 iteration: 3007000 loss: 0.0013 lr: 0.001 2025-06-12 07:17:02 iteration: 3008000 loss: 0.0012 lr: 0.001 2025-06-12 07:17:22 iteration: 3009000 loss: 0.0012 lr: 0.001 2025-06-12 07:17:42 iteration: 3010000 loss: 0.0012 lr: 0.001 2025-06-12 07:18:02 iteration: 3011000 loss: 0.0012 lr: 0.001 2025-06-12 07:18:23 iteration: 3012000 loss: 0.0012 lr: 0.001 2025-06-12 07:18:42 iteration: 3013000 loss: 0.0012 lr: 0.001 2025-06-12 07:19:01 iteration: 3014000 loss: 0.0012 lr: 0.001 2025-06-12 07:19:21 iteration: 3015000 loss: 0.0012 lr: 0.001 2025-06-12 07:19:40 iteration: 3016000 loss: 0.0011 lr: 0.001 2025-06-12 07:20:01 iteration: 3017000 loss: 0.0011 lr: 0.001 2025-06-12 07:20:20 iteration: 3018000 loss: 0.0011 lr: 0.001 2025-06-12 07:20:39 iteration: 3019000 loss: 0.0011 lr: 0.001 2025-06-12 07:20:58 iteration: 3020000 loss: 0.0012 lr: 0.001 2025-06-12 07:21:18 iteration: 3021000 loss: 0.0011 lr: 0.001 2025-06-12 07:21:37 iteration: 3022000 loss: 0.0011 lr: 0.001 2025-06-12 07:21:56 iteration: 3023000 loss: 0.0011 lr: 0.001 2025-06-12 07:22:16 iteration: 3024000 loss: 0.0011 lr: 0.001 2025-06-12 07:22:36 iteration: 3025000 loss: 0.0011 lr: 0.001 2025-06-12 07:22:56 iteration: 3026000 loss: 0.0011 lr: 0.001 2025-06-12 07:23:15 iteration: 3027000 loss: 0.0011 lr: 0.001 2025-06-12 07:23:35 iteration: 3028000 loss: 0.0011 lr: 0.001 2025-06-12 07:23:54 iteration: 3029000 loss: 0.0011 lr: 0.001 2025-06-12 07:24:14 iteration: 3030000 loss: 0.0011 lr: 0.001 2025-06-12 07:24:33 iteration: 3031000 loss: 0.0011 lr: 0.001 2025-06-12 07:24:53 iteration: 3032000 loss: 0.0011 lr: 0.001 2025-06-12 07:25:12 iteration: 3033000 loss: 0.0011 lr: 0.001 2025-06-12 07:25:32 iteration: 3034000 loss: 0.0011 lr: 0.001 2025-06-12 07:25:52 iteration: 3035000 loss: 0.0011 lr: 0.001 2025-06-12 07:26:11 iteration: 3036000 loss: 0.0011 lr: 0.001 2025-06-12 07:26:31 iteration: 3037000 loss: 0.0010 lr: 0.001 2025-06-12 07:26:50 iteration: 3038000 loss: 0.0011 lr: 0.001 2025-06-12 07:27:09 iteration: 3039000 loss: 0.0011 lr: 0.001 2025-06-12 07:27:29 iteration: 3040000 loss: 0.0011 lr: 0.001 2025-06-12 07:27:49 iteration: 3041000 loss: 0.0011 lr: 0.001 2025-06-12 07:28:08 iteration: 3042000 loss: 0.0011 lr: 0.001 2025-06-12 07:28:27 iteration: 3043000 loss: 0.0010 lr: 0.001 2025-06-12 07:28:47 iteration: 3044000 loss: 0.0010 lr: 0.001 2025-06-12 07:29:07 iteration: 3045000 loss: 0.0010 lr: 0.001 2025-06-12 07:29:27 iteration: 3046000 loss: 0.0011 lr: 0.001 2025-06-12 07:29:46 iteration: 3047000 loss: 0.0011 lr: 0.001 2025-06-12 07:30:06 iteration: 3048000 loss: 0.0011 lr: 0.001 2025-06-12 07:30:25 iteration: 3049000 loss: 0.0011 lr: 0.001 2025-06-12 07:30:45 iteration: 3050000 loss: 0.0011 lr: 0.001 2025-06-12 07:31:05 iteration: 3051000 loss: 0.0011 lr: 0.001 2025-06-12 07:31:24 iteration: 3052000 loss: 0.0011 lr: 0.001 2025-06-12 07:31:43 iteration: 3053000 loss: 0.0010 lr: 0.001 2025-06-12 07:32:02 iteration: 3054000 loss: 0.0011 lr: 0.001 2025-06-12 07:32:22 iteration: 3055000 loss: 0.0011 lr: 0.001 2025-06-12 07:32:42 iteration: 3056000 loss: 0.0011 lr: 0.001 2025-06-12 07:33:01 iteration: 3057000 loss: 0.0010 lr: 0.001 2025-06-12 07:33:20 iteration: 3058000 loss: 0.0010 lr: 0.001 2025-06-12 07:33:39 iteration: 3059000 loss: 0.0010 lr: 0.001 2025-06-12 07:33:59 iteration: 3060000 loss: 0.0010 lr: 0.001 2025-06-12 07:34:19 iteration: 3061000 loss: 0.0010 lr: 0.001 2025-06-12 07:34:38 iteration: 3062000 loss: 0.0011 lr: 0.001 2025-06-12 07:34:58 iteration: 3063000 loss: 0.0010 lr: 0.001 2025-06-12 07:35:18 iteration: 3064000 loss: 0.0010 lr: 0.001 2025-06-12 07:35:37 iteration: 3065000 loss: 0.0011 lr: 0.001 2025-06-12 07:35:57 iteration: 3066000 loss: 0.0010 lr: 0.001 2025-06-12 07:36:17 iteration: 3067000 loss: 0.0011 lr: 0.001 2025-06-12 07:36:36 iteration: 3068000 loss: 0.0010 lr: 0.001 2025-06-12 07:36:56 iteration: 3069000 loss: 0.0010 lr: 0.001 2025-06-12 07:37:15 iteration: 3070000 loss: 0.0011 lr: 0.001 2025-06-12 07:37:35 iteration: 3071000 loss: 0.0010 lr: 0.001 2025-06-12 07:37:54 iteration: 3072000 loss: 0.0010 lr: 0.001 2025-06-12 07:38:13 iteration: 3073000 loss: 0.0010 lr: 0.001 2025-06-12 07:38:33 iteration: 3074000 loss: 0.0011 lr: 0.001 2025-06-12 07:38:53 iteration: 3075000 loss: 0.0011 lr: 0.001 2025-06-12 07:39:13 iteration: 3076000 loss: 0.0010 lr: 0.001 2025-06-12 07:39:33 iteration: 3077000 loss: 0.0010 lr: 0.001 2025-06-12 07:39:53 iteration: 3078000 loss: 0.0010 lr: 0.001 2025-06-12 07:40:12 iteration: 3079000 loss: 0.0011 lr: 0.001 2025-06-12 07:40:32 iteration: 3080000 loss: 0.0010 lr: 0.001 2025-06-12 07:40:53 iteration: 3081000 loss: 0.0010 lr: 0.001 2025-06-12 07:41:12 iteration: 3082000 loss: 0.0010 lr: 0.001 2025-06-12 07:41:32 iteration: 3083000 loss: 0.0010 lr: 0.001 2025-06-12 07:41:51 iteration: 3084000 loss: 0.0011 lr: 0.001 2025-06-12 07:42:11 iteration: 3085000 loss: 0.0010 lr: 0.001 2025-06-12 07:42:30 iteration: 3086000 loss: 0.0010 lr: 0.001 2025-06-12 07:42:51 iteration: 3087000 loss: 0.0010 lr: 0.001 2025-06-12 07:43:10 iteration: 3088000 loss: 0.0010 lr: 0.001 2025-06-12 07:43:30 iteration: 3089000 loss: 0.0010 lr: 0.001 2025-06-12 07:43:50 iteration: 3090000 loss: 0.0010 lr: 0.001 2025-06-12 07:44:09 iteration: 3091000 loss: 0.0010 lr: 0.001 2025-06-12 07:44:28 iteration: 3092000 loss: 0.0010 lr: 0.001 2025-06-12 07:44:48 iteration: 3093000 loss: 0.0010 lr: 0.001 2025-06-12 07:45:08 iteration: 3094000 loss: 0.0010 lr: 0.001 2025-06-12 07:45:28 iteration: 3095000 loss: 0.0010 lr: 0.001 2025-06-12 07:45:48 iteration: 3096000 loss: 0.0010 lr: 0.001 2025-06-12 07:46:08 iteration: 3097000 loss: 0.0010 lr: 0.001 2025-06-12 07:46:28 iteration: 3098000 loss: 0.0010 lr: 0.001 2025-06-12 07:46:47 iteration: 3099000 loss: 0.0010 lr: 0.001 2025-06-12 07:47:08 iteration: 3100000 loss: 0.0010 lr: 0.001 2025-06-12 07:47:08 Config: {'all_joints': [[0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12], [13], [14], [15], [16], [17], [18], [19], [20], [21], [22], [23], [24], [25], [26], [27]], 'all_joints_names': ['nose(bottom)', 'nose(tip)', 'nose(top)', 'pad(top)(left)', 'pad(side)(left)', 'pad(center)', 'pad(top)(right)', 'pad(side)(right)', 'lowerlip', 'upperlip(left)', 'upperlip(right)', 'eye(front)(left)', 'eye(top)(left)', 'eye(back)(left)', 'eye(bottom)(left)', 'eye(front)(right)', 'eye(top)(right)', 'eye(back)(right)', 'eye(bottom)(right)', 'ear(base)(left)', 'ear(top)(left)', 'ear(tip)(left)', 'ear(bottom)(left)', 'ear(base)(right)', 'ear(top)(right)', 'ear(tip)(right)', 'ear(bottom)(right)', 'ref(head-post)'], 'batch_size': 1, 'crop_pad': 0, 'dataset': 'training-datasets/iteration-1/UnaugmentedDataSet_rig1_aug2May28/rig1_aug2_houlab95shuffle1.mat', 'dataset_type': 'imgaug', 'deterministic': False, 'fg_fraction': 0.25, 'global_scale': 0.8, 'init_weights': '/home/daruwal/micromamba/envs/mouse-fe-analysis/lib/python3.10/site-packages/deeplabcut/pose_estimation_tensorflow/models/pretrained/resnet_v1_50.ckpt', 'intermediate_supervision': False, 'intermediate_supervision_layer': 12, 'location_refinement': True, 'locref_huber_loss': True, 'locref_loss_weight': 1.0, 'locref_stdev': 7.2801, 'log_dir': 'log', 'mean_pixel': [123.68, 116.779, 103.939], 'mirror': False, 'net_type': 'resnet_50', 'num_joints': 28, 'optimizer': 'sgd', 'pairwise_huber_loss': True, 'pairwise_predict': False, 'partaffinityfield_predict': False, 'regularize': False, 'scoremap_dir': 'test', 'shuffle': True, 'snapshot_prefix': '/data/disk1/daruwal/repos/mouse-fe-analysis/dlc-training/projects/rig1_aug2-houlab-2025-05-28/dlc-models/iteration-1/rig1_aug2May28-trainset95shuffle1/test/snapshot', 'stride': 8.0, 'weigh_negatives': False, 'weigh_only_present_joints': False, 'weigh_part_predictions': False, 'weight_decay': 0.0001}