(base) shuyu@manoli-gpu:~$ free total used free shared buff/cache available Mem: 65779352 4386272 51628984 107860 9764096 60555672 Swap: 31228924 3281152 27947772 (base) shuyu@manoli-gpu:~$ conda activate sleap (sleap) shuyu@manoli-gpu:~$ cd SLEAP (sleap) shuyu@manoli-gpu:~/SLEAP$ ls 2022-06-16-sleap-bottomup-11pt_D 2022-06-26-sleap-bottomup-11pt_D 2022-07-18-sleap-bottomup-11pt_D-others 2022-07-25-sleap-bottomup-11pt_D-white 2022-09-02-sleap-bottomup-mass-predict 2022-09-02-sleap-bottomup-newvid 2022-09-09-sleap-bottomup-e06 2022-09-13-sleap-bottomup-add-e06labels 2022-11-06-sleap-bottomup-e06 2022-11-09-sleap-bottomup-e06-extra 2022-11-29-sleap-bottomup-e10 2022-12-01-sleap-bottomup-e04 2022-12-02-sleap-bottomup-e05 2022-12-03-sleap-bottomup-e08 2022-12-04-sleap-bottomup-e12 2022-12-06-sleap-bottomup-e13 2022-12-07-sleap-bottomup-e14 2022-12-08-sleap-bottomup-e07 2022-12-09-sleap-bottomup-NS25-network 2023-01-06-sleap-bottomup-add-AJlabels 2023-01-10-sleap-bottomup-add-AJlabels-test 2023-01-17-sleap-bottomup-add-AJlabels-more 2023-01-18-sleap-bottomup-add-AJlabels-more-test 2023-01-19-sleap-bottomup-add-AJlabels-more-test Code Data logfiles Skeletons SLEAP_manipulations temp (sleap) shuyu@manoli-gpu:~/SLEAP$ sleap-label INFO:numexpr.utils:NumExpr defaulting to 6 threads. Saving config: /home/shuyu/.sleap/1.2.9/preferences.yaml Restoring GUI state... 2023-01-20 17:40:28.866062: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2023-01-20 17:40:28.902416: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2023-01-20 17:40:28.902602: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero Software versions: SLEAP: 1.2.9 TensorFlow: 2.6.3 Numpy: 1.19.5 Python: 3.7.12 OS: Linux-5.4.0-132-generic-x86_64-with-debian-buster-sid Happy SLEAPing! :) Resetting monitor window. Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Start training centroid... ['sleap-train', '/tmp/tmpjxt8we32/230120_193352_training_job.json', '/home/shuyu/SLEAP/2023-01-20-sleap-topdown/sleap-1627-add228.slp', '--zmq', '--save_viz'] INFO:numexpr.utils:NumExpr defaulting to 6 threads. INFO:sleap.nn.training:Versions: SLEAP: 1.2.9 TensorFlow: 2.6.3 Numpy: 1.19.5 Python: 3.7.12 OS: Linux-5.4.0-132-generic-x86_64-with-debian-buster-sid INFO:sleap.nn.training:Training labels file: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/sleap-1627-add228.slp INFO:sleap.nn.training:Training profile: /tmp/tmpjxt8we32/230120_193352_training_job.json INFO:sleap.nn.training: INFO:sleap.nn.training:Arguments: INFO:sleap.nn.training:{ "training_job_path": "/tmp/tmpjxt8we32/230120_193352_training_job.json", "labels_path": "/home/shuyu/SLEAP/2023-01-20-sleap-topdown/sleap-1627-add228.slp", "video_paths": [ "" ], "val_labels": null, "test_labels": null, "tensorboard": false, "save_viz": true, "zmq": true, "run_name": "", "prefix": "", "suffix": "", "cpu": false, "first_gpu": false, "last_gpu": false, "gpu": "auto" } INFO:sleap.nn.training: INFO:sleap.nn.training:Training job: INFO:sleap.nn.training:{ "data": { "labels": { "training_labels": null, "validation_labels": null, "validation_fraction": 0.1, "test_labels": null, "split_by_inds": false, "training_inds": null, "validation_inds": null, "test_inds": null, "search_path_hints": [], "skeletons": [] }, "preprocessing": { "ensure_rgb": false, "ensure_grayscale": false, "imagenet_mode": null, "input_scaling": 0.75, "pad_to_stride": null, "resize_and_pad_to_target": true, "target_height": null, "target_width": null }, "instance_cropping": { "center_on_part": null, "crop_size": null, "crop_size_detection_padding": 16 } }, "model": { "backbone": { "leap": null, "unet": { "stem_stride": null, "max_stride": 64, "output_stride": 2, "filters": 16, "filters_rate": 2.0, "middle_block": true, "up_interpolate": true, "stacks": 1 }, "hourglass": null, "resnet": null, "pretrained_encoder": null }, "heads": { "single_instance": null, "centroid": { "anchor_part": null, "sigma": 10.0, "output_stride": 2, "loss_weight": 1.0, "offset_refinement": false }, "centered_instance": null, "multi_instance": null, "multi_class_bottomup": null, "multi_class_topdown": null } }, "optimization": { "preload_data": true, "augmentation_config": { "rotate": true, "rotation_min_angle": -180.0, "rotation_max_angle": 180.0, "translate": false, "translate_min": -5, "translate_max": 5, "scale": true, "scale_min": 0.7, "scale_max": 1.3, "uniform_noise": false, "uniform_noise_min_val": 0.0, "uniform_noise_max_val": 10.0, "gaussian_noise": true, "gaussian_noise_mean": 5.0, "gaussian_noise_stddev": 1.0, "contrast": true, "contrast_min_gamma": 0.5, "contrast_max_gamma": 2.0, "brightness": true, "brightness_min_val": 0.0, "brightness_max_val": 10.0, "random_crop": false, "random_crop_height": 256, "random_crop_width": 256, "random_flip": false, "flip_horizontal": true }, "online_shuffling": true, "shuffle_buffer_size": 128, "prefetch": true, "batch_size": 4, "batches_per_epoch": null, "min_batches_per_epoch": 200, "val_batches_per_epoch": null, "min_val_batches_per_epoch": 10, "epochs": 200, "optimizer": "adam", "initial_learning_rate": 0.0001, "learning_rate_schedule": { "reduce_on_plateau": true, "reduction_factor": 0.5, "plateau_min_delta": 1e-06, "plateau_patience": 5, "plateau_cooldown": 3, "min_learning_rate": 1e-08 }, "hard_keypoint_mining": { "online_mining": false, "hard_to_easy_ratio": 2.0, "min_hard_keypoints": 2, "max_hard_keypoints": null, "loss_scale": 5.0 }, "early_stopping": { "stop_training_on_plateau": true, "plateau_min_delta": 1e-08, "plateau_patience": 20 } }, "outputs": { "save_outputs": true, "run_name": "230120_193352.centroid.n=1855", "run_name_prefix": "exp20_", "run_name_suffix": "", "runs_folder": "/home/shuyu/SLEAP/2023-01-20-sleap-topdown/models", "tags": [ "" ], "save_visualizations": true, "delete_viz_images": true, "zip_outputs": false, "log_to_csv": true, "checkpointing": { "initial_model": false, "best_model": true, "every_epoch": false, "latest_model": false, "final_model": false }, "tensorboard": { "write_logs": false, "loss_frequency": "epoch", "architecture_graph": false, "profile_graph": false, "visualizations": true }, "zmq": { "subscribe_to_controller": true, "controller_address": "tcp://127.0.0.1:9000", "controller_polling_timeout": 10, "publish_updates": true, "publish_address": "tcp://127.0.0.1:9001" } }, "name": "", "description": "", "sleap_version": "1.2.9", "filename": "/tmp/tmpjxt8we32/230120_193352_training_job.json" } INFO:sleap.nn.training: 2023-01-20 19:33:55.028486: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2023-01-20 19:33:55.033724: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2023-01-20 19:33:55.033905: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero INFO:sleap.nn.training:Auto-selected GPU 0 with 10646 MiB of free memory. INFO:sleap.nn.training:Using GPU 0 for acceleration. INFO:sleap.nn.training:Disabled GPU memory pre-allocation. INFO:sleap.nn.training:System: GPUs: 1/1 available Device: /physical_device:GPU:0 Available: True Initalized: False Memory growth: True INFO:sleap.nn.training: INFO:sleap.nn.training:Initializing trainer... INFO:sleap.nn.training:Loading training labels from: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/sleap-1627-add228.slp INFO:sleap.nn.training:Creating training and validation splits from validation fraction: 0.1 INFO:sleap.nn.training: Splits: Training = 1669 / Validation = 186. INFO:sleap.nn.training:Setting up for training... INFO:sleap.nn.training:Setting up pipeline builders... INFO:sleap.nn.training:Setting up model... INFO:sleap.nn.training:Building test pipeline... 2023-01-20 19:34:01.348632: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. 2023-01-20 19:34:01.348990: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2023-01-20 19:34:01.349187: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2023-01-20 19:34:01.349323: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2023-01-20 19:34:02.103489: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2023-01-20 19:34:02.103766: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2023-01-20 19:34:02.103991: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2023-01-20 19:34:02.104409: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 9291 MB memory: -> device: 0, name: NVIDIA GeForce RTX 2080 Ti, pci bus id: 0000:01:00.0, compute capability: 7.5 2023-01-20 19:34:02.567668: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2) INFO:sleap.nn.training:Loaded test example. [6.694s] INFO:sleap.nn.training: Input shape: (1152, 1536, 3) INFO:sleap.nn.training:Created Keras model. INFO:sleap.nn.training: Backbone: UNet(stacks=1, filters=16, filters_rate=2.0, kernel_size=3, stem_kernel_size=7, convs_per_block=2, stem_blocks=0, down_blocks=6, middle_block=True, up_blocks=5, up_interpolate=True, block_contraction=False) INFO:sleap.nn.training: Max stride: 64 INFO:sleap.nn.training: Parameters: 31,449,201 INFO:sleap.nn.training: Heads: INFO:sleap.nn.training: [0] = CentroidConfmapsHead(anchor_part=None, sigma=10.0, output_stride=2, loss_weight=1.0) INFO:sleap.nn.training: Outputs: INFO:sleap.nn.training: [0] = KerasTensor(type_spec=TensorSpec(shape=(None, 576, 768, 1), dtype=tf.float32, name=None), name='CentroidConfmapsHead/BiasAdd:0', description="created by layer 'CentroidConfmapsHead'") INFO:sleap.nn.training:Setting up data pipelines... INFO:sleap.nn.training:Training set: n = 1669 INFO:sleap.nn.training:Validation set: n = 186 INFO:sleap.nn.training:Setting up optimization... INFO:sleap.nn.training: Learning rate schedule: LearningRateScheduleConfig(reduce_on_plateau=True, reduction_factor=0.5, plateau_min_delta=1e-06, plateau_patience=5, plateau_cooldown=3, min_learning_rate=1e-08) INFO:sleap.nn.training: Early stopping: EarlyStoppingConfig(stop_training_on_plateau=True, plateau_min_delta=1e-08, plateau_patience=20) INFO:sleap.nn.training:Setting up outputs... INFO:sleap.nn.callbacks:Training controller subscribed to: tcp://127.0.0.1:9000 (topic: ) INFO:sleap.nn.training: ZMQ controller subcribed to: tcp://127.0.0.1:9000 INFO:sleap.nn.callbacks:Progress reporter publishing on: tcp://127.0.0.1:9001 for: not_set INFO:sleap.nn.training: ZMQ progress reporter publish on: tcp://127.0.0.1:9001 INFO:sleap.nn.training:Created run path: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855 INFO:sleap.nn.training:Setting up visualization... 2023-01-20 19:34:07.438464: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: -34 } dim { size: -35 } dim { size: -36 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "GPU" vendor: "NVIDIA" model: "NVIDIA GeForce RTX 2080 Ti" frequency: 1545 num_cores: 68 environment { key: "architecture" value: "7.5" } environment { key: "cuda" value: "11020" } environment { key: "cudnn" value: "8100" } num_registers: 65536 l1_cache_size: 24576 l2_cache_size: 5767168 shared_memory_size_per_multiprocessor: 65536 memory_size: 9742909440 bandwidth: 616000000 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: -37 } dim { size: -38 } dim { size: 1 } } } 2023-01-20 19:34:09.230672: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: -34 } dim { size: -35 } dim { size: -36 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "GPU" vendor: "NVIDIA" model: "NVIDIA GeForce RTX 2080 Ti" frequency: 1545 num_cores: 68 environment { key: "architecture" value: "7.5" } environment { key: "cuda" value: "11020" } environment { key: "cudnn" value: "8100" } num_registers: 65536 l1_cache_size: 24576 l2_cache_size: 5767168 shared_memory_size_per_multiprocessor: 65536 memory_size: 9742909440 bandwidth: 616000000 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: -37 } dim { size: -38 } dim { size: 1 } } } INFO:sleap.nn.training:Finished trainer set up. [11.5s] INFO:sleap.nn.training:Creating tf.data.Datasets for training data generation... INFO:sleap.nn.training:Finished creating training datasets. [343.5s] INFO:sleap.nn.training:Starting training loop... Epoch 1/200 2023-01-20 19:39:55.164551: I tensorflow/stream_executor/cuda/cuda_dnn.cc:369] Loaded cuDNN version 8201 2023-01-20 19:40:00.158419: W tensorflow/core/common_runtime/bfc_allocator.cc:272] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.81GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. 2023-01-20 19:40:00.158497: W tensorflow/core/common_runtime/bfc_allocator.cc:272] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.81GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. 2023-01-20 19:40:00.899045: W tensorflow/core/common_runtime/bfc_allocator.cc:272] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.81GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. 2023-01-20 19:40:00.899118: W tensorflow/core/common_runtime/bfc_allocator.cc:272] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.81GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. 2023-01-20 19:40:01.809365: W tensorflow/core/common_runtime/bfc_allocator.cc:272] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.81GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. 2023-01-20 19:40:01.809466: W tensorflow/core/common_runtime/bfc_allocator.cc:272] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.81GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1032s vs `on_train_batch_end` time: 0.2048s). Check your callbacks. 417/417 - 309s - loss: 0.0013 - val_loss: 0.0012 2023-01-20 19:45:04.248327: W tensorflow/core/common_runtime/bfc_allocator.cc:338] Garbage collection: deallocate free memory regions (i.e., allocations) so that we can re-allocate a larger region to avoid OOM due to memory fragmentation. If you see this message frequently, you are running near the threshold of the available device memory and re-allocation may incur great performance overhead. You may try smaller batch sizes to observe the performance impact. Set TF_ENABLE_GPU_GARBAGE_COLLECTION=false if you'd like to disable this feature. 2023-01-20 19:45:04.778851: W tensorflow/core/common_runtime/bfc_allocator.cc:272] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.22GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. 2023-01-20 19:45:04.817372: W tensorflow/core/common_runtime/bfc_allocator.cc:272] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.22GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. 2023-01-20 19:45:04.817479: W tensorflow/core/common_runtime/bfc_allocator.cc:272] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.22GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. 2023-01-20 19:45:07.282366: W tensorflow/core/common_runtime/bfc_allocator.cc:272] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.81GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. Epoch 2/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 292s - loss: 0.0012 - val_loss: 0.0011 Epoch 3/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 291s - loss: 9.9204e-04 - val_loss: 9.5534e-04 Epoch 4/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 289s - loss: 9.2403e-04 - val_loss: 9.3935e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Epoch 5/200 417/417 - 283s - loss: 9.1476e-04 - val_loss: 9.1127e-04 Epoch 6/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 283s - loss: 8.9041e-04 - val_loss: 9.1418e-04 Epoch 7/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 279s - loss: 8.7426e-04 - val_loss: 8.8192e-04 Epoch 8/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 290s - loss: 8.6949e-04 - val_loss: 8.5781e-04 Epoch 9/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 280s - loss: 8.4462e-04 - val_loss: 8.1740e-04 Epoch 10/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 265s - loss: 8.2515e-04 - val_loss: 8.2176e-04 Epoch 11/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 278s - loss: 7.9611e-04 - val_loss: 8.3332e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Epoch 12/200 417/417 - 279s - loss: 7.9297e-04 - val_loss: 7.4152e-04 Epoch 13/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 295s - loss: 7.6525e-04 - val_loss: 7.9638e-04 Epoch 14/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 277s - loss: 7.6565e-04 - val_loss: 7.7685e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Epoch 15/200 417/417 - 259s - loss: 7.4523e-04 - val_loss: 7.7733e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Epoch 16/200 417/417 - 279s - loss: 7.4416e-04 - val_loss: 7.5004e-04 Epoch 17/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 291s - loss: 7.2879e-04 - val_loss: 7.1444e-04 Epoch 18/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 257s - loss: 7.2984e-04 - val_loss: 7.5298e-04 Epoch 19/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 292s - loss: 7.2059e-04 - val_loss: 6.9392e-04 Epoch 20/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 259s - loss: 7.1280e-04 - val_loss: 7.2249e-04 Epoch 21/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 295s - loss: 6.9575e-04 - val_loss: 6.9741e-04 Epoch 22/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 265s - loss: 6.8220e-04 - val_loss: 6.8356e-04 Epoch 23/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 277s - loss: 6.8179e-04 - val_loss: 6.5719e-04 Epoch 24/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 283s - loss: 6.5860e-04 - val_loss: 6.9156e-04 Epoch 25/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 296s - loss: 6.3121e-04 - val_loss: 6.4107e-04 Epoch 26/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 266s - loss: 6.2182e-04 - val_loss: 6.6905e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Epoch 27/200 417/417 - 293s - loss: 6.0661e-04 - val_loss: 6.0162e-04 Epoch 28/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 268s - loss: 5.8125e-04 - val_loss: 5.7244e-04 Epoch 29/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 282s - loss: 5.6461e-04 - val_loss: 5.9861e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Epoch 30/200 417/417 - 281s - loss: 5.4323e-04 - val_loss: 5.8597e-04 Epoch 31/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 295s - loss: 5.2825e-04 - val_loss: 5.6261e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Epoch 32/200 417/417 - 269s - loss: 5.1420e-04 - val_loss: 5.4449e-04 Epoch 33/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 279s - loss: 5.0968e-04 - val_loss: 5.1478e-04 Epoch 34/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 292s - loss: 4.9704e-04 - val_loss: 4.7701e-04 Epoch 35/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 282s - loss: 4.8564e-04 - val_loss: 4.7206e-04 Epoch 36/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 281s - loss: 4.7401e-04 - val_loss: 4.9955e-04 Epoch 37/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 264s - loss: 4.6802e-04 - val_loss: 4.7869e-04 Epoch 38/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 282s - loss: 4.6911e-04 - val_loss: 4.9804e-04 Epoch 39/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 287s - loss: 4.6784e-04 - val_loss: 4.7713e-04 Epoch 40/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 296s - loss: 4.5995e-04 - val_loss: 4.7305e-04 Epoch 00040: ReduceLROnPlateau reducing learning rate to 4.999999873689376e-05. Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Epoch 41/200 417/417 - 264s - loss: 4.2016e-04 - val_loss: 4.5944e-04 Epoch 42/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 278s - loss: 4.1212e-04 - val_loss: 4.4854e-04 Epoch 43/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 283s - loss: 4.0011e-04 - val_loss: 4.4964e-04 Epoch 44/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 281s - loss: 4.0447e-04 - val_loss: 4.2751e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Epoch 45/200 417/417 - 280s - loss: 3.9507e-04 - val_loss: 4.5947e-04 Epoch 46/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 280s - loss: 3.9264e-04 - val_loss: 4.3268e-04 Epoch 47/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 284s - loss: 4.0162e-04 - val_loss: 4.3859e-04 Epoch 48/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 281s - loss: 3.8626e-04 - val_loss: 4.1238e-04 Epoch 49/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 281s - loss: 3.8675e-04 - val_loss: 4.1599e-04 Epoch 50/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 283s - loss: 3.8678e-04 - val_loss: 4.2478e-04 Epoch 51/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 283s - loss: 3.9259e-04 - val_loss: 4.1941e-04 Epoch 52/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 282s - loss: 3.8345e-04 - val_loss: 4.2548e-04 Epoch 53/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 280s - loss: 3.7503e-04 - val_loss: 4.1544e-04 Epoch 00053: ReduceLROnPlateau reducing learning rate to 2.499999936844688e-05. Epoch 54/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 299s - loss: 3.5880e-04 - val_loss: 3.6920e-04 Epoch 55/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 270s - loss: 3.5433e-04 - val_loss: 4.1573e-04 Epoch 56/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 282s - loss: 3.5639e-04 - val_loss: 3.9354e-04 Epoch 57/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 296s - loss: 3.5319e-04 - val_loss: 3.9887e-04 Epoch 58/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 270s - loss: 3.4182e-04 - val_loss: 3.6333e-04 Epoch 59/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 282s - loss: 3.4461e-04 - val_loss: 4.0702e-04 Epoch 60/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 281s - loss: 3.4265e-04 - val_loss: 3.8081e-04 Epoch 61/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 293s - loss: 3.4454e-04 - val_loss: 3.7173e-04 Epoch 62/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 273s - loss: 3.4049e-04 - val_loss: 3.8618e-04 Epoch 63/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 285s - loss: 3.4454e-04 - val_loss: 3.9053e-04 Epoch 00063: ReduceLROnPlateau reducing learning rate to 1.249999968422344e-05. Epoch 64/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 281s - loss: 3.2972e-04 - val_loss: 3.7200e-04 Epoch 65/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 294s - loss: 3.2610e-04 - val_loss: 3.8121e-04 Epoch 66/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 270s - loss: 3.2248e-04 - val_loss: 3.5325e-04 Epoch 67/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 286s - loss: 3.2953e-04 - val_loss: 3.8028e-04 Epoch 68/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 296s - loss: 3.1524e-04 - val_loss: 3.7079e-04 Epoch 69/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 264s - loss: 3.2900e-04 - val_loss: 3.4512e-04 Epoch 70/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 287s - loss: 3.1861e-04 - val_loss: 3.6857e-04 Epoch 71/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 285s - loss: 3.1800e-04 - val_loss: 3.6372e-04 Epoch 72/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 283s - loss: 3.1867e-04 - val_loss: 3.7326e-04 Epoch 73/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 280s - loss: 3.1516e-04 - val_loss: 3.6831e-04 Epoch 74/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 282s - loss: 3.2453e-04 - val_loss: 3.7136e-04 Epoch 00074: ReduceLROnPlateau reducing learning rate to 6.24999984211172e-06. Epoch 75/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 284s - loss: 3.0793e-04 - val_loss: 3.6358e-04 Epoch 76/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 296s - loss: 3.1208e-04 - val_loss: 3.5091e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Epoch 77/200 417/417 - 264s - loss: 3.1230e-04 - val_loss: 3.6751e-04 Epoch 78/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 300s - loss: 3.0829e-04 - val_loss: 3.6524e-04 Epoch 79/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 269s - loss: 3.1718e-04 - val_loss: 3.6212e-04 Epoch 80/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 282s - loss: 3.0895e-04 - val_loss: 3.4275e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Epoch 81/200 417/417 - 294s - loss: 3.0895e-04 - val_loss: 3.5143e-04 Epoch 82/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 272s - loss: 3.1169e-04 - val_loss: 3.5219e-04 Epoch 83/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 283s - loss: 3.0397e-04 - val_loss: 3.5349e-04 Epoch 84/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 293s - loss: 3.0675e-04 - val_loss: 3.5111e-04 Epoch 85/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 264s - loss: 3.0516e-04 - val_loss: 3.5877e-04 Epoch 00085: ReduceLROnPlateau reducing learning rate to 3.12499992105586e-06. Epoch 86/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 298s - loss: 2.9454e-04 - val_loss: 3.6866e-04 Epoch 87/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 281s - loss: 2.9954e-04 - val_loss: 3.4402e-04 Epoch 88/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 276s - loss: 2.9957e-04 - val_loss: 3.2807e-04 Epoch 89/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 263s - loss: 2.9695e-04 - val_loss: 3.4392e-04 Epoch 90/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 283s - loss: 2.9907e-04 - val_loss: 3.3452e-04 Epoch 91/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 283s - loss: 2.9621e-04 - val_loss: 3.6869e-04 Epoch 92/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 295s - loss: 2.9608e-04 - val_loss: 3.7355e-04 Epoch 93/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 269s - loss: 2.9519e-04 - val_loss: 3.7993e-04 Epoch 00093: ReduceLROnPlateau reducing learning rate to 1.56249996052793e-06. Epoch 94/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 283s - loss: 2.9400e-04 - val_loss: 3.3240e-04 Epoch 95/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 281s - loss: 2.9978e-04 - val_loss: 3.5758e-04 Epoch 96/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 281s - loss: 2.9177e-04 - val_loss: 3.3807e-04 Epoch 97/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 285s - loss: 3.0380e-04 - val_loss: 3.5898e-04 Epoch 98/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 283s - loss: 2.9795e-04 - val_loss: 3.4043e-04 Epoch 99/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 294s - loss: 2.9647e-04 - val_loss: 3.6266e-04 Epoch 100/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 263s - loss: 3.0195e-04 - val_loss: 3.5765e-04 Epoch 00100: ReduceLROnPlateau reducing learning rate to 7.81249980263965e-07. Epoch 101/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 285s - loss: 2.9664e-04 - val_loss: 3.5308e-04 Epoch 102/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 294s - loss: 2.9148e-04 - val_loss: 3.5601e-04 Epoch 103/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 277s - loss: 2.9283e-04 - val_loss: 3.6882e-04 Epoch 104/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 271s - loss: 2.9078e-04 - val_loss: 3.4547e-04 Epoch 105/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 285s - loss: 2.9580e-04 - val_loss: 3.4908e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Epoch 106/200 417/417 - 281s - loss: 2.9245e-04 - val_loss: 3.6540e-04 Epoch 107/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 283s - loss: 2.9154e-04 - val_loss: 3.6698e-04 Epoch 00107: ReduceLROnPlateau reducing learning rate to 3.906249901319825e-07. Epoch 108/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png 417/417 - 295s - loss: 2.9762e-04 - val_loss: 3.4061e-04 Epoch 00108: early stopping INFO:sleap.nn.training:Finished training loop. [511.9 min] INFO:sleap.nn.training:Deleting visualization directory: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png INFO:sleap.nn.training:Saving evaluation metrics to model folder... Predicting... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 0% ETA: -:--:-- ?2023-01-21 04:12:19.661312: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: -86 } dim { size: -87 } dim { size: -88 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -5 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -5 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "GPU" vendor: "NVIDIA" model: "NVIDIA GeForce RTX 2080 Ti" frequency: 1545 num_cores: 68 environment { key: "architecture" value: "7.5" } environment { key: "cuda" value: "11020" } environment { key: "cudnn" value: "8100" } num_registers: 65536 l1_cache_size: 24576 l2_cache_size: 5767168 shared_memory_size_per_multiprocessor: 65536 memory_size: 9742909440 bandwidth: 616000000 } outputs { dtype: DT_FLOAT shape { dim { size: -5 } dim { size: -89 } dim { size: -90 } dim { size: 1 } } } 2023-01-21 04:12:19.661562: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_UINT8 } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_UINT8 shape { dim { size: 4 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -5 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -5 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -5 } dim { size: -94 } dim { size: -95 } dim { size: 3 } } } Predicting... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╸ 100% ETA: 0:00:01 8.2 FPS2023-01-21 04:16:24.069141: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: -86 } dim { size: -87 } dim { size: -88 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -5 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -5 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "GPU" vendor: "NVIDIA" model: "NVIDIA GeForce RTX 2080 Ti" frequency: 1545 num_cores: 68 environment { key: "architecture" value: "7.5" } environment { key: "cuda" value: "11020" } environment { key: "cudnn" value: "8100" } num_registers: 65536 l1_cache_size: 24576 l2_cache_size: 5767168 shared_memory_size_per_multiprocessor: 65536 memory_size: 9742909440 bandwidth: 616000000 } outputs { dtype: DT_FLOAT shape { dim { size: -5 } dim { size: -89 } dim { size: -90 } dim { size: 1 } } } 2023-01-21 04:16:24.069369: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_UINT8 } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_UINT8 shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -5 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -5 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -5 } dim { size: -94 } dim { size: -95 } dim { size: 3 } } } Predicting... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% ETA: 0:00:00 6.8 FPS INFO:sleap.nn.evals:Saved predictions: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/labels_pr.train.slp INFO:sleap.nn.evals:Saved metrics: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/metrics.train.npz INFO:sleap.nn.evals:OKS mAP: 0.533664 Predicting... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 0% ETA: -:--:-- ?2023-01-21 04:16:29.048140: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: -86 } dim { size: -87 } dim { size: -88 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -5 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -5 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "GPU" vendor: "NVIDIA" model: "NVIDIA GeForce RTX 2080 Ti" frequency: 1545 num_cores: 68 environment { key: "architecture" value: "7.5" } environment { key: "cuda" value: "11020" } environment { key: "cudnn" value: "8100" } num_registers: 65536 l1_cache_size: 24576 l2_cache_size: 5767168 shared_memory_size_per_multiprocessor: 65536 memory_size: 9742909440 bandwidth: 616000000 } outputs { dtype: DT_FLOAT shape { dim { size: -5 } dim { size: -89 } dim { size: -90 } dim { size: 1 } } } 2023-01-21 04:16:29.048392: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_UINT8 } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_UINT8 shape { dim { size: 4 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -5 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -5 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -5 } dim { size: -94 } dim { size: -95 } dim { size: 3 } } } Predicting... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╸ 99% ETA: 0:00:01 5.7 FPS2023-01-21 04:16:55.859578: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: -86 } dim { size: -87 } dim { size: -88 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -5 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -5 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "GPU" vendor: "NVIDIA" model: "NVIDIA GeForce RTX 2080 Ti" frequency: 1545 num_cores: 68 environment { key: "architecture" value: "7.5" } environment { key: "cuda" value: "11020" } environment { key: "cudnn" value: "8100" } num_registers: 65536 l1_cache_size: 24576 l2_cache_size: 5767168 shared_memory_size_per_multiprocessor: 65536 memory_size: 9742909440 bandwidth: 616000000 } outputs { dtype: DT_FLOAT shape { dim { size: -5 } dim { size: -89 } dim { size: -90 } dim { size: 1 } } } 2023-01-21 04:16:55.859802: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_UINT8 } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_UINT8 shape { dim { size: 2 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -5 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -5 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -5 } dim { size: -94 } dim { size: -95 } dim { size: 3 } } } Predicting... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% ETA: 0:00:00 2.0 FPS INFO:sleap.nn.evals:Saved predictions: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/labels_pr.val.slp INFO:sleap.nn.evals:Saved metrics: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/metrics.val.npz INFO:sleap.nn.evals:OKS mAP: 0.546813 INFO:sleap.nn.callbacks:Closing the reporter controller/context. INFO:sleap.nn.callbacks:Closing the training controller socket/context. Run Path: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855 Finished training centroid. Resetting monitor window. Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png Start training centered_instance... ['sleap-train', '/tmp/tmp13jcw5cv/230121_041700_training_job.json', '/home/shuyu/SLEAP/2023-01-20-sleap-topdown/sleap-1627-add228.slp', '--zmq', '--save_viz'] INFO:numexpr.utils:NumExpr defaulting to 6 threads. INFO:sleap.nn.training:Versions: SLEAP: 1.2.9 TensorFlow: 2.6.3 Numpy: 1.19.5 Python: 3.7.12 OS: Linux-5.4.0-132-generic-x86_64-with-debian-buster-sid INFO:sleap.nn.training:Training labels file: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/sleap-1627-add228.slp INFO:sleap.nn.training:Training profile: /tmp/tmp13jcw5cv/230121_041700_training_job.json INFO:sleap.nn.training: INFO:sleap.nn.training:Arguments: INFO:sleap.nn.training:{ "training_job_path": "/tmp/tmp13jcw5cv/230121_041700_training_job.json", "labels_path": "/home/shuyu/SLEAP/2023-01-20-sleap-topdown/sleap-1627-add228.slp", "video_paths": [ "" ], "val_labels": null, "test_labels": null, "tensorboard": false, "save_viz": true, "zmq": true, "run_name": "", "prefix": "", "suffix": "", "cpu": false, "first_gpu": false, "last_gpu": false, "gpu": "auto" } INFO:sleap.nn.training: INFO:sleap.nn.training:Training job: INFO:sleap.nn.training:{ "data": { "labels": { "training_labels": null, "validation_labels": null, "validation_fraction": 0.1, "test_labels": null, "split_by_inds": false, "training_inds": null, "validation_inds": null, "test_inds": null, "search_path_hints": [], "skeletons": [] }, "preprocessing": { "ensure_rgb": false, "ensure_grayscale": false, "imagenet_mode": null, "input_scaling": 1.0, "pad_to_stride": null, "resize_and_pad_to_target": true, "target_height": null, "target_width": null }, "instance_cropping": { "center_on_part": null, "crop_size": null, "crop_size_detection_padding": 16 } }, "model": { "backbone": { "leap": null, "unet": { "stem_stride": null, "max_stride": 64, "output_stride": 4, "filters": 24, "filters_rate": 2.0, "middle_block": true, "up_interpolate": true, "stacks": 1 }, "hourglass": null, "resnet": null, "pretrained_encoder": null }, "heads": { "single_instance": null, "centroid": null, "centered_instance": { "anchor_part": null, "part_names": null, "sigma": 5.0, "output_stride": 4, "loss_weight": 1.0, "offset_refinement": false }, "multi_instance": null, "multi_class_bottomup": null, "multi_class_topdown": null } }, "optimization": { "preload_data": true, "augmentation_config": { "rotate": true, "rotation_min_angle": -180.0, "rotation_max_angle": 180.0, "translate": false, "translate_min": -5, "translate_max": 5, "scale": true, "scale_min": 0.7, "scale_max": 1.3, "uniform_noise": false, "uniform_noise_min_val": 0.0, "uniform_noise_max_val": 10.0, "gaussian_noise": true, "gaussian_noise_mean": 5.0, "gaussian_noise_stddev": 1.0, "contrast": true, "contrast_min_gamma": 0.5, "contrast_max_gamma": 2.0, "brightness": true, "brightness_min_val": 0.0, "brightness_max_val": 10.0, "random_crop": false, "random_crop_height": 256, "random_crop_width": 256, "random_flip": false, "flip_horizontal": true }, "online_shuffling": true, "shuffle_buffer_size": 128, "prefetch": true, "batch_size": 4, "batches_per_epoch": null, "min_batches_per_epoch": 200, "val_batches_per_epoch": null, "min_val_batches_per_epoch": 10, "epochs": 200, "optimizer": "adam", "initial_learning_rate": 0.0001, "learning_rate_schedule": { "reduce_on_plateau": true, "reduction_factor": 0.5, "plateau_min_delta": 1e-06, "plateau_patience": 5, "plateau_cooldown": 3, "min_learning_rate": 1e-08 }, "hard_keypoint_mining": { "online_mining": false, "hard_to_easy_ratio": 2.0, "min_hard_keypoints": 2, "max_hard_keypoints": null, "loss_scale": 5.0 }, "early_stopping": { "stop_training_on_plateau": true, "plateau_min_delta": 1e-08, "plateau_patience": 10 } }, "outputs": { "save_outputs": true, "run_name": "230121_041700.centered_instance.n=1855", "run_name_prefix": "exp20_", "run_name_suffix": "", "runs_folder": "/home/shuyu/SLEAP/2023-01-20-sleap-topdown/models", "tags": [ "" ], "save_visualizations": true, "delete_viz_images": true, "zip_outputs": false, "log_to_csv": true, "checkpointing": { "initial_model": false, "best_model": true, "every_epoch": false, "latest_model": false, "final_model": false }, "tensorboard": { "write_logs": false, "loss_frequency": "epoch", "architecture_graph": false, "profile_graph": false, "visualizations": true }, "zmq": { "subscribe_to_controller": true, "controller_address": "tcp://127.0.0.1:9000", "controller_polling_timeout": 10, "publish_updates": true, "publish_address": "tcp://127.0.0.1:9001" } }, "name": "", "description": "", "sleap_version": "1.2.9", "filename": "/tmp/tmp13jcw5cv/230121_041700_training_job.json" } INFO:sleap.nn.training: 2023-01-21 04:17:03.188264: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2023-01-21 04:17:03.199257: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2023-01-21 04:17:03.199429: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero INFO:sleap.nn.training:Auto-selected GPU 0 with 10623 MiB of free memory. INFO:sleap.nn.training:Using GPU 0 for acceleration. INFO:sleap.nn.training:Disabled GPU memory pre-allocation. INFO:sleap.nn.training:System: GPUs: 1/1 available Device: /physical_device:GPU:0 Available: True Initalized: False Memory growth: True INFO:sleap.nn.training: INFO:sleap.nn.training:Initializing trainer... INFO:sleap.nn.training:Loading training labels from: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/sleap-1627-add228.slp INFO:sleap.nn.training:Creating training and validation splits from validation fraction: 0.1 INFO:sleap.nn.training: Splits: Training = 1669 / Validation = 186. INFO:sleap.nn.training:Setting up for training... INFO:sleap.nn.training:Setting up pipeline builders... INFO:sleap.nn.training:Setting up model... INFO:sleap.nn.training:Building test pipeline... 2023-01-21 04:17:09.668249: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. 2023-01-21 04:17:09.668614: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2023-01-21 04:17:09.668822: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2023-01-21 04:17:09.668959: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2023-01-21 04:17:10.104007: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2023-01-21 04:17:10.104191: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2023-01-21 04:17:10.104323: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2023-01-21 04:17:10.104425: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 9268 MB memory: -> device: 0, name: NVIDIA GeForce RTX 2080 Ti, pci bus id: 0000:01:00.0, compute capability: 7.5 2023-01-21 04:17:10.514811: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2) 2023-01-21 04:17:12.668916: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } INFO:sleap.nn.training:Loaded test example. [7.307s] INFO:sleap.nn.training: Input shape: (1152, 1152, 3) INFO:sleap.nn.training:Created Keras model. INFO:sleap.nn.training: Backbone: UNet(stacks=1, filters=24, filters_rate=2.0, kernel_size=3, stem_kernel_size=7, convs_per_block=2, stem_blocks=0, down_blocks=6, middle_block=True, up_blocks=4, up_interpolate=True, block_contraction=False) INFO:sleap.nn.training: Max stride: 64 INFO:sleap.nn.training: Parameters: 70,673,795 INFO:sleap.nn.training: Heads: INFO:sleap.nn.training: [0] = CenteredInstanceConfmapsHead(part_names=['nose', 'leftEar', 'rightEar', 'spine1', 'spine2', 'spine3', 'leftHip', 'rightHip', 'tailBase', 'tailEnd', 'rump'], anchor_part=None, sigma=5.0, output_stride=4, loss_weight=1.0) INFO:sleap.nn.training: Outputs: INFO:sleap.nn.training: [0] = KerasTensor(type_spec=TensorSpec(shape=(None, 288, 288, 11), dtype=tf.float32, name=None), name='CenteredInstanceConfmapsHead/BiasAdd:0', description="created by layer 'CenteredInstanceConfmapsHead'") INFO:sleap.nn.training:Setting up data pipelines... INFO:sleap.nn.training:Training set: n = 1669 INFO:sleap.nn.training:Validation set: n = 186 INFO:sleap.nn.training:Setting up optimization... INFO:sleap.nn.training: Learning rate schedule: LearningRateScheduleConfig(reduce_on_plateau=True, reduction_factor=0.5, plateau_min_delta=1e-06, plateau_patience=5, plateau_cooldown=3, min_learning_rate=1e-08) INFO:sleap.nn.training: Early stopping: EarlyStoppingConfig(stop_training_on_plateau=True, plateau_min_delta=1e-08, plateau_patience=10) INFO:sleap.nn.training:Setting up outputs... INFO:sleap.nn.callbacks:Training controller subscribed to: tcp://127.0.0.1:9000 (topic: ) INFO:sleap.nn.training: ZMQ controller subcribed to: tcp://127.0.0.1:9000 INFO:sleap.nn.callbacks:Progress reporter publishing on: tcp://127.0.0.1:9001 for: not_set INFO:sleap.nn.training: ZMQ progress reporter publish on: tcp://127.0.0.1:9001 INFO:sleap.nn.training:Created run path: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855 INFO:sleap.nn.training:Setting up visualization... 2023-01-21 04:17:15.775985: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 2023-01-21 04:17:17.386180: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } INFO:sleap.nn.training:Finished trainer set up. [11.7s] INFO:sleap.nn.training:Creating tf.data.Datasets for training data generation... 2023-01-21 04:21:42.073818: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 2023-01-21 04:22:13.479458: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } INFO:sleap.nn.training:Finished creating training datasets. [296.4s] INFO:sleap.nn.training:Starting training loop... 2023-01-21 04:22:14.055939: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } Epoch 1/200 2023-01-21 04:22:15.952305: I tensorflow/stream_executor/cuda/cuda_dnn.cc:369] Loaded cuDNN version 8201 2023-01-21 04:22:21.160236: W tensorflow/core/common_runtime/bfc_allocator.cc:272] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.22GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. 2023-01-21 04:22:21.160296: W tensorflow/core/common_runtime/bfc_allocator.cc:272] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.22GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0182s vs `on_train_batch_end` time: 0.3181s). Check your callbacks. 2023-01-21 04:29:20.912960: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 451s - loss: 8.4602e-04 - nose: 9.0624e-04 - leftEar: 8.6067e-04 - rightEar: 8.6052e-04 - spine1: 9.0186e-04 - spine2: 8.9277e-04 - spine3: 8.8463e-04 - leftHip: 8.3906e-04 - rightHip: 8.4893e-04 - tailBase: 7.1160e-04 - tailEnd: 7.0154e-04 - rump: 8.9843e-04 - val_loss: 8.4500e-04 - val_nose: 9.1623e-04 - val_leftEar: 8.6688e-04 - val_rightEar: 8.7146e-04 - val_spine1: 8.9279e-04 - val_spine2: 8.8591e-04 - val_spine3: 8.8270e-04 - val_leftHip: 8.3167e-04 - val_rightHip: 8.5943e-04 - val_tailBase: 6.9533e-04 - val_tailEnd: 6.8958e-04 - val_rump: 9.0299e-04 2023-01-21 04:29:47.093152: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 11 } dim { size: 288 } dim { size: 288 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "GPU" vendor: "NVIDIA" model: "NVIDIA GeForce RTX 2080 Ti" frequency: 1545 num_cores: 68 environment { key: "architecture" value: "7.5" } environment { key: "cuda" value: "11020" } environment { key: "cudnn" value: "8100" } num_registers: 65536 l1_cache_size: 24576 l2_cache_size: 5767168 shared_memory_size_per_multiprocessor: 65536 memory_size: 9719119872 bandwidth: 616000000 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: -5 } dim { size: -6 } dim { size: 1 } } } Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png Epoch 2/200 2023-01-21 04:36:49.594408: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 443s - loss: 8.3101e-04 - nose: 8.9815e-04 - leftEar: 8.4941e-04 - rightEar: 8.4966e-04 - spine1: 8.7565e-04 - spine2: 8.6244e-04 - spine3: 8.5995e-04 - leftHip: 8.2947e-04 - rightHip: 8.3875e-04 - tailBase: 6.9924e-04 - tailEnd: 7.0026e-04 - rump: 8.7811e-04 - val_loss: 8.2067e-04 - val_nose: 9.0782e-04 - val_leftEar: 8.4841e-04 - val_rightEar: 8.5411e-04 - val_spine1: 8.3923e-04 - val_spine2: 8.1544e-04 - val_spine3: 8.5413e-04 - val_leftHip: 8.1393e-04 - val_rightHip: 8.4448e-04 - val_tailBase: 6.8325e-04 - val_tailEnd: 6.8882e-04 - val_rump: 8.7771e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png Epoch 3/200 2023-01-21 04:44:07.257143: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 435s - loss: 8.0637e-04 - nose: 8.8492e-04 - leftEar: 8.3708e-04 - rightEar: 8.3667e-04 - spine1: 8.3353e-04 - spine2: 8.1192e-04 - spine3: 8.1436e-04 - leftHip: 8.0988e-04 - rightHip: 8.1765e-04 - tailBase: 6.7929e-04 - tailEnd: 6.9775e-04 - rump: 8.4702e-04 - val_loss: 7.9124e-04 - val_nose: 8.8406e-04 - val_leftEar: 8.3992e-04 - val_rightEar: 8.4371e-04 - val_spine1: 8.1458e-04 - val_spine2: 7.9218e-04 - val_spine3: 7.6937e-04 - val_leftHip: 7.9206e-04 - val_rightHip: 8.1432e-04 - val_tailBase: 6.4437e-04 - val_tailEnd: 6.8155e-04 - val_rump: 8.2754e-04 Epoch 4/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 04:51:21.647138: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 435s - loss: 7.6627e-04 - nose: 8.4108e-04 - leftEar: 8.1258e-04 - rightEar: 8.1032e-04 - spine1: 7.8915e-04 - spine2: 7.8028e-04 - spine3: 7.3814e-04 - leftHip: 7.7488e-04 - rightHip: 7.7830e-04 - tailBase: 6.3456e-04 - tailEnd: 6.8816e-04 - rump: 7.8146e-04 - val_loss: 7.4157e-04 - val_nose: 8.0461e-04 - val_leftEar: 8.0258e-04 - val_rightEar: 7.9749e-04 - val_spine1: 7.6038e-04 - val_spine2: 7.5402e-04 - val_spine3: 6.9191e-04 - val_leftHip: 7.5363e-04 - val_rightHip: 7.6882e-04 - val_tailBase: 5.9852e-04 - val_tailEnd: 6.6793e-04 - val_rump: 7.5744e-04 Epoch 5/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 04:58:37.713081: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 435s - loss: 7.1426e-04 - nose: 7.6531e-04 - leftEar: 7.7313e-04 - rightEar: 7.7306e-04 - spine1: 7.4646e-04 - spine2: 7.3781e-04 - spine3: 6.5983e-04 - leftHip: 7.3228e-04 - rightHip: 7.3913e-04 - tailBase: 5.7232e-04 - tailEnd: 6.5995e-04 - rump: 6.9757e-04 - val_loss: 6.9927e-04 - val_nose: 7.3547e-04 - val_leftEar: 7.7592e-04 - val_rightEar: 7.7017e-04 - val_spine1: 7.5207e-04 - val_spine2: 7.2379e-04 - val_spine3: 6.6392e-04 - val_leftHip: 7.2553e-04 - val_rightHip: 7.4687e-04 - val_tailBase: 5.1373e-04 - val_tailEnd: 6.1945e-04 - val_rump: 6.6501e-04 Epoch 6/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 05:05:50.281347: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 430s - loss: 6.5309e-04 - nose: 6.9791e-04 - leftEar: 7.2940e-04 - rightEar: 7.2416e-04 - spine1: 7.1315e-04 - spine2: 6.9659e-04 - spine3: 6.0466e-04 - leftHip: 6.7467e-04 - rightHip: 6.7858e-04 - tailBase: 4.7062e-04 - tailEnd: 5.7351e-04 - rump: 6.2072e-04 - val_loss: 6.2582e-04 - val_nose: 6.7641e-04 - val_leftEar: 7.2101e-04 - val_rightEar: 7.1168e-04 - val_spine1: 6.9260e-04 - val_spine2: 6.4751e-04 - val_spine3: 5.6925e-04 - val_leftHip: 6.4550e-04 - val_rightHip: 6.6175e-04 - val_tailBase: 4.0618e-04 - val_tailEnd: 5.4739e-04 - val_rump: 6.0470e-04 Epoch 7/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 05:12:59.355294: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 425s - loss: 6.0326e-04 - nose: 6.5562e-04 - leftEar: 6.9083e-04 - rightEar: 6.8877e-04 - spine1: 6.8145e-04 - spine2: 6.5559e-04 - spine3: 5.6527e-04 - leftHip: 6.2258e-04 - rightHip: 6.2196e-04 - tailBase: 3.9304e-04 - tailEnd: 4.9224e-04 - rump: 5.6852e-04 - val_loss: 5.9888e-04 - val_nose: 6.4876e-04 - val_leftEar: 6.9947e-04 - val_rightEar: 6.7095e-04 - val_spine1: 6.6696e-04 - val_spine2: 6.4909e-04 - val_spine3: 5.7713e-04 - val_leftHip: 6.2826e-04 - val_rightHip: 6.2616e-04 - val_tailBase: 3.8089e-04 - val_tailEnd: 4.5759e-04 - val_rump: 5.8246e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png Epoch 8/200 2023-01-21 05:20:09.272385: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 427s - loss: 5.7081e-04 - nose: 6.2685e-04 - leftEar: 6.5882e-04 - rightEar: 6.5618e-04 - spine1: 6.5519e-04 - spine2: 6.2523e-04 - spine3: 5.4189e-04 - leftHip: 5.8765e-04 - rightHip: 5.8692e-04 - tailBase: 3.5816e-04 - tailEnd: 4.3983e-04 - rump: 5.4220e-04 - val_loss: 5.5454e-04 - val_nose: 6.3738e-04 - val_leftEar: 6.6182e-04 - val_rightEar: 6.4347e-04 - val_spine1: 6.3283e-04 - val_spine2: 5.7423e-04 - val_spine3: 5.2744e-04 - val_leftHip: 5.4911e-04 - val_rightHip: 5.6571e-04 - val_tailBase: 3.3399e-04 - val_tailEnd: 4.1090e-04 - val_rump: 5.6301e-04 Epoch 9/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 05:27:20.298523: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 426s - loss: 5.4454e-04 - nose: 6.0368e-04 - leftEar: 6.4033e-04 - rightEar: 6.3287e-04 - spine1: 6.3772e-04 - spine2: 6.0082e-04 - spine3: 5.1865e-04 - leftHip: 5.6069e-04 - rightHip: 5.6121e-04 - tailBase: 3.2885e-04 - tailEnd: 3.8765e-04 - rump: 5.1743e-04 - val_loss: 5.3006e-04 - val_nose: 6.0665e-04 - val_leftEar: 6.4426e-04 - val_rightEar: 6.2455e-04 - val_spine1: 6.1891e-04 - val_spine2: 5.7432e-04 - val_spine3: 5.0764e-04 - val_leftHip: 5.3629e-04 - val_rightHip: 5.5110e-04 - val_tailBase: 2.9697e-04 - val_tailEnd: 3.5216e-04 - val_rump: 5.1785e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png Epoch 10/200 2023-01-21 05:34:31.769623: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 429s - loss: 5.3023e-04 - nose: 5.8739e-04 - leftEar: 6.2480e-04 - rightEar: 6.2120e-04 - spine1: 6.2636e-04 - spine2: 5.8812e-04 - spine3: 5.0694e-04 - leftHip: 5.4929e-04 - rightHip: 5.4452e-04 - tailBase: 3.1411e-04 - tailEnd: 3.6830e-04 - rump: 5.0153e-04 - val_loss: 5.2041e-04 - val_nose: 6.0482e-04 - val_leftEar: 6.3119e-04 - val_rightEar: 6.0721e-04 - val_spine1: 6.1193e-04 - val_spine2: 5.6293e-04 - val_spine3: 4.9793e-04 - val_leftHip: 5.2091e-04 - val_rightHip: 5.3664e-04 - val_tailBase: 2.7651e-04 - val_tailEnd: 3.4957e-04 - val_rump: 5.2491e-04 Epoch 11/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 05:41:43.353803: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 429s - loss: 5.1663e-04 - nose: 5.7667e-04 - leftEar: 6.1345e-04 - rightEar: 6.0528e-04 - spine1: 6.1512e-04 - spine2: 5.7087e-04 - spine3: 4.9410e-04 - leftHip: 5.3282e-04 - rightHip: 5.3195e-04 - tailBase: 3.0120e-04 - tailEnd: 3.5099e-04 - rump: 4.9049e-04 - val_loss: 5.1773e-04 - val_nose: 5.8960e-04 - val_leftEar: 6.2260e-04 - val_rightEar: 5.9952e-04 - val_spine1: 6.0370e-04 - val_spine2: 5.5124e-04 - val_spine3: 5.0260e-04 - val_leftHip: 5.3792e-04 - val_rightHip: 5.3348e-04 - val_tailBase: 2.9390e-04 - val_tailEnd: 3.3328e-04 - val_rump: 5.2722e-04 Epoch 12/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 05:48:55.566945: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 427s - loss: 5.0430e-04 - nose: 5.6949e-04 - leftEar: 5.9965e-04 - rightEar: 5.9086e-04 - spine1: 6.0009e-04 - spine2: 5.6029e-04 - spine3: 4.8762e-04 - leftHip: 5.2040e-04 - rightHip: 5.2457e-04 - tailBase: 2.8801e-04 - tailEnd: 3.2838e-04 - rump: 4.7791e-04 - val_loss: 4.9306e-04 - val_nose: 5.8200e-04 - val_leftEar: 5.7844e-04 - val_rightEar: 5.7244e-04 - val_spine1: 5.8141e-04 - val_spine2: 5.3122e-04 - val_spine3: 4.8853e-04 - val_leftHip: 5.1936e-04 - val_rightHip: 5.2611e-04 - val_tailBase: 2.6679e-04 - val_tailEnd: 2.9160e-04 - val_rump: 4.8571e-04 Epoch 13/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 05:56:05.579818: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 427s - loss: 4.9052e-04 - nose: 5.5532e-04 - leftEar: 5.8055e-04 - rightEar: 5.7647e-04 - spine1: 5.8606e-04 - spine2: 5.4541e-04 - spine3: 4.7096e-04 - leftHip: 5.1190e-04 - rightHip: 5.0733e-04 - tailBase: 2.7939e-04 - tailEnd: 3.1891e-04 - rump: 4.6344e-04 - val_loss: 4.8789e-04 - val_nose: 5.5120e-04 - val_leftEar: 5.8420e-04 - val_rightEar: 5.5753e-04 - val_spine1: 5.7434e-04 - val_spine2: 5.2683e-04 - val_spine3: 4.7521e-04 - val_leftHip: 5.0030e-04 - val_rightHip: 5.2054e-04 - val_tailBase: 2.6527e-04 - val_tailEnd: 3.1230e-04 - val_rump: 4.9910e-04 Epoch 14/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 06:03:20.321437: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 431s - loss: 4.8426e-04 - nose: 5.4695e-04 - leftEar: 5.7323e-04 - rightEar: 5.6339e-04 - spine1: 5.8371e-04 - spine2: 5.4150e-04 - spine3: 4.6635e-04 - leftHip: 5.0554e-04 - rightHip: 5.0420e-04 - tailBase: 2.7501e-04 - tailEnd: 3.0859e-04 - rump: 4.5836e-04 - val_loss: 4.8673e-04 - val_nose: 5.5257e-04 - val_leftEar: 5.7266e-04 - val_rightEar: 5.6778e-04 - val_spine1: 5.8902e-04 - val_spine2: 5.4164e-04 - val_spine3: 4.7787e-04 - val_leftHip: 4.9260e-04 - val_rightHip: 5.3101e-04 - val_tailBase: 2.5887e-04 - val_tailEnd: 2.8743e-04 - val_rump: 4.8252e-04 Epoch 15/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 06:10:37.858990: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 434s - loss: 4.6886e-04 - nose: 5.3209e-04 - leftEar: 5.5510e-04 - rightEar: 5.4787e-04 - spine1: 5.6989e-04 - spine2: 5.2424e-04 - spine3: 4.4992e-04 - leftHip: 4.8827e-04 - rightHip: 4.8883e-04 - tailBase: 2.6361e-04 - tailEnd: 2.9641e-04 - rump: 4.4119e-04 - val_loss: 4.7443e-04 - val_nose: 5.5353e-04 - val_leftEar: 5.8815e-04 - val_rightEar: 5.5656e-04 - val_spine1: 5.6132e-04 - val_spine2: 5.0796e-04 - val_spine3: 4.5027e-04 - val_leftHip: 4.8724e-04 - val_rightHip: 4.9049e-04 - val_tailBase: 2.5450e-04 - val_tailEnd: 2.9729e-04 - val_rump: 4.7138e-04 Epoch 16/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 06:17:52.576758: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 431s - loss: 4.6386e-04 - nose: 5.2891e-04 - leftEar: 5.4861e-04 - rightEar: 5.4250e-04 - spine1: 5.6306e-04 - spine2: 5.1901e-04 - spine3: 4.4590e-04 - leftHip: 4.8259e-04 - rightHip: 4.8107e-04 - tailBase: 2.6129e-04 - tailEnd: 2.9306e-04 - rump: 4.3644e-04 - val_loss: 4.6175e-04 - val_nose: 5.1289e-04 - val_leftEar: 5.5336e-04 - val_rightEar: 5.2811e-04 - val_spine1: 5.4678e-04 - val_spine2: 5.0103e-04 - val_spine3: 4.6315e-04 - val_leftHip: 4.7770e-04 - val_rightHip: 4.8841e-04 - val_tailBase: 2.5367e-04 - val_tailEnd: 2.8930e-04 - val_rump: 4.6485e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png Epoch 17/200 2023-01-21 06:25:11.298209: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 436s - loss: 4.5800e-04 - nose: 5.2321e-04 - leftEar: 5.4014e-04 - rightEar: 5.3639e-04 - spine1: 5.5802e-04 - spine2: 5.1061e-04 - spine3: 4.4121e-04 - leftHip: 4.7873e-04 - rightHip: 4.7653e-04 - tailBase: 2.5628e-04 - tailEnd: 2.8619e-04 - rump: 4.3069e-04 - val_loss: 4.5831e-04 - val_nose: 5.3507e-04 - val_leftEar: 5.3767e-04 - val_rightEar: 5.1481e-04 - val_spine1: 5.3169e-04 - val_spine2: 5.0655e-04 - val_spine3: 4.6029e-04 - val_leftHip: 4.7291e-04 - val_rightHip: 5.0182e-04 - val_tailBase: 2.4395e-04 - val_tailEnd: 2.7100e-04 - val_rump: 4.6570e-04 Epoch 18/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 06:32:26.824836: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 432s - loss: 4.4857e-04 - nose: 5.1226e-04 - leftEar: 5.2486e-04 - rightEar: 5.2399e-04 - spine1: 5.4902e-04 - spine2: 5.0021e-04 - spine3: 4.3380e-04 - leftHip: 4.6749e-04 - rightHip: 4.6778e-04 - tailBase: 2.4863e-04 - tailEnd: 2.7974e-04 - rump: 4.2646e-04 - val_loss: 4.4754e-04 - val_nose: 5.2361e-04 - val_leftEar: 5.2649e-04 - val_rightEar: 5.0757e-04 - val_spine1: 5.3465e-04 - val_spine2: 4.8451e-04 - val_spine3: 4.3380e-04 - val_leftHip: 4.7716e-04 - val_rightHip: 4.6087e-04 - val_tailBase: 2.5022e-04 - val_tailEnd: 2.7609e-04 - val_rump: 4.4797e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png Epoch 19/200 2023-01-21 06:39:40.973677: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 432s - loss: 4.4004e-04 - nose: 5.0505e-04 - leftEar: 5.1816e-04 - rightEar: 5.0969e-04 - spine1: 5.3340e-04 - spine2: 4.8860e-04 - spine3: 4.2578e-04 - leftHip: 4.6069e-04 - rightHip: 4.5595e-04 - tailBase: 2.4702e-04 - tailEnd: 2.7913e-04 - rump: 4.1701e-04 - val_loss: 4.4685e-04 - val_nose: 5.1717e-04 - val_leftEar: 5.3816e-04 - val_rightEar: 5.0296e-04 - val_spine1: 5.3259e-04 - val_spine2: 4.9608e-04 - val_spine3: 4.4514e-04 - val_leftHip: 4.6644e-04 - val_rightHip: 4.8409e-04 - val_tailBase: 2.2827e-04 - val_tailEnd: 2.5414e-04 - val_rump: 4.5030e-04 Epoch 20/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 06:46:53.177564: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 439s - loss: 4.3147e-04 - nose: 4.9500e-04 - leftEar: 5.0380e-04 - rightEar: 4.9844e-04 - spine1: 5.2471e-04 - spine2: 4.8094e-04 - spine3: 4.1625e-04 - leftHip: 4.5251e-04 - rightHip: 4.5237e-04 - tailBase: 2.4026e-04 - tailEnd: 2.7359e-04 - rump: 4.0830e-04 - val_loss: 4.4768e-04 - val_nose: 4.9844e-04 - val_leftEar: 5.1388e-04 - val_rightEar: 5.0496e-04 - val_spine1: 5.5454e-04 - val_spine2: 5.1598e-04 - val_spine3: 4.5641e-04 - val_leftHip: 4.6453e-04 - val_rightHip: 4.8063e-04 - val_tailBase: 2.2751e-04 - val_tailEnd: 2.7465e-04 - val_rump: 4.3290e-04 Epoch 21/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 06:54:10.054415: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 427s - loss: 4.2547e-04 - nose: 4.9109e-04 - leftEar: 4.9353e-04 - rightEar: 4.8761e-04 - spine1: 5.1690e-04 - spine2: 4.7158e-04 - spine3: 4.0804e-04 - leftHip: 4.4467e-04 - rightHip: 4.4461e-04 - tailBase: 2.4088e-04 - tailEnd: 2.7494e-04 - rump: 4.0636e-04 - val_loss: 4.3225e-04 - val_nose: 5.0199e-04 - val_leftEar: 5.1274e-04 - val_rightEar: 4.8667e-04 - val_spine1: 5.2276e-04 - val_spine2: 4.8062e-04 - val_spine3: 4.2646e-04 - val_leftHip: 4.4463e-04 - val_rightHip: 4.7007e-04 - val_tailBase: 2.3064e-04 - val_tailEnd: 2.5934e-04 - val_rump: 4.1881e-04 Epoch 22/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 07:01:25.651524: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 430s - loss: 4.2057e-04 - nose: 4.8127e-04 - leftEar: 4.8738e-04 - rightEar: 4.8482e-04 - spine1: 5.1148e-04 - spine2: 4.6892e-04 - spine3: 4.0504e-04 - leftHip: 4.4373e-04 - rightHip: 4.4169e-04 - tailBase: 2.3335e-04 - tailEnd: 2.6446e-04 - rump: 4.0418e-04 - val_loss: 4.2050e-04 - val_nose: 5.0445e-04 - val_leftEar: 4.7700e-04 - val_rightEar: 4.6946e-04 - val_spine1: 4.8966e-04 - val_spine2: 4.4754e-04 - val_spine3: 4.1063e-04 - val_leftHip: 4.2643e-04 - val_rightHip: 4.5182e-04 - val_tailBase: 2.3420e-04 - val_tailEnd: 2.6667e-04 - val_rump: 4.4766e-04 Epoch 23/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 07:08:43.259472: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 431s - loss: 4.1357e-04 - nose: 4.7818e-04 - leftEar: 4.7589e-04 - rightEar: 4.7707e-04 - spine1: 5.0367e-04 - spine2: 4.6075e-04 - spine3: 4.0355e-04 - leftHip: 4.3400e-04 - rightHip: 4.3351e-04 - tailBase: 2.2988e-04 - tailEnd: 2.5667e-04 - rump: 3.9611e-04 - val_loss: 4.2120e-04 - val_nose: 4.9945e-04 - val_leftEar: 4.9851e-04 - val_rightEar: 4.8570e-04 - val_spine1: 5.0762e-04 - val_spine2: 4.5679e-04 - val_spine3: 4.0309e-04 - val_leftHip: 4.3933e-04 - val_rightHip: 4.4612e-04 - val_tailBase: 2.2314e-04 - val_tailEnd: 2.6615e-04 - val_rump: 4.0734e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png Epoch 24/200 2023-01-21 07:15:58.204241: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 443s - loss: 4.0795e-04 - nose: 4.6777e-04 - leftEar: 4.7292e-04 - rightEar: 4.7031e-04 - spine1: 4.9835e-04 - spine2: 4.5592e-04 - spine3: 3.9750e-04 - leftHip: 4.3077e-04 - rightHip: 4.2735e-04 - tailBase: 2.2481e-04 - tailEnd: 2.5333e-04 - rump: 3.8836e-04 - val_loss: 4.2085e-04 - val_nose: 4.8864e-04 - val_leftEar: 5.0152e-04 - val_rightEar: 4.6528e-04 - val_spine1: 4.9750e-04 - val_spine2: 4.6915e-04 - val_spine3: 4.2320e-04 - val_leftHip: 4.2336e-04 - val_rightHip: 4.6654e-04 - val_tailBase: 2.2174e-04 - val_tailEnd: 2.4656e-04 - val_rump: 4.2592e-04 Epoch 25/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 07:23:15.483460: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 430s - loss: 4.0320e-04 - nose: 4.6173e-04 - leftEar: 4.6012e-04 - rightEar: 4.5908e-04 - spine1: 4.9352e-04 - spine2: 4.5419e-04 - spine3: 3.9073e-04 - leftHip: 4.2552e-04 - rightHip: 4.2496e-04 - tailBase: 2.2648e-04 - tailEnd: 2.5463e-04 - rump: 3.8419e-04 - val_loss: 4.1050e-04 - val_nose: 4.8228e-04 - val_leftEar: 4.6502e-04 - val_rightEar: 4.6994e-04 - val_spine1: 4.9090e-04 - val_spine2: 4.4429e-04 - val_spine3: 4.0730e-04 - val_leftHip: 4.2904e-04 - val_rightHip: 4.5257e-04 - val_tailBase: 2.1820e-04 - val_tailEnd: 2.4250e-04 - val_rump: 4.1345e-04 Epoch 26/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 07:30:41.352042: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 470s - loss: 3.9899e-04 - nose: 4.5752e-04 - leftEar: 4.5608e-04 - rightEar: 4.5751e-04 - spine1: 4.8618e-04 - spine2: 4.4578e-04 - spine3: 3.9175e-04 - leftHip: 4.2398e-04 - rightHip: 4.1918e-04 - tailBase: 2.2199e-04 - tailEnd: 2.4609e-04 - rump: 3.8283e-04 - val_loss: 4.0556e-04 - val_nose: 4.8109e-04 - val_leftEar: 4.5742e-04 - val_rightEar: 4.4302e-04 - val_spine1: 4.8297e-04 - val_spine2: 4.5491e-04 - val_spine3: 4.1991e-04 - val_leftHip: 4.3331e-04 - val_rightHip: 4.5186e-04 - val_tailBase: 2.0265e-04 - val_tailEnd: 2.2621e-04 - val_rump: 4.0779e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png Epoch 27/200 2023-01-21 07:38:28.855373: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 431s - loss: 3.9434e-04 - nose: 4.5179e-04 - leftEar: 4.4883e-04 - rightEar: 4.4979e-04 - spine1: 4.7922e-04 - spine2: 4.4299e-04 - spine3: 3.8855e-04 - leftHip: 4.1572e-04 - rightHip: 4.1823e-04 - tailBase: 2.1778e-04 - tailEnd: 2.4521e-04 - rump: 3.7964e-04 - val_loss: 4.0453e-04 - val_nose: 4.5806e-04 - val_leftEar: 4.7035e-04 - val_rightEar: 4.4677e-04 - val_spine1: 4.9480e-04 - val_spine2: 4.5456e-04 - val_spine3: 4.0355e-04 - val_leftHip: 4.3097e-04 - val_rightHip: 4.3512e-04 - val_tailBase: 2.0877e-04 - val_tailEnd: 2.3369e-04 - val_rump: 4.1316e-04 Epoch 28/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 07:45:47.893381: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 435s - loss: 3.9102e-04 - nose: 4.4822e-04 - leftEar: 4.4613e-04 - rightEar: 4.4495e-04 - spine1: 4.7389e-04 - spine2: 4.3846e-04 - spine3: 3.8340e-04 - leftHip: 4.1343e-04 - rightHip: 4.1075e-04 - tailBase: 2.1893e-04 - tailEnd: 2.4591e-04 - rump: 3.7710e-04 - val_loss: 3.9600e-04 - val_nose: 4.6014e-04 - val_leftEar: 4.6114e-04 - val_rightEar: 4.4011e-04 - val_spine1: 4.7778e-04 - val_spine2: 4.3809e-04 - val_spine3: 3.9157e-04 - val_leftHip: 4.1709e-04 - val_rightHip: 4.2550e-04 - val_tailBase: 2.1405e-04 - val_tailEnd: 2.2930e-04 - val_rump: 4.0120e-04 Epoch 29/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 07:53:02.361588: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 428s - loss: 3.9053e-04 - nose: 4.4592e-04 - leftEar: 4.4440e-04 - rightEar: 4.4750e-04 - spine1: 4.7401e-04 - spine2: 4.3739e-04 - spine3: 3.8114e-04 - leftHip: 4.1153e-04 - rightHip: 4.1548e-04 - tailBase: 2.1936e-04 - tailEnd: 2.4235e-04 - rump: 3.7671e-04 - val_loss: 4.0213e-04 - val_nose: 4.6923e-04 - val_leftEar: 4.7499e-04 - val_rightEar: 4.5326e-04 - val_spine1: 4.8758e-04 - val_spine2: 4.4164e-04 - val_spine3: 3.9435e-04 - val_leftHip: 4.2154e-04 - val_rightHip: 4.4926e-04 - val_tailBase: 2.0615e-04 - val_tailEnd: 2.2591e-04 - val_rump: 3.9950e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png Epoch 30/200 2023-01-21 08:00:13.673700: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 429s - loss: 3.8170e-04 - nose: 4.3718e-04 - leftEar: 4.3752e-04 - rightEar: 4.3647e-04 - spine1: 4.6712e-04 - spine2: 4.2636e-04 - spine3: 3.7483e-04 - leftHip: 4.0294e-04 - rightHip: 4.0406e-04 - tailBase: 2.1228e-04 - tailEnd: 2.3248e-04 - rump: 3.6748e-04 - val_loss: 4.0162e-04 - val_nose: 4.5833e-04 - val_leftEar: 4.5702e-04 - val_rightEar: 4.4459e-04 - val_spine1: 4.7511e-04 - val_spine2: 4.4266e-04 - val_spine3: 3.9773e-04 - val_leftHip: 4.4646e-04 - val_rightHip: 4.4130e-04 - val_tailBase: 2.1929e-04 - val_tailEnd: 2.2896e-04 - val_rump: 4.0631e-04 Epoch 31/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 08:07:23.350304: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 430s - loss: 3.8016e-04 - nose: 4.3456e-04 - leftEar: 4.3451e-04 - rightEar: 4.3586e-04 - spine1: 4.6507e-04 - spine2: 4.2446e-04 - spine3: 3.7241e-04 - leftHip: 4.0281e-04 - rightHip: 4.0419e-04 - tailBase: 2.1142e-04 - tailEnd: 2.3367e-04 - rump: 3.6283e-04 - val_loss: 4.0138e-04 - val_nose: 4.5763e-04 - val_leftEar: 4.7126e-04 - val_rightEar: 4.3813e-04 - val_spine1: 4.8448e-04 - val_spine2: 4.4122e-04 - val_spine3: 3.9558e-04 - val_leftHip: 4.3877e-04 - val_rightHip: 4.4990e-04 - val_tailBase: 2.1376e-04 - val_tailEnd: 2.3252e-04 - val_rump: 3.9193e-04 Epoch 32/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 08:14:31.931844: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 431s - loss: 3.7766e-04 - nose: 4.3000e-04 - leftEar: 4.2984e-04 - rightEar: 4.3121e-04 - spine1: 4.6109e-04 - spine2: 4.2117e-04 - spine3: 3.7091e-04 - leftHip: 4.0194e-04 - rightHip: 3.9942e-04 - tailBase: 2.0885e-04 - tailEnd: 2.3439e-04 - rump: 3.6549e-04 - val_loss: 3.8844e-04 - val_nose: 4.5035e-04 - val_leftEar: 4.4582e-04 - val_rightEar: 4.2503e-04 - val_spine1: 4.6047e-04 - val_spine2: 4.2561e-04 - val_spine3: 3.8682e-04 - val_leftHip: 4.0922e-04 - val_rightHip: 4.1591e-04 - val_tailBase: 2.1823e-04 - val_tailEnd: 2.4107e-04 - val_rump: 3.9426e-04 Epoch 33/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 08:21:43.103636: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 426s - loss: 3.7144e-04 - nose: 4.1957e-04 - leftEar: 4.2150e-04 - rightEar: 4.2300e-04 - spine1: 4.5288e-04 - spine2: 4.1712e-04 - spine3: 3.6496e-04 - leftHip: 3.9509e-04 - rightHip: 3.9500e-04 - tailBase: 2.0761e-04 - tailEnd: 2.2982e-04 - rump: 3.5927e-04 - val_loss: 3.8562e-04 - val_nose: 4.3899e-04 - val_leftEar: 4.2782e-04 - val_rightEar: 4.2723e-04 - val_spine1: 4.6208e-04 - val_spine2: 4.2334e-04 - val_spine3: 3.7257e-04 - val_leftHip: 3.9758e-04 - val_rightHip: 4.3914e-04 - val_tailBase: 2.0678e-04 - val_tailEnd: 2.5589e-04 - val_rump: 3.9036e-04 Epoch 34/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 08:28:54.035615: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 435s - loss: 3.6959e-04 - nose: 4.1446e-04 - leftEar: 4.1795e-04 - rightEar: 4.1998e-04 - spine1: 4.4812e-04 - spine2: 4.1363e-04 - spine3: 3.6298e-04 - leftHip: 3.9472e-04 - rightHip: 3.9539e-04 - tailBase: 2.1160e-04 - tailEnd: 2.2835e-04 - rump: 3.5833e-04 - val_loss: 3.8998e-04 - val_nose: 4.3827e-04 - val_leftEar: 4.4266e-04 - val_rightEar: 4.3001e-04 - val_spine1: 4.6511e-04 - val_spine2: 4.3551e-04 - val_spine3: 3.8929e-04 - val_leftHip: 4.2095e-04 - val_rightHip: 4.2516e-04 - val_tailBase: 2.2187e-04 - val_tailEnd: 2.3192e-04 - val_rump: 3.8907e-04 Epoch 35/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 08:36:11.336696: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 425s - loss: 3.6523e-04 - nose: 4.0771e-04 - leftEar: 4.1451e-04 - rightEar: 4.1334e-04 - spine1: 4.4645e-04 - spine2: 4.1029e-04 - spine3: 3.5742e-04 - leftHip: 3.8918e-04 - rightHip: 3.8922e-04 - tailBase: 2.0459e-04 - tailEnd: 2.2860e-04 - rump: 3.5626e-04 - val_loss: 3.8448e-04 - val_nose: 4.2915e-04 - val_leftEar: 4.5184e-04 - val_rightEar: 4.2478e-04 - val_spine1: 4.6553e-04 - val_spine2: 4.2018e-04 - val_spine3: 3.8304e-04 - val_leftHip: 4.1523e-04 - val_rightHip: 4.1454e-04 - val_tailBase: 2.1149e-04 - val_tailEnd: 2.2093e-04 - val_rump: 3.9256e-04 Epoch 36/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 08:43:23.957608: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 476s - loss: 3.6114e-04 - nose: 4.0846e-04 - leftEar: 4.0505e-04 - rightEar: 4.1078e-04 - spine1: 4.4182e-04 - spine2: 4.0472e-04 - spine3: 3.5636e-04 - leftHip: 3.8468e-04 - rightHip: 3.8467e-04 - tailBase: 1.9925e-04 - tailEnd: 2.2485e-04 - rump: 3.5191e-04 - val_loss: 3.6160e-04 - val_nose: 4.0346e-04 - val_leftEar: 4.1014e-04 - val_rightEar: 3.9587e-04 - val_spine1: 4.4557e-04 - val_spine2: 3.9958e-04 - val_spine3: 3.6446e-04 - val_leftHip: 3.7780e-04 - val_rightHip: 4.0730e-04 - val_tailBase: 2.0148e-04 - val_tailEnd: 2.0316e-04 - val_rump: 3.6880e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png Epoch 37/200 2023-01-21 08:51:22.905822: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 462s - loss: 3.5790e-04 - nose: 3.9916e-04 - leftEar: 4.0130e-04 - rightEar: 4.0040e-04 - spine1: 4.3829e-04 - spine2: 4.0090e-04 - spine3: 3.5335e-04 - leftHip: 3.8381e-04 - rightHip: 3.8042e-04 - tailBase: 2.0556e-04 - tailEnd: 2.2322e-04 - rump: 3.5045e-04 - val_loss: 3.8239e-04 - val_nose: 4.2962e-04 - val_leftEar: 4.3861e-04 - val_rightEar: 4.4287e-04 - val_spine1: 4.5361e-04 - val_spine2: 4.2589e-04 - val_spine3: 3.8534e-04 - val_leftHip: 4.0313e-04 - val_rightHip: 4.3331e-04 - val_tailBase: 1.8747e-04 - val_tailEnd: 2.1516e-04 - val_rump: 3.9131e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png Epoch 38/200 2023-01-21 08:58:57.840600: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 412s - loss: 3.5682e-04 - nose: 3.9827e-04 - leftEar: 4.0440e-04 - rightEar: 3.9873e-04 - spine1: 4.3475e-04 - spine2: 4.0140e-04 - spine3: 3.5565e-04 - leftHip: 3.8272e-04 - rightHip: 3.8552e-04 - tailBase: 1.9666e-04 - tailEnd: 2.2076e-04 - rump: 3.4615e-04 - val_loss: 3.7808e-04 - val_nose: 4.3507e-04 - val_leftEar: 4.4365e-04 - val_rightEar: 4.1729e-04 - val_spine1: 4.4224e-04 - val_spine2: 4.0178e-04 - val_spine3: 3.7557e-04 - val_leftHip: 3.8888e-04 - val_rightHip: 4.1055e-04 - val_tailBase: 2.2306e-04 - val_tailEnd: 2.4002e-04 - val_rump: 3.8073e-04 Epoch 39/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 09:06:03.823164: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 424s - loss: 3.5609e-04 - nose: 3.9422e-04 - leftEar: 4.0259e-04 - rightEar: 4.0036e-04 - spine1: 4.3704e-04 - spine2: 4.0284e-04 - spine3: 3.5476e-04 - leftHip: 3.8111e-04 - rightHip: 3.8132e-04 - tailBase: 1.9774e-04 - tailEnd: 2.1782e-04 - rump: 3.4714e-04 - val_loss: 3.6509e-04 - val_nose: 4.2051e-04 - val_leftEar: 4.1769e-04 - val_rightEar: 4.0364e-04 - val_spine1: 4.3909e-04 - val_spine2: 4.0603e-04 - val_spine3: 3.6407e-04 - val_leftHip: 3.8066e-04 - val_rightHip: 4.0700e-04 - val_tailBase: 1.9837e-04 - val_tailEnd: 2.0994e-04 - val_rump: 3.6897e-04 Epoch 40/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 09:13:11.532530: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 420s - loss: 3.5223e-04 - nose: 3.9473e-04 - leftEar: 4.0004e-04 - rightEar: 4.0059e-04 - spine1: 4.2980e-04 - spine2: 3.9304e-04 - spine3: 3.4834e-04 - leftHip: 3.7859e-04 - rightHip: 3.7583e-04 - tailBase: 1.9514e-04 - tailEnd: 2.1653e-04 - rump: 3.4192e-04 - val_loss: 3.5675e-04 - val_nose: 3.9852e-04 - val_leftEar: 4.0833e-04 - val_rightEar: 3.9859e-04 - val_spine1: 4.1742e-04 - val_spine2: 3.8931e-04 - val_spine3: 3.6784e-04 - val_leftHip: 3.7942e-04 - val_rightHip: 3.9275e-04 - val_tailBase: 1.9890e-04 - val_tailEnd: 2.0810e-04 - val_rump: 3.6506e-04 Epoch 41/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 09:20:24.409403: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 426s - loss: 3.4642e-04 - nose: 3.8688e-04 - leftEar: 3.9009e-04 - rightEar: 3.8995e-04 - spine1: 4.2188e-04 - spine2: 3.8864e-04 - spine3: 3.4204e-04 - leftHip: 3.7322e-04 - rightHip: 3.7443e-04 - tailBase: 1.9203e-04 - tailEnd: 2.1353e-04 - rump: 3.3791e-04 - val_loss: 3.6882e-04 - val_nose: 4.1297e-04 - val_leftEar: 4.1619e-04 - val_rightEar: 3.9573e-04 - val_spine1: 4.4569e-04 - val_spine2: 4.1963e-04 - val_spine3: 3.8844e-04 - val_leftHip: 3.9514e-04 - val_rightHip: 4.0562e-04 - val_tailBase: 1.9182e-04 - val_tailEnd: 2.0201e-04 - val_rump: 3.8375e-04 Epoch 42/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 09:27:31.913962: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 427s - loss: 3.4733e-04 - nose: 3.8252e-04 - leftEar: 3.8769e-04 - rightEar: 3.8929e-04 - spine1: 4.2564e-04 - spine2: 3.9120e-04 - spine3: 3.4791e-04 - leftHip: 3.7610e-04 - rightHip: 3.7221e-04 - tailBase: 1.9588e-04 - tailEnd: 2.1126e-04 - rump: 3.4091e-04 - val_loss: 3.5978e-04 - val_nose: 4.1505e-04 - val_leftEar: 4.0299e-04 - val_rightEar: 3.8644e-04 - val_spine1: 4.3728e-04 - val_spine2: 4.0861e-04 - val_spine3: 3.6727e-04 - val_leftHip: 3.9025e-04 - val_rightHip: 3.9992e-04 - val_tailBase: 1.8752e-04 - val_tailEnd: 1.9261e-04 - val_rump: 3.6967e-04 Epoch 43/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 09:34:41.148090: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 431s - loss: 3.4210e-04 - nose: 3.7633e-04 - leftEar: 3.8261e-04 - rightEar: 3.8943e-04 - spine1: 4.1915e-04 - spine2: 3.8113e-04 - spine3: 3.3702e-04 - leftHip: 3.6518e-04 - rightHip: 3.6975e-04 - tailBase: 1.9597e-04 - tailEnd: 2.1278e-04 - rump: 3.3376e-04 - val_loss: 3.5917e-04 - val_nose: 4.0149e-04 - val_leftEar: 4.1073e-04 - val_rightEar: 3.8493e-04 - val_spine1: 4.2837e-04 - val_spine2: 4.0349e-04 - val_spine3: 3.7077e-04 - val_leftHip: 3.8014e-04 - val_rightHip: 3.9696e-04 - val_tailBase: 1.9550e-04 - val_tailEnd: 2.1300e-04 - val_rump: 3.6554e-04 Epoch 44/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 09:41:49.168388: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 419s - loss: 3.4270e-04 - nose: 3.7718e-04 - leftEar: 3.8163e-04 - rightEar: 3.8679e-04 - spine1: 4.1871e-04 - spine2: 3.8092e-04 - spine3: 3.4069e-04 - leftHip: 3.6689e-04 - rightHip: 3.7059e-04 - tailBase: 1.9504e-04 - tailEnd: 2.1084e-04 - rump: 3.4036e-04 - val_loss: 3.8503e-04 - val_nose: 4.4134e-04 - val_leftEar: 4.3321e-04 - val_rightEar: 4.1617e-04 - val_spine1: 4.6201e-04 - val_spine2: 4.3484e-04 - val_spine3: 3.8731e-04 - val_leftHip: 4.0907e-04 - val_rightHip: 4.2925e-04 - val_tailBase: 2.0428e-04 - val_tailEnd: 2.1990e-04 - val_rump: 3.9796e-04 Epoch 45/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 09:48:56.711736: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 435s - loss: 3.3700e-04 - nose: 3.7429e-04 - leftEar: 3.7570e-04 - rightEar: 3.8165e-04 - spine1: 4.0924e-04 - spine2: 3.7631e-04 - spine3: 3.3299e-04 - leftHip: 3.6420e-04 - rightHip: 3.6064e-04 - tailBase: 1.8678e-04 - tailEnd: 2.1174e-04 - rump: 3.3343e-04 - val_loss: 3.7075e-04 - val_nose: 3.9408e-04 - val_leftEar: 4.1202e-04 - val_rightEar: 4.1084e-04 - val_spine1: 4.5390e-04 - val_spine2: 4.2066e-04 - val_spine3: 3.8801e-04 - val_leftHip: 3.9635e-04 - val_rightHip: 4.1468e-04 - val_tailBase: 1.9745e-04 - val_tailEnd: 2.0456e-04 - val_rump: 3.8570e-04 Epoch 00045: ReduceLROnPlateau reducing learning rate to 4.999999873689376e-05. Epoch 46/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 09:56:08.352631: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 430s - loss: 3.1469e-04 - nose: 3.4525e-04 - leftEar: 3.5445e-04 - rightEar: 3.5905e-04 - spine1: 3.8504e-04 - spine2: 3.5125e-04 - spine3: 3.1059e-04 - leftHip: 3.4292e-04 - rightHip: 3.4428e-04 - tailBase: 1.7440e-04 - tailEnd: 1.9012e-04 - rump: 3.0418e-04 - val_loss: 3.4460e-04 - val_nose: 3.9277e-04 - val_leftEar: 3.9116e-04 - val_rightEar: 3.8466e-04 - val_spine1: 4.1611e-04 - val_spine2: 3.8229e-04 - val_spine3: 3.5426e-04 - val_leftHip: 3.7469e-04 - val_rightHip: 3.7869e-04 - val_tailBase: 1.7383e-04 - val_tailEnd: 1.9493e-04 - val_rump: 3.4720e-04 Epoch 47/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 10:03:21.730597: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 423s - loss: 3.1143e-04 - nose: 3.4241e-04 - leftEar: 3.4738e-04 - rightEar: 3.5110e-04 - spine1: 3.7820e-04 - spine2: 3.4304e-04 - spine3: 3.0930e-04 - leftHip: 3.3925e-04 - rightHip: 3.3835e-04 - tailBase: 1.7652e-04 - tailEnd: 1.9334e-04 - rump: 3.0687e-04 - val_loss: 3.4378e-04 - val_nose: 3.6991e-04 - val_leftEar: 3.8193e-04 - val_rightEar: 3.7361e-04 - val_spine1: 4.1051e-04 - val_spine2: 3.8500e-04 - val_spine3: 3.5963e-04 - val_leftHip: 3.7518e-04 - val_rightHip: 3.8893e-04 - val_tailBase: 1.9109e-04 - val_tailEnd: 1.8872e-04 - val_rump: 3.5706e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png Epoch 48/200 2023-01-21 10:10:42.670407: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 429s - loss: 3.1359e-04 - nose: 3.4471e-04 - leftEar: 3.4845e-04 - rightEar: 3.5229e-04 - spine1: 3.8340e-04 - spine2: 3.4712e-04 - spine3: 3.1415e-04 - leftHip: 3.4052e-04 - rightHip: 3.3803e-04 - tailBase: 1.7722e-04 - tailEnd: 1.9495e-04 - rump: 3.0865e-04 - val_loss: 3.3048e-04 - val_nose: 3.7569e-04 - val_leftEar: 3.7613e-04 - val_rightEar: 3.7033e-04 - val_spine1: 3.8636e-04 - val_spine2: 3.6900e-04 - val_spine3: 3.4282e-04 - val_leftHip: 3.5256e-04 - val_rightHip: 3.6761e-04 - val_tailBase: 1.7844e-04 - val_tailEnd: 1.8328e-04 - val_rump: 3.3310e-04 Epoch 49/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 10:19:41.876139: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 569s - loss: 3.1179e-04 - nose: 3.4226e-04 - leftEar: 3.4624e-04 - rightEar: 3.5227e-04 - spine1: 3.8627e-04 - spine2: 3.5003e-04 - spine3: 3.0781e-04 - leftHip: 3.3894e-04 - rightHip: 3.4209e-04 - tailBase: 1.7209e-04 - tailEnd: 1.9043e-04 - rump: 3.0124e-04 - val_loss: 3.3708e-04 - val_nose: 3.6038e-04 - val_leftEar: 3.7725e-04 - val_rightEar: 3.5847e-04 - val_spine1: 4.0980e-04 - val_spine2: 3.7885e-04 - val_spine3: 3.3932e-04 - val_leftHip: 3.7380e-04 - val_rightHip: 3.8485e-04 - val_tailBase: 1.8540e-04 - val_tailEnd: 1.8533e-04 - val_rump: 3.5447e-04 Epoch 50/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 10:29:16.372260: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 560s - loss: 3.0904e-04 - nose: 3.3882e-04 - leftEar: 3.4487e-04 - rightEar: 3.5247e-04 - spine1: 3.7727e-04 - spine2: 3.4465e-04 - spine3: 3.0595e-04 - leftHip: 3.3653e-04 - rightHip: 3.3650e-04 - tailBase: 1.7312e-04 - tailEnd: 1.8743e-04 - rump: 3.0186e-04 - val_loss: 3.3339e-04 - val_nose: 3.7571e-04 - val_leftEar: 3.7235e-04 - val_rightEar: 3.6159e-04 - val_spine1: 3.9962e-04 - val_spine2: 3.7266e-04 - val_spine3: 3.3765e-04 - val_leftHip: 3.6139e-04 - val_rightHip: 3.7134e-04 - val_tailBase: 1.8351e-04 - val_tailEnd: 1.8727e-04 - val_rump: 3.4418e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png Epoch 51/200 2023-01-21 10:40:21.920553: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 656s - loss: 3.0851e-04 - nose: 3.3950e-04 - leftEar: 3.4766e-04 - rightEar: 3.5334e-04 - spine1: 3.7552e-04 - spine2: 3.4184e-04 - spine3: 3.0501e-04 - leftHip: 3.3718e-04 - rightHip: 3.3212e-04 - tailBase: 1.7437e-04 - tailEnd: 1.8758e-04 - rump: 2.9948e-04 - val_loss: 3.3837e-04 - val_nose: 3.7172e-04 - val_leftEar: 3.7054e-04 - val_rightEar: 3.7477e-04 - val_spine1: 4.0732e-04 - val_spine2: 3.8455e-04 - val_spine3: 3.5105e-04 - val_leftHip: 3.8384e-04 - val_rightHip: 3.7438e-04 - val_tailBase: 1.7144e-04 - val_tailEnd: 1.8608e-04 - val_rump: 3.4638e-04 Epoch 52/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 10:48:10.225331: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 463s - loss: 3.0663e-04 - nose: 3.3090e-04 - leftEar: 3.4257e-04 - rightEar: 3.4646e-04 - spine1: 3.7976e-04 - spine2: 3.4367e-04 - spine3: 3.0508e-04 - leftHip: 3.3591e-04 - rightHip: 3.3322e-04 - tailBase: 1.6868e-04 - tailEnd: 1.8785e-04 - rump: 2.9880e-04 - val_loss: 3.3845e-04 - val_nose: 3.9123e-04 - val_leftEar: 3.8484e-04 - val_rightEar: 3.5910e-04 - val_spine1: 4.0189e-04 - val_spine2: 3.7776e-04 - val_spine3: 3.4856e-04 - val_leftHip: 3.6795e-04 - val_rightHip: 3.6981e-04 - val_tailBase: 1.8562e-04 - val_tailEnd: 1.9047e-04 - val_rump: 3.4569e-04 Epoch 53/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 10:58:48.181921: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 640s - loss: 3.0413e-04 - nose: 3.3075e-04 - leftEar: 3.3911e-04 - rightEar: 3.4333e-04 - spine1: 3.7124e-04 - spine2: 3.3766e-04 - spine3: 3.0315e-04 - leftHip: 3.2895e-04 - rightHip: 3.3217e-04 - tailBase: 1.7154e-04 - tailEnd: 1.8743e-04 - rump: 3.0017e-04 - val_loss: 3.3198e-04 - val_nose: 3.7031e-04 - val_leftEar: 3.7312e-04 - val_rightEar: 3.5725e-04 - val_spine1: 4.0102e-04 - val_spine2: 3.7046e-04 - val_spine3: 3.4459e-04 - val_leftHip: 3.5995e-04 - val_rightHip: 3.7389e-04 - val_tailBase: 1.7389e-04 - val_tailEnd: 1.7637e-04 - val_rump: 3.5093e-04 Epoch 00053: ReduceLROnPlateau reducing learning rate to 2.499999936844688e-05. Epoch 54/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 11:07:20.266181: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 495s - loss: 2.9432e-04 - nose: 3.1954e-04 - leftEar: 3.3022e-04 - rightEar: 3.3387e-04 - spine1: 3.6359e-04 - spine2: 3.2404e-04 - spine3: 2.9157e-04 - leftHip: 3.2062e-04 - rightHip: 3.1937e-04 - tailBase: 1.6601e-04 - tailEnd: 1.8251e-04 - rump: 2.8612e-04 - val_loss: 3.2378e-04 - val_nose: 3.5063e-04 - val_leftEar: 3.5097e-04 - val_rightEar: 3.5321e-04 - val_spine1: 3.9182e-04 - val_spine2: 3.6144e-04 - val_spine3: 3.3189e-04 - val_leftHip: 3.4615e-04 - val_rightHip: 3.6669e-04 - val_tailBase: 1.8946e-04 - val_tailEnd: 1.8893e-04 - val_rump: 3.3040e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png Epoch 55/200 2023-01-21 11:14:53.170388: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 450s - loss: 2.9073e-04 - nose: 3.1949e-04 - leftEar: 3.2528e-04 - rightEar: 3.2852e-04 - spine1: 3.5538e-04 - spine2: 3.2396e-04 - spine3: 2.8856e-04 - leftHip: 3.1673e-04 - rightHip: 3.1663e-04 - tailBase: 1.6588e-04 - tailEnd: 1.7690e-04 - rump: 2.8068e-04 - val_loss: 3.1044e-04 - val_nose: 3.2975e-04 - val_leftEar: 3.4894e-04 - val_rightEar: 3.4524e-04 - val_spine1: 3.8172e-04 - val_spine2: 3.4796e-04 - val_spine3: 3.2064e-04 - val_leftHip: 3.2486e-04 - val_rightHip: 3.5596e-04 - val_tailBase: 1.6758e-04 - val_tailEnd: 1.7315e-04 - val_rump: 3.1905e-04 Epoch 56/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 11:22:23.778560: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 450s - loss: 2.8836e-04 - nose: 3.1276e-04 - leftEar: 3.2333e-04 - rightEar: 3.2458e-04 - spine1: 3.5521e-04 - spine2: 3.1979e-04 - spine3: 2.8428e-04 - leftHip: 3.1371e-04 - rightHip: 3.1666e-04 - tailBase: 1.6386e-04 - tailEnd: 1.7650e-04 - rump: 2.8134e-04 - val_loss: 3.1719e-04 - val_nose: 3.5108e-04 - val_leftEar: 3.5618e-04 - val_rightEar: 3.4573e-04 - val_spine1: 3.7868e-04 - val_spine2: 3.5169e-04 - val_spine3: 3.2230e-04 - val_leftHip: 3.4348e-04 - val_rightHip: 3.5060e-04 - val_tailBase: 1.7675e-04 - val_tailEnd: 1.8503e-04 - val_rump: 3.2760e-04 Epoch 57/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 11:29:47.816364: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 447s - loss: 2.8889e-04 - nose: 3.1191e-04 - leftEar: 3.2263e-04 - rightEar: 3.2785e-04 - spine1: 3.5136e-04 - spine2: 3.1594e-04 - spine3: 2.8752e-04 - leftHip: 3.1623e-04 - rightHip: 3.1652e-04 - tailBase: 1.6483e-04 - tailEnd: 1.8084e-04 - rump: 2.8213e-04 - val_loss: 3.2391e-04 - val_nose: 3.2877e-04 - val_leftEar: 3.6457e-04 - val_rightEar: 3.5606e-04 - val_spine1: 3.9460e-04 - val_spine2: 3.6394e-04 - val_spine3: 3.2872e-04 - val_leftHip: 3.5766e-04 - val_rightHip: 3.7398e-04 - val_tailBase: 1.7813e-04 - val_tailEnd: 1.8584e-04 - val_rump: 3.3072e-04 Epoch 58/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 11:37:16.409942: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 448s - loss: 2.8574e-04 - nose: 3.1445e-04 - leftEar: 3.2193e-04 - rightEar: 3.2239e-04 - spine1: 3.5283e-04 - spine2: 3.1489e-04 - spine3: 2.8120e-04 - leftHip: 3.1478e-04 - rightHip: 3.1079e-04 - tailBase: 1.5953e-04 - tailEnd: 1.7452e-04 - rump: 2.7583e-04 - val_loss: 3.0794e-04 - val_nose: 3.3666e-04 - val_leftEar: 3.4503e-04 - val_rightEar: 3.3707e-04 - val_spine1: 3.7431e-04 - val_spine2: 3.4106e-04 - val_spine3: 3.0915e-04 - val_leftHip: 3.3340e-04 - val_rightHip: 3.6082e-04 - val_tailBase: 1.6146e-04 - val_tailEnd: 1.6519e-04 - val_rump: 3.2323e-04 Epoch 59/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 11:44:41.735301: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 437s - loss: 2.8460e-04 - nose: 3.0705e-04 - leftEar: 3.1607e-04 - rightEar: 3.2247e-04 - spine1: 3.4867e-04 - spine2: 3.1399e-04 - spine3: 2.7927e-04 - leftHip: 3.1095e-04 - rightHip: 3.1245e-04 - tailBase: 1.6320e-04 - tailEnd: 1.7713e-04 - rump: 2.7937e-04 - val_loss: 3.0192e-04 - val_nose: 3.4384e-04 - val_leftEar: 3.5039e-04 - val_rightEar: 3.3005e-04 - val_spine1: 3.5104e-04 - val_spine2: 3.2138e-04 - val_spine3: 3.0532e-04 - val_leftHip: 3.2929e-04 - val_rightHip: 3.4371e-04 - val_tailBase: 1.6189e-04 - val_tailEnd: 1.6640e-04 - val_rump: 3.1780e-04 Epoch 60/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 11:52:06.829552: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 462s - loss: 2.8639e-04 - nose: 3.1362e-04 - leftEar: 3.2299e-04 - rightEar: 3.2676e-04 - spine1: 3.5647e-04 - spine2: 3.1378e-04 - spine3: 2.8126e-04 - leftHip: 3.1358e-04 - rightHip: 3.1248e-04 - tailBase: 1.5733e-04 - tailEnd: 1.7302e-04 - rump: 2.7904e-04 - val_loss: 3.2654e-04 - val_nose: 3.3828e-04 - val_leftEar: 3.5784e-04 - val_rightEar: 3.5541e-04 - val_spine1: 3.9142e-04 - val_spine2: 3.6685e-04 - val_spine3: 3.3674e-04 - val_leftHip: 3.6222e-04 - val_rightHip: 3.6208e-04 - val_tailBase: 1.7967e-04 - val_tailEnd: 1.8987e-04 - val_rump: 3.5156e-04 Epoch 61/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 12:00:07.648123: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 456s - loss: 2.8074e-04 - nose: 3.0415e-04 - leftEar: 3.1565e-04 - rightEar: 3.1833e-04 - spine1: 3.4747e-04 - spine2: 3.1045e-04 - spine3: 2.7450e-04 - leftHip: 3.1095e-04 - rightHip: 3.0556e-04 - tailBase: 1.5727e-04 - tailEnd: 1.7187e-04 - rump: 2.7198e-04 - val_loss: 3.0420e-04 - val_nose: 3.1920e-04 - val_leftEar: 3.4348e-04 - val_rightEar: 3.2330e-04 - val_spine1: 3.7098e-04 - val_spine2: 3.4076e-04 - val_spine3: 3.1103e-04 - val_leftHip: 3.1986e-04 - val_rightHip: 3.4363e-04 - val_tailBase: 1.7138e-04 - val_tailEnd: 1.7628e-04 - val_rump: 3.2627e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png Epoch 62/200 2023-01-21 12:07:55.219677: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 526s - loss: 2.8299e-04 - nose: 3.0793e-04 - leftEar: 3.1561e-04 - rightEar: 3.2392e-04 - spine1: 3.4457e-04 - spine2: 3.1267e-04 - spine3: 2.7998e-04 - leftHip: 3.0944e-04 - rightHip: 3.1415e-04 - tailBase: 1.5823e-04 - tailEnd: 1.7273e-04 - rump: 2.7362e-04 - val_loss: 3.0885e-04 - val_nose: 3.5010e-04 - val_leftEar: 3.4099e-04 - val_rightEar: 3.2940e-04 - val_spine1: 3.6684e-04 - val_spine2: 3.3973e-04 - val_spine3: 3.1220e-04 - val_leftHip: 3.3925e-04 - val_rightHip: 3.4387e-04 - val_tailBase: 1.6940e-04 - val_tailEnd: 1.7626e-04 - val_rump: 3.2933e-04 Epoch 63/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 12:15:50.255857: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 412s - loss: 2.7776e-04 - nose: 3.0419e-04 - leftEar: 3.1206e-04 - rightEar: 3.1746e-04 - spine1: 3.4186e-04 - spine2: 3.0553e-04 - spine3: 2.7118e-04 - leftHip: 3.0648e-04 - rightHip: 3.0509e-04 - tailBase: 1.5356e-04 - tailEnd: 1.6893e-04 - rump: 2.6903e-04 - val_loss: 3.0870e-04 - val_nose: 3.3783e-04 - val_leftEar: 3.4155e-04 - val_rightEar: 3.3659e-04 - val_spine1: 3.7292e-04 - val_spine2: 3.4030e-04 - val_spine3: 3.1313e-04 - val_leftHip: 3.4109e-04 - val_rightHip: 3.4630e-04 - val_tailBase: 1.6851e-04 - val_tailEnd: 1.7541e-04 - val_rump: 3.2206e-04 Epoch 64/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 12:23:40.311234: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 473s - loss: 2.7751e-04 - nose: 2.9951e-04 - leftEar: 3.0839e-04 - rightEar: 3.1813e-04 - spine1: 3.3821e-04 - spine2: 3.0484e-04 - spine3: 2.7304e-04 - leftHip: 3.0327e-04 - rightHip: 3.0205e-04 - tailBase: 1.5968e-04 - tailEnd: 1.7418e-04 - rump: 2.7133e-04 - val_loss: 3.0578e-04 - val_nose: 3.4790e-04 - val_leftEar: 3.4908e-04 - val_rightEar: 3.2611e-04 - val_spine1: 3.6777e-04 - val_spine2: 3.4157e-04 - val_spine3: 3.0927e-04 - val_leftHip: 3.2775e-04 - val_rightHip: 3.3847e-04 - val_tailBase: 1.6422e-04 - val_tailEnd: 1.6948e-04 - val_rump: 3.2197e-04 Epoch 00064: ReduceLROnPlateau reducing learning rate to 1.249999968422344e-05. Epoch 65/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 12:31:06.632724: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 442s - loss: 2.7397e-04 - nose: 2.9584e-04 - leftEar: 3.0593e-04 - rightEar: 3.1062e-04 - spine1: 3.3708e-04 - spine2: 3.0126e-04 - spine3: 2.6975e-04 - leftHip: 3.0425e-04 - rightHip: 3.0205e-04 - tailBase: 1.5369e-04 - tailEnd: 1.6566e-04 - rump: 2.6752e-04 - val_loss: 2.9858e-04 - val_nose: 3.1804e-04 - val_leftEar: 3.3436e-04 - val_rightEar: 3.2037e-04 - val_spine1: 3.6538e-04 - val_spine2: 3.2545e-04 - val_spine3: 3.0368e-04 - val_leftHip: 3.2618e-04 - val_rightHip: 3.3061e-04 - val_tailBase: 1.7297e-04 - val_tailEnd: 1.7156e-04 - val_rump: 3.1577e-04 Epoch 66/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 12:38:27.158710: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 434s - loss: 2.7247e-04 - nose: 2.9661e-04 - leftEar: 3.0904e-04 - rightEar: 3.0925e-04 - spine1: 3.3444e-04 - spine2: 2.9836e-04 - spine3: 2.6660e-04 - leftHip: 2.9887e-04 - rightHip: 3.0188e-04 - tailBase: 1.5232e-04 - tailEnd: 1.6690e-04 - rump: 2.6290e-04 - val_loss: 3.0374e-04 - val_nose: 3.2731e-04 - val_leftEar: 3.3192e-04 - val_rightEar: 3.3433e-04 - val_spine1: 3.6052e-04 - val_spine2: 3.3554e-04 - val_spine3: 3.0470e-04 - val_leftHip: 3.3433e-04 - val_rightHip: 3.4118e-04 - val_tailBase: 1.7256e-04 - val_tailEnd: 1.7782e-04 - val_rump: 3.2086e-04 Epoch 67/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 12:45:44.707101: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 438s - loss: 2.7051e-04 - nose: 2.9663e-04 - leftEar: 3.0255e-04 - rightEar: 3.0672e-04 - spine1: 3.2941e-04 - spine2: 2.9564e-04 - spine3: 2.6500e-04 - leftHip: 2.9696e-04 - rightHip: 2.9841e-04 - tailBase: 1.5211e-04 - tailEnd: 1.7051e-04 - rump: 2.6168e-04 - val_loss: 3.0993e-04 - val_nose: 3.2559e-04 - val_leftEar: 3.4811e-04 - val_rightEar: 3.4070e-04 - val_spine1: 3.8289e-04 - val_spine2: 3.4987e-04 - val_spine3: 3.1534e-04 - val_leftHip: 3.3959e-04 - val_rightHip: 3.5079e-04 - val_tailBase: 1.6993e-04 - val_tailEnd: 1.6637e-04 - val_rump: 3.2010e-04 Epoch 68/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 12:53:00.588049: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 430s - loss: 2.6861e-04 - nose: 2.8903e-04 - leftEar: 3.0202e-04 - rightEar: 3.0641e-04 - spine1: 3.2718e-04 - spine2: 2.9162e-04 - spine3: 2.6160e-04 - leftHip: 2.9759e-04 - rightHip: 2.9423e-04 - tailBase: 1.5267e-04 - tailEnd: 1.6846e-04 - rump: 2.6387e-04 - val_loss: 3.0516e-04 - val_nose: 3.2263e-04 - val_leftEar: 3.4860e-04 - val_rightEar: 3.4065e-04 - val_spine1: 3.6518e-04 - val_spine2: 3.4148e-04 - val_spine3: 3.0649e-04 - val_leftHip: 3.2858e-04 - val_rightHip: 3.3996e-04 - val_tailBase: 1.6230e-04 - val_tailEnd: 1.7750e-04 - val_rump: 3.2342e-04 Epoch 69/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 13:00:20.080289: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 449s - loss: 2.6854e-04 - nose: 2.9224e-04 - leftEar: 2.9814e-04 - rightEar: 3.0094e-04 - spine1: 3.3005e-04 - spine2: 2.9554e-04 - spine3: 2.6364e-04 - leftHip: 2.9947e-04 - rightHip: 2.9392e-04 - tailBase: 1.5218e-04 - tailEnd: 1.6687e-04 - rump: 2.6097e-04 - val_loss: 3.0585e-04 - val_nose: 3.2715e-04 - val_leftEar: 3.6215e-04 - val_rightEar: 3.3919e-04 - val_spine1: 3.7222e-04 - val_spine2: 3.4582e-04 - val_spine3: 3.1095e-04 - val_leftHip: 3.3664e-04 - val_rightHip: 3.4379e-04 - val_tailBase: 1.5956e-04 - val_tailEnd: 1.6282e-04 - val_rump: 3.0403e-04 Epoch 70/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 13:07:34.972977: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 433s - loss: 2.6731e-04 - nose: 2.8887e-04 - leftEar: 3.0512e-04 - rightEar: 3.0383e-04 - spine1: 3.2828e-04 - spine2: 2.9072e-04 - spine3: 2.6277e-04 - leftHip: 2.9757e-04 - rightHip: 2.9190e-04 - tailBase: 1.4854e-04 - tailEnd: 1.6431e-04 - rump: 2.5852e-04 - val_loss: 3.0528e-04 - val_nose: 3.4912e-04 - val_leftEar: 3.4924e-04 - val_rightEar: 3.3092e-04 - val_spine1: 3.6728e-04 - val_spine2: 3.2813e-04 - val_spine3: 3.0324e-04 - val_leftHip: 3.3645e-04 - val_rightHip: 3.3884e-04 - val_tailBase: 1.6330e-04 - val_tailEnd: 1.7372e-04 - val_rump: 3.1784e-04 Epoch 71/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 13:14:49.379889: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 422s - loss: 2.6666e-04 - nose: 2.8729e-04 - leftEar: 2.9853e-04 - rightEar: 3.0121e-04 - spine1: 3.2686e-04 - spine2: 2.9035e-04 - spine3: 2.5774e-04 - leftHip: 2.9641e-04 - rightHip: 2.9568e-04 - tailBase: 1.5290e-04 - tailEnd: 1.6964e-04 - rump: 2.5661e-04 - val_loss: 3.0543e-04 - val_nose: 3.3053e-04 - val_leftEar: 3.4023e-04 - val_rightEar: 3.3789e-04 - val_spine1: 3.6656e-04 - val_spine2: 3.4200e-04 - val_spine3: 3.1390e-04 - val_leftHip: 3.3404e-04 - val_rightHip: 3.5014e-04 - val_tailBase: 1.6104e-04 - val_tailEnd: 1.7082e-04 - val_rump: 3.1259e-04 Epoch 00071: ReduceLROnPlateau reducing learning rate to 6.24999984211172e-06. Epoch 72/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 13:22:27.919938: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 463s - loss: 2.6369e-04 - nose: 2.8320e-04 - leftEar: 2.9794e-04 - rightEar: 3.0106e-04 - spine1: 3.2266e-04 - spine2: 2.8918e-04 - spine3: 2.5648e-04 - leftHip: 2.9245e-04 - rightHip: 2.9312e-04 - tailBase: 1.4768e-04 - tailEnd: 1.6205e-04 - rump: 2.5475e-04 - val_loss: 3.0063e-04 - val_nose: 3.2258e-04 - val_leftEar: 3.3708e-04 - val_rightEar: 3.2922e-04 - val_spine1: 3.5948e-04 - val_spine2: 3.3074e-04 - val_spine3: 3.0414e-04 - val_leftHip: 3.2337e-04 - val_rightHip: 3.4356e-04 - val_tailBase: 1.6338e-04 - val_tailEnd: 1.7769e-04 - val_rump: 3.1568e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png Epoch 73/200 2023-01-21 13:29:59.910678: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 438s - loss: 2.6146e-04 - nose: 2.8435e-04 - leftEar: 2.9195e-04 - rightEar: 2.9754e-04 - spine1: 3.1790e-04 - spine2: 2.8398e-04 - spine3: 2.5357e-04 - leftHip: 2.8989e-04 - rightHip: 2.8912e-04 - tailBase: 1.4618e-04 - tailEnd: 1.6576e-04 - rump: 2.5580e-04 - val_loss: 3.0059e-04 - val_nose: 3.2370e-04 - val_leftEar: 3.3003e-04 - val_rightEar: 3.2769e-04 - val_spine1: 3.5527e-04 - val_spine2: 3.3523e-04 - val_spine3: 3.0851e-04 - val_leftHip: 3.3825e-04 - val_rightHip: 3.5592e-04 - val_tailBase: 1.6066e-04 - val_tailEnd: 1.6793e-04 - val_rump: 3.0335e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png Epoch 74/200 2023-01-21 13:37:49.719610: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 504s - loss: 2.6323e-04 - nose: 2.8178e-04 - leftEar: 2.9162e-04 - rightEar: 3.0210e-04 - spine1: 3.1809e-04 - spine2: 2.8552e-04 - spine3: 2.6235e-04 - leftHip: 2.9148e-04 - rightHip: 2.9030e-04 - tailBase: 1.5151e-04 - tailEnd: 1.6129e-04 - rump: 2.5948e-04 - val_loss: 2.9469e-04 - val_nose: 3.1438e-04 - val_leftEar: 3.3852e-04 - val_rightEar: 3.1848e-04 - val_spine1: 3.5832e-04 - val_spine2: 3.2445e-04 - val_spine3: 3.0123e-04 - val_leftHip: 3.2478e-04 - val_rightHip: 3.2470e-04 - val_tailBase: 1.6269e-04 - val_tailEnd: 1.6409e-04 - val_rump: 3.0990e-04 Epoch 75/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 13:46:25.010512: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 482s - loss: 2.6186e-04 - nose: 2.8704e-04 - leftEar: 2.9440e-04 - rightEar: 3.0190e-04 - spine1: 3.1841e-04 - spine2: 2.8588e-04 - spine3: 2.5604e-04 - leftHip: 2.9331e-04 - rightHip: 2.8791e-04 - tailBase: 1.4578e-04 - tailEnd: 1.5894e-04 - rump: 2.5079e-04 - val_loss: 2.9145e-04 - val_nose: 3.1243e-04 - val_leftEar: 3.2921e-04 - val_rightEar: 3.0156e-04 - val_spine1: 3.5255e-04 - val_spine2: 3.2397e-04 - val_spine3: 2.9804e-04 - val_leftHip: 3.2707e-04 - val_rightHip: 3.3214e-04 - val_tailBase: 1.5541e-04 - val_tailEnd: 1.6064e-04 - val_rump: 3.1294e-04 Epoch 76/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 13:53:53.893778: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 442s - loss: 2.6181e-04 - nose: 2.8351e-04 - leftEar: 2.9380e-04 - rightEar: 2.9675e-04 - spine1: 3.2095e-04 - spine2: 2.8512e-04 - spine3: 2.5705e-04 - leftHip: 2.9030e-04 - rightHip: 2.8569e-04 - tailBase: 1.4747e-04 - tailEnd: 1.6312e-04 - rump: 2.5620e-04 - val_loss: 2.9661e-04 - val_nose: 3.1451e-04 - val_leftEar: 3.3219e-04 - val_rightEar: 3.1567e-04 - val_spine1: 3.6679e-04 - val_spine2: 3.3991e-04 - val_spine3: 3.1068e-04 - val_leftHip: 3.3088e-04 - val_rightHip: 3.4222e-04 - val_tailBase: 1.5080e-04 - val_tailEnd: 1.5084e-04 - val_rump: 3.0822e-04 Epoch 77/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 14:01:18.822158: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 456s - loss: 2.6154e-04 - nose: 2.8462e-04 - leftEar: 2.9724e-04 - rightEar: 2.9632e-04 - spine1: 3.2029e-04 - spine2: 2.8338e-04 - spine3: 2.5582e-04 - leftHip: 2.8867e-04 - rightHip: 2.8661e-04 - tailBase: 1.4682e-04 - tailEnd: 1.6122e-04 - rump: 2.5602e-04 - val_loss: 3.1373e-04 - val_nose: 3.2408e-04 - val_leftEar: 3.3444e-04 - val_rightEar: 3.4657e-04 - val_spine1: 3.8224e-04 - val_spine2: 3.5244e-04 - val_spine3: 3.1551e-04 - val_leftHip: 3.4665e-04 - val_rightHip: 3.5787e-04 - val_tailBase: 1.8428e-04 - val_tailEnd: 1.8263e-04 - val_rump: 3.2428e-04 Epoch 78/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 14:09:00.168128: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 445s - loss: 2.5793e-04 - nose: 2.7628e-04 - leftEar: 2.8544e-04 - rightEar: 2.9074e-04 - spine1: 3.1370e-04 - spine2: 2.8178e-04 - spine3: 2.5376e-04 - leftHip: 2.9034e-04 - rightHip: 2.8647e-04 - tailBase: 1.4657e-04 - tailEnd: 1.5979e-04 - rump: 2.5234e-04 - val_loss: 3.0392e-04 - val_nose: 3.1738e-04 - val_leftEar: 3.3156e-04 - val_rightEar: 3.3055e-04 - val_spine1: 3.5894e-04 - val_spine2: 3.4706e-04 - val_spine3: 3.1396e-04 - val_leftHip: 3.3308e-04 - val_rightHip: 3.4214e-04 - val_tailBase: 1.7021e-04 - val_tailEnd: 1.7481e-04 - val_rump: 3.2344e-04 Epoch 79/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 14:16:28.141238: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 440s - loss: 2.6316e-04 - nose: 2.8276e-04 - leftEar: 2.9579e-04 - rightEar: 3.0242e-04 - spine1: 3.2149e-04 - spine2: 2.8736e-04 - spine3: 2.5984e-04 - leftHip: 2.9034e-04 - rightHip: 2.9165e-04 - tailBase: 1.4568e-04 - tailEnd: 1.6209e-04 - rump: 2.5531e-04 - val_loss: 3.0287e-04 - val_nose: 3.2991e-04 - val_leftEar: 3.4070e-04 - val_rightEar: 3.3088e-04 - val_spine1: 3.5079e-04 - val_spine2: 3.2955e-04 - val_spine3: 3.1830e-04 - val_leftHip: 3.2780e-04 - val_rightHip: 3.4988e-04 - val_tailBase: 1.6433e-04 - val_tailEnd: 1.7089e-04 - val_rump: 3.1852e-04 Epoch 80/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 14:23:45.590964: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 455s - loss: 2.6317e-04 - nose: 2.8520e-04 - leftEar: 2.9441e-04 - rightEar: 3.0133e-04 - spine1: 3.2250e-04 - spine2: 2.8528e-04 - spine3: 2.5977e-04 - leftHip: 2.8962e-04 - rightHip: 2.8885e-04 - tailBase: 1.4882e-04 - tailEnd: 1.6360e-04 - rump: 2.5550e-04 - val_loss: 3.0909e-04 - val_nose: 3.2644e-04 - val_leftEar: 3.4197e-04 - val_rightEar: 3.3875e-04 - val_spine1: 3.8000e-04 - val_spine2: 3.4591e-04 - val_spine3: 3.1563e-04 - val_leftHip: 3.4198e-04 - val_rightHip: 3.5470e-04 - val_tailBase: 1.6640e-04 - val_tailEnd: 1.7440e-04 - val_rump: 3.1381e-04 Epoch 00080: ReduceLROnPlateau reducing learning rate to 3.12499992105586e-06. Epoch 81/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 14:31:18.608693: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 432s - loss: 2.6022e-04 - nose: 2.7792e-04 - leftEar: 2.9304e-04 - rightEar: 3.0028e-04 - spine1: 3.1688e-04 - spine2: 2.8269e-04 - spine3: 2.5596e-04 - leftHip: 2.8755e-04 - rightHip: 2.8585e-04 - tailBase: 1.4819e-04 - tailEnd: 1.5893e-04 - rump: 2.5518e-04 - val_loss: 3.0608e-04 - val_nose: 3.3558e-04 - val_leftEar: 3.4569e-04 - val_rightEar: 3.3001e-04 - val_spine1: 3.8116e-04 - val_spine2: 3.4284e-04 - val_spine3: 3.1632e-04 - val_leftHip: 3.2343e-04 - val_rightHip: 3.5985e-04 - val_tailBase: 1.5708e-04 - val_tailEnd: 1.6177e-04 - val_rump: 3.1311e-04 Epoch 82/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 14:38:36.075815: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 435s - loss: 2.5978e-04 - nose: 2.8298e-04 - leftEar: 2.8905e-04 - rightEar: 2.9783e-04 - spine1: 3.1642e-04 - spine2: 2.8400e-04 - spine3: 2.5351e-04 - leftHip: 2.8778e-04 - rightHip: 2.8692e-04 - tailBase: 1.4490e-04 - tailEnd: 1.6199e-04 - rump: 2.5219e-04 - val_loss: 3.1005e-04 - val_nose: 3.2309e-04 - val_leftEar: 3.4095e-04 - val_rightEar: 3.3251e-04 - val_spine1: 3.8201e-04 - val_spine2: 3.5322e-04 - val_spine3: 3.1674e-04 - val_leftHip: 3.3895e-04 - val_rightHip: 3.6639e-04 - val_tailBase: 1.6867e-04 - val_tailEnd: 1.7127e-04 - val_rump: 3.1673e-04 Epoch 83/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 14:45:56.249363: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 445s - loss: 2.5859e-04 - nose: 2.8511e-04 - leftEar: 2.9335e-04 - rightEar: 2.9905e-04 - spine1: 3.1506e-04 - spine2: 2.8116e-04 - spine3: 2.5189e-04 - leftHip: 2.8709e-04 - rightHip: 2.8664e-04 - tailBase: 1.4277e-04 - tailEnd: 1.5499e-04 - rump: 2.4740e-04 - val_loss: 3.0189e-04 - val_nose: 3.2898e-04 - val_leftEar: 3.3978e-04 - val_rightEar: 3.1873e-04 - val_spine1: 3.6958e-04 - val_spine2: 3.4486e-04 - val_spine3: 3.0057e-04 - val_leftHip: 3.3223e-04 - val_rightHip: 3.5227e-04 - val_tailBase: 1.6066e-04 - val_tailEnd: 1.6599e-04 - val_rump: 3.0719e-04 Epoch 84/200 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png 2023-01-21 14:53:20.080000: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 445s - loss: 2.5811e-04 - nose: 2.7944e-04 - leftEar: 2.9389e-04 - rightEar: 2.9680e-04 - spine1: 3.1809e-04 - spine2: 2.8262e-04 - spine3: 2.5205e-04 - leftHip: 2.8705e-04 - rightHip: 2.8635e-04 - tailBase: 1.3935e-04 - tailEnd: 1.5583e-04 - rump: 2.4777e-04 - val_loss: 3.0212e-04 - val_nose: 3.1172e-04 - val_leftEar: 3.2734e-04 - val_rightEar: 3.2141e-04 - val_spine1: 3.7344e-04 - val_spine2: 3.4940e-04 - val_spine3: 3.1403e-04 - val_leftHip: 3.2969e-04 - val_rightHip: 3.4590e-04 - val_tailBase: 1.6145e-04 - val_tailEnd: 1.7587e-04 - val_rump: 3.1308e-04 Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png Epoch 85/200 2023-01-21 15:00:42.470131: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 1152 } dim { size: 1152 } dim { size: 3 } } } 833/833 - 427s - loss: 2.5663e-04 - nose: 2.7817e-04 - leftEar: 2.9073e-04 - rightEar: 2.9357e-04 - spine1: 3.1714e-04 - spine2: 2.8020e-04 - spine3: 2.4892e-04 - leftHip: 2.8834e-04 - rightHip: 2.8531e-04 - tailBase: 1.3934e-04 - tailEnd: 1.5449e-04 - rump: 2.4670e-04 - val_loss: 3.0067e-04 - val_nose: 3.4150e-04 - val_leftEar: 3.4163e-04 - val_rightEar: 3.4214e-04 - val_spine1: 3.6921e-04 - val_spine2: 3.3091e-04 - val_spine3: 2.9470e-04 - val_leftHip: 3.3364e-04 - val_rightHip: 3.3258e-04 - val_tailBase: 1.5959e-04 - val_tailEnd: 1.5887e-04 - val_rump: 3.0264e-04 Epoch 00085: early stopping INFO:sleap.nn.training:Finished training loop. [639.0 min] INFO:sleap.nn.training:Deleting visualization directory: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz INFO:sleap.nn.training:Saving evaluation metrics to model folder... Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230120_193352.centroid.n=1855/viz/validation.*.png Polling: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855/viz/validation.*.png Predicting... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 0% ETA: -:--:-- ?2023-01-21 15:01:19.161663: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_UINT8 } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_UINT8 shape { dim { size: 4 } dim { size: 1536 } dim { size: 2048 } dim { size: 3 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "110" frequency: 3699 num_cores: 6 environment { key: "cpu_instruction_set" value: "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2" } environment { key: "eigen" value: "3.3.90" } l1_cache_size: 32768 l2_cache_size: 262144 l3_cache_size: 9437184 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: -26 } dim { size: -27 } dim { size: 3 } } } 2023-01-21 15:01:19.168301: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: -61 } dim { size: -62 } dim { size: -63 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -9 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -9 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "GPU" vendor: "NVIDIA" model: "NVIDIA GeForce RTX 2080 Ti" frequency: 1545 num_cores: 68 environment { key: "architecture" value: "7.5" } environment { key: "cuda" value: "11020" } environment { key: "cudnn" value: "8100" } num_registers: 65536 l1_cache_size: 24576 l2_cache_size: 5767168 shared_memory_size_per_multiprocessor: 65536 memory_size: 9719119872 bandwidth: 616000000 } outputs { dtype: DT_FLOAT shape { dim { size: -9 } dim { size: -64 } dim { size: -65 } dim { size: 1 } } } Predicting... ━━━━━━━━━╸━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 25% ETA: 0:04:53 4.3 FPSRun Path: /home/shuyu/SLEAP/2023-01-20-sleap-topdown/models/exp20_230121_041700.centered_instance.n=1855