File size: 14,247 Bytes
c011401 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 |
Gcs Debug state: GcsNodeManager: - RegisterNode request count: 1 - DrainNode request count: 0 - GetAllNodeInfo request count: 25 GcsActorManager: - RegisterActor request count: 0 - CreateActor request count: 0 - GetActorInfo request count: 0 - GetNamedActorInfo request count: 0 - GetAllActorInfo request count: 0 - KillActor request count: 0 - ListNamedActors request count: 0 - Registered actors count: 0 - Destroyed actors count: 0 - Named actors count: 0 - Unresolved actors count: 0 - Pending actors count: 0 - Created actors count: 0 - owners_: 0 - actor_to_register_callbacks_: 0 - actor_to_restart_callbacks_: 0 - actor_to_create_callbacks_: 0 - sorted_destroyed_actor_list_: 0 GcsResourceManager: - GetAllAvailableResources request count: 0 - GetAllTotalResources request count: 0 - GetAllResourceUsage request count: 748 GcsPlacementGroupManager: - CreatePlacementGroup request count: 0 - RemovePlacementGroup request count: 0 - GetPlacementGroup request count: 0 - GetAllPlacementGroup request count: 0 - WaitPlacementGroupUntilReady request count: 0 - GetNamedPlacementGroup request count: 0 - Scheduling pending placement group count: 0 - Registered placement groups count: 0 - Named placement group count: 0 - Pending placement groups count: 0 - Infeasible placement groups count: 0 Publisher: GCS_NODE_INFO_CHANNEL - cumulative published messages: 2 - cumulative published bytes: 635 - current buffered bytes: 101 RAY_LOG_CHANNEL - cumulative published messages: 824 - cumulative published bytes: 627009 - current buffered bytes: 819 GCS_JOB_CHANNEL - cumulative published messages: 2 - cumulative published bytes: 522 - current buffered bytes: 266 RAY_ERROR_INFO_CHANNEL - cumulative published messages: 1 - cumulative published bytes: 294 - current buffered bytes: 294 GCS_WORKER_DELTA_CHANNEL - cumulative published messages: 1 - cumulative published bytes: 98 - current buffered bytes: 98 [runtime env manager] ID to URIs table: [runtime env manager] URIs reference table: GcsTaskManager: -Total num task events reported: 466 -Total num status task events dropped: 0 -Total num profile events dropped: 0 -Current num of task events stored: 150 -Total num of actor creation tasks: 0 -Total num of actor tasks: 0 -Total num of normal tasks: 149 -Total num of driver tasks: 1 GcsAutoscalerStateManager: - last_seen_autoscaler_state_version_: 0 - last_cluster_resource_state_version_: 0 - pending demands: Global stats: 45490 total (4 active) Queueing time: mean = 133.668 us, max = 265.104 ms, min = -0.000 s, total = 6.081 s Execution time: mean = 312.315 us, total = 14.207 s Event stats: event_loop_lag_probe - 14997 total (0 active), Execution time: mean = 24.107 us, total = 361.534 ms, Queueing time: mean = 10.476 us, max = 13.561 ms, min = 1.478 us, total = 157.114 ms GcsInMemoryStore.Put - 6046 total (0 active), Execution time: mean = 50.027 us, total = 302.466 ms, Queueing time: mean = 349.990 us, max = 264.647 ms, min = 1.657 us, total = 2.116 s RayletLoadPulled - 3750 total (1 active), Execution time: mean = 254.185 us, total = 953.195 ms, Queueing time: mean = 94.780 us, max = 61.400 ms, min = 8.098 us, total = 355.424 ms NodeManagerService.grpc_client.GetResourceLoad.OnReplyReceived - 3748 total (0 active), Execution time: mean = 78.284 us, total = 293.409 ms, Queueing time: mean = 108.430 us, max = 1.754 ms, min = 12.805 us, total = 406.396 ms NodeManagerService.grpc_client.GetResourceLoad - 3748 total (0 active), Execution time: mean = 1.533 ms, total = 5.744 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s InternalKVGcsService.grpc_server.InternalKVPut - 1511 total (0 active), Execution time: mean = 540.931 us, total = 817.346 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s InternalKVGcsService.grpc_server.InternalKVPut.HandleRequestImpl - 1511 total (0 active), Execution time: mean = 40.426 us, total = 61.084 ms, Queueing time: mean = 97.664 us, max = 215.796 us, min = 10.684 us, total = 147.571 ms NodeInfoGcsService.grpc_server.CheckAlive - 1500 total (0 active), Execution time: mean = 574.693 us, total = 862.040 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeInfoGcsService.grpc_server.CheckAlive.HandleRequestImpl - 1500 total (0 active), Execution time: mean = 38.480 us, total = 57.720 ms, Queueing time: mean = 108.518 us, max = 380.880 us, min = 15.404 us, total = 162.777 ms ClusterResourceManager.ResetRemoteNodeView - 1251 total (1 active), Execution time: mean = 10.600 us, total = 13.261 ms, Queueing time: mean = 74.829 us, max = 452.755 us, min = 13.442 us, total = 93.610 ms HealthCheck - 1248 total (0 active), Execution time: mean = 7.903 us, total = 9.863 ms, Queueing time: mean = 98.362 us, max = 362.825 us, min = 10.802 us, total = 122.755 ms GcsInMemoryStore.Get - 847 total (0 active), Execution time: mean = 343.197 us, total = 290.688 ms, Queueing time: mean = 1.514 ms, max = 173.127 ms, min = 2.684 us, total = 1.282 s InternalKVGcsService.grpc_server.InternalKVGet.HandleRequestImpl - 844 total (0 active), Execution time: mean = 51.862 us, total = 43.772 ms, Queueing time: mean = 495.735 us, max = 28.461 ms, min = 8.082 us, total = 418.401 ms InternalKVGcsService.grpc_server.InternalKVGet - 844 total (0 active), Execution time: mean = 3.713 ms, total = 3.134 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage - 748 total (0 active), Execution time: mean = 647.082 us, total = 484.017 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage.HandleRequestImpl - 748 total (0 active), Execution time: mean = 92.925 us, total = 69.508 ms, Queueing time: mean = 109.174 us, max = 357.554 us, min = 18.650 us, total = 81.662 ms GCSServer.deadline_timer.debug_state_dump - 375 total (1 active, 1 running), Execution time: mean = 1.409 ms, total = 528.459 ms, Queueing time: mean = 70.235 us, max = 190.421 us, min = -0.000 s, total = 26.338 ms GcsResourceManager::Update - 65 total (0 active), Execution time: mean = 112.509 us, total = 7.313 ms, Queueing time: mean = 2.281 ms, max = 141.699 ms, min = 21.745 us, total = 148.267 ms GCSServer.deadline_timer.debug_state_event_stats_print - 63 total (1 active), Execution time: mean = 1.580 ms, total = 99.519 ms, Queueing time: mean = 57.218 us, max = 147.377 us, min = 12.098 us, total = 3.605 ms NodeInfoGcsService.grpc_server.GetAllNodeInfo.HandleRequestImpl - 25 total (0 active), Execution time: mean = 45.294 us, total = 1.132 ms, Queueing time: mean = 113.000 us, max = 177.345 us, min = 5.865 us, total = 2.825 ms NodeInfoGcsService.grpc_server.GetAllNodeInfo - 25 total (0 active), Execution time: mean = 637.760 us, total = 15.944 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s WorkerInfoGcsService.grpc_server.AddWorkerInfo - 21 total (0 active), Execution time: mean = 628.481 us, total = 13.198 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s WorkerInfoGcsService.grpc_server.AddWorkerInfo.HandleRequestImpl - 21 total (0 active), Execution time: mean = 33.851 us, total = 710.872 us, Queueing time: mean = 118.271 us, max = 219.457 us, min = 30.057 us, total = 2.484 ms GcsInMemoryStore.GetAll - 7 total (0 active), Execution time: mean = 15.493 us, total = 108.452 us, Queueing time: mean = 50.219 us, max = 89.318 us, min = 12.959 us, total = 351.533 us NodeInfoGcsService.grpc_server.GetClusterId - 5 total (0 active), Execution time: mean = 3.596 ms, total = 17.982 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeInfoGcsService.grpc_server.GetClusterId.HandleRequestImpl - 5 total (0 active), Execution time: mean = 630.534 us, total = 3.153 ms, Queueing time: mean = 1.617 ms, max = 7.601 ms, min = 97.713 us, total = 8.084 ms PeriodicalRunner.RunFnPeriodically - 4 total (0 active), Execution time: mean = 278.951 us, total = 1.116 ms, Queueing time: mean = 134.239 ms, max = 265.104 ms, min = 3.060 ms, total = 536.955 ms GcsInMemoryStore.Keys - 3 total (0 active), Execution time: mean = 954.434 us, total = 2.863 ms, Queueing time: mean = 1.949 ms, max = 2.902 ms, min = 46.240 us, total = 5.847 ms InternalKVGcsService.grpc_server.InternalKVExists.HandleRequestImpl - 2 total (0 active), Execution time: mean = 74.728 us, total = 149.457 us, Queueing time: mean = 123.106 us, max = 149.669 us, min = 96.542 us, total = 246.211 us InternalKVGcsService.grpc_server.InternalKVExists - 2 total (0 active), Execution time: mean = 822.436 us, total = 1.645 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s GcsInMemoryStore.Exists - 2 total (0 active), Execution time: mean = 18.626 us, total = 37.252 us, Queueing time: mean = 62.010 us, max = 113.193 us, min = 10.827 us, total = 124.020 us NodeInfoGcsService.grpc_server.UnregisterNode.HandleRequestImpl - 1 total (0 active), Execution time: mean = 592.358 us, total = 592.358 us, Queueing time: mean = 119.924 us, max = 119.924 us, min = 119.924 us, total = 119.924 us WorkerInfoGcsService.grpc_server.ReportWorkerFailure - 1 total (0 active), Execution time: mean = 3.576 ms, total = 3.576 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s JobInfoGcsService.grpc_server.AddJob.HandleRequestImpl - 1 total (0 active), Execution time: mean = 163.616 us, total = 163.616 us, Queueing time: mean = 116.688 us, max = 116.688 us, min = 116.688 us, total = 116.688 us InternalKVGcsService.grpc_server.InternalKVDel - 1 total (0 active), Execution time: mean = 652.924 us, total = 652.924 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s InternalKVGcsService.grpc_server.GetInternalConfig - 1 total (0 active), Execution time: mean = 728.591 us, total = 728.591 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s JobInfoGcsService.grpc_server.MarkJobFinished.HandleRequestImpl - 1 total (0 active), Execution time: mean = 22.929 us, total = 22.929 us, Queueing time: mean = 75.837 us, max = 75.837 us, min = 75.837 us, total = 75.837 us GcsInMemoryStore.BatchDelete - 1 total (0 active), Execution time: mean = 2.453 us, total = 2.453 us, Queueing time: mean = 23.770 us, max = 23.770 us, min = 23.770 us, total = 23.770 us InternalKVGcsService.grpc_server.InternalKVDel.HandleRequestImpl - 1 total (0 active), Execution time: mean = 76.847 us, total = 76.847 us, Queueing time: mean = 104.159 us, max = 104.159 us, min = 104.159 us, total = 104.159 us InternalKVGcsService.grpc_server.GetInternalConfig.HandleRequestImpl - 1 total (0 active), Execution time: mean = 51.411 us, total = 51.411 us, Queueing time: mean = 144.072 us, max = 144.072 us, min = 144.072 us, total = 144.072 us JobInfoGcsService.grpc_server.GetNextJobID - 1 total (0 active), Execution time: mean = 479.420 us, total = 479.420 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s JobInfoGcsService.grpc_server.ReportJobError.HandleRequestImpl - 1 total (0 active), Execution time: mean = 95.871 us, total = 95.871 us, Queueing time: mean = 192.950 us, max = 192.950 us, min = 192.950 us, total = 192.950 us JobInfoGcsService.grpc_server.GetNextJobID.HandleRequestImpl - 1 total (0 active), Execution time: mean = 27.913 us, total = 27.913 us, Queueing time: mean = 133.835 us, max = 133.835 us, min = 133.835 us, total = 133.835 us NodeInfoGcsService.grpc_server.RegisterNode.HandleRequestImpl - 1 total (0 active), Execution time: mean = 224.478 us, total = 224.478 us, Queueing time: mean = 118.485 us, max = 118.485 us, min = 118.485 us, total = 118.485 us WorkerInfoGcsService.grpc_server.ReportWorkerFailure.HandleRequestImpl - 1 total (0 active), Execution time: mean = 78.971 us, total = 78.971 us, Queueing time: mean = 143.920 us, max = 143.920 us, min = 143.920 us, total = 143.920 us GcsHealthCheckManager::RemoveNode - 1 total (0 active), Execution time: mean = 974.000 ns, total = 974.000 ns, Queueing time: mean = 664.000 ns, max = 664.000 ns, min = 664.000 ns, total = 664.000 ns JobInfoGcsService.grpc_server.ReportJobError - 1 total (0 active), Execution time: mean = 865.448 us, total = 865.448 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s GcsInMemoryStore.Delete - 1 total (0 active), Execution time: mean = 22.833 us, total = 22.833 us, Queueing time: mean = 12.860 us, max = 12.860 us, min = 12.860 us, total = 12.860 us NodeInfoGcsService.grpc_server.UnregisterNode - 1 total (0 active), Execution time: mean = 2.174 ms, total = 2.174 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s GcsHealthCheckManager::AddNode - 1 total (0 active), Execution time: mean = 23.993 us, total = 23.993 us, Queueing time: mean = 920.000 ns, max = 920.000 ns, min = 920.000 ns, total = 920.000 ns JobInfoGcsService.grpc_server.MarkJobFinished - 1 total (0 active), Execution time: mean = 3.478 ms, total = 3.478 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s JobInfoGcsService.grpc_server.GetAllJobInfo.HandleRequestImpl - 1 total (0 active), Execution time: mean = 30.137 us, total = 30.137 us, Queueing time: mean = 129.348 us, max = 129.348 us, min = 129.348 us, total = 129.348 us JobInfoGcsService.grpc_server.AddJob - 1 total (0 active), Execution time: mean = 823.170 us, total = 823.170 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s JobInfoGcsService.grpc_server.GetAllJobInfo - 1 total (0 active), Execution time: mean = 487.884 us, total = 487.884 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeInfoGcsService.grpc_server.RegisterNode - 1 total (0 active), Execution time: mean = 1.269 ms, total = 1.269 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s |