File size: 15,303 Bytes
c011401 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 |
Gcs Debug state: GcsNodeManager: - RegisterNode request count: 1 - DrainNode request count: 0 - GetAllNodeInfo request count: 115 GcsActorManager: - RegisterActor request count: 3 - CreateActor request count: 2 - GetActorInfo request count: 22 - GetNamedActorInfo request count: 26 - GetAllActorInfo request count: 0 - KillActor request count: 0 - ListNamedActors request count: 0 - Registered actors count: 2 - Destroyed actors count: 0 - Named actors count: 2 - Unresolved actors count: 0 - Pending actors count: 0 - Created actors count: 1 - owners_: 0 - actor_to_register_callbacks_: 0 - actor_to_restart_callbacks_: 0 - actor_to_create_callbacks_: 0 - sorted_destroyed_actor_list_: 0 GcsResourceManager: - GetAllAvailableResources request count: 22 - GetAllTotalResources request count: 224 - GetAllResourceUsage request count: 10 GcsPlacementGroupManager: - CreatePlacementGroup request count: 0 - RemovePlacementGroup request count: 0 - GetPlacementGroup request count: 0 - GetAllPlacementGroup request count: 0 - WaitPlacementGroupUntilReady request count: 0 - GetNamedPlacementGroup request count: 0 - Scheduling pending placement group count: 0 - Registered placement groups count: 0 - Named placement group count: 0 - Pending placement groups count: 0 - Infeasible placement groups count: 0 Publisher: GCS_NODE_INFO_CHANNEL - cumulative published messages: 1 - cumulative published bytes: 532 - current buffered bytes: 0 RAY_LOG_CHANNEL - cumulative published messages: 245 - cumulative published bytes: 58937 - current buffered bytes: 148 GCS_JOB_CHANNEL - cumulative published messages: 1 - cumulative published bytes: 286 - current buffered bytes: 286 GCS_ACTOR_CHANNEL - cumulative published messages: 6 - cumulative published bytes: 1857 - current buffered bytes: 363 [runtime env manager] ID to URIs table: [runtime env manager] URIs reference table: GcsTaskManager: -Total num task events reported: 6423 -Total num status task events dropped: 0 -Total num profile events dropped: 0 -Current num of task events stored: 3140 -Total num of actor creation tasks: 3 -Total num of actor tasks: 629 -Total num of normal tasks: 2310 -Total num of driver tasks: 1 GcsAutoscalerStateManager: - last_seen_autoscaler_state_version_: 0 - last_cluster_resource_state_version_: 0 - pending demands: Global stats: 4907 total (4 active) Queueing time: mean = 629.752 us, max = 271.493 ms, min = 1.037 us, total = 3.090 s Execution time: mean = 956.066 us, total = 4.691 s Event stats: GcsInMemoryStore.Put - 602 total (0 active), Execution time: mean = 474.337 us, total = 285.551 ms, Queueing time: mean = 3.168 ms, max = 270.926 ms, min = 1.507 us, total = 1.907 s GcsInMemoryStore.Get - 479 total (0 active), Execution time: mean = 24.913 us, total = 11.933 ms, Queueing time: mean = 16.697 us, max = 116.362 us, min = 1.807 us, total = 7.998 ms InternalKVGcsService.grpc_server.InternalKVGet - 478 total (0 active), Execution time: mean = 1.614 ms, total = 771.684 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s InternalKVGcsService.grpc_server.InternalKVGet.HandleRequestImpl - 478 total (0 active), Execution time: mean = 42.767 us, total = 20.442 ms, Queueing time: mean = 560.317 us, max = 83.308 ms, min = 9.088 us, total = 267.832 ms InternalKVGcsService.grpc_server.InternalKVPut - 426 total (0 active), Execution time: mean = 743.705 us, total = 316.818 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s InternalKVGcsService.grpc_server.InternalKVPut.HandleRequestImpl - 426 total (0 active), Execution time: mean = 45.138 us, total = 19.229 ms, Queueing time: mean = 114.488 us, max = 292.757 us, min = 4.927 us, total = 48.772 ms GcsResourceManager::Update - 327 total (0 active), Execution time: mean = 106.183 us, total = 34.722 ms, Queueing time: mean = 117.754 us, max = 320.996 us, min = 12.910 us, total = 38.505 ms NodeResourceInfoGcsService.grpc_server.GetAllTotalResources.HandleRequestImpl - 224 total (0 active), Execution time: mean = 62.595 us, total = 14.021 ms, Queueing time: mean = 122.089 us, max = 239.383 us, min = 26.000 us, total = 27.348 ms NodeResourceInfoGcsService.grpc_server.GetAllTotalResources - 224 total (0 active), Execution time: mean = 603.841 us, total = 135.260 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s event_loop_lag_probe - 199 total (0 active), Execution time: mean = 28.262 us, total = 5.624 ms, Queueing time: mean = 82.689 us, max = 13.707 ms, min = 4.696 us, total = 16.455 ms NodeInfoGcsService.grpc_server.GetAllNodeInfo - 115 total (0 active), Execution time: mean = 873.825 us, total = 100.490 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeInfoGcsService.grpc_server.GetAllNodeInfo.HandleRequestImpl - 115 total (0 active), Execution time: mean = 48.430 us, total = 5.569 ms, Queueing time: mean = 72.265 us, max = 441.236 us, min = 10.604 us, total = 8.311 ms WorkerInfoGcsService.grpc_server.AddWorkerInfo - 97 total (0 active), Execution time: mean = 2.858 ms, total = 277.243 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s WorkerInfoGcsService.grpc_server.AddWorkerInfo.HandleRequestImpl - 97 total (0 active), Execution time: mean = 36.002 us, total = 3.492 ms, Queueing time: mean = 81.703 us, max = 282.349 us, min = 15.472 us, total = 7.925 ms GcsInMemoryStore.Exists - 52 total (0 active), Execution time: mean = 19.776 us, total = 1.028 ms, Queueing time: mean = 13.098 us, max = 35.020 us, min = 3.347 us, total = 681.120 us InternalKVGcsService.grpc_server.InternalKVExists.HandleRequestImpl - 52 total (0 active), Execution time: mean = 44.770 us, total = 2.328 ms, Queueing time: mean = 106.373 us, max = 190.116 us, min = 25.777 us, total = 5.531 ms InternalKVGcsService.grpc_server.InternalKVExists - 52 total (0 active), Execution time: mean = 593.657 us, total = 30.870 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s RayletLoadPulled - 50 total (1 active), Execution time: mean = 262.707 us, total = 13.135 ms, Queueing time: mean = 1.766 ms, max = 83.628 ms, min = 24.706 us, total = 88.324 ms NodeManagerService.grpc_client.GetResourceLoad.OnReplyReceived - 48 total (0 active), Execution time: mean = 87.863 us, total = 4.217 ms, Queueing time: mean = 123.095 us, max = 387.905 us, min = 29.445 us, total = 5.909 ms NodeManagerService.grpc_client.GetResourceLoad - 48 total (0 active), Execution time: mean = 18.396 ms, total = 883.022 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s ActorInfoGcsService.grpc_server.GetNamedActorInfo - 26 total (0 active), Execution time: mean = 617.850 us, total = 16.064 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s ActorInfoGcsService.grpc_server.GetNamedActorInfo.HandleRequestImpl - 26 total (0 active), Execution time: mean = 72.910 us, total = 1.896 ms, Queueing time: mean = 102.222 us, max = 183.274 us, min = 30.041 us, total = 2.658 ms ActorInfoGcsService.grpc_server.GetActorInfo.HandleRequestImpl - 22 total (0 active), Execution time: mean = 44.716 us, total = 983.752 us, Queueing time: mean = 97.833 us, max = 366.966 us, min = 28.822 us, total = 2.152 ms NodeResourceInfoGcsService.grpc_server.GetAllAvailableResources.HandleRequestImpl - 22 total (0 active), Execution time: mean = 57.924 us, total = 1.274 ms, Queueing time: mean = 98.393 us, max = 192.996 us, min = 21.778 us, total = 2.165 ms ActorInfoGcsService.grpc_server.GetActorInfo - 22 total (0 active), Execution time: mean = 539.093 us, total = 11.860 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeResourceInfoGcsService.grpc_server.GetAllAvailableResources - 22 total (0 active), Execution time: mean = 550.275 us, total = 12.106 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeInfoGcsService.grpc_server.CheckAlive - 20 total (0 active), Execution time: mean = 566.117 us, total = 11.322 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeInfoGcsService.grpc_server.CheckAlive.HandleRequestImpl - 20 total (0 active), Execution time: mean = 33.609 us, total = 672.184 us, Queueing time: mean = 104.099 us, max = 182.667 us, min = 28.908 us, total = 2.082 ms NodeInfoGcsService.grpc_server.GetClusterId - 19 total (0 active), Execution time: mean = 1.107 ms, total = 21.038 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeInfoGcsService.grpc_server.GetClusterId.HandleRequestImpl - 19 total (0 active), Execution time: mean = 237.287 us, total = 4.508 ms, Queueing time: mean = 407.387 us, max = 6.222 ms, min = 15.457 us, total = 7.740 ms ClusterResourceManager.ResetRemoteNodeView - 17 total (1 active), Execution time: mean = 11.467 us, total = 194.932 us, Queueing time: mean = 5.054 ms, max = 84.269 ms, min = 39.270 us, total = 85.915 ms HealthCheck - 15 total (0 active), Execution time: mean = 9.616 us, total = 144.239 us, Queueing time: mean = 103.855 us, max = 278.656 us, min = 18.668 us, total = 1.558 ms NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage - 10 total (0 active), Execution time: mean = 725.990 us, total = 7.260 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage.HandleRequestImpl - 10 total (0 active), Execution time: mean = 83.800 us, total = 838.000 us, Queueing time: mean = 119.545 us, max = 160.968 us, min = 46.601 us, total = 1.195 ms GcsInMemoryStore.GetAll - 6 total (0 active), Execution time: mean = 10.177 us, total = 61.064 us, Queueing time: mean = 43.354 us, max = 51.164 us, min = 10.926 us, total = 260.123 us GCSServer.deadline_timer.debug_state_dump - 5 total (1 active, 1 running), Execution time: mean = 1.377 ms, total = 6.883 ms, Queueing time: mean = 72.032 us, max = 164.743 us, min = 49.822 us, total = 360.160 us PeriodicalRunner.RunFnPeriodically - 4 total (0 active), Execution time: mean = 356.910 us, total = 1.428 ms, Queueing time: mean = 137.951 ms, max = 271.493 ms, min = 4.103 ms, total = 551.805 ms ActorInfoGcsService.grpc_server.RegisterActor - 3 total (0 active), Execution time: mean = 835.235 us, total = 2.506 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s ActorInfoGcsService.grpc_server.RegisterActor.HandleRequestImpl - 3 total (0 active), Execution time: mean = 270.190 us, total = 810.571 us, Queueing time: mean = 151.661 us, max = 311.345 us, min = 26.759 us, total = 454.984 us ActorInfoGcsService.grpc_server.CreateActor - 2 total (0 active), Execution time: mean = 412.544 ms, total = 825.088 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeManagerService.grpc_client.RequestWorkerLease - 2 total (0 active), Execution time: mean = 1.676 ms, total = 3.351 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeManagerService.grpc_client.RequestWorkerLease.OnReplyReceived - 2 total (0 active), Execution time: mean = 380.474 us, total = 760.947 us, Queueing time: mean = 54.748 us, max = 96.340 us, min = 13.156 us, total = 109.496 us CoreWorkerService.grpc_client.PushTask.OnReplyReceived - 2 total (0 active), Execution time: mean = 554.228 us, total = 1.108 ms, Queueing time: mean = 217.080 us, max = 264.592 us, min = 169.568 us, total = 434.160 us CoreWorkerService.grpc_client.PushTask - 2 total (0 active), Execution time: mean = 408.406 ms, total = 816.812 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s ActorInfoGcsService.grpc_server.CreateActor.HandleRequestImpl - 2 total (0 active), Execution time: mean = 510.305 us, total = 1.021 ms, Queueing time: mean = 146.537 us, max = 157.315 us, min = 135.760 us, total = 293.075 us JobInfoGcsService.grpc_server.GetAllJobInfo - 1 total (0 active), Execution time: mean = 256.088 us, total = 256.088 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeInfoGcsService.grpc_server.RegisterNode.HandleRequestImpl - 1 total (0 active), Execution time: mean = 238.095 us, total = 238.095 us, Queueing time: mean = 35.010 us, max = 35.010 us, min = 35.010 us, total = 35.010 us JobInfoGcsService.grpc_server.AddJob - 1 total (0 active), Execution time: mean = 1.078 ms, total = 1.078 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s GcsHealthCheckManager::AddNode - 1 total (0 active), Execution time: mean = 137.504 us, total = 137.504 us, Queueing time: mean = 1.037 us, max = 1.037 us, min = 1.037 us, total = 1.037 us InternalKVGcsService.grpc_server.InternalKVDel - 1 total (0 active), Execution time: mean = 830.474 us, total = 830.474 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeInfoGcsService.grpc_server.RegisterNode - 1 total (0 active), Execution time: mean = 1.216 ms, total = 1.216 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s JobInfoGcsService.grpc_server.GetNextJobID.HandleRequestImpl - 1 total (0 active), Execution time: mean = 11.378 us, total = 11.378 us, Queueing time: mean = 13.907 us, max = 13.907 us, min = 13.907 us, total = 13.907 us JobInfoGcsService.grpc_server.GetNextJobID - 1 total (0 active), Execution time: mean = 212.029 us, total = 212.029 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s GCSServer.deadline_timer.debug_state_event_stats_print - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s InternalKVGcsService.grpc_server.InternalKVDel.HandleRequestImpl - 1 total (0 active), Execution time: mean = 86.056 us, total = 86.056 us, Queueing time: mean = 117.455 us, max = 117.455 us, min = 117.455 us, total = 117.455 us InternalKVGcsService.grpc_server.GetInternalConfig - 1 total (0 active), Execution time: mean = 378.249 us, total = 378.249 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s JobInfoGcsService.grpc_server.AddJob.HandleRequestImpl - 1 total (0 active), Execution time: mean = 217.287 us, total = 217.287 us, Queueing time: mean = 32.449 us, max = 32.449 us, min = 32.449 us, total = 32.449 us JobInfoGcsService.grpc_server.GetAllJobInfo.HandleRequestImpl - 1 total (0 active), Execution time: mean = 27.145 us, total = 27.145 us, Queueing time: mean = 26.197 us, max = 26.197 us, min = 26.197 us, total = 26.197 us GcsInMemoryStore.Delete - 1 total (0 active), Execution time: mean = 23.886 us, total = 23.886 us, Queueing time: mean = 12.124 us, max = 12.124 us, min = 12.124 us, total = 12.124 us InternalKVGcsService.grpc_server.GetInternalConfig.HandleRequestImpl - 1 total (0 active), Execution time: mean = 29.427 us, total = 29.427 us, Queueing time: mean = 18.424 us, max = 18.424 us, min = 18.424 us, total = 18.424 us |