File size: 15,317 Bytes
c011401 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 |
Gcs Debug state: GcsNodeManager: - RegisterNode request count: 1 - DrainNode request count: 0 - GetAllNodeInfo request count: 125 GcsActorManager: - RegisterActor request count: 3 - CreateActor request count: 2 - GetActorInfo request count: 22 - GetNamedActorInfo request count: 26 - GetAllActorInfo request count: 0 - KillActor request count: 0 - ListNamedActors request count: 0 - Registered actors count: 2 - Destroyed actors count: 0 - Named actors count: 2 - Unresolved actors count: 0 - Pending actors count: 0 - Created actors count: 1 - owners_: 0 - actor_to_register_callbacks_: 0 - actor_to_restart_callbacks_: 0 - actor_to_create_callbacks_: 0 - sorted_destroyed_actor_list_: 0 GcsResourceManager: - GetAllAvailableResources request count: 22 - GetAllTotalResources request count: 224 - GetAllResourceUsage request count: 10 GcsPlacementGroupManager: - CreatePlacementGroup request count: 0 - RemovePlacementGroup request count: 0 - GetPlacementGroup request count: 0 - GetAllPlacementGroup request count: 0 - WaitPlacementGroupUntilReady request count: 0 - GetNamedPlacementGroup request count: 0 - Scheduling pending placement group count: 0 - Registered placement groups count: 0 - Named placement group count: 0 - Pending placement groups count: 0 - Infeasible placement groups count: 0 Publisher: GCS_JOB_CHANNEL - cumulative published messages: 1 - cumulative published bytes: 286 - current buffered bytes: 286 GCS_ACTOR_CHANNEL - cumulative published messages: 6 - cumulative published bytes: 1857 - current buffered bytes: 363 RAY_LOG_CHANNEL - cumulative published messages: 235 - cumulative published bytes: 56914 - current buffered bytes: 137 GCS_NODE_INFO_CHANNEL - cumulative published messages: 1 - cumulative published bytes: 534 - current buffered bytes: 0 [runtime env manager] ID to URIs table: [runtime env manager] URIs reference table: GcsTaskManager: -Total num task events reported: 6451 -Total num status task events dropped: 0 -Total num profile events dropped: 0 -Current num of task events stored: 3137 -Total num of actor creation tasks: 3 -Total num of actor tasks: 631 -Total num of normal tasks: 2305 -Total num of driver tasks: 1 GcsAutoscalerStateManager: - last_seen_autoscaler_state_version_: 0 - last_cluster_resource_state_version_: 0 - pending demands: Global stats: 4908 total (4 active) Queueing time: mean = 708.220 us, max = 271.392 ms, min = 793.000 ns, total = 3.476 s Execution time: mean = 1.346 ms, total = 6.605 s Event stats: GcsInMemoryStore.Put - 597 total (0 active), Execution time: mean = 475.658 us, total = 283.968 ms, Queueing time: mean = 3.183 ms, max = 270.469 ms, min = 2.416 us, total = 1.900 s GcsInMemoryStore.Get - 491 total (0 active), Execution time: mean = 24.004 us, total = 11.786 ms, Queueing time: mean = 19.817 us, max = 381.568 us, min = 2.391 us, total = 9.730 ms InternalKVGcsService.grpc_server.InternalKVGet.HandleRequestImpl - 490 total (0 active), Execution time: mean = 41.024 us, total = 20.102 ms, Queueing time: mean = 560.224 us, max = 83.420 ms, min = 5.975 us, total = 274.510 ms InternalKVGcsService.grpc_server.InternalKVGet - 490 total (0 active), Execution time: mean = 1.681 ms, total = 823.540 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s InternalKVGcsService.grpc_server.InternalKVPut - 421 total (0 active), Execution time: mean = 446.717 us, total = 188.068 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s InternalKVGcsService.grpc_server.InternalKVPut.HandleRequestImpl - 421 total (0 active), Execution time: mean = 37.707 us, total = 15.875 ms, Queueing time: mean = 78.718 us, max = 377.007 us, min = 10.899 us, total = 33.140 ms GcsResourceManager::Update - 306 total (0 active), Execution time: mean = 93.758 us, total = 28.690 ms, Queueing time: mean = 494.481 us, max = 62.937 ms, min = 3.779 us, total = 151.311 ms NodeResourceInfoGcsService.grpc_server.GetAllTotalResources - 224 total (0 active), Execution time: mean = 435.864 us, total = 97.633 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeResourceInfoGcsService.grpc_server.GetAllTotalResources.HandleRequestImpl - 224 total (0 active), Execution time: mean = 52.480 us, total = 11.755 ms, Queueing time: mean = 77.235 us, max = 341.266 us, min = 15.261 us, total = 17.301 ms event_loop_lag_probe - 197 total (0 active), Execution time: mean = 26.511 us, total = 5.223 ms, Queueing time: mean = 73.175 us, max = 11.825 ms, min = 4.497 us, total = 14.415 ms NodeInfoGcsService.grpc_server.GetAllNodeInfo.HandleRequestImpl - 125 total (0 active), Execution time: mean = 44.998 us, total = 5.625 ms, Queueing time: mean = 86.838 us, max = 342.788 us, min = 9.170 us, total = 10.855 ms NodeInfoGcsService.grpc_server.GetAllNodeInfo - 125 total (0 active), Execution time: mean = 1.507 ms, total = 188.317 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s WorkerInfoGcsService.grpc_server.AddWorkerInfo - 97 total (0 active), Execution time: mean = 4.799 ms, total = 465.502 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s WorkerInfoGcsService.grpc_server.AddWorkerInfo.HandleRequestImpl - 97 total (0 active), Execution time: mean = 40.254 us, total = 3.905 ms, Queueing time: mean = 2.670 ms, max = 83.381 ms, min = 19.803 us, total = 258.967 ms RayletLoadPulled - 50 total (1 active), Execution time: mean = 253.048 us, total = 12.652 ms, Queueing time: mean = 1.544 ms, max = 73.702 ms, min = 20.864 us, total = 77.200 ms GcsInMemoryStore.Exists - 47 total (0 active), Execution time: mean = 19.969 us, total = 938.555 us, Queueing time: mean = 14.887 us, max = 66.824 us, min = 3.962 us, total = 699.709 us InternalKVGcsService.grpc_server.InternalKVExists.HandleRequestImpl - 47 total (0 active), Execution time: mean = 47.783 us, total = 2.246 ms, Queueing time: mean = 99.976 us, max = 238.760 us, min = 15.927 us, total = 4.699 ms InternalKVGcsService.grpc_server.InternalKVExists - 47 total (0 active), Execution time: mean = 557.546 us, total = 26.205 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeManagerService.grpc_client.GetResourceLoad.OnReplyReceived - 47 total (0 active), Execution time: mean = 77.214 us, total = 3.629 ms, Queueing time: mean = 87.050 us, max = 244.734 us, min = 17.626 us, total = 4.091 ms NodeManagerService.grpc_client.GetResourceLoad - 47 total (0 active), Execution time: mean = 39.961 ms, total = 1.878 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s ActorInfoGcsService.grpc_server.GetNamedActorInfo - 26 total (0 active), Execution time: mean = 603.905 us, total = 15.702 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s ActorInfoGcsService.grpc_server.GetNamedActorInfo.HandleRequestImpl - 26 total (0 active), Execution time: mean = 79.795 us, total = 2.075 ms, Queueing time: mean = 97.576 us, max = 230.552 us, min = 26.116 us, total = 2.537 ms NodeResourceInfoGcsService.grpc_server.GetAllAvailableResources - 22 total (0 active), Execution time: mean = 485.434 us, total = 10.680 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeResourceInfoGcsService.grpc_server.GetAllAvailableResources.HandleRequestImpl - 22 total (0 active), Execution time: mean = 51.103 us, total = 1.124 ms, Queueing time: mean = 104.260 us, max = 213.968 us, min = 28.161 us, total = 2.294 ms ActorInfoGcsService.grpc_server.GetActorInfo - 22 total (0 active), Execution time: mean = 566.460 us, total = 12.462 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s ActorInfoGcsService.grpc_server.GetActorInfo.HandleRequestImpl - 22 total (0 active), Execution time: mean = 47.605 us, total = 1.047 ms, Queueing time: mean = 132.479 us, max = 380.477 us, min = 22.337 us, total = 2.915 ms NodeInfoGcsService.grpc_server.CheckAlive.HandleRequestImpl - 20 total (0 active), Execution time: mean = 31.789 us, total = 635.787 us, Queueing time: mean = 86.134 us, max = 167.333 us, min = 14.055 us, total = 1.723 ms NodeInfoGcsService.grpc_server.CheckAlive - 20 total (0 active), Execution time: mean = 805.763 us, total = 16.115 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeInfoGcsService.grpc_server.GetClusterId.HandleRequestImpl - 19 total (0 active), Execution time: mean = 214.035 us, total = 4.067 ms, Queueing time: mean = 345.569 us, max = 4.585 ms, min = 16.281 us, total = 6.566 ms NodeInfoGcsService.grpc_server.GetClusterId - 19 total (0 active), Execution time: mean = 1.232 ms, total = 23.412 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s ClusterResourceManager.ResetRemoteNodeView - 17 total (1 active), Execution time: mean = 11.298 us, total = 192.068 us, Queueing time: mean = 4.492 ms, max = 73.887 ms, min = 31.581 us, total = 76.362 ms HealthCheck - 15 total (0 active), Execution time: mean = 7.556 us, total = 113.337 us, Queueing time: mean = 90.192 us, max = 233.009 us, min = 18.453 us, total = 1.353 ms NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage.HandleRequestImpl - 10 total (0 active), Execution time: mean = 76.652 us, total = 766.521 us, Queueing time: mean = 84.347 us, max = 158.172 us, min = 22.543 us, total = 843.469 us NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage - 10 total (0 active), Execution time: mean = 611.555 us, total = 6.116 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s GcsInMemoryStore.GetAll - 6 total (0 active), Execution time: mean = 21.034 us, total = 126.202 us, Queueing time: mean = 99.547 us, max = 129.597 us, min = 10.150 us, total = 597.282 us GCSServer.deadline_timer.debug_state_dump - 5 total (1 active, 1 running), Execution time: mean = 14.559 ms, total = 72.795 ms, Queueing time: mean = 71.886 us, max = 138.889 us, min = 34.744 us, total = 359.430 us PeriodicalRunner.RunFnPeriodically - 4 total (0 active), Execution time: mean = 172.632 us, total = 690.529 us, Queueing time: mean = 137.594 ms, max = 271.392 ms, min = 3.709 ms, total = 550.375 ms ActorInfoGcsService.grpc_server.RegisterActor.HandleRequestImpl - 3 total (0 active), Execution time: mean = 324.645 us, total = 973.934 us, Queueing time: mean = 23.879 ms, max = 71.278 ms, min = 113.476 us, total = 71.637 ms ActorInfoGcsService.grpc_server.RegisterActor - 3 total (0 active), Execution time: mean = 24.663 ms, total = 73.990 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s ActorInfoGcsService.grpc_server.CreateActor.HandleRequestImpl - 2 total (0 active), Execution time: mean = 556.494 us, total = 1.113 ms, Queueing time: mean = 32.989 us, max = 35.504 us, min = 30.473 us, total = 65.977 us CoreWorkerService.grpc_client.PushTask.OnReplyReceived - 2 total (0 active), Execution time: mean = 625.774 us, total = 1.252 ms, Queueing time: mean = 181.541 us, max = 235.735 us, min = 127.347 us, total = 363.082 us ActorInfoGcsService.grpc_server.CreateActor - 2 total (0 active), Execution time: mean = 571.032 ms, total = 1.142 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeManagerService.grpc_client.RequestWorkerLease - 2 total (0 active), Execution time: mean = 1.578 ms, total = 3.156 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeManagerService.grpc_client.RequestWorkerLease.OnReplyReceived - 2 total (0 active), Execution time: mean = 425.116 us, total = 850.233 us, Queueing time: mean = 89.821 us, max = 151.067 us, min = 28.575 us, total = 179.642 us CoreWorkerService.grpc_client.PushTask - 2 total (0 active), Execution time: mean = 567.017 ms, total = 1.134 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeInfoGcsService.grpc_server.RegisterNode.HandleRequestImpl - 1 total (0 active), Execution time: mean = 211.981 us, total = 211.981 us, Queueing time: mean = 126.035 us, max = 126.035 us, min = 126.035 us, total = 126.035 us InternalKVGcsService.grpc_server.GetInternalConfig - 1 total (0 active), Execution time: mean = 727.502 us, total = 727.502 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s JobInfoGcsService.grpc_server.GetAllJobInfo.HandleRequestImpl - 1 total (0 active), Execution time: mean = 22.867 us, total = 22.867 us, Queueing time: mean = 134.802 us, max = 134.802 us, min = 134.802 us, total = 134.802 us GCSServer.deadline_timer.debug_state_event_stats_print - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s JobInfoGcsService.grpc_server.GetAllJobInfo - 1 total (0 active), Execution time: mean = 837.667 us, total = 837.667 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeInfoGcsService.grpc_server.RegisterNode - 1 total (0 active), Execution time: mean = 1.248 ms, total = 1.248 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s JobInfoGcsService.grpc_server.GetNextJobID.HandleRequestImpl - 1 total (0 active), Execution time: mean = 29.410 us, total = 29.410 us, Queueing time: mean = 109.675 us, max = 109.675 us, min = 109.675 us, total = 109.675 us GcsHealthCheckManager::AddNode - 1 total (0 active), Execution time: mean = 18.414 us, total = 18.414 us, Queueing time: mean = 793.000 ns, max = 793.000 ns, min = 793.000 ns, total = 793.000 ns JobInfoGcsService.grpc_server.GetNextJobID - 1 total (0 active), Execution time: mean = 412.411 us, total = 412.411 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s JobInfoGcsService.grpc_server.AddJob.HandleRequestImpl - 1 total (0 active), Execution time: mean = 153.268 us, total = 153.268 us, Queueing time: mean = 132.867 us, max = 132.867 us, min = 132.867 us, total = 132.867 us InternalKVGcsService.grpc_server.InternalKVDel.HandleRequestImpl - 1 total (0 active), Execution time: mean = 70.713 us, total = 70.713 us, Queueing time: mean = 156.019 us, max = 156.019 us, min = 156.019 us, total = 156.019 us InternalKVGcsService.grpc_server.InternalKVDel - 1 total (0 active), Execution time: mean = 872.181 us, total = 872.181 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s JobInfoGcsService.grpc_server.AddJob - 1 total (0 active), Execution time: mean = 845.585 us, total = 845.585 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s InternalKVGcsService.grpc_server.GetInternalConfig.HandleRequestImpl - 1 total (0 active), Execution time: mean = 52.216 us, total = 52.216 us, Queueing time: mean = 128.857 us, max = 128.857 us, min = 128.857 us, total = 128.857 us GcsInMemoryStore.Delete - 1 total (0 active), Execution time: mean = 22.629 us, total = 22.629 us, Queueing time: mean = 11.404 us, max = 11.404 us, min = 11.404 us, total = 11.404 us |