File size: 14,828 Bytes
c011401 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 |
Gcs Debug state: GcsNodeManager: - RegisterNode request count: 1 - DrainNode request count: 0 - GetAllNodeInfo request count: 101 GcsActorManager: - RegisterActor request count: 3 - CreateActor request count: 2 - GetActorInfo request count: 22 - GetNamedActorInfo request count: 26 - GetAllActorInfo request count: 0 - KillActor request count: 0 - ListNamedActors request count: 0 - Registered actors count: 2 - Destroyed actors count: 0 - Named actors count: 2 - Unresolved actors count: 0 - Pending actors count: 0 - Created actors count: 1 - owners_: 0 - actor_to_register_callbacks_: 0 - actor_to_restart_callbacks_: 0 - actor_to_create_callbacks_: 0 - sorted_destroyed_actor_list_: 0 GcsResourceManager: - GetAllAvailableResources request count: 0 - GetAllTotalResources request count: 163 - GetAllResourceUsage request count: 8 GcsPlacementGroupManager: - CreatePlacementGroup request count: 0 - RemovePlacementGroup request count: 0 - GetPlacementGroup request count: 0 - GetAllPlacementGroup request count: 0 - WaitPlacementGroupUntilReady request count: 0 - GetNamedPlacementGroup request count: 0 - Scheduling pending placement group count: 0 - Registered placement groups count: 0 - Named placement group count: 0 - Pending placement groups count: 0 - Infeasible placement groups count: 0 Publisher: GCS_JOB_CHANNEL - cumulative published messages: 1 - cumulative published bytes: 287 - current buffered bytes: 287 GCS_ACTOR_CHANNEL - cumulative published messages: 6 - cumulative published bytes: 1859 - current buffered bytes: 365 GCS_NODE_INFO_CHANNEL - cumulative published messages: 1 - cumulative published bytes: 536 - current buffered bytes: 0 RAY_LOG_CHANNEL - cumulative published messages: 168 - cumulative published bytes: 47241 - current buffered bytes: 825 [runtime env manager] ID to URIs table: [runtime env manager] URIs reference table: GcsTaskManager: -Total num task events reported: 4631 -Total num status task events dropped: 0 -Total num profile events dropped: 0 -Current num of task events stored: 2230 -Total num of actor creation tasks: 3 -Total num of actor tasks: 440 -Total num of normal tasks: 1620 -Total num of driver tasks: 1 GcsAutoscalerStateManager: - last_seen_autoscaler_state_version_: 0 - last_cluster_resource_state_version_: 0 - pending demands: Global stats: 4192 total (4 active) Queueing time: mean = 870.149 us, max = 275.096 ms, min = 813.000 ns, total = 3.648 s Execution time: mean = 1.523 ms, total = 6.386 s Event stats: GcsInMemoryStore.Put - 491 total (0 active), Execution time: mean = 599.695 us, total = 294.450 ms, Queueing time: mean = 3.930 ms, max = 274.410 ms, min = 2.082 us, total = 1.929 s GcsInMemoryStore.Get - 481 total (0 active), Execution time: mean = 27.795 us, total = 13.370 ms, Queueing time: mean = 22.736 us, max = 441.066 us, min = 2.231 us, total = 10.936 ms InternalKVGcsService.grpc_server.InternalKVGet - 480 total (0 active), Execution time: mean = 2.241 ms, total = 1.076 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s InternalKVGcsService.grpc_server.InternalKVGet.HandleRequestImpl - 480 total (0 active), Execution time: mean = 40.158 us, total = 19.276 ms, Queueing time: mean = 1.237 ms, max = 77.964 ms, min = 2.610 us, total = 593.802 ms InternalKVGcsService.grpc_server.InternalKVPut - 327 total (0 active), Execution time: mean = 557.854 us, total = 182.418 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s InternalKVGcsService.grpc_server.InternalKVPut.HandleRequestImpl - 327 total (0 active), Execution time: mean = 44.133 us, total = 14.431 ms, Queueing time: mean = 103.612 us, max = 846.080 us, min = 12.657 us, total = 33.881 ms GcsResourceManager::Update - 241 total (0 active), Execution time: mean = 105.292 us, total = 25.375 ms, Queueing time: mean = 107.693 us, max = 239.851 us, min = 19.542 us, total = 25.954 ms NodeResourceInfoGcsService.grpc_server.GetAllTotalResources - 163 total (0 active), Execution time: mean = 588.138 us, total = 95.867 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeResourceInfoGcsService.grpc_server.GetAllTotalResources.HandleRequestImpl - 163 total (0 active), Execution time: mean = 61.723 us, total = 10.061 ms, Queueing time: mean = 112.771 us, max = 249.748 us, min = 19.558 us, total = 18.382 ms event_loop_lag_probe - 156 total (0 active), Execution time: mean = 30.906 us, total = 4.821 ms, Queueing time: mean = 105.250 us, max = 12.751 ms, min = 3.704 us, total = 16.419 ms NodeInfoGcsService.grpc_server.GetAllNodeInfo.HandleRequestImpl - 101 total (0 active), Execution time: mean = 59.160 us, total = 5.975 ms, Queueing time: mean = 2.348 ms, max = 78.339 ms, min = 5.083 us, total = 237.114 ms NodeInfoGcsService.grpc_server.GetAllNodeInfo - 101 total (0 active), Execution time: mean = 3.523 ms, total = 355.797 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s WorkerInfoGcsService.grpc_server.AddWorkerInfo - 97 total (0 active), Execution time: mean = 2.592 ms, total = 251.443 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s WorkerInfoGcsService.grpc_server.AddWorkerInfo.HandleRequestImpl - 97 total (0 active), Execution time: mean = 34.195 us, total = 3.317 ms, Queueing time: mean = 1.143 ms, max = 72.073 ms, min = 9.242 us, total = 110.877 ms InternalKVGcsService.grpc_server.InternalKVExists - 48 total (0 active), Execution time: mean = 553.421 us, total = 26.564 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s GcsInMemoryStore.Exists - 48 total (0 active), Execution time: mean = 23.276 us, total = 1.117 ms, Queueing time: mean = 12.890 us, max = 56.874 us, min = 5.605 us, total = 618.708 us InternalKVGcsService.grpc_server.InternalKVExists.HandleRequestImpl - 48 total (0 active), Execution time: mean = 50.355 us, total = 2.417 ms, Queueing time: mean = 87.671 us, max = 302.313 us, min = 19.482 us, total = 4.208 ms RayletLoadPulled - 40 total (1 active), Execution time: mean = 286.444 us, total = 11.458 ms, Queueing time: mean = 1.371 ms, max = 35.530 ms, min = 53.555 us, total = 54.849 ms NodeManagerService.grpc_client.GetResourceLoad.OnReplyReceived - 38 total (0 active), Execution time: mean = 80.582 us, total = 3.062 ms, Queueing time: mean = 87.203 us, max = 197.411 us, min = 23.064 us, total = 3.314 ms NodeManagerService.grpc_client.GetResourceLoad - 38 total (0 active), Execution time: mean = 55.544 ms, total = 2.111 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s ActorInfoGcsService.grpc_server.GetNamedActorInfo - 26 total (0 active), Execution time: mean = 617.256 us, total = 16.049 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s ActorInfoGcsService.grpc_server.GetNamedActorInfo.HandleRequestImpl - 26 total (0 active), Execution time: mean = 76.694 us, total = 1.994 ms, Queueing time: mean = 183.215 us, max = 2.717 ms, min = 18.899 us, total = 4.764 ms ActorInfoGcsService.grpc_server.GetActorInfo.HandleRequestImpl - 22 total (0 active), Execution time: mean = 48.615 us, total = 1.070 ms, Queueing time: mean = 78.206 us, max = 340.820 us, min = 21.158 us, total = 1.721 ms ActorInfoGcsService.grpc_server.GetActorInfo - 22 total (0 active), Execution time: mean = 543.757 us, total = 11.963 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeInfoGcsService.grpc_server.CheckAlive - 16 total (0 active), Execution time: mean = 521.860 us, total = 8.350 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeInfoGcsService.grpc_server.CheckAlive.HandleRequestImpl - 16 total (0 active), Execution time: mean = 33.763 us, total = 540.212 us, Queueing time: mean = 108.176 us, max = 262.219 us, min = 24.074 us, total = 1.731 ms ClusterResourceManager.ResetRemoteNodeView - 14 total (1 active), Execution time: mean = 11.610 us, total = 162.539 us, Queueing time: mean = 2.708 ms, max = 35.736 ms, min = 48.108 us, total = 37.916 ms HealthCheck - 12 total (0 active), Execution time: mean = 10.666 us, total = 127.996 us, Queueing time: mean = 47.138 us, max = 112.410 us, min = 25.013 us, total = 565.653 us NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage.HandleRequestImpl - 8 total (0 active), Execution time: mean = 89.585 us, total = 716.680 us, Queueing time: mean = 72.853 us, max = 143.954 us, min = 24.993 us, total = 582.820 us NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage - 8 total (0 active), Execution time: mean = 638.277 us, total = 5.106 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s GcsInMemoryStore.GetAll - 6 total (0 active), Execution time: mean = 15.876 us, total = 95.258 us, Queueing time: mean = 70.714 us, max = 88.340 us, min = 14.699 us, total = 424.283 us NodeInfoGcsService.grpc_server.GetClusterId - 5 total (0 active), Execution time: mean = 2.705 ms, total = 13.527 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeInfoGcsService.grpc_server.GetClusterId.HandleRequestImpl - 5 total (0 active), Execution time: mean = 345.403 us, total = 1.727 ms, Queueing time: mean = 831.840 us, max = 3.746 ms, min = 47.767 us, total = 4.159 ms GCSServer.deadline_timer.debug_state_dump - 4 total (1 active, 1 running), Execution time: mean = 8.555 ms, total = 34.218 ms, Queueing time: mean = 82.004 us, max = 193.615 us, min = 63.246 us, total = 328.018 us PeriodicalRunner.RunFnPeriodically - 4 total (0 active), Execution time: mean = 393.388 us, total = 1.574 ms, Queueing time: mean = 138.572 ms, max = 275.096 ms, min = 1.639 ms, total = 554.288 ms ActorInfoGcsService.grpc_server.RegisterActor - 3 total (0 active), Execution time: mean = 845.530 us, total = 2.537 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s ActorInfoGcsService.grpc_server.RegisterActor.HandleRequestImpl - 3 total (0 active), Execution time: mean = 313.724 us, total = 941.173 us, Queueing time: mean = 72.886 us, max = 178.148 us, min = 19.402 us, total = 218.657 us ActorInfoGcsService.grpc_server.CreateActor.HandleRequestImpl - 2 total (0 active), Execution time: mean = 607.790 us, total = 1.216 ms, Queueing time: mean = 91.809 us, max = 140.101 us, min = 43.518 us, total = 183.619 us NodeManagerService.grpc_client.RequestWorkerLease.OnReplyReceived - 2 total (0 active), Execution time: mean = 462.858 us, total = 925.715 us, Queueing time: mean = 58.031 us, max = 87.177 us, min = 28.885 us, total = 116.062 us ActorInfoGcsService.grpc_server.CreateActor - 2 total (0 active), Execution time: mean = 460.347 ms, total = 920.695 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s CoreWorkerService.grpc_client.PushTask - 2 total (0 active), Execution time: mean = 383.978 ms, total = 767.956 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s CoreWorkerService.grpc_client.PushTask.OnReplyReceived - 2 total (0 active), Execution time: mean = 674.872 us, total = 1.350 ms, Queueing time: mean = 122.320 us, max = 208.950 us, min = 35.691 us, total = 244.641 us NodeManagerService.grpc_client.RequestWorkerLease - 2 total (0 active), Execution time: mean = 39.925 ms, total = 79.849 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s InternalKVGcsService.grpc_server.InternalKVDel.HandleRequestImpl - 1 total (0 active), Execution time: mean = 72.308 us, total = 72.308 us, Queueing time: mean = 122.507 us, max = 122.507 us, min = 122.507 us, total = 122.507 us JobInfoGcsService.grpc_server.AddJob - 1 total (0 active), Execution time: mean = 1.122 ms, total = 1.122 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s JobInfoGcsService.grpc_server.GetAllJobInfo - 1 total (0 active), Execution time: mean = 694.348 us, total = 694.348 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s JobInfoGcsService.grpc_server.AddJob.HandleRequestImpl - 1 total (0 active), Execution time: mean = 176.640 us, total = 176.640 us, Queueing time: mean = 42.257 us, max = 42.257 us, min = 42.257 us, total = 42.257 us NodeInfoGcsService.grpc_server.RegisterNode - 1 total (0 active), Execution time: mean = 1.080 ms, total = 1.080 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s InternalKVGcsService.grpc_server.GetInternalConfig - 1 total (0 active), Execution time: mean = 814.710 us, total = 814.710 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s JobInfoGcsService.grpc_server.GetAllJobInfo.HandleRequestImpl - 1 total (0 active), Execution time: mean = 31.207 us, total = 31.207 us, Queueing time: mean = 131.149 us, max = 131.149 us, min = 131.149 us, total = 131.149 us NodeInfoGcsService.grpc_server.RegisterNode.HandleRequestImpl - 1 total (0 active), Execution time: mean = 174.000 us, total = 174.000 us, Queueing time: mean = 98.431 us, max = 98.431 us, min = 98.431 us, total = 98.431 us GcsHealthCheckManager::AddNode - 1 total (0 active), Execution time: mean = 22.569 us, total = 22.569 us, Queueing time: mean = 813.000 ns, max = 813.000 ns, min = 813.000 ns, total = 813.000 ns GcsInMemoryStore.Delete - 1 total (0 active), Execution time: mean = 21.629 us, total = 21.629 us, Queueing time: mean = 11.954 us, max = 11.954 us, min = 11.954 us, total = 11.954 us GCSServer.deadline_timer.debug_state_event_stats_print - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s InternalKVGcsService.grpc_server.GetInternalConfig.HandleRequestImpl - 1 total (0 active), Execution time: mean = 82.134 us, total = 82.134 us, Queueing time: mean = 122.005 us, max = 122.005 us, min = 122.005 us, total = 122.005 us JobInfoGcsService.grpc_server.GetNextJobID.HandleRequestImpl - 1 total (0 active), Execution time: mean = 28.810 us, total = 28.810 us, Queueing time: mean = 106.520 us, max = 106.520 us, min = 106.520 us, total = 106.520 us JobInfoGcsService.grpc_server.GetNextJobID - 1 total (0 active), Execution time: mean = 649.864 us, total = 649.864 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s InternalKVGcsService.grpc_server.InternalKVDel - 1 total (0 active), Execution time: mean = 858.033 us, total = 858.033 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s |