File size: 12,183 Bytes
c011401 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 |
Gcs Debug state: GcsNodeManager: - RegisterNode request count: 1 - DrainNode request count: 0 - GetAllNodeInfo request count: 30 GcsActorManager: - RegisterActor request count: 0 - CreateActor request count: 0 - GetActorInfo request count: 0 - GetNamedActorInfo request count: 0 - GetAllActorInfo request count: 0 - KillActor request count: 0 - ListNamedActors request count: 0 - Registered actors count: 0 - Destroyed actors count: 0 - Named actors count: 0 - Unresolved actors count: 0 - Pending actors count: 0 - Created actors count: 0 - owners_: 0 - actor_to_register_callbacks_: 0 - actor_to_restart_callbacks_: 0 - actor_to_create_callbacks_: 0 - sorted_destroyed_actor_list_: 0 GcsResourceManager: - GetAllAvailableResources request count: 0 - GetAllTotalResources request count: 0 - GetAllResourceUsage request count: 42 GcsPlacementGroupManager: - CreatePlacementGroup request count: 0 - RemovePlacementGroup request count: 0 - GetPlacementGroup request count: 0 - GetAllPlacementGroup request count: 0 - WaitPlacementGroupUntilReady request count: 0 - GetNamedPlacementGroup request count: 0 - Scheduling pending placement group count: 0 - Registered placement groups count: 0 - Named placement group count: 0 - Pending placement groups count: 0 - Infeasible placement groups count: 0 Publisher: GCS_NODE_INFO_CHANNEL - cumulative published messages: 1 - cumulative published bytes: 534 - current buffered bytes: 0 GCS_JOB_CHANNEL - cumulative published messages: 1 - cumulative published bytes: 256 - current buffered bytes: 256 RAY_ERROR_INFO_CHANNEL - cumulative published messages: 1 - cumulative published bytes: 293 - current buffered bytes: 293 RAY_LOG_CHANNEL - cumulative published messages: 44 - cumulative published bytes: 35848 - current buffered bytes: 819 [runtime env manager] ID to URIs table: [runtime env manager] URIs reference table: GcsTaskManager: -Total num task events reported: 13 -Total num status task events dropped: 0 -Total num profile events dropped: 0 -Current num of task events stored: 4 -Total num of actor creation tasks: 0 -Total num of actor tasks: 0 -Total num of normal tasks: 3 -Total num of driver tasks: 1 GcsAutoscalerStateManager: - last_seen_autoscaler_state_version_: 0 - last_cluster_resource_state_version_: 0 - pending demands: Global stats: 2878 total (4 active) Queueing time: mean = 902.805 us, max = 267.816 ms, min = 357.000 ns, total = 2.598 s Execution time: mean = 435.689 us, total = 1.254 s Event stats: event_loop_lag_probe - 842 total (0 active), Execution time: mean = 25.910 us, total = 21.817 ms, Queueing time: mean = 25.011 us, max = 12.416 ms, min = 2.508 us, total = 21.059 ms GcsInMemoryStore.Put - 378 total (0 active), Execution time: mean = 716.427 us, total = 270.809 ms, Queueing time: mean = 4.974 ms, max = 266.951 ms, min = 3.711 us, total = 1.880 s RayletLoadPulled - 210 total (1 active), Execution time: mean = 275.153 us, total = 57.782 ms, Queueing time: mean = 86.795 us, max = 1.396 ms, min = 19.899 us, total = 18.227 ms NodeManagerService.grpc_client.GetResourceLoad.OnReplyReceived - 207 total (0 active), Execution time: mean = 82.498 us, total = 17.077 ms, Queueing time: mean = 117.846 us, max = 1.637 ms, min = 24.459 us, total = 24.394 ms NodeManagerService.grpc_client.GetResourceLoad - 207 total (0 active), Execution time: mean = 1.621 ms, total = 335.462 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s InternalKVGcsService.grpc_server.InternalKVPut - 98 total (0 active), Execution time: mean = 615.753 us, total = 60.344 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s InternalKVGcsService.grpc_server.InternalKVPut.HandleRequestImpl - 98 total (0 active), Execution time: mean = 43.200 us, total = 4.234 ms, Queueing time: mean = 109.419 us, max = 172.888 us, min = 27.863 us, total = 10.723 ms GcsInMemoryStore.Get - 96 total (0 active), Execution time: mean = 507.114 us, total = 48.683 ms, Queueing time: mean = 163.787 us, max = 14.513 ms, min = 2.841 us, total = 15.724 ms InternalKVGcsService.grpc_server.InternalKVGet - 95 total (0 active), Execution time: mean = 2.725 ms, total = 258.902 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s InternalKVGcsService.grpc_server.InternalKVGet.HandleRequestImpl - 95 total (0 active), Execution time: mean = 47.513 us, total = 4.514 ms, Queueing time: mean = 438.621 us, max = 16.520 ms, min = 18.847 us, total = 41.669 ms NodeInfoGcsService.grpc_server.CheckAlive - 84 total (0 active), Execution time: mean = 602.639 us, total = 50.622 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeInfoGcsService.grpc_server.CheckAlive.HandleRequestImpl - 84 total (0 active), Execution time: mean = 35.591 us, total = 2.990 ms, Queueing time: mean = 124.114 us, max = 221.259 us, min = 32.156 us, total = 10.426 ms ClusterResourceManager.ResetRemoteNodeView - 71 total (1 active), Execution time: mean = 11.111 us, total = 788.884 us, Queueing time: mean = 132.926 us, max = 2.065 ms, min = 25.295 us, total = 9.438 ms HealthCheck - 68 total (0 active), Execution time: mean = 9.257 us, total = 629.495 us, Queueing time: mean = 104.827 us, max = 153.218 us, min = 22.549 us, total = 7.128 ms NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage.HandleRequestImpl - 42 total (0 active), Execution time: mean = 92.387 us, total = 3.880 ms, Queueing time: mean = 123.926 us, max = 170.050 us, min = 37.097 us, total = 5.205 ms NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage - 42 total (0 active), Execution time: mean = 701.889 us, total = 29.479 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s NodeInfoGcsService.grpc_server.GetAllNodeInfo.HandleRequestImpl - 30 total (0 active), Execution time: mean = 32.936 us, total = 988.074 us, Queueing time: mean = 84.904 us, max = 215.695 us, min = 18.831 us, total = 2.547 ms NodeInfoGcsService.grpc_server.GetAllNodeInfo - 30 total (0 active), Execution time: mean = 540.758 us, total = 16.223 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s GCSServer.deadline_timer.debug_state_dump - 21 total (1 active, 1 running), Execution time: mean = 1.395 ms, total = 29.303 ms, Queueing time: mean = 79.677 us, max = 199.609 us, min = 49.496 us, total = 1.673 ms WorkerInfoGcsService.grpc_server.AddWorkerInfo.HandleRequestImpl - 17 total (0 active), Execution time: mean = 29.256 us, total = 497.351 us, Queueing time: mean = 91.417 us, max = 193.387 us, min = 20.348 us, total = 1.554 ms WorkerInfoGcsService.grpc_server.AddWorkerInfo - 17 total (0 active), Execution time: mean = 516.336 us, total = 8.778 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s GcsInMemoryStore.GetAll - 6 total (0 active), Execution time: mean = 17.923 us, total = 107.536 us, Queueing time: mean = 98.775 us, max = 129.467 us, min = 6.087 us, total = 592.650 us NodeInfoGcsService.grpc_server.GetClusterId.HandleRequestImpl - 5 total (0 active), Execution time: mean = 574.346 us, total = 2.872 ms, Queueing time: mean = 855.005 us, max = 3.968 ms, min = 44.233 us, total = 4.275 ms NodeInfoGcsService.grpc_server.GetClusterId - 5 total (0 active), Execution time: mean = 2.929 ms, total = 14.644 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s PeriodicalRunner.RunFnPeriodically - 4 total (0 active), Execution time: mean = 315.651 us, total = 1.263 ms, Queueing time: mean = 135.502 ms, max = 267.816 ms, min = 2.806 ms, total = 542.009 ms GCSServer.deadline_timer.debug_state_event_stats_print - 4 total (1 active), Execution time: mean = 1.227 ms, total = 4.907 ms, Queueing time: mean = 78.329 us, max = 137.413 us, min = 79.763 us, total = 313.316 us GcsResourceManager::Update - 3 total (0 active), Execution time: mean = 80.802 us, total = 242.406 us, Queueing time: mean = 103.159 us, max = 155.159 us, min = 42.737 us, total = 309.476 us NodeInfoGcsService.grpc_server.RegisterNode.HandleRequestImpl - 1 total (0 active), Execution time: mean = 141.861 us, total = 141.861 us, Queueing time: mean = 31.046 us, max = 31.046 us, min = 31.046 us, total = 31.046 us JobInfoGcsService.grpc_server.AddJob.HandleRequestImpl - 1 total (0 active), Execution time: mean = 173.259 us, total = 173.259 us, Queueing time: mean = 131.708 us, max = 131.708 us, min = 131.708 us, total = 131.708 us InternalKVGcsService.grpc_server.InternalKVExists.HandleRequestImpl - 1 total (0 active), Execution time: mean = 66.447 us, total = 66.447 us, Queueing time: mean = 183.030 us, max = 183.030 us, min = 183.030 us, total = 183.030 us NodeInfoGcsService.grpc_server.RegisterNode - 1 total (0 active), Execution time: mean = 872.630 us, total = 872.630 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s JobInfoGcsService.grpc_server.ReportJobError.HandleRequestImpl - 1 total (0 active), Execution time: mean = 62.728 us, total = 62.728 us, Queueing time: mean = 8.177 us, max = 8.177 us, min = 8.177 us, total = 8.177 us JobInfoGcsService.grpc_server.GetNextJobID.HandleRequestImpl - 1 total (0 active), Execution time: mean = 23.396 us, total = 23.396 us, Queueing time: mean = 130.133 us, max = 130.133 us, min = 130.133 us, total = 130.133 us InternalKVGcsService.grpc_server.InternalKVDel - 1 total (0 active), Execution time: mean = 850.019 us, total = 850.019 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s JobInfoGcsService.grpc_server.GetNextJobID - 1 total (0 active), Execution time: mean = 563.161 us, total = 563.161 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s GcsHealthCheckManager::AddNode - 1 total (0 active), Execution time: mean = 8.999 us, total = 8.999 us, Queueing time: mean = 357.000 ns, max = 357.000 ns, min = 357.000 ns, total = 357.000 ns GcsInMemoryStore.Exists - 1 total (0 active), Execution time: mean = 24.296 us, total = 24.296 us, Queueing time: mean = 13.277 us, max = 13.277 us, min = 13.277 us, total = 13.277 us JobInfoGcsService.grpc_server.GetAllJobInfo.HandleRequestImpl - 1 total (0 active), Execution time: mean = 15.523 us, total = 15.523 us, Queueing time: mean = 61.169 us, max = 61.169 us, min = 61.169 us, total = 61.169 us InternalKVGcsService.grpc_server.GetInternalConfig - 1 total (0 active), Execution time: mean = 605.772 us, total = 605.772 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s InternalKVGcsService.grpc_server.GetInternalConfig.HandleRequestImpl - 1 total (0 active), Execution time: mean = 61.798 us, total = 61.798 us, Queueing time: mean = 133.774 us, max = 133.774 us, min = 133.774 us, total = 133.774 us InternalKVGcsService.grpc_server.InternalKVExists - 1 total (0 active), Execution time: mean = 798.934 us, total = 798.934 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s JobInfoGcsService.grpc_server.ReportJobError - 1 total (0 active), Execution time: mean = 434.188 us, total = 434.188 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s InternalKVGcsService.grpc_server.InternalKVDel.HandleRequestImpl - 1 total (0 active), Execution time: mean = 70.044 us, total = 70.044 us, Queueing time: mean = 121.379 us, max = 121.379 us, min = 121.379 us, total = 121.379 us GcsInMemoryStore.Delete - 1 total (0 active), Execution time: mean = 25.580 us, total = 25.580 us, Queueing time: mean = 11.240 us, max = 11.240 us, min = 11.240 us, total = 11.240 us JobInfoGcsService.grpc_server.GetAllJobInfo - 1 total (0 active), Execution time: mean = 387.969 us, total = 387.969 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s JobInfoGcsService.grpc_server.AddJob - 1 total (0 active), Execution time: mean = 890.187 us, total = 890.187 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s |