I0507 11:59:44.523502 4727 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-x82q" status="Running" XMT: Solicit on eth0, interval 117950ms. time="2024-05-07T11:59:43.954655670Z" level=info msg="StartContainer for \"93fa5decd62691912f90c9b27526f5e00183239bfa4d3f4ea8578a7873b9c2b4\"" time="2024-05-07T11:59:43.954289531Z" level=info msg="CreateContainer within sandbox \"ee9dc07bca79ef7dffe2a6eb326e27236e9e97c35913c7aae16ee0a62632fc25\" for &ContainerMetadata{Name:cortex-gw,Attempt:1660,} returns container id \"93fa5decd62691912f90c9b27526f5e00183239bfa4d3f4ea8578a7873b9c2b4\"" time="2024-05-07T11:59:43.941729092Z" level=info msg="CreateContainer within sandbox \"ee9dc07bca79ef7dffe2a6eb326e27236e9e97c35913c7aae16ee0a62632fc25\" for container &ContainerMetadata{Name:cortex-gw,Attempt:1660,}" I0507 11:59:43.939053 3659 scope.go:117] "RemoveContainer" containerID="9940112c30fda42aa2b814faddfc969d9a2328ae70ecb9b858d75aa6f8b61483" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36674 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36674 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36674 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" E0507 11:59:43.923954 4643 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=benchloadtestingxxl2-grafana-56b7c6b6df-j7jh6_hosted-grafana(83fb0f38-728e-4050-9500-6ac9fc9f21c8)\"" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-j7jh6" podUID="83fb0f38-728e-4050-9500-6ac9fc9f21c8" I0507 11:59:43.923422 4643 scope.go:117] "RemoveContainer" containerID="a85b6a771be0a2165463617e0c7a4f5b42dbb5c232c57166f32a72d969a25bf1" I0507 11:59:43.910438 4624 kubelet.go:2498] "SyncLoop (probe)" probe="readiness" status="ready" pod="hosted-grafana/ephemeral1511182183108soniaag-grafana-66698879db-52td7" E0507 11:59:43.910202 4624 prober.go:239] "Unable to write all bytes from execInContainer" err="short write" expectedBytes=12133 actualBytes=10240 I0507 11:59:43.794276 4775 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-nvpf" status="Running" E0507 11:59:43.743153 4601 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=benchloadtestingxxl2-grafana-5bb9765dd8-ktf2b_hosted-grafana(e8405a93-3a4c-4074-909d-661219c1f849)\"" pod="hosted-grafana/benchloadtestingxxl2-grafana-5bb9765dd8-ktf2b" podUID="e8405a93-3a4c-4074-909d-661219c1f849" I0507 11:59:43.742649 4601 scope.go:117] "RemoveContainer" containerID="8dbc699386128aa4e4af25beb0ea7e7ecad1b2d5e829061a04ff808054f050aa" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36672 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36672 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36672 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" E0507 11:59:43.704880 4602 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=benchloadtestingxxl2-grafana-56b7c6b6df-4gclf_hosted-grafana(fe493f66-8d1f-4435-9208-0304fd499ee1)\"" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-4gclf" podUID="fe493f66-8d1f-4435-9208-0304fd499ee1" I0507 11:59:43.704288 4602 scope.go:117] "RemoveContainer" containerID="2773338620ccfb32536d17788865e6fd4c7de7250ab31a7922195ffc1387ee5f" I0507 11:59:43.644447 4755 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-1-main-n2s16-1-1dd-97837cc3-lpds" status="Running" E0507 11:59:43.643392 4592 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=benchloadtestingxxl2-grafana-56b7c6b6df-vqkzm_hosted-grafana(d3742b42-2b35-4c32-8267-7cf79bbcb441)\"" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-vqkzm" podUID="d3742b42-2b35-4c32-8267-7cf79bbcb441" I0507 11:59:43.642869 4592 scope.go:117] "RemoveContainer" containerID="d53fbb23caf1e92d73b7ccf3a991c2ccd1d1b1ef072bfb1f6798a781d3809679" I0507 11:59:43.642392 4592 kubelet.go:2498] "SyncLoop (probe)" probe="liveness" status="unhealthy" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-vqkzm" I0507 11:59:43.520806 4724 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-j5wp" status="Running" I0507 11:59:43.503115 4773 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-dqf8" status="Running" 2024-05-07T11:59:43.499167Z INFO TelemetryEventsCollector ExtHandler Collected 2 events for extension: Microsoft.Azure.Extensions.CustomScript 2024-05-07T11:59:43.484606Z INFO ExtHandler ExtHandler Downloading agent manifest I0507 11:59:43.476936 4734 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-vvgr" status="Running" E0507 11:59:43.469105 3315 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"gcom-sync\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/kubernetes-dev/frontend-monitoring:6a8eb5a\\\"\"" pod="faro/update-usage-28487080-9sqzn" podUID="2cc85139-2f31-44ae-a308-3dc0df893592" I0507 11:59:43.455842 4729 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-v5z4" status="Running" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36670 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36670 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36670 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" E0507 11:59:43.362209 4601 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-85282-20418-1\\\"\"" pod="hosted-grafana/ephemeral1511182185282svenner-grafana-6f6b6f4d85-9xlcc" podUID="fee4a5b2-d22d-4d80-8041-8796a997679a" I0507 11:59:43.321744 4731 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-x24s" status="Running" I0507 11:59:43.306213 4736 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-78d9" status="Running" I0507 11:59:43.223958 4731 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-x24s" status="Running" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36668 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36668 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36668 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" I0507 11:59:43.157806 4724 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-t8zj" status="Running" XMT: Solicit on eth0, interval 126130ms. E0507 11:59:43.151914 4572 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=k6testslow2-grafana-7b64f97bd7-t22zk_hosted-grafana(9890650a-e338-4648-be7a-bb7f9726aa46)\"" pod="hosted-grafana/k6testslow2-grafana-7b64f97bd7-t22zk" podUID="9890650a-e338-4648-be7a-bb7f9726aa46" I0507 11:59:43.151045 4572 scope.go:117] "RemoveContainer" containerID="885a879aadfd9cb0665b3cf90d203d82dd9a1da0a75fe01d8b01f5cd49efd910" I0507 11:59:43.119017 4736 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-dhdn" status="Running" I0507 11:59:43.091861 4740 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-1-main-n2s16-1-1dd-97837cc3-cfqk" status="Running" I0507 11:59:43.048506 4590 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-hg-n2s8-6-1dd39c-3bfd06e9-5jwm" status="Running" I0507 11:59:43.041723 4589 kubelet_pods.go:906] "Unable to retrieve pull secret, the image pull may not succeed." pod="grafana-apps/bigquery-datasource-grafana-app-fast-7c94f74fd6-9n6t7" secret="" err="secret \"dockerhub\" not found" I0507 11:59:42.939527 4773 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-1-main-n2s16-1-1dd-97837cc3-4t6k" status="Running" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36657 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36657 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36657 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" E0507 11:59:42.925813 4733 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=bryan-prometheus-0_bryan-prometheus(6dadfe71-eb19-4231-a96e-c64bb5499a1e)\"" pod="bryan-prometheus/bryan-prometheus-0" podUID="6dadfe71-eb19-4231-a96e-c64bb5499a1e" I0507 11:59:42.925068 4733 scope.go:117] "RemoveContainer" containerID="f0f5ac8b5f4dba0a416c838dd7ccfa903bd1ca22e36ebc4d98a29b4e646063c6" I0507 11:59:42.923268 4731 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-7c5w" status="Running" E0507 11:59:42.886785 4597 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.0.2\\\"\"" pod="hosted-grafana/johangrafana10-grafana-69c6449bbd-k2bgp" podUID="bb953c26-c201-4082-9b56-85ab12c1d0e1" I0507 11:59:42.870523 4708 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-cache-n2hc8-1-1d-61155fd9-kqpq" status="Running" 2024-05-07T11:59:42.753652Z INFO TelemetryEventsCollector ExtHandler Collected 2 events for extension: Microsoft.Azure.Extensions.CustomScript audit: type=1400 audit(1715083182.707:151): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36655 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36655 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36655 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36655 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" I0507 11:59:42.683728 4736 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-78d9" status="Running" I0507 11:59:42.668601 4727 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-rr2n" status="Running" E0507 11:59:42.540931 4590 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-85282-20418-1\\\"\"" pod="hosted-grafana/ephemeral1511182185282svenner-grafana-6944cbdfcc-64z2p" podUID="1abeccba-cc20-47a4-b55c-fff4b7decbe1" I0507 11:59:42.473132 4761 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-1-main-n2s16-1-1dd-97837cc3-84qv" status="Running" I0507 11:59:42.461420 4733 kubelet_pods.go:906] "Unable to retrieve pull secret, the image pull may not succeed." pod="grafana-apps/query-grafana-app-fast-7d6dfcc787-t626q" secret="" err="secret \"dockerhub\" not found" I0507 11:59:42.452711 4610 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-perf-n2s8-0-1dd3-91689928-hl8m" status="Running" audit: type=1400 audit(1715083182.427:150): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36645 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083182.427:149): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36645 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083182.427:148): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36645 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36645 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36645 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36645 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" E0507 11:59:42.390135 4726 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"support-agent\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=support-agent pod=support-agent-557dff8b77-sx6hb_support-agent(f7b72dbb-4f3a-45b1-88c0-62337a3e8d3d)\"" pod="support-agent/support-agent-557dff8b77-sx6hb" podUID="f7b72dbb-4f3a-45b1-88c0-62337a3e8d3d" I0507 11:59:42.389722 4726 scope.go:117] "RemoveContainer" containerID="b35f99aea28d40fc317084351108a819285a8c62583dab13a9dc1a35e150715d" E0507 11:59:42.363245 4601 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=benchloadtestingxxl2-grafana-56b7c6b6df-l2ck2_hosted-grafana(aa7ee40a-4ed2-456b-86bc-a6b77d48fa82)\"" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-l2ck2" podUID="aa7ee40a-4ed2-456b-86bc-a6b77d48fa82" E0507 11:59:42.362999 4601 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.0.2\\\"\"" pod="hosted-grafana/johan6-grafana-85546bbbf5-xbkrj" podUID="a1ca81cd-1fd3-4f14-b6a5-a129930ba761" I0507 11:59:42.361599 4601 scope.go:117] "RemoveContainer" containerID="3c87e154309d0cdeea32be3c09f3cac0965efff7f1775be65bfef8fbc925782d" I0507 11:59:42.325378 4581 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-hg-n2s8-6-1dd39c-3bfd06e9-4hrn" status="Running" audit: type=1400 audit(1715083182.207:147): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36643 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083182.207:146): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36643 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083182.207:145): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36643 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36643 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36643 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36643 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" E0507 11:59:42.183133 4578 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=oncalldev-grafana-7b88d9459-fvqtx_hosted-grafana(fc7753d0-4067-4626-b539-5fd27ded163b)\"" pod="hosted-grafana/oncalldev-grafana-7b88d9459-fvqtx" podUID="fc7753d0-4067-4626-b539-5fd27ded163b" E0507 11:59:42.183013 4578 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=benchloadtestingxxl2-grafana-56b7c6b6df-5chwh_hosted-grafana(d349a15d-9276-457b-8e62-7d35f1bf81c0)\"" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-5chwh" podUID="d349a15d-9276-457b-8e62-7d35f1bf81c0" I0507 11:59:42.182460 4578 scope.go:117] "RemoveContainer" containerID="633c86551db397ab1ff61c06ae2334cfd3d76e9152cd24012569a2671774acbb" I0507 11:59:42.182336 4578 scope.go:117] "RemoveContainer" containerID="11b59ae6f9e2827841afd016742b48dc692f5756819d08f4cd123fba98fd732d" I0507 11:59:42.044224 4729 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-6vzp" status="Running" E0507 11:59:42.042844 4589 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=ephemeral1180076306267marefr-grafana-5d979884b7-lnsq2_hosted-grafana(78337132-8e81-47cc-8772-51f6e72e7927)\"" pod="hosted-grafana/ephemeral1180076306267marefr-grafana-5d979884b7-lnsq2" podUID="78337132-8e81-47cc-8772-51f6e72e7927" E0507 11:59:42.042747 4589 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=benchloadtestingxxl2-grafana-56b7c6b6df-nj27g_hosted-grafana(212d6baa-7068-4ad2-9617-f67f010e866d)\"" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-nj27g" podUID="212d6baa-7068-4ad2-9617-f67f010e866d" I0507 11:59:42.042088 4589 scope.go:117] "RemoveContainer" containerID="efb5462666d496e154e0477e0540b5325157c76f784e16834d1ab78c4fce2815" I0507 11:59:42.041815 4589 scope.go:117] "RemoveContainer" containerID="b22ba3d04cca58dd06227978cad02c4d11287b17e97aa2c9ed2a324204455fdf" I0507 11:59:41.978947 4736 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-lw2b" status="Running" E0507 11:59:41.965110 4731 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"overrides-exporter\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/kubernetes-dev/enterprise-logs:callum-shard-firstlast-08\\\"\"" pod="loki-dev-010/overrides-exporter-98c77fd66-6zj6m" podUID="1ff5bf3e-5856-4f6f-ae04-273f2dee170b" audit: type=1400 audit(1715083181.923:144): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36641 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083181.923:143): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36641 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083181.923:142): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36641 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" kauditd_printk_skb: 62 callbacks suppressed AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36641 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36641 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36641 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" I0507 11:59:41.844213 4732 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-vpgr" status="Running" I0507 11:59:41.844328 4639 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-spot-n2s8-0-1dd3-f81338c4-lnkb" status="Running" I0507 11:59:41.819406 4775 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-f2n6" status="Running" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36639 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36639 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36639 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" I0507 11:59:41.619645 6247 prober.go:107] "Probe failed" probeType="Readiness" pod="grafana-agent/grafana-agent-helm-4" podUID="c36c5200-1cd6-4093-893c-c022f91af996" containerName="grafana-agent" probeResult="failure" output="Get \"http://10.0.99.125:3090/-/ready\": dial tcp 10.0.99.125:3090: connect: connection refused" E0507 11:59:41.604606 4586 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6432-916-1\\\"\"" pod="hosted-grafana/ephemeral1180076306432stephan-grafana-696d787664-jftqh" podUID="41fba902-127b-4514-b1ca-ed431bc59a6c" E0507 11:59:41.604446 4586 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=dev05devuseast0test-grafana-6cb68b9788-v8dgd_hosted-grafana(59ef7574-134f-4888-826e-9a22062f29f8)\"" pod="hosted-grafana/dev05devuseast0test-grafana-6cb68b9788-v8dgd" podUID="59ef7574-134f-4888-826e-9a22062f29f8" I0507 11:59:41.603726 4586 scope.go:117] "RemoveContainer" containerID="c1992a17a0b5dc3d80080fcc1602d9481f2b4259ab708628828de7f34211f199" E0507 11:59:41.601058 4624 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=benchloadtestingxxl2-grafana-56b7c6b6df-8rbvv_hosted-grafana(55c96163-2915-466e-a1e7-38faf29e5f57)\"" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-8rbvv" podUID="55c96163-2915-466e-a1e7-38faf29e5f57" I0507 11:59:41.600163 4746 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-1-main-n2s16-1-1dd-97837cc3-sjzb" status="Running" I0507 11:59:41.599190 4624 scope.go:117] "RemoveContainer" containerID="db587d00d587cb056ec6251bcea2fb39b19a5862db5d637d9980c88ea624c88c" E0507 11:59:41.575311 4578 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 2m40s restarting failed container=grafana pod=k6testslow3-grafana-dcb88d9f9-h74pm_hosted-grafana(01d5760f-8e5c-41f2-ac50-49eda7752498)\"" pod="hosted-grafana/k6testslow3-grafana-dcb88d9f9-h74pm" podUID="01d5760f-8e5c-41f2-ac50-49eda7752498" I0507 11:59:41.574530 4578 scope.go:117] "RemoveContainer" containerID="bb168ccb972acd75ea8ada775d9453e1478cb408bdb2b1a9fdb660a1ae30a1a1" E0507 11:59:41.538570 4590 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=k6teststeady6-grafana-67b679bd8c-l7knf_hosted-grafana(c5975fd5-22d7-4efb-a6b6-3064876188c1)\"" pod="hosted-grafana/k6teststeady6-grafana-67b679bd8c-l7knf" podUID="c5975fd5-22d7-4efb-a6b6-3064876188c1" I0507 11:59:41.537978 4590 scope.go:117] "RemoveContainer" containerID="5b8aad8ab95e5f4201702424140d73f5cc582d6d48583a31ca0b0dabea27d806" E0507 11:59:41.484796 4588 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=benchloadtestingxxl2-grafana-56b7c6b6df-kkfm4_hosted-grafana(8d976e31-c3bb-489b-bb66-eb16be45f1e2)\"" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-kkfm4" podUID="8d976e31-c3bb-489b-bb66-eb16be45f1e2" I0507 11:59:41.484251 4588 scope.go:117] "RemoveContainer" containerID="b491b9e12e1ac8e16fd091f5dd23a3b892c443352b1a7f2952d49dd83c09b0d6" I0507 11:59:41.481102 4644 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-spot-n2s8-0-1dd3-f81338c4-kdh7" status="Running" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36637 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36637 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36637 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" E0507 11:59:41.375710 4736 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"ruler\" with CreateContainerConfigError: \"secret \\\"ruler-alertmanager-token\\\" not found\"" pod="ge-metrics-federation/gem-mimir-ruler-5f56f7846b-fgxdm" podUID="07c06e21-137b-4fdd-b7d3-703f0a567720" E0507 11:59:41.375655 4736 kuberuntime_manager.go:1256] container &Container{Name:ruler,Image:grafana/enterprise-metrics:v2.12.0,Command:[],Args:[-target=ruler -config.expand-env=true -config.file=/etc/mimir/mimir.yaml -distributor.remote-timeout=10s],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:http-metrics,HostPort:0,ContainerPort:8080,Protocol:TCP,HostIP:,},ContainerPort{Name:grpc,HostPort:0,ContainerPort:9095,Protocol:TCP,HostIP:,},ContainerPort{Name:memberlist,HostPort:0,ContainerPort:7946,Protocol:TCP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:JAEGER_AGENT_HOST,Value:alloy-otlp.alloy-otlp.svc.cluster.local.,ValueFrom:nil,},EnvVar{Name:JAEGER_TAGS,Value:namespace=ge-metrics-federation,cluster=dev-us-central-0,ValueFrom:nil,},EnvVar{Name:JAEGER_SAMPLER_MANAGER_HOST_PORT,Value:http://alloy-otlp.alloy-otlp.svc.cluster.local.:5778/sampling,ValueFrom:nil,},EnvVar{Name:GOOGLE_APPLICATION_CREDENTIALS,Value:/var/secrets/google/credentials.json,ValueFrom:nil,},EnvVar{Name:AM_TOKEN,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:ruler-alertmanager-token,},Key:token,Optional:nil,},},},EnvVar{Name:JAEGER_REPORTER_MAX_QUEUE_SIZE,Value:1000,ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{cpu: {{100 -3} {} 100m DecimalSI},memory: {{134217728 0} {} BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:gcs-credentials,ReadOnly:false,MountPath:/var/secrets/google/,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:config,ReadOnly:false,MountPath:/etc/mimir,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:license,ReadOnly:false,MountPath:/license,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:runtime-config,ReadOnly:false,MountPath:/var/mimir,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:storage,ReadOnly:false,MountPath:/data,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:active-queries,ReadOnly:false,MountPath:/active-query-tracker,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-jtnbs,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/ready,Port:{1 0 http-metrics},Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:45,TimeoutSeconds:1,PeriodSeconds:10,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:*true,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod gem-mimir-ruler-5f56f7846b-fgxdm_ge-metrics-federation(07c06e21-137b-4fdd-b7d3-703f0a567720): CreateContainerConfigError: secret "ruler-alertmanager-token" not found I0507 11:59:41.373523 4646 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-spot-n2s8-0-1dd3-f81338c4-m75z" status="Running" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36635 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36635 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36635 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" time="2024-05-07T11:59:41.183044974Z" level=info msg="RemoveContainer for \"8d94f2aa54fdb8f6e4ddfed0f7db95a8f719e0eeccbe440b54033c29c7138cca\" returns successfully" E0507 11:59:41.179412 2776 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=dafdeveuwest2-grafana-7845d969b5-f8h5q_hosted-grafana(14ac9939-b36a-40d7-9ca9-a0367aab99d8)\"" pod="hosted-grafana/dafdeveuwest2-grafana-7845d969b5-f8h5q" podUID="14ac9939-b36a-40d7-9ca9-a0367aab99d8" time="2024-05-07T11:59:41.179375449Z" level=info msg="RemoveContainer for \"8d94f2aa54fdb8f6e4ddfed0f7db95a8f719e0eeccbe440b54033c29c7138cca\"" I0507 11:59:41.178454 2776 scope.go:117] "RemoveContainer" containerID="eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef" I0507 11:59:41.178001 2776 scope.go:117] "RemoveContainer" containerID="8d94f2aa54fdb8f6e4ddfed0f7db95a8f719e0eeccbe440b54033c29c7138cca" I0507 11:59:41.177954 2776 kubelet.go:2421] "SyncLoop (PLEG): event for pod" pod="hosted-grafana/dafdeveuwest2-grafana-7845d969b5-f8h5q" event={"ID":"14ac9939-b36a-40d7-9ca9-a0367aab99d8","Type":"ContainerDied","Data":"eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef"} I0507 11:59:41.177905 2776 generic.go:334] "Generic (PLEG): container finished" podID="14ac9939-b36a-40d7-9ca9-a0367aab99d8" containerID="eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef" exitCode=1 E0507 11:59:41.152034 4572 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6439-933-1\\\"\"" pod="hosted-grafana/ephemeral1180076306439dafyddt-grafana-9769b9f5-g5qqf" podUID="e6633496-a926-4a28-8db8-6405d33cb4bc" E0507 11:59:41.151972 4572 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-enterprise-6372-855-1\\\"\"" pod="hosted-grafana/ephemeral1180076306372jacobso-grafana-7f66f49b8d-kzhxd" podUID="7ac84154-783b-4672-b865-f728da592129" E0507 11:59:41.042036 4589 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6432-916-1\\\"\"" pod="hosted-grafana/ephemeral1180076306432stephan-grafana-6486f498c4-79z9j" podUID="301e3aba-d59e-4699-9e89-41507660c707" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36619 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36619 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36619 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" E0507 11:59:40.886783 4597 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77487-8287-1\\\"\"" pod="hosted-grafana/ephemeral1511182177487torkelo-grafana-79dd77959f-2l2kd" podUID="4d3be4e9-d8c5-487f-a292-ecb699c3aaad" E0507 11:59:40.886262 4597 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-84322-18859-1\\\"\"" pod="hosted-grafana/ephemeral1511182184322yuritce-grafana-c6bc4dcc7-zffvh" podUID="fd6e5a63-416f-4ec0-9372-99da41bdffe5" I0507 11:59:40.813204 4731 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-7c5w" status="Running" E0507 11:59:40.787727 2776 prober.go:104] "Probe errored" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef not found: not found" probeType="Readiness" pod="hosted-grafana/dafdeveuwest2-grafana-7845d969b5-f8h5q" podUID="14ac9939-b36a-40d7-9ca9-a0367aab99d8" containerName="grafana" E0507 11:59:40.787679 2776 remote_runtime.go:496] "ExecSync cmd from runtime service failed" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef not found: not found" containerID="eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef" cmd=["/bin/hgrun","check"] time="2024-05-07T11:59:40.787470004Z" level=error msg="ExecSync for \"eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef\" failed" error="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef not found: not found" E0507 11:59:40.786721 2776 remote_runtime.go:496] "ExecSync cmd from runtime service failed" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef not found: not found" containerID="eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef" cmd=["/bin/hgrun","check"] time="2024-05-07T11:59:40.786428397Z" level=error msg="ExecSync for \"eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef\" failed" error="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef not found: not found" time="2024-05-07T11:59:40.785339490Z" level=error msg="ExecSync for \"eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef\" failed" error="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef not found: not found" E0507 11:59:40.785696 2776 remote_runtime.go:496] "ExecSync cmd from runtime service failed" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef not found: not found" containerID="eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef" cmd=["/bin/hgrun","check"] > I0507 11:59:40.784002 2776 prober.go:107] "Probe failed" probeType="Readiness" pod="hosted-grafana/dafdeveuwest2-grafana-7845d969b5-f8h5q" podUID="14ac9939-b36a-40d7-9ca9-a0367aab99d8" containerName="grafana" probeResult="failure" output=< ts=2024-05-07T11:59:40.579977004Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.452 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:40.165927901Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.452 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:40.095094601Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.452 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:39.968003625Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.452 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:39.859160404Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.452 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:39.73942929Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.452 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:39.597629189Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.452 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:39.513020473Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.452 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:39.430133771Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.452 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:39.357435956Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.452 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:39.309688652Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.452 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:39.237760243Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.452 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:39.214283344Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.452 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health time="2024-05-07T11:59:40.783954281Z" level=warning msg="cleaning up after shim disconnected" id=eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef namespace=k8s.io time="2024-05-07T11:59:40.783971781Z" level=info msg="cleaning up dead shim" namespace=k8s.io time="2024-05-07T11:59:40.783846880Z" level=info msg="shim disconnected" id=eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef namespace=k8s.io run-containerd-io.containerd.runtime.v2.task-k8s.io-eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef-rootfs.mount: Deactivated successfully. I0507 11:59:40.768911 4618 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-eu-west-3-main-n2s8-1-1dd39c-d1c92061-kvt4" status="Running" E0507 11:59:40.761832 3303 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"cortex-gw\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=cortex-gw pod=cortex-gw-78bc9b5ccc-8hkmp_faro(44b54226-b4bd-46e0-a3f0-257cb44d9ea8)\"" pod="faro/cortex-gw-78bc9b5ccc-8hkmp" podUID="44b54226-b4bd-46e0-a3f0-257cb44d9ea8" cri-containerd-eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef.scope: Deactivated successfully. I0507 11:59:40.761377 3303 scope.go:117] "RemoveContainer" containerID="9f3955a57aa496cb888a35102ef0ee777d6a75cdc12addbdafc2d9b3fb9cc080" E0507 11:59:40.722802 4732 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"support-agent\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=support-agent pod=support-agent-557dff8b77-c6f8b_support-agent(ede5a224-96fb-45d0-b452-1eb2de73cf19)\"" pod="support-agent/support-agent-557dff8b77-c6f8b" podUID="ede5a224-96fb-45d0-b452-1eb2de73cf19" I0507 11:59:40.722255 4732 scope.go:117] "RemoveContainer" containerID="e0a235a59cc57d2dbbcab276b25c7bb1bab9cecc37697779748125072457736f" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36617 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36617 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36617 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" I0507 11:59:40.582478 4586 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-hg-n2s8-6-1dd39c-3bfd06e9-tjd7" status="Running" I0507 11:59:40.574065 4647 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-spot-n2s8-0-1dd3-f81338c4-86xp" status="Running" E0507 11:59:40.570675 4595 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=k6testslow4-grafana-bc95d7c87-55jh7_hosted-grafana(69c825b9-ceee-41b6-861c-a9695a3b7771)\"" pod="hosted-grafana/k6testslow4-grafana-bc95d7c87-55jh7" podUID="69c825b9-ceee-41b6-861c-a9695a3b7771" I0507 11:59:40.569595 4595 scope.go:117] "RemoveContainer" containerID="d72ee3acf4ba575ccbe6544c8fcfef41f73a5120459124bb518dbd9f589891e7" I0507 11:59:40.566172 4647 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-spot-n2s8-0-1dd3-f81338c4-gmgr" status="Running" E0507 11:59:40.525566 4615 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1\\\"\"" pod="hosted-grafana/ephemeral1180076306436hairyhe-grafana-6fb647ccf6-n9f7m" podUID="37586611-dc03-4f8b-8bce-80dadde5d571" I0507 11:59:40.505981 4603 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-perf-n2s8-0-1dd3-91689928-mm2d" status="Running" I0507 11:59:40.480336 4777 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-dhh8" status="Running" I0507 11:59:40.447535 4726 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-1-main-n2s16-1-1dd-97837cc3-5k89" status="Running" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36599 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36599 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36599 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" E0507 11:59:40.363392 4601 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-80141-12952-1\\\"\"" pod="hosted-grafana/ephemeral1511182180141joshhun-grafana-7dbb6d4777-rzgds" podUID="c644591b-d4d7-4bef-913f-00b9f96539d4" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36597 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36597 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36597 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" E0507 11:59:40.183052 4578 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6433-919-1\\\"\"" pod="hosted-grafana/ephemeral1180076306433stephan-grafana-596f864fd8-wmfmg" podUID="49c2807d-900e-4029-804b-93c078a398f6" E0507 11:59:40.087432 4730 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"agent\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=agent pod=jaeger-agent-856f67c6d7-6xj9z_jaeger(1a240429-7c6f-4c4c-8c4e-d2579a6e737e)\"" pod="jaeger/jaeger-agent-856f67c6d7-6xj9z" podUID="1a240429-7c6f-4c4c-8c4e-d2579a6e737e" I0507 11:59:40.087072 4730 scope.go:117] "RemoveContainer" containerID="7518b1ab8a429bc205618da216992759f7c641bb6a7feea31daa6fc52170f1db" I0507 11:59:40.005475 4734 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-t2kf" status="Running" I0507 11:59:39.995966 4726 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-7h6b" status="Running" E0507 11:59:39.925282 4733 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"pdc\" with ErrImageNeverPull: \"Container image \\\"us.gcr.io/hosted-grafana/pdc:0.1.415\\\" is not present with pull policy of Never\"" pod="pdc/private-datasource-connect-564fb6cfbb-l8pgv" podUID="57e4a0cb-5e77-47bd-b277-70f4b1512c44" E0507 11:59:39.925221 4733 kuberuntime_manager.go:1256] container &Container{Name:pdc,Image:us.gcr.io/hosted-grafana/pdc:0.1.415,Command:[],Args:[-proxy.auth.ca-keys-dir=/var/run/secrets/pdc-certs -proxy.socks-server.addr=:10443 -proxy.ssh-server.addr=:2222 -proxy.use-socks-username-for-routing -proxy.api.http-address=:9182 -proxy.check-connpool-address-in-ring -memberlist.join=dns+gossip-ring.pdc.svc.cluster.local:7946 -api.http-address=:11443 -distributor.enabled=true -distributor.addr=:10444 -distributor.use-socks-username-for-routing -gateway.enabled=true -gateway.addr=:2244 -log.level=debug -certs.ca-private-key-file=/var/run/secrets/pdc-certs/ca.key -certs.ca-cert-file=/var/run/secrets/pdc-certs/ca.crt -certs.ca-pub-file=/var/run/secrets/pdc-certs/ca.pub -certs.cluster=local-k8s -shard-size=3 -graceful-shutdown-period=30s -enable-multiple-networks],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:socks,HostPort:0,ContainerPort:10443,Protocol:TCP,HostIP:,},ContainerPort{Name:ssh,HostPort:0,ContainerPort:2222,Protocol:TCP,HostIP:,},ContainerPort{Name:distributor,HostPort:0,ContainerPort:10444,Protocol:TCP,HostIP:,},ContainerPort{Name:gateway,HostPort:0,ContainerPort:2244,Protocol:TCP,HostIP:,},ContainerPort{Name:api,HostPort:0,ContainerPort:11443,Protocol:TCP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:POD_NAME,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.name,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},},Resources:ResourceRequirements{Limits:ResourceList{cpu: {{500 -3} {} 500m DecimalSI},memory: {{134217728 0} {} BinarySI},},Requests:ResourceList{cpu: {{250 -3} {} 250m DecimalSI},memory: {{67108864 0} {} BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:pdc-certs,ReadOnly:true,MountPath:/var/run/secrets/pdc-certs,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-wcbmb,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/ready,Port:{0 11443 },Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:40,TimeoutSeconds:1,PeriodSeconds:5,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:&Lifecycle{PostStart:nil,PreStop:&LifecycleHandler{Exec:&ExecAction{Command:[/bin/sleep 5],},HTTPGet:nil,TCPSocket:nil,},},TerminationMessagePath:/dev/termination-log,ImagePullPolicy:Never,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod private-datasource-connect-564fb6cfbb-l8pgv_pdc(57e4a0cb-5e77-47bd-b277-70f4b1512c44): ErrImageNeverPull: Container image "us.gcr.io/hosted-grafana/pdc:0.1.415" is not present with pull policy of Never AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36593 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36593 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36593 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" I0507 11:59:39.848842 4597 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-hg-n2s8-6-1dd39c-3bfd06e9-4jtn" status="Running" E0507 11:59:39.831408 4591 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.0.2\\\"\"" pod="hosted-grafana/grafana10-grafana-78d6685854-sx786" podUID="c6bea181-911d-41f1-9008-c07eb94d5d9d" E0507 11:59:39.706381 4602 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=k6testslow4-grafana-bc95d7c87-b596x_hosted-grafana(3eeb0d67-7e96-4a25-bde3-99dd64c0d7d1)\"" pod="hosted-grafana/k6testslow4-grafana-bc95d7c87-b596x" podUID="3eeb0d67-7e96-4a25-bde3-99dd64c0d7d1" I0507 11:59:39.705664 4602 scope.go:117] "RemoveContainer" containerID="f127301dfd17ffb654fd37d23c3b47994165737a57d5e1ccbc5169cec18da4d7" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36591 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36591 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36591 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" E0507 11:59:39.690250 4733 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"ruler\" with CreateContainerConfigError: \"secret \\\"ruler-alertmanager-token\\\" not found\"" pod="ge-metrics-federation/gem-mimir-ruler-8c54cd69f-27jpq" podUID="0a159d8c-5540-44c2-a592-f43db7a1aae6" E0507 11:59:39.690197 4733 kuberuntime_manager.go:1256] container &Container{Name:ruler,Image:grafana/enterprise-metrics:v2.11.1,Command:[],Args:[-target=ruler -config.expand-env=true -config.file=/etc/mimir/mimir.yaml],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:http-metrics,HostPort:0,ContainerPort:8080,Protocol:TCP,HostIP:,},ContainerPort{Name:grpc,HostPort:0,ContainerPort:9095,Protocol:TCP,HostIP:,},ContainerPort{Name:memberlist,HostPort:0,ContainerPort:7946,Protocol:TCP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:JAEGER_AGENT_HOST,Value:alloy-otlp.alloy-otlp.svc.cluster.local.,ValueFrom:nil,},EnvVar{Name:JAEGER_TAGS,Value:namespace=ge-metrics-federation,cluster=dev-us-central-0,ValueFrom:nil,},EnvVar{Name:JAEGER_SAMPLER_MANAGER_HOST_PORT,Value:http://alloy-otlp.alloy-otlp.svc.cluster.local.:5778/sampling,ValueFrom:nil,},EnvVar{Name:GOOGLE_APPLICATION_CREDENTIALS,Value:/var/secrets/google/credentials.json,ValueFrom:nil,},EnvVar{Name:AM_TOKEN,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:ruler-alertmanager-token,},Key:token,Optional:nil,},},},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{cpu: {{100 -3} {} 100m DecimalSI},memory: {{134217728 0} {} BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:gcs-credentials,ReadOnly:false,MountPath:/var/secrets/google/,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:config,ReadOnly:false,MountPath:/etc/mimir,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:license,ReadOnly:false,MountPath:/license,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:runtime-config,ReadOnly:false,MountPath:/var/mimir,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:storage,ReadOnly:false,MountPath:/data,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:active-queries,ReadOnly:false,MountPath:/active-query-tracker,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-qp8rd,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/ready,Port:{1 0 http-metrics},Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:45,TimeoutSeconds:1,PeriodSeconds:10,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:*true,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod gem-mimir-ruler-8c54cd69f-27jpq_ge-metrics-federation(0a159d8c-5540-44c2-a592-f43db7a1aae6): CreateContainerConfigError: secret "ruler-alertmanager-token" not found I0507 11:59:39.611540 4773 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-vf2b" status="Running" I0507 11:59:39.571623 4737 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-mwjl" status="Running" E0507 11:59:39.570301 4595 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=k6testslow1-grafana-54499558d4-fvfn2_hosted-grafana(d9130e10-120d-4288-9a2a-27a11c3d5fb5)\"" pod="hosted-grafana/k6testslow1-grafana-54499558d4-fvfn2" podUID="d9130e10-120d-4288-9a2a-27a11c3d5fb5" I0507 11:59:39.569666 4595 scope.go:117] "RemoveContainer" containerID="172037953598baa976c96c3b22a935cb88d55ed99211307c20c2f5a0ca7049ef" I0507 11:59:39.560605 4739 kubelet_pods.go:906] "Unable to retrieve pull secret, the image pull may not succeed." pod="logs-endpoint-dev-005/kafka-exporter-766c6757b5-bggf6" secret="" err="secret \"not-needed\" not found" I0507 11:59:39.431762 4589 kubelet_pods.go:906] "Unable to retrieve pull secret, the image pull may not succeed." pod="kafka/kafka-controller-2" secret="" err="secret \"gcr\" not found" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36589 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36589 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36589 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" I0507 11:59:39.397972 4737 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-cndv" status="Running" E0507 11:59:39.370058 4601 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6433-919-1\\\"\"" pod="hosted-grafana/ephemeral1180076306433stephan-grafana-6b47bdf747-5vlqc" podUID="1ad10dd5-1033-44c1-a8f0-3bd5f04f9396" E0507 11:59:39.370037 4601 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.0.2\\\"\"" pod="hosted-grafana/johangrafana10-grafana-6b6b6954cc-gst92" podUID="c9a01896-0ab8-496e-b51d-a9c542f87965" I0507 11:59:39.216314 4737 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-6bbf" status="Running" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36587 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36587 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36587 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" time="2024-05-07T11:59:39.180955622Z" level=info msg="RemoveContainer for \"011391a46c9e71c4dd0e38155b5439bf8dfd38d967f29995d1856e8688423c3f\" returns successfully" E0507 11:59:39.176592 4631 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=benchloadtestingxxl2-grafana-56b7c6b6df-7q7fd_hosted-grafana(5f7c88ef-55b9-4d72-8a2f-c9f2fa39f273)\"" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-7q7fd" podUID="5f7c88ef-55b9-4d72-8a2f-c9f2fa39f273" time="2024-05-07T11:59:39.175742342Z" level=info msg="RemoveContainer for \"011391a46c9e71c4dd0e38155b5439bf8dfd38d967f29995d1856e8688423c3f\"" I0507 11:59:39.171822 4631 scope.go:117] "RemoveContainer" containerID="6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408" I0507 11:59:39.171341 4631 scope.go:117] "RemoveContainer" containerID="011391a46c9e71c4dd0e38155b5439bf8dfd38d967f29995d1856e8688423c3f" I0507 11:59:39.171313 4631 kubelet.go:2426] "SyncLoop (PLEG): event for pod" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-7q7fd" event={"ID":"5f7c88ef-55b9-4d72-8a2f-c9f2fa39f273","Type":"ContainerDied","Data":"6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408"} I0507 11:59:39.171274 4631 generic.go:334] "Generic (PLEG): container finished" podID="5f7c88ef-55b9-4d72-8a2f-c9f2fa39f273" containerID="6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408" exitCode=1 I0507 11:59:39.168633 2776 kubelet.go:2493] "SyncLoop (probe)" probe="readiness" status="" pod="hosted-grafana/dafdeveuwest2-grafana-7845d969b5-f8h5q" I0507 11:59:39.168215 2776 kubelet.go:2421] "SyncLoop (PLEG): event for pod" pod="hosted-grafana/dafdeveuwest2-grafana-7845d969b5-f8h5q" event={"ID":"14ac9939-b36a-40d7-9ca9-a0367aab99d8","Type":"ContainerStarted","Data":"eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef"} E0507 11:59:39.152534 4572 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2\\\"\"" pod="hosted-grafana/ephemeral1511182177506ashharr-grafana-9446fd844-7nzkd" podUID="cc369230-6dd4-43de-8a99-b900823fc053" E0507 11:59:39.152408 4572 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=k6testslow5-grafana-6b4464c649-qgbsd_hosted-grafana(7576d413-e89a-47d5-84e3-697803ff3819)\"" pod="hosted-grafana/k6testslow5-grafana-6b4464c649-qgbsd" podUID="7576d413-e89a-47d5-84e3-697803ff3819" E0507 11:59:39.152231 4572 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=legacyalerting-grafana-767589fc8d-6nm2t_hosted-grafana(2ddc4db1-70a6-41fe-a272-b5986a846e3a)\"" pod="hosted-grafana/legacyalerting-grafana-767589fc8d-6nm2t" podUID="2ddc4db1-70a6-41fe-a272-b5986a846e3a" I0507 11:59:39.151757 4572 scope.go:117] "RemoveContainer" containerID="2e88c55b76390a7f9f0383f808e012173d5e761d2f50663f05e595d53a1605f9" I0507 11:59:39.151671 4572 scope.go:117] "RemoveContainer" containerID="321219eadc6cdc30f7a3e6d9c9adc1f5852d72f80bdc13cb1870d2ef24d6a8db" E0507 11:59:39.149450 4729 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"cluster-agent\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=cluster-agent pod=appdynamics-cluster-agent-appdynamics-cluster-agent-56667dmbnkv_integration(69bc5e6c-0451-443e-af8a-c831871afbb8)\"" pod="integration/appdynamics-cluster-agent-appdynamics-cluster-agent-56667dmbnkv" podUID="69bc5e6c-0451-443e-af8a-c831871afbb8" I0507 11:59:39.148673 4729 scope.go:117] "RemoveContainer" containerID="e887493c6e0eba98c77d48f6440bced72b79684e24aeef0a09a1cdebd9dbe85e" E0507 11:59:39.044392 4589 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=k6testslow2-grafana-6f8cdc574c-vkwn9_hosted-grafana(25bd0e02-eade-4034-93b0-1b212199e949)\"" pod="hosted-grafana/k6testslow2-grafana-6f8cdc574c-vkwn9" podUID="25bd0e02-eade-4034-93b0-1b212199e949" I0507 11:59:39.043426 4589 scope.go:117] "RemoveContainer" containerID="b50868147571f34e653fc07591105b455a08c4658b575711ffcd9749591a40be" I0507 11:59:38.994979 4768 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-9jbm" status="Running" I0507 11:59:38.950175 4732 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-462z" status="Running" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36576 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36576 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36576 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" I0507 11:59:38.801070 4592 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-hg-n2s8-6-1dd39c-3bfd06e9-lqg5" status="Running" E0507 11:59:38.778703 4734 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"agent\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=agent pod=jaeger-agent-856f67c6d7-tcsmd_jaeger(9121c1a3-6d79-4411-be8e-41406c88944a)\"" pod="jaeger/jaeger-agent-856f67c6d7-tcsmd" podUID="9121c1a3-6d79-4411-be8e-41406c88944a" I0507 11:59:38.778026 4734 scope.go:117] "RemoveContainer" containerID="11f6b0c150af531670af31444292720785e10c6fbe5a00b7ef85dd062ac22ffe" E0507 11:59:38.773774 4625 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=benchloadtestingxxl2-grafana-56b7c6b6df-kzlv4_hosted-grafana(db10d4d9-7c00-4757-9bca-fa2c735f6595)\"" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-kzlv4" podUID="db10d4d9-7c00-4757-9bca-fa2c735f6595" I0507 11:59:38.773181 4625 scope.go:117] "RemoveContainer" containerID="32f138fccb479a3484193f15220d1d6bef16a750071446fd9a6b1288c31d8d83" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68547 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68547 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68547 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083178.748:507): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68546 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083178.748:506): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68545 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083178.748:505): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68545 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083178.748:504): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68545 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083178.748:503): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68543 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083178.748:502): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68543 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083178.748:501): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68543 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68546 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68546 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68546 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68545 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68545 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68545 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68543 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68543 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68543 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083178.744:500): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68542 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083178.744:499): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68542 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083178.744:498): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68542 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" kauditd_printk_skb: 5 callbacks suppressed time="2024-05-07T11:59:38.748149797Z" level=info msg="StartContainer for \"eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef\" returns successfully" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68542 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68542 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=68542 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" I0507 11:59:38.738752 4777 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-z6f6" status="Running" I0507 11:59:38.725225 4744 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-1-main-n2s16-1-1dd-97837cc3-xhp4" status="Running" Started libcontainer container eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef. E0507 11:59:38.704674 4602 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-80141-12952-1\\\"\"" pod="hosted-grafana/ephemeral1511182180141joshhun-grafana-744f5cfd67-6w9jb" podUID="1e2a9789-01a5-4ab0-97d4-06dee391f43f" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36574 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36574 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36574 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" time="2024-05-07T11:59:38.694650245Z" level=info msg="StartContainer for \"eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef\"" var-lib-containerd-tmpmounts-containerd\x2dmount777551232.mount: Deactivated successfully. time="2024-05-07T11:59:38.693858938Z" level=info msg="CreateContainer within sandbox \"10f321d50b51389b7845c1c5393aff42e209a45fcee6fec6d74c8b60dd5266ad\" for &ContainerMetadata{Name:grafana,Attempt:7,} returns container id \"eeccb21da13bfae40b1a01984522c7a8f8dcb65dba3cc1cc2f2ba73381d9eaef\"" time="2024-05-07T11:59:38.671307647Z" level=info msg="CreateContainer within sandbox \"10f321d50b51389b7845c1c5393aff42e209a45fcee6fec6d74c8b60dd5266ad\" for container &ContainerMetadata{Name:grafana,Attempt:7,}" I0507 11:59:38.667270 2776 scope.go:117] "RemoveContainer" containerID="8d94f2aa54fdb8f6e4ddfed0f7db95a8f719e0eeccbe440b54033c29c7138cca" I0507 11:59:38.638688 4776 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-qmgr" status="Running" I0507 11:59:38.539722 4771 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-7lvz" status="Running" E0507 11:59:38.539381 4590 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6433-919-1\\\"\"" pod="hosted-grafana/ephemeral1180076306433stephan-grafana-74cf7bb5b7-qmfgj" podUID="0b654534-e87f-4660-a634-3aa23906f24d" I0507 11:59:38.497645 4735 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-trgv" status="Running" E0507 11:59:38.489629 4631 prober.go:104] "Probe errored" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task 6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408 not found: not found" probeType="Readiness" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-7q7fd" podUID="5f7c88ef-55b9-4d72-8a2f-c9f2fa39f273" containerName="grafana" E0507 11:59:38.489587 4631 remote_runtime.go:496] "ExecSync cmd from runtime service failed" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task 6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408 not found: not found" containerID="6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408" cmd=["/bin/hgrun","check"] time="2024-05-07T11:59:38.489392061Z" level=error msg="ExecSync for \"6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408\" failed" error="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task 6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408 not found: not found" E0507 11:59:38.488880 4631 remote_runtime.go:496] "ExecSync cmd from runtime service failed" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task 6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408 not found: not found" containerID="6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408" cmd=["/bin/hgrun","check"] time="2024-05-07T11:59:38.488693040Z" level=error msg="ExecSync for \"6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408\" failed" error="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task 6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408 not found: not found" E0507 11:59:38.488155 4631 remote_runtime.go:496] "ExecSync cmd from runtime service failed" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task 6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408 not found: not found" containerID="6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408" cmd=["/bin/hgrun","check"] time="2024-05-07T11:59:38.487998369Z" level=error msg="ExecSync for \"6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408\" failed" error="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task 6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408 not found: not found" E0507 11:59:38.487509 4631 prober.go:104] "Probe errored" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task 6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408 not found: not found" probeType="Readiness" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-7q7fd" podUID="5f7c88ef-55b9-4d72-8a2f-c9f2fa39f273" containerName="grafana" E0507 11:59:38.487460 4631 remote_runtime.go:496] "ExecSync cmd from runtime service failed" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task 6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408 not found: not found" containerID="6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408" cmd=["/bin/hgrun","check"] time="2024-05-07T11:59:38.487245276Z" level=error msg="ExecSync for \"6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408\" failed" error="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task 6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408 not found: not found" E0507 11:59:38.486679 4631 remote_runtime.go:496] "ExecSync cmd from runtime service failed" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task 6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408 not found: not found" containerID="6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408" cmd=["/bin/hgrun","check"] time="2024-05-07T11:59:38.486488188Z" level=error msg="ExecSync for \"6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408\" failed" error="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task 6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408 not found: not found" time="2024-05-07T11:59:38.485721949Z" level=error msg="ExecSync for \"6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408\" failed" error="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task 6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408 not found: not found" > E0507 11:59:38.485960 4631 remote_runtime.go:496] "ExecSync cmd from runtime service failed" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task 6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408 not found: not found" containerID="6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408" cmd=["/bin/hgrun","check"] ts=2024-05-07T11:59:32.025687537Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request ts=2024-05-07T11:59:31.47288066Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:30.25205688Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:29.489758832Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:28.89372222Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:27.549765307Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:27.111040682Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:25.771680727Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:25.398875396Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:24.719782935Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:24.258151357Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:23.238736396Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:22.763876203Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:22.337877023Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:21.976968587Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:21.412826453Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:20.542485218Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health I0507 11:59:38.485111 4631 prober.go:107] "Probe failed" probeType="Readiness" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-7q7fd" podUID="5f7c88ef-55b9-4d72-8a2f-c9f2fa39f273" containerName="grafana" probeResult="failure" output=< E0507 11:59:38.485042 4631 prober.go:239] "Unable to write all bytes from execInContainer" err="short write" expectedBytes=11555 actualBytes=10240 ts=2024-05-07T11:59:20.133570834Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:19.968957316Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:19.615197163Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:19.34348473Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:18.888685553Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:18.342780761Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:17.847204341Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:17.651565636Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:17.380926872Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:17.139181882Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:16.971537625Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:16.843946945Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:16.798396868Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:16.588703925Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:16.524179416Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:16.399564514Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:16.274970494Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:16.185392265Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:16.167132483Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health time="2024-05-07T11:59:38.484586527Z" level=error msg="Failed to delete exec process \"d9e0a1867ce73695ad859f2b0a76fe8f5053db8a5e49142d747e53a445729bd4\" for container \"6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408\"" error="ttrpc: closed: unknown" time="2024-05-07T11:59:38.484606284Z" level=warning msg="cleaning up after shim disconnected" id=6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408 namespace=k8s.io time="2024-05-07T11:59:38.484616161Z" level=info msg="cleaning up dead shim" namespace=k8s.io time="2024-05-07T11:59:38.484530585Z" level=info msg="shim disconnected" id=6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408 namespace=k8s.io AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36564 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36564 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36564 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" I0507 11:59:38.411649 4775 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-t94m" status="Running" time="2024-05-07T11:59:38.385332298Z" level=info msg="RemoveContainer for \"f4fe841b1c27d602f09fc97c821d9cbc9654349f07afc532685d1a6a2ed887ed\" returns successfully" E0507 11:59:38.377710 4589 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=k6teststeady3-grafana-659d5ff58d-h4lmj_hosted-grafana(85274c17-190e-4275-a8f3-6e111cd833bf)\"" pod="hosted-grafana/k6teststeady3-grafana-659d5ff58d-h4lmj" podUID="85274c17-190e-4275-a8f3-6e111cd833bf" time="2024-05-07T11:59:38.377259687Z" level=info msg="RemoveContainer for \"f4fe841b1c27d602f09fc97c821d9cbc9654349f07afc532685d1a6a2ed887ed\"" I0507 11:59:38.376714 4589 scope.go:117] "RemoveContainer" containerID="fc7a558bca122d6b5fb9aa81e62a87053c8a6a84945fd7a5fd4508d7cbc0878a" I0507 11:59:38.375936 4589 scope.go:117] "RemoveContainer" containerID="f4fe841b1c27d602f09fc97c821d9cbc9654349f07afc532685d1a6a2ed887ed" I0507 11:59:38.375903 4589 kubelet.go:2426] "SyncLoop (PLEG): event for pod" pod="hosted-grafana/k6teststeady3-grafana-659d5ff58d-h4lmj" event={"ID":"85274c17-190e-4275-a8f3-6e111cd833bf","Type":"ContainerDied","Data":"fc7a558bca122d6b5fb9aa81e62a87053c8a6a84945fd7a5fd4508d7cbc0878a"} I0507 11:59:38.375858 4589 generic.go:334] "Generic (PLEG): container finished" podID="85274c17-190e-4275-a8f3-6e111cd833bf" containerID="fc7a558bca122d6b5fb9aa81e62a87053c8a6a84945fd7a5fd4508d7cbc0878a" exitCode=1 I0507 11:59:38.242879 4603 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-perf-n2s8-0-1dd3-91689928-9f87" status="Running" I0507 11:59:38.197379 2791 kubelet.go:2421] "SyncLoop (PLEG): event for pod" pod="hosted-grafana/dafdeveuwest2-grafana-546fbd789d-czx47" event={"ID":"fc6ba4ea-9950-4999-8ad2-bdc9a577fb34","Type":"ContainerStarted","Data":"4346b5c7d89f0d5b24a4b190740ed7327d12365cbd86dd55f845a8416cb1824e"} AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36561 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36561 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36561 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" I0507 11:59:38.195145 2791 kubelet.go:2421] "SyncLoop (PLEG): event for pod" pod="hosted-grafana/victor-grafana-7b7bb568cc-grflq" event={"ID":"1803645b-5526-41b4-bf88-271be4827277","Type":"ContainerStarted","Data":"a63e882bf89caa0e0f3027fc2c068801b36e5b3386b4d4f2570cb4f4d77298f4"} AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36559 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36559 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36559 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36558 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36558 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36558 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" E0507 11:59:38.183090 4578 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=grafanawargame1-grafana-5cb77d49cd-kxj2w_hosted-grafana(ce1d40ca-a97a-4e10-b362-dae978b1723f)\"" pod="hosted-grafana/grafanawargame1-grafana-5cb77d49cd-kxj2w" podUID="ce1d40ca-a97a-4e10-b362-dae978b1723f" I0507 11:59:38.182549 4578 scope.go:117] "RemoveContainer" containerID="a30ce77f9f4670e75bd7ceb718e65c4a191d953fdfecc0e5c25086f212bd0a33" time="2024-05-07T11:59:38.178929893Z" level=info msg="StartContainer for \"a63e882bf89caa0e0f3027fc2c068801b36e5b3386b4d4f2570cb4f4d77298f4\" returns successfully" Started libcontainer container a63e882bf89caa0e0f3027fc2c068801b36e5b3386b4d4f2570cb4f4d77298f4. E0507 11:59:38.152209 4572 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=k6testslow1-grafana-5bf579b6cc-cqp4w_hosted-grafana(e56accb0-e2cc-41ba-b7c8-91d0ab3c1b8d)\"" pod="hosted-grafana/k6testslow1-grafana-5bf579b6cc-cqp4w" podUID="e56accb0-e2cc-41ba-b7c8-91d0ab3c1b8d" I0507 11:59:38.151368 4572 scope.go:117] "RemoveContainer" containerID="0e8924bd8e1c9018a410b5b69f9ad729c09b60fa5e4257f17783ab7bbb5f949a" I0507 11:59:38.131862 4738 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-j94k" status="Running" time="2024-05-07T11:59:38.129348185Z" level=info msg="StartContainer for \"a63e882bf89caa0e0f3027fc2c068801b36e5b3386b4d4f2570cb4f4d77298f4\"" time="2024-05-07T11:59:38.128540975Z" level=info msg="CreateContainer within sandbox \"346dc5e0c503a6ca1f0281e3b6f8e32563ebf8a61ed467b4d0dc5b4030a115b9\" for &ContainerMetadata{Name:hgrun,Attempt:0,} returns container id \"a63e882bf89caa0e0f3027fc2c068801b36e5b3386b4d4f2570cb4f4d77298f4\"" time="2024-05-07T11:59:38.118600653Z" level=info msg="CreateContainer within sandbox \"346dc5e0c503a6ca1f0281e3b6f8e32563ebf8a61ed467b4d0dc5b4030a115b9\" for container &ContainerMetadata{Name:hgrun,Attempt:0,}" time="2024-05-07T11:59:38.117772842Z" level=info msg="PullImage \"us.gcr.io/hosted-grafana/hg-plugins:2024-05-07-v545244-f51851984\"" I0507 11:59:38.116658 2791 azure_credentials.go:220] image(us.gcr.io/hosted-grafana/hg-plugins) is not from ACR, return empty authentication time="2024-05-07T11:59:38.116099322Z" level=info msg="PullImage \"us.gcr.io/hosted-grafana/hgrun:0.1.452\" returns image reference \"sha256:9fb1bce3e4a228f50768d21842cd7d7fafc1d586eaa0326c9d3c86d79a36868a\"" time="2024-05-07T11:59:38.116062821Z" level=info msg="Pulled image \"us.gcr.io/hosted-grafana/hgrun:0.1.452\" with image id \"sha256:9fb1bce3e4a228f50768d21842cd7d7fafc1d586eaa0326c9d3c86d79a36868a\", repo tag \"us.gcr.io/hosted-grafana/hgrun:0.1.452\", repo digest \"us.gcr.io/hosted-grafana/hgrun@sha256:b492dbbbee9faf9dba63c9fd89e6f9e148239765454c6a54c4284a2828dec153\", size \"19109699\" in 781.210092ms" time="2024-05-07T11:59:38.115073809Z" level=info msg="ImageUpdate event name:\"us.gcr.io/hosted-grafana/hgrun@sha256:b492dbbbee9faf9dba63c9fd89e6f9e148239765454c6a54c4284a2828dec153\" labels:{key:\"io.cri-containerd.image\" value:\"managed\"}" time="2024-05-07T11:59:38.113482790Z" level=info msg="ImageUpdate event name:\"us.gcr.io/hosted-grafana/hgrun:0.1.452\" labels:{key:\"io.cri-containerd.image\" value:\"managed\"}" time="2024-05-07T11:59:38.111878670Z" level=info msg="ImageUpdate event name:\"sha256:9fb1bce3e4a228f50768d21842cd7d7fafc1d586eaa0326c9d3c86d79a36868a\" labels:{key:\"io.cri-containerd.image\" value:\"managed\"}" time="2024-05-07T11:59:38.111105661Z" level=info msg="stop pulling image us.gcr.io/hosted-grafana/hgrun:0.1.452: active requests=0, bytes read=6766" time="2024-05-07T11:59:38.110778357Z" level=info msg="ImageUpdate event name:\"us.gcr.io/hosted-grafana/hgrun:0.1.452\" labels:{key:\"io.cri-containerd.image\" value:\"managed\"}" I0507 11:59:38.092172 4527 kubelet.go:2426] "SyncLoop (PLEG): event for pod" pod="otel-demo/otel-demo-dev-checkoutservice-6ddf9b978b-zqrsr" event={"ID":"f263b787-926e-459a-95a0-f9ef8e4e9bc2","Type":"ContainerStarted","Data":"95bf586cd79d43120ff44582d4dbd2476de61744411f8515b9b2c527a41fd5d9"} E0507 11:59:38.083847 4589 prober.go:104] "Probe errored" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task fc7a558bca122d6b5fb9aa81e62a87053c8a6a84945fd7a5fd4508d7cbc0878a not found: not found" probeType="Readiness" pod="hosted-grafana/k6teststeady3-grafana-659d5ff58d-h4lmj" podUID="85274c17-190e-4275-a8f3-6e111cd833bf" containerName="grafana" E0507 11:59:38.083788 4589 remote_runtime.go:496] "ExecSync cmd from runtime service failed" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task fc7a558bca122d6b5fb9aa81e62a87053c8a6a84945fd7a5fd4508d7cbc0878a not found: not found" containerID="fc7a558bca122d6b5fb9aa81e62a87053c8a6a84945fd7a5fd4508d7cbc0878a" cmd=["/bin/hgrun","check"] time="2024-05-07T11:59:38.083577708Z" level=error msg="ExecSync for \"fc7a558bca122d6b5fb9aa81e62a87053c8a6a84945fd7a5fd4508d7cbc0878a\" failed" error="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task fc7a558bca122d6b5fb9aa81e62a87053c8a6a84945fd7a5fd4508d7cbc0878a not found: not found" E0507 11:59:38.083085 4589 remote_runtime.go:496] "ExecSync cmd from runtime service failed" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task fc7a558bca122d6b5fb9aa81e62a87053c8a6a84945fd7a5fd4508d7cbc0878a not found: not found" containerID="fc7a558bca122d6b5fb9aa81e62a87053c8a6a84945fd7a5fd4508d7cbc0878a" cmd=["/bin/hgrun","check"] time="2024-05-07T11:59:38.082843516Z" level=error msg="ExecSync for \"fc7a558bca122d6b5fb9aa81e62a87053c8a6a84945fd7a5fd4508d7cbc0878a\" failed" error="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task fc7a558bca122d6b5fb9aa81e62a87053c8a6a84945fd7a5fd4508d7cbc0878a not found: not found" time="2024-05-07T11:59:38.081886661Z" level=error msg="ExecSync for \"fc7a558bca122d6b5fb9aa81e62a87053c8a6a84945fd7a5fd4508d7cbc0878a\" failed" error="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task fc7a558bca122d6b5fb9aa81e62a87053c8a6a84945fd7a5fd4508d7cbc0878a not found: not found" > E0507 11:59:38.082179 4589 remote_runtime.go:496] "ExecSync cmd from runtime service failed" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task fc7a558bca122d6b5fb9aa81e62a87053c8a6a84945fd7a5fd4508d7cbc0878a not found: not found" containerID="fc7a558bca122d6b5fb9aa81e62a87053c8a6a84945fd7a5fd4508d7cbc0878a" cmd=["/bin/hgrun","check"] ts=2024-05-07T11:59:37.257054006Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:36.616408464Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:35.943353082Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:35.651524459Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:34.604654159Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:33.75455853Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health I0507 11:59:38.081101 4589 prober.go:107] "Probe failed" probeType="Readiness" pod="hosted-grafana/k6teststeady3-grafana-659d5ff58d-h4lmj" podUID="85274c17-190e-4275-a8f3-6e111cd833bf" containerName="grafana" probeResult="failure" output=< ts=2024-05-07T11:59:32.904157283Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:32.40247514Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:32.084670264Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:31.541769769Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:31.195900943Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:30.90418468Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:30.441210172Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:30.279567335Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:30.151869927Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:29.762204853Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:29.416349977Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:29.317686715Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:29.076543925Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:28.879830344Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:28.760910211Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:28.624184207Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:28.464985015Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:28.414503386Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:28.338847511Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:28.318295189Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-62-g2605e8595 msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health time="2024-05-07T11:59:38.080784392Z" level=warning msg="cleaning up after shim disconnected" id=fc7a558bca122d6b5fb9aa81e62a87053c8a6a84945fd7a5fd4508d7cbc0878a namespace=k8s.io time="2024-05-07T11:59:38.080799009Z" level=info msg="cleaning up dead shim" namespace=k8s.io time="2024-05-07T11:59:38.080699151Z" level=info msg="shim disconnected" id=fc7a558bca122d6b5fb9aa81e62a87053c8a6a84945fd7a5fd4508d7cbc0878a namespace=k8s.io time="2024-05-07T11:59:38.080668278Z" level=info msg="StartContainer for \"95bf586cd79d43120ff44582d4dbd2476de61744411f8515b9b2c527a41fd5d9\" returns successfully" I0507 11:59:37.961573 4624 kubelet.go:2498] "SyncLoop (probe)" probe="readiness" status="ready" pod="hosted-grafana/ephemeral1511182180263lucyche-grafana-575d4b9448-r7sp7" E0507 11:59:37.960437 4624 prober.go:239] "Unable to write all bytes from execInContainer" err="short write" expectedBytes=11843 actualBytes=10240 I0507 11:59:37.915108 4726 prober.go:107] "Probe failed" probeType="Readiness" pod="agent-management-dev-002/agent-management-api-7ff7b9b9-k9nft" podUID="9893f9ac-f3e4-41fb-8da7-592061d2386c" containerName="agent-management-api" probeResult="failure" output="HTTP probe failed with statuscode: 400" I0507 11:59:37.913494 4727 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-h9bx" status="Running" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36499 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36499 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36499 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" run-containerd-io.containerd.runtime.v2.task-k8s.io-fc7a558bca122d6b5fb9aa81e62a87053c8a6a84945fd7a5fd4508d7cbc0878a-rootfs.mount: Deactivated successfully. cri-containerd-fc7a558bca122d6b5fb9aa81e62a87053c8a6a84945fd7a5fd4508d7cbc0878a.scope: Consumed 16.023s CPU time. cri-containerd-fc7a558bca122d6b5fb9aa81e62a87053c8a6a84945fd7a5fd4508d7cbc0878a.scope: Deactivated successfully. I0507 11:59:37.726128 3089 kubelet_pods.go:906] "Unable to retrieve pull secret, the image pull may not succeed." pod="kafka/kafka-broker-1" secret="" err="secret \"gcr\" not found" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36497 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36497 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36497 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" E0507 11:59:37.667321 2776 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=victor-grafana-5f4c7cbdf7-bwfdl_hosted-grafana(525f5872-d605-4bd1-93b5-2ac890004c88)\"" pod="hosted-grafana/victor-grafana-5f4c7cbdf7-bwfdl" podUID="525f5872-d605-4bd1-93b5-2ac890004c88" I0507 11:59:37.666642 2776 scope.go:117] "RemoveContainer" containerID="34da1ae22805bba50bdf08001da7492e265427af617701a829edfe1ca14fc152" I0507 11:59:37.628819 4775 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-xzrv" status="Running" E0507 11:59:37.604573 4586 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1\\\"\"" pod="hosted-grafana/ephemeral1180076306436hairyhe-grafana-58c766bdfb-87bmx" podUID="93917033-fece-4ffc-b04e-5dbdfc534657" E0507 11:59:37.574330 4578 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=benchloadtestingxxl2-grafana-56b7c6b6df-4kt2b_hosted-grafana(4fcb02de-77ce-4324-b879-5ad84d1ba5a4)\"" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-4kt2b" podUID="4fcb02de-77ce-4324-b879-5ad84d1ba5a4" I0507 11:59:37.573819 4578 scope.go:117] "RemoveContainer" containerID="7ecf9a69091f8fbee2555f938ad0737da2887c0b0188bbcea8c5736e8fa628ff" E0507 11:59:37.570564 4595 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=pyroscopecanaries-grafana-5fb895ff5c-qg8fq_hosted-grafana(6c11e383-592a-4250-9bea-ac8003a507b9)\"" pod="hosted-grafana/pyroscopecanaries-grafana-5fb895ff5c-qg8fq" podUID="6c11e383-592a-4250-9bea-ac8003a507b9" E0507 11:59:37.570104 4595 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=benchloadtestingxxl2-grafana-759c97b668-s8bxs_hosted-grafana(bfa4c7a9-f32e-4075-b044-efc1342b6234)\"" pod="hosted-grafana/benchloadtestingxxl2-grafana-759c97b668-s8bxs" podUID="bfa4c7a9-f32e-4075-b044-efc1342b6234" I0507 11:59:37.569718 4595 scope.go:117] "RemoveContainer" containerID="2bcb4df11d62f3277600ff9d5680d84674ea998b36e762750e2fa214f96ec43a" I0507 11:59:37.569430 4595 scope.go:117] "RemoveContainer" containerID="c4b8dcfc6d5278f921fd4d84b7cb58190be6f914af5f93fb9f306bd4714d7664" E0507 11:59:37.526480 4615 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=dev05devuseast0test-grafana-74755ff84c-8glnl_hosted-grafana(fa354582-d6f8-49fe-b75f-7b27131dc41f)\"" pod="hosted-grafana/dev05devuseast0test-grafana-74755ff84c-8glnl" podUID="fa354582-d6f8-49fe-b75f-7b27131dc41f" I0507 11:59:37.525021 4615 scope.go:117] "RemoveContainer" containerID="37e8f7a22bb264a186c54dd50c0abbdf94f56c024a357e26a462160835aa224e" E0507 11:59:37.414253 4589 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ErrImagePull: \"[rpc error: code = NotFound desc = failed to pull and unpack image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2\\\": failed to resolve reference \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2\\\": us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2: not found, failed to pull and unpack image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2\\\": failed to resolve reference \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2\\\": unexpected status from HEAD request to https://us.gcr.io/v2/hosted-grafana/hosted-grafana-pro/manifests/10.1.0-ephemeral-oss-77506-8314-2: 403 Forbidden]\"" pod="hosted-grafana/ephemeral1511182177506ashharr-grafana-7666b574d5-22xlb" podUID="dd9f7420-7d07-4def-b1b1-5f1d0a8a7ca8" while [ "$(pidof plugins-pause)" = "" ]; do sleep 0.5; done; ln --force -s /proc/$(pidof hgrun-pause)/root/bin/hgrun /bin/hgrun; E0507 11:59:37.414181 4589 kuberuntime_manager.go:1256] container &Container{Name:grafana,Image:us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2,Command:[/bin/sh],Args:[-c set -e; while [ "$(pidof hgrun-pause)" = "" ]; do sleep 0.5; done; exec /bin/hgrun -log.level=debug launch -bundledPluginsManifest /proc/$(pidof plugins-pause)/root/manifest.json -bundledPluginsDir /proc/$(pidof plugins-pause)/root/plugins],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:http-metrics,HostPort:0,ContainerPort:80,Protocol:TCP,HostIP:,},ContainerPort{Name:grpc,HostPort:0,ContainerPort:10000,Protocol:TCP,HostIP:,},ContainerPort{Name:profiling,HostPort:0,ContainerPort:6060,Protocol:TCP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:HG_API,Value:http://hosted-grafana-api,ValueFrom:nil,},EnvVar{Name:HG_INSTANCE_SLUG,Value:ephemeral1511182177506ashharr,ValueFrom:nil,},EnvVar{Name:HG_INSTANCE_SECRET,Value:dea83588a727490d3795d736e53d48bbee310ae2,ValueFrom:nil,},EnvVar{Name:EXTRA_OPTIONS,Value:-profile -profile-port=6060 -profile-addr=0.0.0.0,ValueFrom:nil,},EnvVar{Name:HG_CREATE_TIME_MS,Value:1715081284620,ValueFrom:nil,},EnvVar{Name:HG_PULL_POLICY,Value:Always,ValueFrom:nil,},EnvVar{Name:HG_START_REASON,Value:active,ValueFrom:nil,},EnvVar{Name:HGRUN_SECURE_PLUGINS,Value:false,ValueFrom:nil,},EnvVar{Name:HGRUN_PLUGIN_RUNNER_ROOT_CA,Value:false,ValueFrom:nil,},EnvVar{Name:OTEL_EXPORTER_OTLP_TRACES_ENDPOINT,Value:http://jaeger-agent.jaeger.svc.cluster.local:4317,ValueFrom:nil,},EnvVar{Name:JAEGER_SAMPLER_PARAM,Value:1,ValueFrom:nil,},EnvVar{Name:OTEL_RESOURCE_ATTRIBUTES,Value:cluster=dev-us-central-0,namespace=hosted-grafana,ValueFrom:nil,},EnvVar{Name:HG_PROBE_PATH,Value:/api/health,ValueFrom:nil,},EnvVar{Name:HGRUN_EXIT_ON_PLUGIN_FAIL,Value:true,ValueFrom:nil,},EnvVar{Name:HGRUN_PLUGIN_INSTALL_RETRIES,Value:2,ValueFrom:nil,},EnvVar{Name:HGRUN_PLUGIN_INSTALL_CONCURRENCY,Value:1,ValueFrom:nil,},EnvVar{Name:HGRUN_LAUNCH_TIMEOUT,Value:3m0s,ValueFrom:nil,},EnvVar{Name:GOMEMLIMIT,Value:429496730,ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{memory: {{536870912 0} {} BinarySI},},Requests:ResourceList{cpu: {{26 -3} {} 26m DecimalSI},memory: {{293601280 0} {} BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{},LivenessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/api/health,Port:{0 80 },Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:300,TimeoutSeconds:10,PeriodSeconds:30,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},ReadinessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:&ExecAction{Command:[/bin/hgrun check],},HTTPGet:nil,TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:0,TimeoutSeconds:30,PeriodSeconds:30,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:&Lifecycle{PostStart:nil,PreStop:&LifecycleHandler{Exec:&ExecAction{Command:[/bin/hgrun drain -timeout 1m0s -waitTime 55s],},HTTPGet:nil,TCPSocket:nil,},},TerminationMessagePath:/dev/termination-log,ImagePullPolicy:Always,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[SYS_PTRACE],Drop:[],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:nil,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod ephemeral1511182177506ashharr-grafana-7666b574d5-22xlb_hosted-grafana(dd9f7420-7d07-4def-b1b1-5f1d0a8a7ca8): ErrImagePull: [rpc error: code = NotFound desc = failed to pull and unpack image "us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2": failed to resolve reference "us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2": us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2: not found, failed to pull and unpack image "us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2": failed to resolve reference "us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2": unexpected status from HEAD request to https://us.gcr.io/v2/hosted-grafana/hosted-grafana-pro/manifests/10.1.0-ephemeral-oss-77506-8314-2: 403 Forbidden] E0507 11:59:37.413937 4589 remote_image.go:180] "PullImage from image service failed" err="rpc error: code = Unknown desc = failed to pull and unpack image \"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2\": failed to resolve reference \"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2\": unexpected status from HEAD request to https://us.gcr.io/v2/hosted-grafana/hosted-grafana-pro/manifests/10.1.0-ephemeral-oss-77506-8314-2: 403 Forbidden" image="us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2" time="2024-05-07T11:59:37.413721567Z" level=info msg="stop pulling image us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2: active requests=0, bytes read=4401" time="2024-05-07T11:59:37.413617100Z" level=error msg="PullImage \"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2\" failed" error="failed to pull and unpack image \"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2\": failed to resolve reference \"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2\": unexpected status from HEAD request to https://us.gcr.io/v2/hosted-grafana/hosted-grafana-pro/manifests/10.1.0-ephemeral-oss-77506-8314-2: 403 Forbidden" audit: type=1400 audit(1715083177.403:79): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36490 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36492 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36492 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36492 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36490 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36490 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36490 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083177.399:78): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36484 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083177.399:77): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36484 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083177.399:76): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36484 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36484 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36484 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36484 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" time="2024-05-07T11:59:37.393546850Z" level=info msg="StartContainer for \"4346b5c7d89f0d5b24a4b190740ed7327d12365cbd86dd55f845a8416cb1824e\" returns successfully" Started libcontainer container 4346b5c7d89f0d5b24a4b190740ed7327d12365cbd86dd55f845a8416cb1824e. time="2024-05-07T11:59:37.344257745Z" level=info msg="StartContainer for \"4346b5c7d89f0d5b24a4b190740ed7327d12365cbd86dd55f845a8416cb1824e\"" time="2024-05-07T11:59:37.343701838Z" level=info msg="CreateContainer within sandbox \"ac0defb47ab561e39c01453f80823086daf554758865a65d1cb608092c1539d5\" for &ContainerMetadata{Name:hgrun,Attempt:0,} returns container id \"4346b5c7d89f0d5b24a4b190740ed7327d12365cbd86dd55f845a8416cb1824e\"" time="2024-05-07T11:59:37.335849942Z" level=info msg="CreateContainer within sandbox \"ac0defb47ab561e39c01453f80823086daf554758865a65d1cb608092c1539d5\" for container &ContainerMetadata{Name:hgrun,Attempt:0,}" time="2024-05-07T11:59:37.334816429Z" level=info msg="PullImage \"us.gcr.io/hosted-grafana/hgrun:0.1.452\"" I0507 11:59:37.334399 2791 azure_credentials.go:220] image(us.gcr.io/hosted-grafana/hgrun) is not from ACR, return empty authentication time="2024-05-07T11:59:37.333654615Z" level=info msg="PullImage \"us.gcr.io/hosted-grafana/hgrun:0.1.452\" returns image reference \"sha256:9fb1bce3e4a228f50768d21842cd7d7fafc1d586eaa0326c9d3c86d79a36868a\"" time="2024-05-07T11:59:37.333615314Z" level=info msg="Pulled image \"us.gcr.io/hosted-grafana/hgrun:0.1.452\" with image id \"sha256:9fb1bce3e4a228f50768d21842cd7d7fafc1d586eaa0326c9d3c86d79a36868a\", repo tag \"us.gcr.io/hosted-grafana/hgrun:0.1.452\", repo digest \"us.gcr.io/hosted-grafana/hgrun@sha256:b492dbbbee9faf9dba63c9fd89e6f9e148239765454c6a54c4284a2828dec153\", size \"19109699\" in 3.751908524s" time="2024-05-07T11:59:37.332685003Z" level=info msg="ImageCreate event name:\"us.gcr.io/hosted-grafana/hgrun@sha256:b492dbbbee9faf9dba63c9fd89e6f9e148239765454c6a54c4284a2828dec153\" labels:{key:\"io.cri-containerd.image\" value:\"managed\"}" time="2024-05-07T11:59:37.331316686Z" level=info msg="ImageUpdate event name:\"us.gcr.io/hosted-grafana/hgrun:0.1.452\" labels:{key:\"io.cri-containerd.image\" value:\"managed\"}" time="2024-05-07T11:59:37.329165060Z" level=info msg="ImageCreate event name:\"sha256:9fb1bce3e4a228f50768d21842cd7d7fafc1d586eaa0326c9d3c86d79a36868a\" labels:{key:\"io.cri-containerd.image\" value:\"managed\"}" time="2024-05-07T11:59:37.328382550Z" level=info msg="stop pulling image us.gcr.io/hosted-grafana/hgrun:0.1.452: active requests=0, bytes read=15714567" time="2024-05-07T11:59:37.328138947Z" level=info msg="ImageCreate event name:\"us.gcr.io/hosted-grafana/hgrun:0.1.452\" labels:{key:\"io.cri-containerd.image\" value:\"managed\"}" run-containerd-io.containerd.runtime.v2.task-k8s.io-6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408-rootfs.mount: Deactivated successfully. cri-containerd-6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408.scope: Consumed 21.501s CPU time. cri-containerd-6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408.scope: Deactivated successfully. E0507 11:59:37.252214 4736 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"ksm\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=ksm pod=new-relic-nri-bundle-nrk8s-ksm-6c785668f5-jcxh2_integration(f7cc3cca-2ffb-4fde-a73e-a4ba8b0f6b3c)\"" pod="integration/new-relic-nri-bundle-nrk8s-ksm-6c785668f5-jcxh2" podUID="f7cc3cca-2ffb-4fde-a73e-a4ba8b0f6b3c" I0507 11:59:37.251511 4736 scope.go:117] "RemoveContainer" containerID="dd4a230cd9a8d74428dafba521d8e57bdec75e75b6cb2c51d6f7b801d2bf0d0e" time="2024-05-07T11:59:37.204502542Z" level=info msg="PullImage \"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2\"" E0507 11:59:37.204057 4589 remote_image.go:180] "PullImage from image service failed" err="rpc error: code = NotFound desc = failed to pull and unpack image \"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2\": failed to resolve reference \"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2\": us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2: not found" image="us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2" time="2024-05-07T11:59:37.203800966Z" level=info msg="stop pulling image us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2: active requests=0, bytes read=6802" time="2024-05-07T11:59:37.203750262Z" level=error msg="PullImage \"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2\" failed" error="rpc error: code = NotFound desc = failed to pull and unpack image \"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2\": failed to resolve reference \"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2\": us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2: not found" time="2024-05-07T11:59:37.202055445Z" level=info msg="trying next host - response was http.StatusNotFound" host=us.gcr.io audit: type=1400 audit(1715083177.183:75): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36423 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083177.183:74): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36423 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083177.183:73): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36423 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36423 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36423 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36423 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" I0507 11:59:37.145947 4573 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-hg-n2s8-6-1dd39c-3bfd06e9-zbcz" status="Running" I0507 11:59:37.134905 4734 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-5rdf" status="Running" I0507 11:59:37.133005 3782 prober.go:107] "Probe failed" probeType="Readiness" pod="loki-dev-014/loki-dev-014-rollout-operator-58fc68b876-2qhmp" podUID="e6504036-2514-4ecc-b78c-c47061f60c9f" containerName="rollout-operator" probeResult="failure" output="HTTP probe failed with statuscode: 500" run-containerd-runc-k8s.io-e5f17d69eee483ec8d43b26d5d628246984ba92f794ee5f3748935f5b6448b9b-runc.6eAyHn.mount: Deactivated successfully. ll header: 00000000: 42 01 0a 80 00 8f 42 01 0a 80 00 01 08 00 IPv4: martian source 10.132.135.59 from 10.132.135.75, on dev eth0 ll header: 00000000: 42 01 0a 80 00 8f 42 01 0a 80 00 01 08 00 IPv4: martian source 10.132.135.59 from 10.132.135.75, on dev eth0 net_ratelimit: 2 callbacks suppressed time="2024-05-07T11:59:37.051802942Z" level=info msg="PullImage \"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2\"" E0507 11:59:37.041999 4589 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.0.2\\\"\"" pod="hosted-grafana/johan6-grafana-796656fd69-6rfnm" podUID="f6fe6c71-2a0e-4797-bd0c-2b508d1287fe" E0507 11:59:37.025280 4585 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6439-933-1\\\"\"" pod="hosted-grafana/ephemeral1180076306439dafyddt-grafana-85fd4854bb-q6874" podUID="d6e144f9-ae89-463b-bcc4-c52f6eb091f8" E0507 11:59:37.025204 4585 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=pyroscopecanaries-grafana-57b7948fdb-6m6ww_hosted-grafana(2ccf8512-b42a-470c-a988-cc464917d285)\"" pod="hosted-grafana/pyroscopecanaries-grafana-57b7948fdb-6m6ww" podUID="2ccf8512-b42a-470c-a988-cc464917d285" I0507 11:59:37.024249 4585 scope.go:117] "RemoveContainer" containerID="c71d355da3277fec1e90b52152383d7d4579dedbc22016e693d363830632f4c5" E0507 11:59:36.999782 4734 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"pdc\" with ErrImageNeverPull: \"Container image \\\"us.gcr.io/hosted-grafana/pdc:0.1.415\\\" is not present with pull policy of Never\"" pod="pdc/private-datasource-connect-564fb6cfbb-fd2jh" podUID="ac6bc6d0-43a4-4885-9ee4-ba3441b0b527" E0507 11:59:36.999725 4734 kuberuntime_manager.go:1256] container &Container{Name:pdc,Image:us.gcr.io/hosted-grafana/pdc:0.1.415,Command:[],Args:[-proxy.auth.ca-keys-dir=/var/run/secrets/pdc-certs -proxy.socks-server.addr=:10443 -proxy.ssh-server.addr=:2222 -proxy.use-socks-username-for-routing -proxy.api.http-address=:9182 -proxy.check-connpool-address-in-ring -memberlist.join=dns+gossip-ring.pdc.svc.cluster.local:7946 -api.http-address=:11443 -distributor.enabled=true -distributor.addr=:10444 -distributor.use-socks-username-for-routing -gateway.enabled=true -gateway.addr=:2244 -log.level=debug -certs.ca-private-key-file=/var/run/secrets/pdc-certs/ca.key -certs.ca-cert-file=/var/run/secrets/pdc-certs/ca.crt -certs.ca-pub-file=/var/run/secrets/pdc-certs/ca.pub -certs.cluster=local-k8s -shard-size=3 -graceful-shutdown-period=30s -enable-multiple-networks],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:socks,HostPort:0,ContainerPort:10443,Protocol:TCP,HostIP:,},ContainerPort{Name:ssh,HostPort:0,ContainerPort:2222,Protocol:TCP,HostIP:,},ContainerPort{Name:distributor,HostPort:0,ContainerPort:10444,Protocol:TCP,HostIP:,},ContainerPort{Name:gateway,HostPort:0,ContainerPort:2244,Protocol:TCP,HostIP:,},ContainerPort{Name:api,HostPort:0,ContainerPort:11443,Protocol:TCP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:POD_NAME,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.name,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},},Resources:ResourceRequirements{Limits:ResourceList{cpu: {{500 -3} {} 500m DecimalSI},memory: {{134217728 0} {} BinarySI},},Requests:ResourceList{cpu: {{250 -3} {} 250m DecimalSI},memory: {{67108864 0} {} BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:pdc-certs,ReadOnly:true,MountPath:/var/run/secrets/pdc-certs,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-pjvg4,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/ready,Port:{0 11443 },Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:40,TimeoutSeconds:1,PeriodSeconds:5,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:&Lifecycle{PostStart:nil,PreStop:&LifecycleHandler{Exec:&ExecAction{Command:[/bin/sleep 5],},HTTPGet:nil,TCPSocket:nil,},},TerminationMessagePath:/dev/termination-log,ImagePullPolicy:Never,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod private-datasource-connect-564fb6cfbb-fd2jh_pdc(ac6bc6d0-43a4-4885-9ee4-ba3441b0b527): ErrImageNeverPull: Container image "us.gcr.io/hosted-grafana/pdc:0.1.415" is not present with pull policy of Never I0507 11:59:36.923242 4745 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-j2vt" status="Running" audit: type=1400 audit(1715083176.899:72): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36386 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083176.899:71): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36386 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083176.899:70): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36386 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" kauditd_printk_skb: 41 callbacks suppressed AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36386 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36386 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36386 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36384 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36384 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36384 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" E0507 11:59:36.667375 2776 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=edwardtest-grafana-c4849b4b7-56ssx_hosted-grafana(eb8ab8ca-f0ef-4df3-923b-4f17718cd1ad)\"" pod="hosted-grafana/edwardtest-grafana-c4849b4b7-56ssx" podUID="eb8ab8ca-f0ef-4df3-923b-4f17718cd1ad" I0507 11:59:36.666757 2776 scope.go:117] "RemoveContainer" containerID="ebebe8ca1b6122cea402a1cfb6fa758f2f8e849943cbf9c09506c00870d1d732" E0507 11:59:36.604366 4586 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-enterprise-6372-855-1\\\"\"" pod="hosted-grafana/ephemeral1180076306372jacobso-grafana-8659d5f696-t5qdc" podUID="a0b71eed-f8bf-4875-9ddd-52a41e861a56" E0507 11:59:36.604305 4586 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=ephemeral1180076306267marefr-grafana-7b4dcd6ddc-d9dx2_hosted-grafana(929d579c-9422-4ad1-989d-9e1edb203f70)\"" pod="hosted-grafana/ephemeral1180076306267marefr-grafana-7b4dcd6ddc-d9dx2" podUID="929d579c-9422-4ad1-989d-9e1edb203f70" I0507 11:59:36.603479 4586 scope.go:117] "RemoveContainer" containerID="95667fbb3341dec0691e603540df17577797c88cb83c2871b860000d66c95429" E0507 11:59:36.599130 4624 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=k6teststeady6-grafana-676df8f4d-2qkx4_hosted-grafana(b0e5bc5f-fa77-48da-a896-52c578fb915b)\"" pod="hosted-grafana/k6teststeady6-grafana-676df8f4d-2qkx4" podUID="b0e5bc5f-fa77-48da-a896-52c578fb915b" I0507 11:59:36.598142 4624 scope.go:117] "RemoveContainer" containerID="e8101aad8fc0984550fb0d96d57c9e2fbc9938e7bf3fa2c25d1e6748dc3a61fc" I0507 11:59:36.581938 4736 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-gwtz" status="Running" E0507 11:59:36.562067 4595 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 1m20s restarting failed container=grafana pod=ltest-grafana-74d66c7766-s2ftd_hosted-grafana(40e776a7-78a1-44d8-95ac-0c4944bb7737)\"" pod="hosted-grafana/ltest-grafana-74d66c7766-s2ftd" podUID="40e776a7-78a1-44d8-95ac-0c4944bb7737" I0507 11:59:36.561272 4595 scope.go:117] "RemoveContainer" containerID="ea831b36e1cf141ea84a1158e1ac08c42bfe6220a73e5f2074dea1d25e9c8619" I0507 11:59:36.560485 4595 kubelet.go:2498] "SyncLoop (probe)" probe="liveness" status="unhealthy" pod="hosted-grafana/ltest-grafana-74d66c7766-s2ftd" E0507 11:59:36.539121 4590 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 2m40s restarting failed container=grafana pod=k6teststeady2-grafana-7c76656c46-56dmp_hosted-grafana(724dd356-a32b-421e-b29b-b7ce1624a7f6)\"" pod="hosted-grafana/k6teststeady2-grafana-7c76656c46-56dmp" podUID="724dd356-a32b-421e-b29b-b7ce1624a7f6" I0507 11:59:36.538134 4590 scope.go:117] "RemoveContainer" containerID="8f11d74c7851b1769f7304a41ab16112c93df08749d2241c55263fdf3731038d" E0507 11:59:36.526633 4615 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-84322-18859-1\\\"\"" pod="hosted-grafana/ephemeral1511182184322yuritce-grafana-5fc4b65c7d-rj4p4" podUID="909719af-cfb6-4d8a-9893-f87f27af458a" I0507 11:59:36.524260 4733 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-kv65" status="Running" ll header: 00000000: 42 01 0a 80 00 17 42 01 0a 80 00 01 08 00 IPv4: martian source 10.132.141.91 from 10.132.141.80, on dev eth0 XMT: Solicit on eth0, interval 108810ms. AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36373 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36373 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36373 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" I0507 11:59:36.334923 4603 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-perf-n2s8-0-1dd3-91689928-9f87" status="Running" Started cri-containerd-95bf586cd79d43120ff44582d4dbd2476de61744411f8515b9b2c527a41fd5d9.scope. I0507 11:59:36.251643 4763 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-x94l" status="Running" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36367 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36367 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36367 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" time="2024-05-07T11:59:36.179017215Z" level=info msg="StartContainer for \"95bf586cd79d43120ff44582d4dbd2476de61744411f8515b9b2c527a41fd5d9\"" time="2024-05-07T11:59:36.177858616Z" level=info msg="CreateContainer within sandbox \"81e019a0248a0300a328fd59f9939c3eaa1b98aa7f325a7f6e00592633275ef6\" for &ContainerMetadata{Name:checkoutservice,Attempt:3417,} returns container id \"95bf586cd79d43120ff44582d4dbd2476de61744411f8515b9b2c527a41fd5d9\"" I0507 11:59:36.156382 4710 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-cache-n2hc8-1-1d-61155fd9-89hx" status="Running" E0507 11:59:36.151662 4572 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=benchloadtestingxxl2-grafana-56b7c6b6df-2w2wq_hosted-grafana(792f4b46-ea76-486f-917b-9603924d3303)\"" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-2w2wq" podUID="792f4b46-ea76-486f-917b-9603924d3303" I0507 11:59:36.150687 4572 scope.go:117] "RemoveContainer" containerID="064e5b87ec988ee1421ec77749112553895c547dee46288c0a5f0a525c69b0cf" I0507 11:59:36.097430 4734 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-vvgr" status="Running" I0507 11:59:36.050423 4589 kubelet.go:2498] "SyncLoop (probe)" probe="readiness" status="ready" pod="hosted-grafana/ephemeral1511182187382tskarhe-grafana-7c98d5b45f-6zzkh" I0507 11:59:36.014609 3224 kubelet_volumes.go:163] "Cleaned up orphaned pod volumes dir" podUID="25cb986c-3d6c-4ed0-abf3-ee59ed6175f9" path="/var/lib/kubelet/pods/25cb986c-3d6c-4ed0-abf3-ee59ed6175f9/volumes" I0507 11:59:36.006474 4732 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-hmlg" status="Running" I0507 11:59:35.990519 4739 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-h8wf" status="Running" E0507 11:59:35.928465 4734 pod_workers.go:1300] "Error syncing pod, skipping" err="unmounted volumes=[custom-grafana-agent], unattached volumes=[], failed to process volumes=[]: context deadline exceeded" pod="loki-dev-010/custom-grafana-agent-856948968f-6jfks" podUID="17b244cc-ecb9-4fbc-beaa-8fa47fafe013" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36365 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36365 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36365 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" time="2024-05-07T11:59:35.780801331Z" level=info msg="RemoveContainer for \"e3360016e710853e0218cca9ca86c8beda5f9ec6ff20d03df41db53900b992b5\" returns successfully" I0507 11:59:35.776278 4729 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-nrth" status="Running" E0507 11:59:35.772687 4602 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=k6teststeady4-grafana-5c4f6cd55-hvn6k_hosted-grafana(a95be6bc-a7bc-48cb-8935-f7040f91f7f9)\"" pod="hosted-grafana/k6teststeady4-grafana-5c4f6cd55-hvn6k" podUID="a95be6bc-a7bc-48cb-8935-f7040f91f7f9" time="2024-05-07T11:59:35.772296342Z" level=info msg="RemoveContainer for \"e3360016e710853e0218cca9ca86c8beda5f9ec6ff20d03df41db53900b992b5\"" I0507 11:59:35.771829 4602 scope.go:117] "RemoveContainer" containerID="c6da2382101cc3ca3a9a6de7b86f62dfd7b344559c7e17cecfb83f1284783adb" I0507 11:59:35.771222 4602 scope.go:117] "RemoveContainer" containerID="e3360016e710853e0218cca9ca86c8beda5f9ec6ff20d03df41db53900b992b5" I0507 11:59:35.771184 4602 kubelet.go:2426] "SyncLoop (PLEG): event for pod" pod="hosted-grafana/k6teststeady4-grafana-5c4f6cd55-hvn6k" event={"ID":"a95be6bc-a7bc-48cb-8935-f7040f91f7f9","Type":"ContainerDied","Data":"c6da2382101cc3ca3a9a6de7b86f62dfd7b344559c7e17cecfb83f1284783adb"} I0507 11:59:35.771136 4602 generic.go:334] "Generic (PLEG): container finished" podID="a95be6bc-a7bc-48cb-8935-f7040f91f7f9" containerID="c6da2382101cc3ca3a9a6de7b86f62dfd7b344559c7e17cecfb83f1284783adb" exitCode=1 E0507 11:59:35.706327 4602 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-85282-20418-1\\\"\"" pod="hosted-grafana/ephemeral1511182185282svenner-grafana-59d5c4d944-k5bkz" podUID="dab84f6b-fe11-4ee5-931a-332f264b2e9c" E0507 11:59:35.705273 4602 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=k6testslow3-grafana-6b64c68f9-mtbkr_hosted-grafana(024c0105-5d91-4a11-80bf-c8d380cb5fb6)\"" pod="hosted-grafana/k6testslow3-grafana-6b64c68f9-mtbkr" podUID="024c0105-5d91-4a11-80bf-c8d380cb5fb6" I0507 11:59:35.704599 4602 scope.go:117] "RemoveContainer" containerID="d9b2a552b0a50add51394b346ec1f55e55cdec603101ab963aecd48029badf68" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36363 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36363 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36363 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" I0507 11:59:35.674912 4773 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-dqf8" status="Running" E0507 11:59:35.613614 4602 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=ephemeral1511182177667ryantxu-grafana-5d7fbd766b-ddcnh_hosted-grafana(1ecbad00-a542-411b-b8af-323a1f1fba79)\"" pod="hosted-grafana/ephemeral1511182177667ryantxu-grafana-5d7fbd766b-ddcnh" podUID="1ecbad00-a542-411b-b8af-323a1f1fba79" I0507 11:59:35.612867 4602 scope.go:117] "RemoveContainer" containerID="5a800f214c789ee2b7060e17c9fe29e1fe5ed3f7067dd08bba14a005ec2e9bfd" I0507 11:59:35.612332 4602 kubelet.go:2498] "SyncLoop (probe)" probe="liveness" status="unhealthy" pod="hosted-grafana/ephemeral1511182177667ryantxu-grafana-5d7fbd766b-ddcnh" I0507 11:59:35.572523 4770 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-rvzj" status="Running" XMT: Solicit on eth0, interval 117800ms. E0507 11:59:35.487872 4724 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=aws-dev-app-0_integration(f9fb0aca-946a-4fdc-ba53-f48bfbc47103)\"" pod="integration/aws-dev-app-0" podUID="f9fb0aca-946a-4fdc-ba53-f48bfbc47103" I0507 11:59:35.487049 4724 scope.go:117] "RemoveContainer" containerID="2a2e04c10ced384f8ccd26986b172c42fcee911abda737fd8129030cc6a556b3" I0507 11:59:35.461697 4631 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-hg-n2s8-6-1dd39c-3bfd06e9-jfjs" status="Running" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36361 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36361 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36361 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" E0507 11:59:35.349860 4598 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=benchloadtestingxxl2-grafana-645fd656b4-q2qth_hosted-grafana(24c22429-43c3-4ad6-8b79-d42030a9204d)\"" pod="hosted-grafana/benchloadtestingxxl2-grafana-645fd656b4-q2qth" podUID="24c22429-43c3-4ad6-8b79-d42030a9204d" I0507 11:59:35.349342 4598 scope.go:117] "RemoveContainer" containerID="5cf3d07ffea04d45c8d68600b11896b2559d462b331d1a5c628c0163e21537c3" I0507 11:59:35.294796 4772 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-sskn" status="Running" I0507 11:59:35.248171 4735 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-xstn" status="Running" XMT: Solicit on eth0, interval 123690ms. AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36359 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36359 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36359 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" E0507 11:59:35.152016 4572 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 2m40s restarting failed container=grafana pod=ephemeral1511182179279sarahzi-grafana-7f464d7c8d-sqbkq_hosted-grafana(8d5619e6-389c-46a0-996f-8b20cdac2937)\"" pod="hosted-grafana/ephemeral1511182179279sarahzi-grafana-7f464d7c8d-sqbkq" podUID="8d5619e6-389c-46a0-996f-8b20cdac2937" I0507 11:59:35.151138 4572 scope.go:117] "RemoveContainer" containerID="34efa66e260e563bd703e808d5efbda0e6f8d6882a5eb3982bcb1e4fd3826a13" I0507 11:59:35.029584 3224 kubelet.go:2408] "SyncLoop REMOVE" source="api" pods=["hosted-grafana/hosted-grafana-api-7b6bd9b949-9csb4"] E0507 11:59:35.024914 4585 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77487-8287-1\\\"\"" pod="hosted-grafana/ephemeral1511182177487torkelo-grafana-8589fc9df9-7795x" podUID="57dbe932-52d4-4144-9577-ecad74ad8d52" I0507 11:59:35.017930 3224 kubelet.go:2414] "SyncLoop DELETE" source="api" pods=["hosted-grafana/hosted-grafana-api-7b6bd9b949-9csb4"] time="2024-05-07T11:59:35.010488825Z" level=info msg="cleaning up dead shim" namespace=k8s.io time="2024-05-07T11:59:35.010474358Z" level=warning msg="cleaning up after shim disconnected" id=c6da2382101cc3ca3a9a6de7b86f62dfd7b344559c7e17cecfb83f1284783adb namespace=k8s.io time="2024-05-07T11:59:35.010405059Z" level=info msg="shim disconnected" id=c6da2382101cc3ca3a9a6de7b86f62dfd7b344559c7e17cecfb83f1284783adb namespace=k8s.io I0507 11:59:34.965425 3224 pod_container_deletor.go:53] "DeleteContainer returned error" containerID={"Type":"containerd","ID":"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e"} err="failed to get container status \"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\": rpc error: code = NotFound desc = an error occurred when try to find container \"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\": not found" E0507 11:59:34.965384 3224 remote_runtime.go:432] "ContainerStatus from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find container \"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\": not found" containerID="c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e" time="2024-05-07T11:59:34.965205507Z" level=error msg="ContainerStatus for \"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\" failed" error="rpc error: code = NotFound desc = an error occurred when try to find container \"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\": not found" I0507 11:59:34.964947 3224 scope.go:117] "RemoveContainer" containerID="c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e" time="2024-05-07T11:59:34.964793584Z" level=info msg="RemoveContainer for \"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\" returns successfully" Removed slice libcontainer container kubepods-burstable-pod25cb986c_3d6c_4ed0_abf3_ee59ed6175f9.slice. time="2024-05-07T11:59:34.959179289Z" level=info msg="RemoveContainer for \"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\"" I0507 11:59:34.956840 3224 scope.go:117] "RemoveContainer" containerID="c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e" I0507 11:59:34.956820 3224 kubelet.go:2430] "SyncLoop (PLEG): event for pod" pod="hosted-grafana/hosted-grafana-api-7b6bd9b949-9csb4" event={"ID":"25cb986c-3d6c-4ed0-abf3-ee59ed6175f9","Type":"ContainerDied","Data":"c605ad2cdc74c6b5288f2532ad71cce81a28ef6965f97a89ff6609deb825553a"} I0507 11:59:34.956796 3224 kubelet.go:2430] "SyncLoop (PLEG): event for pod" pod="hosted-grafana/hosted-grafana-api-7b6bd9b949-9csb4" event={"ID":"25cb986c-3d6c-4ed0-abf3-ee59ed6175f9","Type":"ContainerDied","Data":"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e"} I0507 11:59:34.956755 3224 generic.go:334] "Generic (PLEG): container finished" podID="25cb986c-3d6c-4ed0-abf3-ee59ed6175f9" containerID="c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e" exitCode=1 I0507 11:59:34.936025 3224 reconciler_common.go:300] "Volume detached for volume \"pdc-certs\" (UniqueName: \"kubernetes.io/secret/25cb986c-3d6c-4ed0-abf3-ee59ed6175f9-pdc-certs\") on node \"ip-10-60-2-58.us-east-2.compute.internal\" DevicePath \"\"" I0507 11:59:34.935988 3224 reconciler_common.go:300] "Volume detached for volume \"kube-api-access-95j2t\" (UniqueName: \"kubernetes.io/projected/25cb986c-3d6c-4ed0-abf3-ee59ed6175f9-kube-api-access-95j2t\") on node \"ip-10-60-2-58.us-east-2.compute.internal\" DevicePath \"\"" I0507 11:59:34.935951 3224 reconciler_common.go:300] "Volume detached for volume \"gcs-serviceaccount\" (UniqueName: \"kubernetes.io/secret/25cb986c-3d6c-4ed0-abf3-ee59ed6175f9-gcs-serviceaccount\") on node \"ip-10-60-2-58.us-east-2.compute.internal\" DevicePath \"\"" E0507 11:59:34.923984 3027 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"mysqld-exporter\" with CreateContainerConfigError: \"secret \\\"testcrossplane-user-exporter\\\" not found\"" pod="crossplane-playground/testcrossplane-exporter-c67cfc58f-vbzl4" podUID="3d49134d-3378-4ec3-824c-5ff4ea2590a5" E0507 11:59:34.923938 3027 kuberuntime_manager.go:1261] container &Container{Name:mysqld-exporter,Image:prom/mysqld-exporter:v0.13.0,Command:[],Args:[--collect.info_schema.innodb_metrics],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:http-metrics,HostPort:0,ContainerPort:9104,Protocol:TCP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:MYSQL_USER,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:testcrossplane-user-exporter,},Key:username,Optional:nil,},},},EnvVar{Name:MYSQL_PASSWORD,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:testcrossplane-user-exporter,},Key:password,Optional:nil,},},},EnvVar{Name:MYSQL_HOST,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:testcrossplane-user-exporter,},Key:endpoint,Optional:nil,},},},EnvVar{Name:MYSQL_PORT,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:testcrossplane-user-exporter,},Key:port,Optional:nil,},},},EnvVar{Name:MYSQL_TLS_MODE,Value:preferred,ValueFrom:nil,},EnvVar{Name:DATA_SOURCE_NAME,Value:$(MYSQL_USER):$(MYSQL_PASSWORD)@tcp($(MYSQL_HOST):$(MYSQL_PORT))/?tls=$(MYSQL_TLS_MODE),ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-dzx7d,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod testcrossplane-exporter-c67cfc58f-vbzl4_crossplane-playground(3d49134d-3378-4ec3-824c-5ff4ea2590a5): CreateContainerConfigError: secret "testcrossplane-user-exporter" not found I0507 11:59:34.921271 4731 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-7jmw" status="Running" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36348 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36348 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36348 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" E0507 11:59:34.856101 4727 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana-render-security\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-security:0.1.181\\\"\"" pod="integration/grafana-render-service-cbff479fc-cj9tp" podUID="0e3114d1-2f3a-49d6-a71d-dbc75050d8e0" I0507 11:59:34.855593 4771 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-s6kw" status="Running" I0507 11:59:34.854084 4727 kubelet_pods.go:906] "Unable to retrieve pull secret, the image pull may not succeed." pod="integration/grafana-render-service-cbff479fc-cj9tp" secret="" err="secret \"us-gcr-io-hosted-grafana\" not found" I0507 11:59:34.841447 3224 operation_generator.go:888] UnmountVolume.TearDown succeeded for volume "kubernetes.io/secret/25cb986c-3d6c-4ed0-abf3-ee59ed6175f9-gcs-serviceaccount" (OuterVolumeSpecName: "gcs-serviceaccount") pod "25cb986c-3d6c-4ed0-abf3-ee59ed6175f9" (UID: "25cb986c-3d6c-4ed0-abf3-ee59ed6175f9"). InnerVolumeSpecName "gcs-serviceaccount". PluginName "kubernetes.io/secret", VolumeGidValue "" I0507 11:59:34.841404 3224 operation_generator.go:888] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/25cb986c-3d6c-4ed0-abf3-ee59ed6175f9-kube-api-access-95j2t" (OuterVolumeSpecName: "kube-api-access-95j2t") pod "25cb986c-3d6c-4ed0-abf3-ee59ed6175f9" (UID: "25cb986c-3d6c-4ed0-abf3-ee59ed6175f9"). InnerVolumeSpecName "kube-api-access-95j2t". PluginName "kubernetes.io/projected", VolumeGidValue "" I0507 11:59:34.836955 3224 operation_generator.go:888] UnmountVolume.TearDown succeeded for volume "kubernetes.io/secret/25cb986c-3d6c-4ed0-abf3-ee59ed6175f9-pdc-certs" (OuterVolumeSpecName: "pdc-certs") pod "25cb986c-3d6c-4ed0-abf3-ee59ed6175f9" (UID: "25cb986c-3d6c-4ed0-abf3-ee59ed6175f9"). InnerVolumeSpecName "pdc-certs". PluginName "kubernetes.io/secret", VolumeGidValue "" I0507 11:59:34.834835 3224 reconciler_common.go:172] "operationExecutor.UnmountVolume started for volume \"gcs-serviceaccount\" (UniqueName: \"kubernetes.io/secret/25cb986c-3d6c-4ed0-abf3-ee59ed6175f9-gcs-serviceaccount\") pod \"25cb986c-3d6c-4ed0-abf3-ee59ed6175f9\" (UID: \"25cb986c-3d6c-4ed0-abf3-ee59ed6175f9\") " I0507 11:59:34.834794 3224 reconciler_common.go:172] "operationExecutor.UnmountVolume started for volume \"pdc-certs\" (UniqueName: \"kubernetes.io/secret/25cb986c-3d6c-4ed0-abf3-ee59ed6175f9-pdc-certs\") pod \"25cb986c-3d6c-4ed0-abf3-ee59ed6175f9\" (UID: \"25cb986c-3d6c-4ed0-abf3-ee59ed6175f9\") " I0507 11:59:34.834734 3224 reconciler_common.go:172] "operationExecutor.UnmountVolume started for volume \"kube-api-access-95j2t\" (UniqueName: \"kubernetes.io/projected/25cb986c-3d6c-4ed0-abf3-ee59ed6175f9-kube-api-access-95j2t\") pod \"25cb986c-3d6c-4ed0-abf3-ee59ed6175f9\" (UID: \"25cb986c-3d6c-4ed0-abf3-ee59ed6175f9\") " run-containerd-io.containerd.runtime.v2.task-k8s.io-c6da2382101cc3ca3a9a6de7b86f62dfd7b344559c7e17cecfb83f1284783adb-rootfs.mount: Deactivated successfully. E0507 11:59:34.739637 4738 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"ruler\" with CreateContainerConfigError: \"secret \\\"ruler-alertmanager-token\\\" not found\"" pod="ge-metrics-federation/gem-mimir-ruler-bd7cbc8cb-fpvxg" podUID="f39fa140-2a71-4cba-bcb7-b37b2fafa343" E0507 11:59:34.739556 4738 kuberuntime_manager.go:1256] container &Container{Name:ruler,Image:grafana/enterprise-metrics:v2.11.1,Command:[],Args:[-target=ruler -config.expand-env=true -config.file=/etc/mimir/mimir.yaml],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:http-metrics,HostPort:0,ContainerPort:8080,Protocol:TCP,HostIP:,},ContainerPort{Name:grpc,HostPort:0,ContainerPort:9095,Protocol:TCP,HostIP:,},ContainerPort{Name:memberlist,HostPort:0,ContainerPort:7946,Protocol:TCP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:JAEGER_AGENT_HOST,Value:jaeger-agent.jaeger.svc.cluster.local.,ValueFrom:nil,},EnvVar{Name:JAEGER_TAGS,Value:namespace=ge-metrics-federation,cluster=dev-us-central-0,ValueFrom:nil,},EnvVar{Name:JAEGER_SAMPLER_MANAGER_HOST_PORT,Value:http://jaeger-agent.jaeger.svc.cluster.local.:5778/sampling,ValueFrom:nil,},EnvVar{Name:GOOGLE_APPLICATION_CREDENTIALS,Value:/var/secrets/google/credentials.json,ValueFrom:nil,},EnvVar{Name:AM_TOKEN,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:ruler-alertmanager-token,},Key:token,Optional:nil,},},},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{cpu: {{100 -3} {} 100m DecimalSI},memory: {{134217728 0} {} BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:gcs-credentials,ReadOnly:false,MountPath:/var/secrets/google/,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:config,ReadOnly:false,MountPath:/etc/mimir,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:license,ReadOnly:false,MountPath:/license,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:runtime-config,ReadOnly:false,MountPath:/var/mimir,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:storage,ReadOnly:false,MountPath:/data,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:active-queries,ReadOnly:false,MountPath:/active-query-tracker,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-xf5ns,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/ready,Port:{1 0 http-metrics},Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:45,TimeoutSeconds:1,PeriodSeconds:10,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:*true,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod gem-mimir-ruler-bd7cbc8cb-fpvxg_ge-metrics-federation(f39fa140-2a71-4cba-bcb7-b37b2fafa343): CreateContainerConfigError: secret "ruler-alertmanager-token" not found cri-containerd-c6da2382101cc3ca3a9a6de7b86f62dfd7b344559c7e17cecfb83f1284783adb.scope: Consumed 15.899s CPU time. cri-containerd-c6da2382101cc3ca3a9a6de7b86f62dfd7b344559c7e17cecfb83f1284783adb.scope: Deactivated successfully. time="2024-05-07T11:59:34.707025668Z" level=info msg="StopPodSandbox for \"c605ad2cdc74c6b5288f2532ad71cce81a28ef6965f97a89ff6609deb825553a\" returns successfully" time="2024-05-07T11:59:34.706960850Z" level=info msg="TearDown network for sandbox \"c605ad2cdc74c6b5288f2532ad71cce81a28ef6965f97a89ff6609deb825553a\" successfully" I0507 11:59:34.703072 3224 kubelet.go:2414] "SyncLoop DELETE" source="api" pods=["hosted-grafana/hosted-grafana-api-7b6bd9b949-9csb4"] AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36346 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36346 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36346 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" time="2024-05-07T11:59:34.624594305Z" level=info msg="cleaning up dead shim" namespace=k8s.io time="2024-05-07T11:59:34.624582051Z" level=warning msg="cleaning up after shim disconnected" id=c605ad2cdc74c6b5288f2532ad71cce81a28ef6965f97a89ff6609deb825553a namespace=k8s.io time="2024-05-07T11:59:34.624527314Z" level=info msg="shim disconnected" id=c605ad2cdc74c6b5288f2532ad71cce81a28ef6965f97a89ff6609deb825553a namespace=k8s.io E0507 11:59:34.604339 4586 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-84322-18859-1\\\"\"" pod="hosted-grafana/ephemeral1511182184322yuritce-grafana-84496d949-4gc7m" podUID="8b778f8a-7749-4b2f-ac6d-aafa8cd32119" E0507 11:59:34.603913 4586 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=ephemeral1511182179279sarahzi-grafana-c5dc5d5-krs7z_hosted-grafana(baa23492-a636-4134-8908-391631902931)\"" pod="hosted-grafana/ephemeral1511182179279sarahzi-grafana-c5dc5d5-krs7z" podUID="baa23492-a636-4134-8908-391631902931" I0507 11:59:34.603362 4586 scope.go:117] "RemoveContainer" containerID="8f3e15e7b4631f5c3b33aa44572fdcc313ba99fcd9e4e1998b464ab93624b2e7" time="2024-05-07T11:59:34.592084495Z" level=info msg="Container to stop \"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\" must be in running or unknown state, current state \"CONTAINER_EXITED\"" time="2024-05-07T11:59:34.592005066Z" level=info msg="StopPodSandbox for \"c605ad2cdc74c6b5288f2532ad71cce81a28ef6965f97a89ff6609deb825553a\"" time="2024-05-07T11:59:34.591282703Z" level=info msg="StopContainer for \"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\" returns successfully" time="2024-05-07T11:59:34.574306831Z" level=info msg="cleaning up dead shim" namespace=k8s.io time="2024-05-07T11:59:34.574295800Z" level=warning msg="cleaning up after shim disconnected" id=c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e namespace=k8s.io time="2024-05-07T11:59:34.574248328Z" level=info msg="shim disconnected" id=c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e namespace=k8s.io E0507 11:59:34.573378 4595 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=k6teststeady2-grafana-bd47b47d4-28b6n_hosted-grafana(91385f28-bb54-4fd7-a913-0dea0a3361d4)\"" pod="hosted-grafana/k6teststeady2-grafana-bd47b47d4-28b6n" podUID="91385f28-bb54-4fd7-a913-0dea0a3361d4" I0507 11:59:34.571158 4595 scope.go:117] "RemoveContainer" containerID="f3b57fb64bedb2f63a9a73a2c7083c0808d8c9e24dc89b6690b1477ae9bb6cab" I0507 11:59:34.538435 4590 kubelet_pods.go:906] "Unable to retrieve pull secret, the image pull may not succeed." pod="grafana-apps/loki-datasource-grafana-app-fast-5f845744dd-tpmtm" secret="" err="secret \"dockerhub\" not found" time="2024-05-07T11:59:34.520032214Z" level=info msg="Stop container \"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\" with signal terminated" time="2024-05-07T11:59:34.519591759Z" level=info msg="StopContainer for \"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\" with timeout 30 (s)" I0507 11:59:34.518822 3224 kuberuntime_container.go:745] "Killing container with a grace period" pod="hosted-grafana/hosted-grafana-api-7b6bd9b949-9csb4" podUID="25cb986c-3d6c-4ed0-abf3-ee59ed6175f9" containerName="hgapi" containerID="containerd://c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e" gracePeriod=30 I0507 11:59:34.518597 3224 kubelet.go:2414] "SyncLoop DELETE" source="api" pods=["hosted-grafana/hosted-grafana-api-7b6bd9b949-9csb4"] I0507 11:59:34.501779 4602 kubelet.go:2498] "SyncLoop (probe)" probe="readiness" status="ready" pod="hosted-grafana/k6teststeady4-grafana-5c4f6cd55-hvn6k" I0507 11:59:34.453214 4736 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-nbp9" status="Running" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36336 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36336 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36336 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" E0507 11:59:34.353776 4585 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ErrImagePull: \"[rpc error: code = NotFound desc = failed to pull and unpack image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1\\\": failed to resolve reference \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1\\\": us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1: not found, failed to pull and unpack image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1\\\": failed to resolve reference \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1\\\": unexpected status from HEAD request to https://us.gcr.io/v2/hosted-grafana/hosted-grafana-pro/manifests/11.1.0-ephemeral-6436-938-1: 403 Forbidden]\"" pod="hosted-grafana/ephemeral1180076306436hairyhe-grafana-86cb5688bb-mtgwf" podUID="640ad907-3089-47b9-89c7-4b59e6b685a5" while [ "$(pidof plugins-pause)" = "" ]; do sleep 0.5; done; ln --force -s /proc/$(pidof hgrun-pause)/root/bin/hgrun /bin/hgrun; E0507 11:59:34.353716 4585 kuberuntime_manager.go:1256] container &Container{Name:grafana,Image:us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1,Command:[/bin/sh],Args:[-c set -e; while [ "$(pidof hgrun-pause)" = "" ]; do sleep 0.5; done; exec /bin/hgrun -log.level=debug launch -bundledPluginsManifest /proc/$(pidof plugins-pause)/root/manifest.json -bundledPluginsDir /proc/$(pidof plugins-pause)/root/plugins],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:http-metrics,HostPort:0,ContainerPort:80,Protocol:TCP,HostIP:,},ContainerPort{Name:grpc,HostPort:0,ContainerPort:10000,Protocol:TCP,HostIP:,},ContainerPort{Name:profiling,HostPort:0,ContainerPort:6060,Protocol:TCP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:HG_API,Value:http://hosted-grafana-api,ValueFrom:nil,},EnvVar{Name:HG_INSTANCE_SLUG,Value:ephemeral1180076306436hairyhe,ValueFrom:nil,},EnvVar{Name:HG_INSTANCE_SECRET,Value:67f2b6eeb2d75a30d764feee937887e7f735a612,ValueFrom:nil,},EnvVar{Name:EXTRA_OPTIONS,Value:-profile -profile-port=6060 -profile-addr=0.0.0.0,ValueFrom:nil,},EnvVar{Name:HG_CREATE_TIME_MS,Value:1715081910810,ValueFrom:nil,},EnvVar{Name:HG_PULL_POLICY,Value:Always,ValueFrom:nil,},EnvVar{Name:HG_START_REASON,Value:active,ValueFrom:nil,},EnvVar{Name:HGRUN_SECURE_PLUGINS,Value:false,ValueFrom:nil,},EnvVar{Name:HGRUN_PLUGIN_RUNNER_ROOT_CA,Value:false,ValueFrom:nil,},EnvVar{Name:OTEL_EXPORTER_OTLP_TRACES_ENDPOINT,Value:http://jaeger-agent.jaeger.svc.cluster.local:4317,ValueFrom:nil,},EnvVar{Name:JAEGER_SAMPLER_PARAM,Value:1,ValueFrom:nil,},EnvVar{Name:OTEL_RESOURCE_ATTRIBUTES,Value:cluster=dev-us-central-0,namespace=hosted-grafana,ValueFrom:nil,},EnvVar{Name:HG_PROBE_PATH,Value:/api/health,ValueFrom:nil,},EnvVar{Name:HGRUN_EXIT_ON_PLUGIN_FAIL,Value:true,ValueFrom:nil,},EnvVar{Name:HGRUN_PLUGIN_INSTALL_RETRIES,Value:2,ValueFrom:nil,},EnvVar{Name:HGRUN_PLUGIN_INSTALL_CONCURRENCY,Value:1,ValueFrom:nil,},EnvVar{Name:HGRUN_LAUNCH_TIMEOUT,Value:3m0s,ValueFrom:nil,},EnvVar{Name:GOMEMLIMIT,Value:429496730,ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{memory: {{536870912 0} {} BinarySI},},Requests:ResourceList{cpu: {{26 -3} {} 26m DecimalSI},memory: {{293601280 0} {} BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{},LivenessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/api/health,Port:{0 80 },Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:300,TimeoutSeconds:10,PeriodSeconds:30,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},ReadinessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:&ExecAction{Command:[/bin/hgrun check],},HTTPGet:nil,TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:0,TimeoutSeconds:30,PeriodSeconds:30,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:&Lifecycle{PostStart:nil,PreStop:&LifecycleHandler{Exec:&ExecAction{Command:[/bin/hgrun drain -timeout 1m0s -waitTime 55s],},HTTPGet:nil,TCPSocket:nil,},},TerminationMessagePath:/dev/termination-log,ImagePullPolicy:Always,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[SYS_PTRACE],Drop:[],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:nil,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod ephemeral1180076306436hairyhe-grafana-86cb5688bb-mtgwf_hosted-grafana(640ad907-3089-47b9-89c7-4b59e6b685a5): ErrImagePull: [rpc error: code = NotFound desc = failed to pull and unpack image "us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1": failed to resolve reference "us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1": us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1: not found, failed to pull and unpack image "us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1": failed to resolve reference "us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1": unexpected status from HEAD request to https://us.gcr.io/v2/hosted-grafana/hosted-grafana-pro/manifests/11.1.0-ephemeral-6436-938-1: 403 Forbidden] E0507 11:59:34.353426 4585 remote_image.go:180] "PullImage from image service failed" err="rpc error: code = Unknown desc = failed to pull and unpack image \"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1\": failed to resolve reference \"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1\": unexpected status from HEAD request to https://us.gcr.io/v2/hosted-grafana/hosted-grafana-pro/manifests/11.1.0-ephemeral-6436-938-1: 403 Forbidden" image="us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1" time="2024-05-07T11:59:34.353141479Z" level=info msg="stop pulling image us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1: active requests=0, bytes read=4402" time="2024-05-07T11:59:34.353117156Z" level=error msg="PullImage \"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1\" failed" error="failed to pull and unpack image \"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1\": failed to resolve reference \"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1\": unexpected status from HEAD request to https://us.gcr.io/v2/hosted-grafana/hosted-grafana-pro/manifests/11.1.0-ephemeral-6436-938-1: 403 Forbidden" E0507 11:59:34.183531 4578 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-80141-12952-1\\\"\"" pod="hosted-grafana/ephemeral1511182180141joshhun-grafana-945f746cd-t2dbn" podUID="5a9ab168-41a1-4427-a3c8-7f8eb23f7491" time="2024-05-07T11:59:34.182459543Z" level=info msg="PullImage \"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1\"" E0507 11:59:34.182064 4585 remote_image.go:180] "PullImage from image service failed" err="rpc error: code = NotFound desc = failed to pull and unpack image \"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1\": failed to resolve reference \"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1\": us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1: not found" image="us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1" time="2024-05-07T11:59:34.181810210Z" level=info msg="stop pulling image us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1: active requests=0, bytes read=6802" time="2024-05-07T11:59:34.181742330Z" level=error msg="PullImage \"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1\" failed" error="rpc error: code = NotFound desc = failed to pull and unpack image \"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1\": failed to resolve reference \"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1\": us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1: not found" I0507 11:59:34.181632 2791 kubelet.go:2421] "SyncLoop (PLEG): event for pod" pod="hosted-grafana/victor-grafana-7b7bb568cc-grflq" event={"ID":"1803645b-5526-41b4-bf88-271be4827277","Type":"ContainerStarted","Data":"8a23f9c88be86595adf482aed0c24902ce397f7b5cf55c300e9aa851a5717a0d"} time="2024-05-07T11:59:34.180182414Z" level=info msg="trying next host - response was http.StatusNotFound" host=us.gcr.io AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36334 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36334 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36334 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" I0507 11:59:34.107464 4767 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-m294" status="Running" E0507 11:59:34.041990 4589 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=securityops-grafana-54d8cdd94c-lcvh2_hosted-grafana(ae03848f-326b-447b-965c-2e247587fef2)\"" pod="hosted-grafana/securityops-grafana-54d8cdd94c-lcvh2" podUID="ae03848f-326b-447b-965c-2e247587fef2" I0507 11:59:34.041393 4589 scope.go:117] "RemoveContainer" containerID="c92811a8440f91cd6b065d841e3d3c8ef00c139b1754d41629933446094f0e5b" E0507 11:59:34.035350 2952 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=edwardtest-grafana-7c84958c48-7qx52_hosted-grafana(b2e4a53a-8d70-4a70-8d03-87fd797e5cab)\"" pod="hosted-grafana/edwardtest-grafana-7c84958c48-7qx52" podUID="b2e4a53a-8d70-4a70-8d03-87fd797e5cab" time="2024-05-07T11:59:34.034812075Z" level=info msg="PullImage \"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6436-938-1\"" I0507 11:59:34.034307 2952 scope.go:117] "RemoveContainer" containerID="731add8ea0b2e9fdf09aebec6431636580ef50b216e74393d429b5ab92e597b4" I0507 11:59:34.033652 6250 kubelet_pods.go:906] "Unable to retrieve pull secret, the image pull may not succeed." pod="kafka/kafka-broker-1" secret="" err="secret \"gcr\" not found" I0507 11:59:33.939013 4589 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-hg-n2s8-6-1dd39c-3bfd06e9-r8v7" status="Running" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36332 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36332 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36332 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" I0507 11:59:33.849663 4730 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-9zxx" status="Running" E0507 11:59:33.830567 4591 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6432-916-1\\\"\"" pod="hosted-grafana/ephemeral1180076306432stephan-grafana-6795bb9d45-sxvzw" podUID="9ed9522d-b242-40b1-907b-8c8e4122c711" E0507 11:59:33.830166 4591 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=oncalldev-grafana-88bf96b75-tfwt2_hosted-grafana(457849b1-2806-4b8c-af1b-ba17047f5234)\"" pod="hosted-grafana/oncalldev-grafana-88bf96b75-tfwt2" podUID="457849b1-2806-4b8c-af1b-ba17047f5234" I0507 11:59:33.829279 4591 scope.go:117] "RemoveContainer" containerID="9f599128c19622f3ecba55323008ef75bfddf1f84fef95a77368a4e46d0ff1f1" I0507 11:59:33.812487 4733 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-lgmg" status="Running" I0507 11:59:33.738098 4624 kubelet.go:2498] "SyncLoop (probe)" probe="readiness" status="ready" pod="hosted-grafana/ephemeral1511182177076papagia-grafana-6b9bb47584-xp7pp" E0507 11:59:33.737125 4624 prober.go:239] "Unable to write all bytes from execInContainer" err="short write" expectedBytes=11846 actualBytes=10240 I0507 11:59:33.699298 4772 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-8nfg" status="Running" time="2024-05-07T11:59:33.674127305Z" level=info msg="StartContainer for \"8a23f9c88be86595adf482aed0c24902ce397f7b5cf55c300e9aa851a5717a0d\" returns successfully" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36330 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36330 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36330 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" Started libcontainer container 8a23f9c88be86595adf482aed0c24902ce397f7b5cf55c300e9aa851a5717a0d. I0507 11:59:33.596434 4737 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-cndv" status="Running" time="2024-05-07T11:59:33.590854700Z" level=info msg="StartContainer for \"8a23f9c88be86595adf482aed0c24902ce397f7b5cf55c300e9aa851a5717a0d\"" time="2024-05-07T11:59:33.590318994Z" level=info msg="CreateContainer within sandbox \"346dc5e0c503a6ca1f0281e3b6f8e32563ebf8a61ed467b4d0dc5b4030a115b9\" for &ContainerMetadata{Name:grafana,Attempt:0,} returns container id \"8a23f9c88be86595adf482aed0c24902ce397f7b5cf55c300e9aa851a5717a0d\"" time="2024-05-07T11:59:33.582971705Z" level=info msg="CreateContainer within sandbox \"346dc5e0c503a6ca1f0281e3b6f8e32563ebf8a61ed467b4d0dc5b4030a115b9\" for container &ContainerMetadata{Name:grafana,Attempt:0,}" time="2024-05-07T11:59:33.581670690Z" level=info msg="PullImage \"us.gcr.io/hosted-grafana/hgrun:0.1.452\"" I0507 11:59:33.580418 2791 azure_credentials.go:220] image(us.gcr.io/hosted-grafana/hgrun) is not from ACR, return empty authentication time="2024-05-07T11:59:33.579768267Z" level=info msg="PullImage \"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-70397\" returns image reference \"sha256:0036b00b52fc547c944c1c820817d91fba6e20775cbf4e6c3e09ad2e682dbd73\"" time="2024-05-07T11:59:33.579726466Z" level=info msg="Pulled image \"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-70397\" with image id \"sha256:0036b00b52fc547c944c1c820817d91fba6e20775cbf4e6c3e09ad2e682dbd73\", repo tag \"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-70397\", repo digest \"us.gcr.io/hosted-grafana/hosted-grafana-pro@sha256:0853965a142fb95648de3281a7c71de0d05fb51616bc32b523dc2f1da6ca06dc\", size \"173405048\" in 1.763700178s" time="2024-05-07T11:59:33.579013658Z" level=info msg="ImageUpdate event name:\"us.gcr.io/hosted-grafana/hosted-grafana-pro@sha256:0853965a142fb95648de3281a7c71de0d05fb51616bc32b523dc2f1da6ca06dc\" labels:{key:\"io.cri-containerd.image\" value:\"managed\"}" time="2024-05-07T11:59:33.577566840Z" level=info msg="ImageUpdate event name:\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-70397\" labels:{key:\"io.cri-containerd.image\" value:\"managed\"}" time="2024-05-07T11:59:33.576106723Z" level=info msg="ImageUpdate event name:\"sha256:0036b00b52fc547c944c1c820817d91fba6e20775cbf4e6c3e09ad2e682dbd73\" labels:{key:\"io.cri-containerd.image\" value:\"managed\"}" time="2024-05-07T11:59:33.575284113Z" level=info msg="stop pulling image us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-70397: active requests=0, bytes read=6802" time="2024-05-07T11:59:33.575105610Z" level=info msg="ImageUpdate event name:\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-70397\" labels:{key:\"io.cri-containerd.image\" value:\"managed\"}" E0507 11:59:33.570763 4595 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77506-8314-2\\\"\"" pod="hosted-grafana/ephemeral1511182177506ashharr-grafana-6898b9c678-h5xpg" podUID="4a9da7ba-6a16-487f-9604-4e89cbd0c918" E0507 11:59:33.538814 4590 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-enterprise-6372-855-1\\\"\"" pod="hosted-grafana/ephemeral1180076306372jacobso-grafana-586dcfb48b-8v6t8" podUID="656b3334-568b-408d-9642-268560046977" I0507 11:59:33.457749 4601 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-hg-n2s8-6-1dd39c-3bfd06e9-n9zk" status="Running" I0507 11:59:33.431432 4645 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-spot-n2s8-0-1dd3-f81338c4-cz4q" status="Running" I0507 11:59:33.422254 1537502 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-x28r" status="Running" audit: type=1400 audit(1715083173.383:28): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36297 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36297 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36297 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36297 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" I0507 11:59:33.376763 4608 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-hg-n2s8-6-1dd39c-3bfd06e9-h7jr" status="Running" E0507 11:59:33.361757 4600 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=benchloadtestingxxl2-grafana-56b7c6b6df-h67p4_hosted-grafana(a3a055b2-b554-4a41-ab79-6f070495296f)\"" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-h67p4" podUID="a3a055b2-b554-4a41-ab79-6f070495296f" I0507 11:59:33.360932 4600 scope.go:117] "RemoveContainer" containerID="d4402a99374d43ba1e3df7dffe05b2fee099671c2e0f07bb729186a41bb549b6" I0507 11:59:33.356171 4645 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-spot-n2s8-0-1dd3-f81338c4-z2rp" status="Running" I0507 11:59:33.219742 4739 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-vlg7" status="Running" E0507 11:59:33.183476 4578 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=ephemeral1511182177667ryantxu-grafana-548f6c9689-4gxks_hosted-grafana(27ef7bb4-75ed-4cd9-9b96-b76cb07bee6d)\"" pod="hosted-grafana/ephemeral1511182177667ryantxu-grafana-548f6c9689-4gxks" podUID="27ef7bb4-75ed-4cd9-9b96-b76cb07bee6d" I0507 11:59:33.182939 4578 scope.go:117] "RemoveContainer" containerID="5ba7a32d2cc7cc82e8a982949ae158d1beb142061bddf3e9f69c2637ea65b1b0" time="2024-05-07T11:59:33.095406969Z" level=info msg="RemoveContainer for \"15651d1ecdf9bd928944f9cfb0523042b518137a440bb97f7d75923beaede053\" returns successfully" E0507 11:59:33.090176 4592 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=benchloadtestingxxl2-grafana-56b7c6b6df-vqkzm_hosted-grafana(d3742b42-2b35-4c32-8267-7cf79bbcb441)\"" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-vqkzm" podUID="d3742b42-2b35-4c32-8267-7cf79bbcb441" time="2024-05-07T11:59:33.090002575Z" level=info msg="RemoveContainer for \"15651d1ecdf9bd928944f9cfb0523042b518137a440bb97f7d75923beaede053\"" I0507 11:59:33.089270 4592 scope.go:117] "RemoveContainer" containerID="d53fbb23caf1e92d73b7ccf3a991c2ccd1d1b1ef072bfb1f6798a781d3809679" I0507 11:59:33.088811 4592 scope.go:117] "RemoveContainer" containerID="15651d1ecdf9bd928944f9cfb0523042b518137a440bb97f7d75923beaede053" I0507 11:59:33.088779 4592 kubelet.go:2426] "SyncLoop (PLEG): event for pod" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-vqkzm" event={"ID":"d3742b42-2b35-4c32-8267-7cf79bbcb441","Type":"ContainerDied","Data":"d53fbb23caf1e92d73b7ccf3a991c2ccd1d1b1ef072bfb1f6798a781d3809679"} I0507 11:59:33.088730 4592 generic.go:334] "Generic (PLEG): container finished" podID="d3742b42-2b35-4c32-8267-7cf79bbcb441" containerID="d53fbb23caf1e92d73b7ccf3a991c2ccd1d1b1ef072bfb1f6798a781d3809679" exitCode=1 I0507 11:59:33.063394 4734 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-dc4k" status="Running" E0507 11:59:33.042338 4589 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.0.2\\\"\"" pod="hosted-grafana/grafana10-grafana-6c9c7c5bc4-4f2gp" podUID="df8fc793-cd73-4984-8c12-9c4d527ff219" I0507 11:59:33.039844 4732 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-9ghc" status="Running" I0507 11:59:32.993425 4640 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-spot-n2s8-0-1dd3-f81338c4-bkfl" status="Running" ll header: 00000000: 42 01 0a 80 00 7c 42 01 0a 80 00 01 08 00 IPv4: martian source 10.132.101.99 from 10.132.101.62, on dev eth0 I0507 11:59:32.964140 4707 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-cache-n2hc8-1-1d-61155fd9-4qc8" status="Running" I0507 11:59:32.940823 4739 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-p4tv" status="Running" audit: type=1400 audit(1715083172.883:27): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36286 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083172.883:26): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36286 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083172.883:25): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36286 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36286 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36286 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36286 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" E0507 11:59:32.873442 3304 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"gcom-sync\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/kubernetes-dev/frontend-monitoring:6a8eb5a\\\"\"" pod="faro/update-usage-28487090-xg5bt" podUID="6e8f7589-7d91-47e6-9128-7ec922779773" E0507 11:59:32.830094 4591 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=k6testslow4-grafana-b5879497c-p4zx9_hosted-grafana(7458c262-83d7-4b7b-b8d4-f95db30c3e39)\"" pod="hosted-grafana/k6testslow4-grafana-b5879497c-p4zx9" podUID="7458c262-83d7-4b7b-b8d4-f95db30c3e39" I0507 11:59:32.829402 4591 scope.go:117] "RemoveContainer" containerID="fbad2b736b62c12c779231631d3eb82fde86d6095a21d2982d457c6801be9293" I0507 11:59:32.776172 4647 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-spot-n2s8-0-1dd3-f81338c4-gmgr" status="Running" time="2024-05-07T11:59:32.755926053Z" level=info msg="CreateContainer within sandbox \"81e019a0248a0300a328fd59f9939c3eaa1b98aa7f325a7f6e00592633275ef6\" for container &ContainerMetadata{Name:checkoutservice,Attempt:3417,}" I0507 11:59:32.739402 4527 scope.go:117] "RemoveContainer" containerID="cdcb0619adb8e55d353b1a804a08de63dd2991fcbb1799d499e675ecae655a8e" E0507 11:59:32.724973 4592 prober.go:104] "Probe errored" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task d53fbb23caf1e92d73b7ccf3a991c2ccd1d1b1ef072bfb1f6798a781d3809679 not found: not found" probeType="Readiness" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-vqkzm" podUID="d3742b42-2b35-4c32-8267-7cf79bbcb441" containerName="grafana" E0507 11:59:32.724932 4592 remote_runtime.go:496] "ExecSync cmd from runtime service failed" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task d53fbb23caf1e92d73b7ccf3a991c2ccd1d1b1ef072bfb1f6798a781d3809679 not found: not found" containerID="d53fbb23caf1e92d73b7ccf3a991c2ccd1d1b1ef072bfb1f6798a781d3809679" cmd=["/bin/hgrun","check"] E0507 11:59:32.724328 4592 remote_runtime.go:496] "ExecSync cmd from runtime service failed" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task d53fbb23caf1e92d73b7ccf3a991c2ccd1d1b1ef072bfb1f6798a781d3809679 not found: not found" containerID="d53fbb23caf1e92d73b7ccf3a991c2ccd1d1b1ef072bfb1f6798a781d3809679" cmd=["/bin/hgrun","check"] time="2024-05-07T11:59:32.724767806Z" level=error msg="ExecSync for \"d53fbb23caf1e92d73b7ccf3a991c2ccd1d1b1ef072bfb1f6798a781d3809679\" failed" error="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task d53fbb23caf1e92d73b7ccf3a991c2ccd1d1b1ef072bfb1f6798a781d3809679 not found: not found" time="2024-05-07T11:59:32.724147410Z" level=error msg="ExecSync for \"d53fbb23caf1e92d73b7ccf3a991c2ccd1d1b1ef072bfb1f6798a781d3809679\" failed" error="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task d53fbb23caf1e92d73b7ccf3a991c2ccd1d1b1ef072bfb1f6798a781d3809679 not found: not found" time="2024-05-07T11:59:32.723400665Z" level=error msg="ExecSync for \"d53fbb23caf1e92d73b7ccf3a991c2ccd1d1b1ef072bfb1f6798a781d3809679\" failed" error="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task d53fbb23caf1e92d73b7ccf3a991c2ccd1d1b1ef072bfb1f6798a781d3809679 not found: not found" > E0507 11:59:32.723675 4592 remote_runtime.go:496] "ExecSync cmd from runtime service failed" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task d53fbb23caf1e92d73b7ccf3a991c2ccd1d1b1ef072bfb1f6798a781d3809679 not found: not found" containerID="d53fbb23caf1e92d73b7ccf3a991c2ccd1d1b1ef072bfb1f6798a781d3809679" cmd=["/bin/hgrun","check"] ts=2024-05-07T11:59:30.205006192Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:28.791795005Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:27.963570749Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:27.061148162Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:26.707407028Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:25.328552026Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:25.041749011Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:24.342974853Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:23.448651822Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:23.116820392Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:22.538898748Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:21.719913728Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:21.1942729Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:20.302909822Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health I0507 11:59:32.722792 4592 prober.go:107] "Probe failed" probeType="Readiness" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-vqkzm" podUID="d3742b42-2b35-4c32-8267-7cf79bbcb441" containerName="grafana" probeResult="failure" output=< ts=2024-05-07T11:59:19.71203793Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:19.309394819Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:18.886442843Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:18.042358889Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:17.766602037Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:17.126424933Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:16.490369589Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:16.198616007Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:15.662620401Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:15.507043882Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:15.45251028Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:15.003702316Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:14.70689258Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:14.669296859Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:14.604778237Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:14.397222806Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:14.240589331Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:14.137013019Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:14.118516695Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health ts=2024-05-07T11:59:14.087202317Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health time="2024-05-07T11:59:32.722555689Z" level=warning msg="cleaning up after shim disconnected" id=d53fbb23caf1e92d73b7ccf3a991c2ccd1d1b1ef072bfb1f6798a781d3809679 namespace=k8s.io time="2024-05-07T11:59:32.722569380Z" level=info msg="cleaning up dead shim" namespace=k8s.io time="2024-05-07T11:59:32.722469678Z" level=info msg="shim disconnected" id=d53fbb23caf1e92d73b7ccf3a991c2ccd1d1b1ef072bfb1f6798a781d3809679 namespace=k8s.io I0507 11:59:32.710370 4646 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-spot-n2s8-0-1dd3-f81338c4-7chh" status="Running" E0507 11:59:32.704958 4602 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77487-8287-1\\\"\"" pod="hosted-grafana/ephemeral1511182177487torkelo-grafana-745789578d-jmj9h" podUID="144f91fd-76a2-4ca1-9e14-ba65fe8113da" I0507 11:59:32.619794 4729 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-6vzp" status="Running" E0507 11:59:32.570199 4595 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 1m20s restarting failed container=grafana pod=ltest-grafana-74d66c7766-s2ftd_hosted-grafana(40e776a7-78a1-44d8-95ac-0c4944bb7737)\"" pod="hosted-grafana/ltest-grafana-74d66c7766-s2ftd" podUID="40e776a7-78a1-44d8-95ac-0c4944bb7737" I0507 11:59:32.569510 4595 scope.go:117] "RemoveContainer" containerID="ea831b36e1cf141ea84a1158e1ac08c42bfe6220a73e5f2074dea1d25e9c8619" E0507 11:59:32.562163 4739 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"pdc\" with ErrImageNeverPull: \"Container image \\\"us.gcr.io/hosted-grafana/pdc:0.1.415\\\" is not present with pull policy of Never\"" pod="pdc/private-datasource-connect-564fb6cfbb-5k5n7" podUID="5ce47ae4-8558-422b-8246-7733512eeb96" E0507 11:59:32.562098 4739 kuberuntime_manager.go:1256] container &Container{Name:pdc,Image:us.gcr.io/hosted-grafana/pdc:0.1.415,Command:[],Args:[-proxy.auth.ca-keys-dir=/var/run/secrets/pdc-certs -proxy.socks-server.addr=:10443 -proxy.ssh-server.addr=:2222 -proxy.use-socks-username-for-routing -proxy.api.http-address=:9182 -proxy.check-connpool-address-in-ring -memberlist.join=dns+gossip-ring.pdc.svc.cluster.local:7946 -api.http-address=:11443 -distributor.enabled=true -distributor.addr=:10444 -distributor.use-socks-username-for-routing -gateway.enabled=true -gateway.addr=:2244 -log.level=debug -certs.ca-private-key-file=/var/run/secrets/pdc-certs/ca.key -certs.ca-cert-file=/var/run/secrets/pdc-certs/ca.crt -certs.ca-pub-file=/var/run/secrets/pdc-certs/ca.pub -certs.cluster=local-k8s -shard-size=3 -graceful-shutdown-period=30s -enable-multiple-networks],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:socks,HostPort:0,ContainerPort:10443,Protocol:TCP,HostIP:,},ContainerPort{Name:ssh,HostPort:0,ContainerPort:2222,Protocol:TCP,HostIP:,},ContainerPort{Name:distributor,HostPort:0,ContainerPort:10444,Protocol:TCP,HostIP:,},ContainerPort{Name:gateway,HostPort:0,ContainerPort:2244,Protocol:TCP,HostIP:,},ContainerPort{Name:api,HostPort:0,ContainerPort:11443,Protocol:TCP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:POD_NAME,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.name,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},},Resources:ResourceRequirements{Limits:ResourceList{cpu: {{500 -3} {} 500m DecimalSI},memory: {{134217728 0} {} BinarySI},},Requests:ResourceList{cpu: {{250 -3} {} 250m DecimalSI},memory: {{67108864 0} {} BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:pdc-certs,ReadOnly:true,MountPath:/var/run/secrets/pdc-certs,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-fcx2w,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/ready,Port:{0 11443 },Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:40,TimeoutSeconds:1,PeriodSeconds:5,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:&Lifecycle{PostStart:nil,PreStop:&LifecycleHandler{Exec:&ExecAction{Command:[/bin/sleep 5],},HTTPGet:nil,TCPSocket:nil,},},TerminationMessagePath:/dev/termination-log,ImagePullPolicy:Never,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod private-datasource-connect-564fb6cfbb-5k5n7_pdc(5ce47ae4-8558-422b-8246-7733512eeb96): ErrImageNeverPull: Container image "us.gcr.io/hosted-grafana/pdc:0.1.415" is not present with pull policy of Never ll header: 00000000: 42 01 0a 80 00 17 42 01 0a 80 00 01 08 00 IPv4: martian source 10.132.141.91 from 10.132.141.80, on dev eth0 I0507 11:59:32.426569 4734 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-dkbf" status="Running" I0507 11:59:32.409568 581823 cache.go:40] re-using cached key and certificate I0507 11:59:32.405906 4578 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-hg-n2s8-6-1dd39c-3bfd06e9-5nj8" status="Running" audit: type=1400 audit(1715083172.379:24): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36276 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083172.379:23): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36276 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083172.379:22): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36276 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36276 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36276 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36276 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" I0507 11:59:32.358966 4732 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-hmlg" status="Running" I0507 11:59:32.223604 4646 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-spot-n2s8-0-1dd3-f81338c4-hzp5" status="Running" I0507 11:59:32.173852 2791 kubelet.go:2421] "SyncLoop (PLEG): event for pod" pod="hosted-grafana/dafdeveuwest2-grafana-546fbd789d-czx47" event={"ID":"fc6ba4ea-9950-4999-8ad2-bdc9a577fb34","Type":"ContainerStarted","Data":"7418e5784964048801b0cb8abacd0a73f4a208454fc6f5418e4f79906761c98d"} E0507 11:59:32.151882 4572 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=e2cmigrationreceiver-grafana-6b9cf7d5c6-wsxm5_hosted-grafana(3f83ab4b-f255-4a22-a690-9f1e9b086226)\"" pod="hosted-grafana/e2cmigrationreceiver-grafana-6b9cf7d5c6-wsxm5" podUID="3f83ab4b-f255-4a22-a690-9f1e9b086226" I0507 11:59:32.151224 4572 scope.go:117] "RemoveContainer" containerID="0339d9823b658e820b2d0535c744d982f15c63ca658cdaa9f690efd7dffdbf8c" I0507 11:59:32.131299 4736 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-gwtz" status="Running" I0507 11:59:32.076846 4726 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-n56x" status="Running" I0507 11:59:32.057206 4726 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-7h6b" status="Running" E0507 11:59:31.990062 2791 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"goldpinger\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=goldpinger pod=goldpinger-69c2w_goldpinger(13276978-61bf-463a-b871-d3b5a2562070)\"" pod="goldpinger/goldpinger-69c2w" podUID="13276978-61bf-463a-b871-d3b5a2562070" I0507 11:59:31.989696 2791 scope.go:117] "RemoveContainer" containerID="188fa7a825c7d671b5c324a4e63725c3039f85bda51fb56794e12823e6d07729" E0507 11:59:31.928148 4734 pod_workers.go:1300] "Error syncing pod, skipping" err="unmounted volumes=[terraform-drift-detector-data], unattached volumes=[terraform-drift-detector-data], failed to process volumes=[]: context deadline exceeded" pod="terraform-drift-detector/terraform-drift-detector-d68b4c545-jg2vj" podUID="6c607496-ef26-454e-b2f2-4cb75b233fa3" E0507 11:59:31.923713 4643 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=benchloadtestingxxl2-grafana-56b7c6b6df-j7jh6_hosted-grafana(83fb0f38-728e-4050-9500-6ac9fc9f21c8)\"" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-j7jh6" podUID="83fb0f38-728e-4050-9500-6ac9fc9f21c8" I0507 11:59:31.923176 4643 scope.go:117] "RemoveContainer" containerID="a85b6a771be0a2165463617e0c7a4f5b42dbb5c232c57166f32a72d969a25bf1" E0507 11:59:31.887809 4597 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.0.2\\\"\"" pod="hosted-grafana/johangrafana10-grafana-69c6449bbd-k2bgp" podUID="bb953c26-c201-4082-9b56-85ab12c1d0e1" E0507 11:59:31.886415 4597 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=k6testslow6-grafana-65f9d6559b-xrs26_hosted-grafana(5c426b18-2b38-44ce-a92e-e5eeadbbb6f0)\"" pod="hosted-grafana/k6testslow6-grafana-65f9d6559b-xrs26" podUID="5c426b18-2b38-44ce-a92e-e5eeadbbb6f0" I0507 11:59:31.885717 4597 scope.go:117] "RemoveContainer" containerID="107669a02b8d89f8f768181e2b8d64c839a1161c10d217fb0c3a2701beb32b72" time="2024-05-07T11:59:31.883468402Z" level=info msg="StartContainer for \"7418e5784964048801b0cb8abacd0a73f4a208454fc6f5418e4f79906761c98d\" returns successfully" audit: type=1400 audit(1715083171.875:21): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36274 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083171.875:20): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36274 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" audit: type=1400 audit(1715083171.875:19): apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36274 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36274 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36274 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=36274 comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined" I0507 11:59:31.879715 4734 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-xwpk" status="Running" Started libcontainer container 7418e5784964048801b0cb8abacd0a73f4a208454fc6f5418e4f79906761c98d. time="2024-05-07T11:59:31.833222796Z" level=info msg="StartContainer for \"7418e5784964048801b0cb8abacd0a73f4a208454fc6f5418e4f79906761c98d\"" time="2024-05-07T11:59:31.832712390Z" level=info msg="CreateContainer within sandbox \"ac0defb47ab561e39c01453f80823086daf554758865a65d1cb608092c1539d5\" for &ContainerMetadata{Name:grafana,Attempt:0,} returns container id \"7418e5784964048801b0cb8abacd0a73f4a208454fc6f5418e4f79906761c98d\"" time="2024-05-07T11:59:31.818485118Z" level=info msg="CreateContainer within sandbox \"ac0defb47ab561e39c01453f80823086daf554758865a65d1cb608092c1539d5\" for container &ContainerMetadata{Name:grafana,Attempt:0,}" time="2024-05-07T11:59:31.815983488Z" level=info msg="PullImage \"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-70397\"" I0507 11:59:31.815514 2791 azure_credentials.go:220] image(us.gcr.io/hosted-grafana/hosted-grafana-pro) is not from ACR, return empty authentication time="2024-05-07T11:59:31.814769473Z" level=info msg="PullImage \"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-70397\" returns image reference \"sha256:0036b00b52fc547c944c1c820817d91fba6e20775cbf4e6c3e09ad2e682dbd73\"" time="2024-05-07T11:59:31.814727873Z" level=info msg="Pulled image \"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-70397\" with image id \"sha256:0036b00b52fc547c944c1c820817d91fba6e20775cbf4e6c3e09ad2e682dbd73\", repo tag \"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-70397\", repo digest \"us.gcr.io/hosted-grafana/hosted-grafana-pro@sha256:0853965a142fb95648de3281a7c71de0d05fb51616bc32b523dc2f1da6ca06dc\", size \"173405048\" in 14.680303992s" time="2024-05-07T11:59:31.813758661Z" level=info msg="ImageCreate event name:\"us.gcr.io/hosted-grafana/hosted-grafana-pro@sha256:0853965a142fb95648de3281a7c71de0d05fb51616bc32b523dc2f1da6ca06dc\" labels:{key:\"io.cri-containerd.image\" value:\"managed\"}" time="2024-05-07T11:59:31.811392832Z" level=info msg="ImageUpdate event name:\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-70397\" labels:{key:\"io.cri-containerd.image\" value:\"managed\"}" time="2024-05-07T11:59:31.809798213Z" level=info msg="ImageCreate event name:\"sha256:0036b00b52fc547c944c1c820817d91fba6e20775cbf4e6c3e09ad2e682dbd73\" labels:{key:\"io.cri-containerd.image\" value:\"managed\"}" time="2024-05-07T11:59:31.808692900Z" level=info msg="stop pulling image us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-70397: active requests=0, bytes read=173418678" time="2024-05-07T11:59:31.807327183Z" level=info msg="ImageCreate event name:\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-70397\" labels:{key:\"io.cri-containerd.image\" value:\"managed\"}" I0507 11:59:31.729843 4735 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-m7wp" status="Running" var-lib-containerd-tmpmounts-containerd\x2dmount4071253084.mount: Deactivated successfully. E0507 11:59:31.705263 4602 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=benchloadtestingxxl2-grafana-56b7c6b6df-4gclf_hosted-grafana(fe493f66-8d1f-4435-9208-0304fd499ee1)\"" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-4gclf" podUID="fe493f66-8d1f-4435-9208-0304fd499ee1" I0507 11:59:31.704465 4602 scope.go:117] "RemoveContainer" containerID="2773338620ccfb32536d17788865e6fd4c7de7250ab31a7922195ffc1387ee5f" I0507 11:59:31.624527 4600 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-hg-n2s8-6-1dd39c-3bfd06e9-v7l7" status="Running" I0507 11:59:31.620358 6247 prober.go:107] "Probe failed" probeType="Readiness" pod="grafana-agent/grafana-agent-helm-4" podUID="c36c5200-1cd6-4093-893c-c022f91af996" containerName="grafana-agent" probeResult="failure" output="Get \"http://10.0.99.125:3090/-/ready\": dial tcp 10.0.99.125:3090: connect: connection refused" I0507 11:59:31.619462 4733 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-dd5b" status="Running" I0507 11:59:31.617463 4733 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-dd5b" status="Running" E0507 11:59:31.554203 4531 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"frontend\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=frontend pod=otel-demo-alt-dev-frontend-79ccf98858-mbj4x_otel-demo-alt(d08e620e-00d0-49f1-a195-820a62e8de8f)\"" pod="otel-demo-alt/otel-demo-alt-dev-frontend-79ccf98858-mbj4x" podUID="d08e620e-00d0-49f1-a195-820a62e8de8f" I0507 11:59:31.553381 4531 scope.go:117] "RemoveContainer" containerID="30500dc79eb03686dd9399cf180582d080070e4a1e9445f98eea7d7867b7bc3d" run-containerd-io.containerd.runtime.v2.task-k8s.io-d53fbb23caf1e92d73b7ccf3a991c2ccd1d1b1ef072bfb1f6798a781d3809679-rootfs.mount: Deactivated successfully. cri-containerd-d53fbb23caf1e92d73b7ccf3a991c2ccd1d1b1ef072bfb1f6798a781d3809679.scope: Consumed 18.147s CPU time. cri-containerd-d53fbb23caf1e92d73b7ccf3a991c2ccd1d1b1ef072bfb1f6798a781d3809679.scope: Deactivated successfully. E0507 11:59:31.468693 3315 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"gcom-sync\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/kubernetes-dev/frontend-monitoring:6a8eb5a\\\"\"" pod="faro/update-usage-28487080-9sqzn" podUID="2cc85139-2f31-44ae-a308-3dc0df893592" E0507 11:59:31.363226 4601 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6439-933-1\\\"\"" pod="hosted-grafana/ephemeral1180076306439dafyddt-grafana-7bcdd45ddc-l5xtr" podUID="57291357-8942-4110-8df1-c23b055d53d6" I0507 11:59:31.298370 4772 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-2dqk" status="Running" I0507 11:59:31.194140 4733 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-lgmg" status="Running" XMT: Solicit on eth0, interval 117900ms. I0507 11:59:31.152952 4764 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-1-main-n2s16-1-1dd-97837cc3-k49c" status="Running" I0507 11:59:31.141456 1970964 cache.go:40] re-using cached key and certificate I0507 11:59:31.014439 4730 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-d88q" status="Running" RCV: Reply message on eth0 from fe80::e9:7eff:fedf:3d37. XMT: Renew on eth0, interval 9700ms. PRC: Renewing lease on eth0. E0507 11:59:30.965946 4731 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"overrides-exporter\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/kubernetes-dev/enterprise-logs:callum-shard-firstlast-08\\\"\"" pod="loki-dev-010/overrides-exporter-98c77fd66-6zj6m" podUID="1ff5bf3e-5856-4f6f-ae04-273f2dee170b" I0507 11:59:30.936319 4607 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-eu-west-3-main-n2s8-1-1dd39c-d1c92061-mzxx" status="Running" E0507 11:59:30.925932 4733 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=bryan-prometheus-0_bryan-prometheus(6dadfe71-eb19-4231-a96e-c64bb5499a1e)\"" pod="bryan-prometheus/bryan-prometheus-0" podUID="6dadfe71-eb19-4231-a96e-c64bb5499a1e" I0507 11:59:30.925416 4733 scope.go:117] "RemoveContainer" containerID="f0f5ac8b5f4dba0a416c838dd7ccfa903bd1ca22e36ebc4d98a29b4e646063c6" I0507 11:59:30.908672 4724 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-j5wp" status="Running" I0507 11:59:30.893285 4737 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-h9sj" status="Running" E0507 11:59:30.886609 4597 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=ephemeral1511182177667ryantxu-grafana-75c5c488b7-4lj5v_hosted-grafana(44a19bfe-3d16-48c2-ad37-08eb74fc6637)\"" pod="hosted-grafana/ephemeral1511182177667ryantxu-grafana-75c5c488b7-4lj5v" podUID="44a19bfe-3d16-48c2-ad37-08eb74fc6637" I0507 11:59:30.885963 4597 scope.go:117] "RemoveContainer" containerID="004f450ab68ac54937e0695bf2ff318d6219fb3fc4afe1b7fae7346c7f7f962d" I0507 11:59:30.862910 4609 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-eu-west-3-main-n2s8-1-1dd39c-d1c92061-kpwx" status="Running" E0507 11:59:30.829525 4591 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=k6teststeady5-grafana-68bd494c65-2q4h8_hosted-grafana(18b85a56-363f-4d78-aef4-541eb20108bf)\"" pod="hosted-grafana/k6teststeady5-grafana-68bd494c65-2q4h8" podUID="18b85a56-363f-4d78-aef4-541eb20108bf" I0507 11:59:30.828924 4591 scope.go:117] "RemoveContainer" containerID="c337186d90d7c7bc46e7ddfed3c4831486b74fa243b590c20dd29bb87bb7b93b" I0507 11:59:30.770697 4739 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-78dr" status="Running" I0507 11:59:30.762441 4773 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-hcwk" status="Running" E0507 11:59:30.604771 4586 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=dev05devuseast0test-grafana-6cb68b9788-v8dgd_hosted-grafana(59ef7574-134f-4888-826e-9a22062f29f8)\"" pod="hosted-grafana/dev05devuseast0test-grafana-6cb68b9788-v8dgd" podUID="59ef7574-134f-4888-826e-9a22062f29f8" I0507 11:59:30.604104 4586 scope.go:117] "RemoveContainer" containerID="c1992a17a0b5dc3d80080fcc1602d9481f2b4259ab708628828de7f34211f199" E0507 11:59:30.570213 4595 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=ephemeral1180076306267marefr-grafana-78764cf8d9-vpztz_hosted-grafana(a78403f0-4ce8-4320-9df1-0d15e427b4a1)\"" pod="hosted-grafana/ephemeral1180076306267marefr-grafana-78764cf8d9-vpztz" podUID="a78403f0-4ce8-4320-9df1-0d15e427b4a1" I0507 11:59:30.569190 4595 scope.go:117] "RemoveContainer" containerID="0b227353407956e7e3fcf7752ca1eec752856d7e36ca37dcc004e2cc7a749079" ll header: 00000000: 42 01 0a 80 00 17 42 01 0a 80 00 01 08 00 IPv4: martian source 10.132.141.91 from 10.132.141.80, on dev eth0 I0507 11:59:30.367585 4601 kubelet_volumes.go:161] "Cleaned up orphaned pod volumes dir" podUID="10bdda8a-7f0b-466e-9c81-045fb5150dc4" path="/var/lib/kubelet/pods/10bdda8a-7f0b-466e-9c81-045fb5150dc4/volumes" E0507 11:59:30.363662 4601 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-85282-20418-1\\\"\"" pod="hosted-grafana/ephemeral1511182185282svenner-grafana-6f6b6f4d85-9xlcc" podUID="fee4a5b2-d22d-4d80-8041-8796a997679a" XMT: Solicit on eth0, interval 130040ms. I0507 11:59:30.304754 4779 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-nx8q" status="Running" I0507 11:59:30.200842 4769 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-kvxj" status="Running" I0507 11:59:30.198170 4748 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-databenchloki-n2-8c6b6266-2tvt" status="Running" E0507 11:59:30.152725 4572 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-enterprise-6372-855-1\\\"\"" pod="hosted-grafana/ephemeral1180076306372jacobso-grafana-7f66f49b8d-kzhxd" podUID="7ac84154-783b-4672-b865-f728da592129" E0507 11:59:30.092903 4724 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"cortex-gw\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=cortex-gw pod=cortex-gw-6f7f764f94-rgtw8_faro(d6bf8bcc-35b9-4c1f-ab69-f857a2328d11)\"" pod="faro/cortex-gw-6f7f764f94-rgtw8" podUID="d6bf8bcc-35b9-4c1f-ab69-f857a2328d11" I0507 11:59:30.092498 4724 scope.go:117] "RemoveContainer" containerID="60da1d466a5340942033d5d688a2f4ad116039a5035b5b6f8233fd240d6472bf" E0507 11:59:30.042197 4589 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=benchloadtestingxxl2-grafana-56b7c6b6df-nj27g_hosted-grafana(212d6baa-7068-4ad2-9617-f67f010e866d)\"" pod="hosted-grafana/benchloadtestingxxl2-grafana-56b7c6b6df-nj27g" podUID="212d6baa-7068-4ad2-9617-f67f010e866d" I0507 11:59:30.041381 4589 scope.go:117] "RemoveContainer" containerID="efb5462666d496e154e0477e0540b5325157c76f784e16834d1ab78c4fce2815" I0507 11:59:29.861354 4531 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-otel-alt-n2s4-0--3cf760c5-s8l4" status="Running" I0507 11:59:29.809996 4602 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-hg-n2s8-6-1dd39c-3bfd06e9-7pn8" status="Running" E0507 11:59:29.725681 3089 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"azure-resourcemanager-exporter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=azure-resourcemanager-exporter pod=azure-resourcemanager-exporter-6b5b58c666-rsttd_infra-exporters(5a95f801-309c-4f33-864a-406262c6ece6)\"" pod="infra-exporters/azure-resourcemanager-exporter-6b5b58c666-rsttd" podUID="5a95f801-309c-4f33-864a-406262c6ece6" I0507 11:59:29.725405 3089 scope.go:117] "RemoveContainer" containerID="fc52eb9945ce8a690b931db46692a6dd0bd9595808feb29e404ffd565f685f84" E0507 11:59:29.722713 4732 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"support-agent\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=support-agent pod=support-agent-557dff8b77-c6f8b_support-agent(ede5a224-96fb-45d0-b452-1eb2de73cf19)\"" pod="support-agent/support-agent-557dff8b77-c6f8b" podUID="ede5a224-96fb-45d0-b452-1eb2de73cf19" I0507 11:59:29.722345 4732 scope.go:117] "RemoveContainer" containerID="e0a235a59cc57d2dbbcab276b25c7bb1bab9cecc37697779748125072457736f" E0507 11:59:29.667989 2776 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"goldpinger\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=goldpinger pod=goldpinger-dw7wf_goldpinger(7dc39876-4602-45e9-a701-c9f8bf8c70b9)\"" pod="goldpinger/goldpinger-dw7wf" podUID="7dc39876-4602-45e9-a701-c9f8bf8c70b9" I0507 11:59:29.667381 2776 scope.go:117] "RemoveContainer" containerID="6f49a440ca8bc4e796384c08cafe8a9402ece7910a5413cb95d8c4fc808e86cd" I0507 11:59:29.656819 4742 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-qqlx" status="Running" E0507 11:59:29.603893 4586 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6432-916-1\\\"\"" pod="hosted-grafana/ephemeral1180076306432stephan-grafana-696d787664-jftqh" podUID="41fba902-127b-4514-b1ca-ed431bc59a6c" E0507 11:59:29.570535 4595 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=ephemeral1511182179279sarahzi-grafana-c8fbf74dd-cnskb_hosted-grafana(b2b8b8d9-9323-467d-99f5-e3289720a333)\"" pod="hosted-grafana/ephemeral1511182179279sarahzi-grafana-c8fbf74dd-cnskb" podUID="b2b8b8d9-9323-467d-99f5-e3289720a333" I0507 11:59:29.569915 4595 scope.go:117] "RemoveContainer" containerID="e223c471263c29a926b1319ae96b0ca116e3668d27011b6bc6fa5adebc0558c5" I0507 11:59:29.550166 4735 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-lngb" status="Running" E0507 11:59:29.538430 4590 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-85282-20418-1\\\"\"" pod="hosted-grafana/ephemeral1511182185282svenner-grafana-6944cbdfcc-64z2p" podUID="1abeccba-cc20-47a4-b55c-fff4b7decbe1" E0507 11:59:29.538063 4590 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=k6teststeady6-grafana-67b679bd8c-l7knf_hosted-grafana(c5975fd5-22d7-4efb-a6b6-3064876188c1)\"" pod="hosted-grafana/k6teststeady6-grafana-67b679bd8c-l7knf" podUID="c5975fd5-22d7-4efb-a6b6-3064876188c1" I0507 11:59:29.537415 4590 scope.go:117] "RemoveContainer" containerID="5b8aad8ab95e5f4201702424140d73f5cc582d6d48583a31ca0b0dabea27d806" I0507 11:59:29.434503 1119040 cache.go:40] re-using cached key and certificate ll header: 00000000: 42 01 0a 80 00 17 42 01 0a 80 00 01 08 00 IPv4: martian source 10.132.141.91 from 10.132.141.80, on dev eth0 I0507 11:59:29.408118 4734 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-databenchloki-n2-8c6b6266-bz78" status="Running" I0507 11:59:29.320184 1537502 kubelet_pods.go:906] "Unable to retrieve pull secret, the image pull may not succeed." pod="logs-endpoint-dev-005/kafka-controller-0" secret="" err="secret \"not-needed\" not found" E0507 11:59:29.151970 4572 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-ephemeral-6439-933-1\\\"\"" pod="hosted-grafana/ephemeral1180076306439dafyddt-grafana-9769b9f5-g5qqf" podUID="e6633496-a926-4a28-8db8-6405d33cb4bc" I0507 11:59:29.142840 4763 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-x94l" status="Running" E0507 11:59:28.939546 3659 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"cortex-gw\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=cortex-gw pod=cortex-gw-74f78948ff-9pcl6_faro(643043e2-707a-4a3f-adf3-08beab1d1ea7)\"" pod="faro/cortex-gw-74f78948ff-9pcl6" podUID="643043e2-707a-4a3f-adf3-08beab1d1ea7" I0507 11:59:28.939111 3659 scope.go:117] "RemoveContainer" containerID="9940112c30fda42aa2b814faddfc969d9a2328ae70ecb9b858d75aa6f8b61483" E0507 11:59:28.925475 4733 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"pdc\" with ErrImageNeverPull: \"Container image \\\"us.gcr.io/hosted-grafana/pdc:0.1.415\\\" is not present with pull policy of Never\"" pod="pdc/private-datasource-connect-564fb6cfbb-l8pgv" podUID="57e4a0cb-5e77-47bd-b277-70f4b1512c44" E0507 11:59:28.925419 4733 kuberuntime_manager.go:1256] container &Container{Name:pdc,Image:us.gcr.io/hosted-grafana/pdc:0.1.415,Command:[],Args:[-proxy.auth.ca-keys-dir=/var/run/secrets/pdc-certs -proxy.socks-server.addr=:10443 -proxy.ssh-server.addr=:2222 -proxy.use-socks-username-for-routing -proxy.api.http-address=:9182 -proxy.check-connpool-address-in-ring -memberlist.join=dns+gossip-ring.pdc.svc.cluster.local:7946 -api.http-address=:11443 -distributor.enabled=true -distributor.addr=:10444 -distributor.use-socks-username-for-routing -gateway.enabled=true -gateway.addr=:2244 -log.level=debug -certs.ca-private-key-file=/var/run/secrets/pdc-certs/ca.key -certs.ca-cert-file=/var/run/secrets/pdc-certs/ca.crt -certs.ca-pub-file=/var/run/secrets/pdc-certs/ca.pub -certs.cluster=local-k8s -shard-size=3 -graceful-shutdown-period=30s -enable-multiple-networks],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:socks,HostPort:0,ContainerPort:10443,Protocol:TCP,HostIP:,},ContainerPort{Name:ssh,HostPort:0,ContainerPort:2222,Protocol:TCP,HostIP:,},ContainerPort{Name:distributor,HostPort:0,ContainerPort:10444,Protocol:TCP,HostIP:,},ContainerPort{Name:gateway,HostPort:0,ContainerPort:2244,Protocol:TCP,HostIP:,},ContainerPort{Name:api,HostPort:0,ContainerPort:11443,Protocol:TCP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:POD_NAME,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.name,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},},Resources:ResourceRequirements{Limits:ResourceList{cpu: {{500 -3} {} 500m DecimalSI},memory: {{134217728 0} {} BinarySI},},Requests:ResourceList{cpu: {{250 -3} {} 250m DecimalSI},memory: {{67108864 0} {} BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:pdc-certs,ReadOnly:true,MountPath:/var/run/secrets/pdc-certs,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-wcbmb,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/ready,Port:{0 11443 },Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:40,TimeoutSeconds:1,PeriodSeconds:5,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:&Lifecycle{PostStart:nil,PreStop:&LifecycleHandler{Exec:&ExecAction{Command:[/bin/sleep 5],},HTTPGet:nil,TCPSocket:nil,},},TerminationMessagePath:/dev/termination-log,ImagePullPolicy:Never,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod private-datasource-connect-564fb6cfbb-l8pgv_pdc(57e4a0cb-5e77-47bd-b277-70f4b1512c44): ErrImageNeverPull: Container image "us.gcr.io/hosted-grafana/pdc:0.1.415" is not present with pull policy of Never E0507 11:59:28.889010 4597 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:10.1.0-ephemeral-oss-77487-8287-1\\\"\"" pod="hosted-grafana/ephemeral1511182177487torkelo-grafana-79dd77959f-2l2kd" podUID="4d3be4e9-d8c5-487f-a292-ecb699c3aaad" E0507 11:59:28.761691 3303 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"cortex-gw\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=cortex-gw pod=cortex-gw-78bc9b5ccc-8hkmp_faro(44b54226-b4bd-46e0-a3f0-257cb44d9ea8)\"" pod="faro/cortex-gw-78bc9b5ccc-8hkmp" podUID="44b54226-b4bd-46e0-a3f0-257cb44d9ea8" I0507 11:59:28.761235 3303 scope.go:117] "RemoveContainer" containerID="9f3955a57aa496cb888a35102ef0ee777d6a75cdc12addbdafc2d9b3fb9cc080" E0507 11:59:28.744029 4601 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=benchloadtestingxxl2-grafana-5bb9765dd8-ktf2b_hosted-grafana(e8405a93-3a4c-4074-909d-661219c1f849)\"" pod="hosted-grafana/benchloadtestingxxl2-grafana-5bb9765dd8-ktf2b" podUID="e8405a93-3a4c-4074-909d-661219c1f849" I0507 11:59:28.743482 4601 scope.go:117] "RemoveContainer" containerID="8dbc699386128aa4e4af25beb0ea7e7ecad1b2d5e829061a04ff808054f050aa" I0507 11:59:28.728222 4586 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-hg-n2s8-6-1dd39c-3bfd06e9-8dwk" status="Running" I0507 11:59:28.706031 4616 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-hg-n2s8-6-1dd39c-3bfd06e9-t4fv" status="Running" I0507 11:59:28.671473 4601 kubelet.go:2404] "SyncLoop REMOVE" source="api" pods=["hosted-grafana/sloappverify-grafana-764f9644df-wzxz5"] E0507 11:59:28.664457 4601 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=grafana pod=ltest-grafana-58869c5dd7-xgm5l_hosted-grafana(4e4c9c69-de88-44dd-bd71-7b8ef56554b1)\"" pod="hosted-grafana/ltest-grafana-58869c5dd7-xgm5l" podUID="4e4c9c69-de88-44dd-bd71-7b8ef56554b1" I0507 11:59:28.663840 4601 scope.go:117] "RemoveContainer" containerID="7dd19cfdca617fbbcacdd9cb716cf62666ab719dac31979615f13c0a7adc87a7" I0507 11:59:28.663309 4601 kubelet.go:2498] "SyncLoop (probe)" probe="liveness" status="unhealthy" pod="hosted-grafana/ltest-grafana-58869c5dd7-xgm5l" I0507 11:59:28.654287 4601 kubelet.go:2410] "SyncLoop DELETE" source="api" pods=["hosted-grafana/sloappverify-grafana-764f9644df-wzxz5"] I0507 11:59:28.639440 4601 pod_container_deletor.go:53] "DeleteContainer returned error" containerID={"Type":"containerd","ID":"ea8c181d2a9baf4e2819046a0699151c11e7d761b3ccdf0b0beaa713ce50fe02"} err="failed to get container status \"ea8c181d2a9baf4e2819046a0699151c11e7d761b3ccdf0b0beaa713ce50fe02\": rpc error: code = NotFound desc = an error occurred when try to find container \"ea8c181d2a9baf4e2819046a0699151c11e7d761b3ccdf0b0beaa713ce50fe02\": not found" E0507 11:59:28.639400 4601 remote_runtime.go:432] "ContainerStatus from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find container \"ea8c181d2a9baf4e2819046a0699151c11e7d761b3ccdf0b0beaa713ce50fe02\": not found" containerID="ea8c181d2a9baf4e2819046a0699151c11e7d761b3ccdf0b0beaa713ce50fe02" time="2024-05-07T11:59:28.639205289Z" level=error msg="ContainerStatus for \"ea8c181d2a9baf4e2819046a0699151c11e7d761b3ccdf0b0beaa713ce50fe02\" failed" error="rpc error: code = NotFound desc = an error occurred when try to find container \"ea8c181d2a9baf4e2819046a0699151c11e7d761b3ccdf0b0beaa713ce50fe02\": not found" I0507 11:59:28.639007 4601 scope.go:117] "RemoveContainer" containerID="ea8c181d2a9baf4e2819046a0699151c11e7d761b3ccdf0b0beaa713ce50fe02" I0507 11:59:28.638984 4601 pod_container_deletor.go:53] "DeleteContainer returned error" containerID={"Type":"containerd","ID":"c8a30401d2ac9c86fdf4db11df6731b750ed2d044efe5757037c04846c0d28c1"} err="failed to get container status \"c8a30401d2ac9c86fdf4db11df6731b750ed2d044efe5757037c04846c0d28c1\": rpc error: code = NotFound desc = an error occurred when try to find container \"c8a30401d2ac9c86fdf4db11df6731b750ed2d044efe5757037c04846c0d28c1\": not found" E0507 11:59:28.638943 4601 remote_runtime.go:432] "ContainerStatus from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find container \"c8a30401d2ac9c86fdf4db11df6731b750ed2d044efe5757037c04846c0d28c1\": not found" containerID="c8a30401d2ac9c86fdf4db11df6731b750ed2d044efe5757037c04846c0d28c1" time="2024-05-07T11:59:28.638744170Z" level=error msg="ContainerStatus for \"c8a30401d2ac9c86fdf4db11df6731b750ed2d044efe5757037c04846c0d28c1\" failed" error="rpc error: code = NotFound desc = an error occurred when try to find container \"c8a30401d2ac9c86fdf4db11df6731b750ed2d044efe5757037c04846c0d28c1\": not found" I0507 11:59:28.638577 4601 scope.go:117] "RemoveContainer" containerID="c8a30401d2ac9c86fdf4db11df6731b750ed2d044efe5757037c04846c0d28c1" I0507 11:59:28.638564 4601 pod_container_deletor.go:53] "DeleteContainer returned error" containerID={"Type":"containerd","ID":"c50338fdb9905376f6d1db35c61599f712f6d3a4b9604b6dc64bf62aea9b3b13"} err="failed to get container status \"c50338fdb9905376f6d1db35c61599f712f6d3a4b9604b6dc64bf62aea9b3b13\": rpc error: code = NotFound desc = an error occurred when try to find container \"c50338fdb9905376f6d1db35c61599f712f6d3a4b9604b6dc64bf62aea9b3b13\": not found" E0507 11:59:28.638532 4601 remote_runtime.go:432] "ContainerStatus from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find container \"c50338fdb9905376f6d1db35c61599f712f6d3a4b9604b6dc64bf62aea9b3b13\": not found" containerID="c50338fdb9905376f6d1db35c61599f712f6d3a4b9604b6dc64bf62aea9b3b13"