ts=2024-04-16T15:10:44.103611953Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/billing_billing-forwarder-5bf8799d98-jx47p_93a99fc6-f43b-44fd-ae92-f1d8e79639ba/billing-forwarder/0.log op=CREATE
ts=2024-04-16T15:10:44.102912583Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/billing_billing-forwarder-5bf8799d98-jx47p_93a99fc6-f43b-44fd-ae92-f1d8e79639ba/billing-forwarder/0.log.20240416-151044 op=CREATE
ts=2024-04-16T15:10:44.097147346Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/billing_billing-forwarder-5bf8799d98-jx47p_93a99fc6-f43b-44fd-ae92-f1d8e79639ba/billing-forwarder/0.log.20240416-151002.gz op=CREATE
ts=2024-04-16T15:10:44.053538158Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:44.040467356Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-d89d8b5fd-lqw92_e60bfc3b-9967-4a9d-8370-ace8d046123b/hosted-grafana-security/0.log to appear..."
ts=2024-04-16T15:10:44.040259872Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-d89d8b5fd-lqw92_e60bfc3b-9967-4a9d-8370-ace8d046123b/hosted-grafana-security/0.log ..."
ts=2024-04-16T15:10:44.020663216Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/grafana-ruler_grafana-ruler-7cb758bfd6-hcq2d_c3259533-8d14-4b65-a6d0-c3880de4feba/grafana-ruler/0.log"
ts=2024-04-16T15:10:44.020589942Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/grafana-ruler_grafana-ruler-7cb758bfd6-hcq2d_c3259533-8d14-4b65-a6d0-c3880de4feba/grafana-ruler/0.log ..."
ts=2024-04-16T15:10:43.995581296Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-bccd55b-fn2h4_0d138adf-7938-4475-8248-05172a562c00/hg-plugins/0.log
ts=2024-04-16T15:10:43.995566217Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-bccd55b-fn2h4_0d138adf-7938-4475-8248-05172a562c00/hg-plugins/0.log - &{Offset:0 Whence:0}"
ts=2024-04-16T15:10:43.995488725Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-bccd55b-fn2h4_0d138adf-7938-4475-8248-05172a562c00/hg-plugins
ts=2024-04-16T15:10:43.994892796Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-867b55884f-nbk4p_18703b31-d5eb-4108-9456-4c937645e5bd/hg-plugins/0.log
ts=2024-04-16T15:10:43.994870631Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-867b55884f-nbk4p_18703b31-d5eb-4108-9456-4c937645e5bd/hg-plugins/0.log - &{Offset:0 Whence:0}"
ts=2024-04-16T15:10:43.994764966Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-867b55884f-nbk4p_18703b31-d5eb-4108-9456-4c937645e5bd/hg-plugins
ts=2024-04-16T15:10:43.994701713Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-6b57fc5bd5-7z5dk_04c29f31-2bfd-463a-8e98-278d15895df8/hg-plugins/0.log
ts=2024-04-16T15:10:43.994682386Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-6b57fc5bd5-7z5dk_04c29f31-2bfd-463a-8e98-278d15895df8/hg-plugins/0.log - &{Offset:0 Whence:0}"
ts=2024-04-16T15:10:43.994616509Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-67fdffbbcd-wrs8v_a1ec727b-6624-42f0-8cc1-b3a07c1bda71/hg-plugins/0.log
ts=2024-04-16T15:10:43.994581626Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-67fdffbbcd-wrs8v_a1ec727b-6624-42f0-8cc1-b3a07c1bda71/hg-plugins/0.log - &{Offset:0 Whence:0}"
ts=2024-04-16T15:10:43.994431069Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-6b57fc5bd5-7z5dk_04c29f31-2bfd-463a-8e98-278d15895df8/hg-plugins
ts=2024-04-16T15:10:43.994354906Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-67fdffbbcd-wrs8v_a1ec727b-6624-42f0-8cc1-b3a07c1bda71/hg-plugins
ts=2024-04-16T15:10:43.991326425Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/grafana-ruler_grafana-ruler-7cb758bfd6-7h75h_3013c1d4-6a76-4548-8eab-d4b71ef0bccd/grafana-ruler/0.log.20240416-150803.tmp op=CREATE
ts=2024-04-16T15:10:43.987816559Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/grafana-ruler_grafana-ruler-7cb758bfd6-hcq2d_c3259533-8d14-4b65-a6d0-c3880de4feba/grafana-ruler/0.log op=CREATE
ts=2024-04-16T15:10:43.987450089Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/grafana-ruler_grafana-ruler-7cb758bfd6-hcq2d_c3259533-8d14-4b65-a6d0-c3880de4feba/grafana-ruler/0.log.20240416-151043 op=CREATE
ts=2024-04-16T15:10:43.985283077Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/grafana-ruler_grafana-ruler-7cb758bfd6-hcq2d_c3259533-8d14-4b65-a6d0-c3880de4feba/grafana-ruler/0.log.20240416-150642.gz op=CREATE
ts=2024-04-16T15:10:43.963516043Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/hosted-grafana_.something-grafana-gateway-78cb874c8f-2pmlc_26b57c76-eabc-4d29-9305-3f034c24e147/istio-proxy/0.log"
ts=2024-04-16T15:10:43.961050802Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-gateway-78cb874c8f-2pmlc_26b57c76-eabc-4d29-9305-3f034c24e147/istio-proxy/0.log ..."
ts=2024-04-16T15:10:43.960201305Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-ffdfcb6f6-vtm7j_1b0ba139-a4c5-4534-98bd-b2c66e4205cb/hg-plugins/0.log - &{Offset:0 Whence:0}"
ts=2024-04-16T15:10:43.960173523Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-ffdfcb6f6-vtm7j_1b0ba139-a4c5-4534-98bd-b2c66e4205cb/hg-plugins/0.log
ts=2024-04-16T15:10:43.960099063Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-ffdfcb6f6-vtm7j_1b0ba139-a4c5-4534-98bd-b2c66e4205cb/hg-plugins
ts=2024-04-16T15:10:43.956496124Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-558c5cff54-58lhv_14f770e0-e9d2-4957-ad3c-51bf7bab9c38/hg-plugins/0.log - &{Offset:0 Whence:0}"
ts=2024-04-16T15:10:43.956459518Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-558c5cff54-58lhv_14f770e0-e9d2-4957-ad3c-51bf7bab9c38/hg-plugins/0.log
ts=2024-04-16T15:10:43.956333382Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-558c5cff54-58lhv_14f770e0-e9d2-4957-ad3c-51bf7bab9c38/hg-plugins
ts=2024-04-16T15:10:43.94285492Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-6f59c6b77d-b4z6m_39598760-48b2-4b37-8693-e511348d3910/hg-plugins/0.log - &{Offset:0 Whence:0}"
ts=2024-04-16T15:10:43.942820378Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-6f59c6b77d-b4z6m_39598760-48b2-4b37-8693-e511348d3910/hg-plugins/0.log
ts=2024-04-16T15:10:43.942720436Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-6f59c6b77d-b4z6m_39598760-48b2-4b37-8693-e511348d3910/hg-plugins
ts=2024-04-16T15:10:43.93643872Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-747b4b8bb6-46kwt_852b076d-c773-4e47-bc55-d7eeae8a5837/hg-plugins/0.log - &{Offset:0 Whence:0}"
ts=2024-04-16T15:10:43.936406569Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-747b4b8bb6-46kwt_852b076d-c773-4e47-bc55-d7eeae8a5837/hg-plugins/0.log
ts=2024-04-16T15:10:43.936280507Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-747b4b8bb6-46kwt_852b076d-c773-4e47-bc55-d7eeae8a5837/hg-plugins
ts=2024-04-16T15:10:43.93512145Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/pyroscope-ebpf_profiler-w87sj_04264a7e-491d-4c22-b109-c97f9b227eb0/profiler/0.log"
ts=2024-04-16T15:10:43.935029359Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/pyroscope-ebpf_profiler-w87sj_04264a7e-491d-4c22-b109-c97f9b227eb0/profiler/0.log ..."
ts=2024-04-16T15:10:43.917279646Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/billing_billing-forwarder-5bf8799d98-jx47p_93a99fc6-f43b-44fd-ae92-f1d8e79639ba/billing-forwarder/0.log.20240416-151002.tmp op=CREATE
ts=2024-04-16T15:10:43.912837401Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-d89d8b5fd-lqw92_e60bfc3b-9967-4a9d-8370-ace8d046123b/hgrun/0.log to appear..."
ts=2024-04-16T15:10:43.912678958Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-d89d8b5fd-lqw92_e60bfc3b-9967-4a9d-8370-ace8d046123b/hgrun/0.log ..."
ts=2024-04-16T15:10:43.913001917Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.910710328Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-10_ruler-c957f4f69-stmzv_cfa34492-c676-42ce-b71c-5793aa5d1488/ruler/0.log op=CREATE
ts=2024-04-16T15:10:43.910230202Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-10_ruler-c957f4f69-stmzv_cfa34492-c676-42ce-b71c-5793aa5d1488/ruler/0.log.20240416-151043 op=CREATE
ts=2024-04-16T15:10:43.90589892Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-10_ruler-c957f4f69-stmzv_cfa34492-c676-42ce-b71c-5793aa5d1488/ruler/0.log.20240416-151033.gz op=CREATE
ts=2024-04-16T15:10:43.902671587Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/pyroscope-ebpf_profiler-w87sj_04264a7e-491d-4c22-b109-c97f9b227eb0/profiler/0.log op=CREATE
ts=2024-04-16T15:10:43.902293197Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/pyroscope-ebpf_profiler-w87sj_04264a7e-491d-4c22-b109-c97f9b227eb0/profiler/0.log.20240416-151043 op=CREATE
ts=2024-04-16T15:10:43.901374194Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/pyroscope-ebpf_profiler-w87sj_04264a7e-491d-4c22-b109-c97f9b227eb0/profiler/0.log.20240416-121049.gz op=CREATE
ts=2024-04-16T15:10:43.886121557Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-d89d8b5fd-lqw92_e60bfc3b-9967-4a9d-8370-ace8d046123b/hg-plugins/0.log to appear..."
ts=2024-04-16T15:10:43.885839882Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-d89d8b5fd-lqw92_e60bfc3b-9967-4a9d-8370-ace8d046123b/hg-plugins/0.log ..."
ts=2024-04-16T15:10:43.882665637Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-gateway-78cb874c8f-2pmlc_26b57c76-eabc-4d29-9305-3f034c24e147/istio-proxy/0.log op=CREATE
ts=2024-04-16T15:10:43.8818563Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-gateway-78cb874c8f-2pmlc_26b57c76-eabc-4d29-9305-3f034c24e147/istio-proxy/0.log.20240416-151043 op=CREATE
ts=2024-04-16T15:10:43.878087537Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-gateway-78cb874c8f-2pmlc_26b57c76-eabc-4d29-9305-3f034c24e147/istio-proxy/0.log.20240416-151013.gz op=CREATE
ts=2024-04-16T15:10:43.870249341Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/hg-plugins/*.log
ts=2024-04-16T15:10:43.870231421Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hg-plugins/0.log
ts=2024-04-16T15:10:43.87016127Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hg-plugins/0.log
ts=2024-04-16T15:10:43.87014674Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hg-plugins/0.log
ts=2024-04-16T15:10:43.870126919Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hg-plugins/0.log reason=null
ts=2024-04-16T15:10:43.870016638Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i2222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"someorg\", plan=\"free\", pod=\"someorg-grafana-666bd48cf9-7zrtv\", pod_template_hash=\"666bd48cf9\", resource_version=\"167212086\", slug=\"someorg\", stackId=\"444444\"}"
ts=2024-04-16T15:10:43.869947507Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/grafana/*.log
ts=2024-04-16T15:10:43.869933646Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/grafana/0.log
ts=2024-04-16T15:10:43.869919827Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/grafana/0.log
ts=2024-04-16T15:10:43.869901596Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/grafana/0.log
ts=2024-04-16T15:10:43.869885086Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/grafana/0.log reason=null
ts=2024-04-16T15:10:43.869833185Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i2222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"someorg\", plan=\"free\", pod=\"someorg-grafana-666bd48cf9-7zrtv\", pod_template_hash=\"666bd48cf9\", resource_version=\"167212086\", slug=\"someorg\", stackId=\"444444\"}"
ts=2024-04-16T15:10:43.869772754Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/hgrun/*.log
ts=2024-04-16T15:10:43.869758435Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hgrun/0.log
ts=2024-04-16T15:10:43.869740834Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hgrun/0.log
ts=2024-04-16T15:10:43.869727154Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hgrun/0.log
ts=2024-04-16T15:10:43.869713063Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hgrun/0.log reason=null
ts=2024-04-16T15:10:43.869672113Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i2222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"someorg\", plan=\"free\", pod=\"someorg-grafana-666bd48cf9-7zrtv\", pod_template_hash=\"666bd48cf9\", resource_version=\"167212086\", slug=\"someorg\", stackId=\"444444\"}"
ts=2024-04-16T15:10:43.869543361Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/hosted-grafana-security/*.log
ts=2024-04-16T15:10:43.869526821Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.869511381Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.869496801Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.869477971Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hosted-grafana-security/0.log reason=null
ts=2024-04-16T15:10:43.869370539Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i2222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"someorg\", plan=\"free\", pod=\"someorg-grafana-666bd48cf9-7zrtv\", pod_template_hash=\"666bd48cf9\", resource_version=\"167212086\", slug=\"someorg\", stackId=\"444444\"}"
ts=2024-04-16T15:10:43.866539497Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod3_distributor-76c559cb69-lbf6f_1b71b005-4a57-4f52-acda-734f074e93c9/distributor/0.log.20240416-150602.tmp op=CREATE
ts=2024-04-16T15:10:43.862521938Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-24_graphite-write-proxy-56c899b9bd-sv5wk_88dab27e-6880-4eab-84ca-23181ddf59f0/graphite-write-proxy/0.log.20240416-150933.tmp op=CREATE
ts=2024-04-16T15:10:43.854830425Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.84883762Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.845881472Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.845202349Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/pyroscope-ebpf_profiler-w87sj_04264a7e-491d-4c22-b109-c97f9b227eb0/profiler/0.log.20240416-121049.tmp op=CREATE
ts=2024-04-16T15:10:43.834940494Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*404e8595-2e9f-4fcf-9495-925f6d245e20/grafana/*.log
ts=2024-04-16T15:10:43.834919864Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/grafana/0.log
ts=2024-04-16T15:10:43.834891984Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/grafana/0.log
ts=2024-04-16T15:10:43.834870753Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/grafana/0.log
ts=2024-04-16T15:10:43.834850373Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/grafana/0.log reason=null
ts=2024-04-16T15:10:43.834799833Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/grafana/0.log
ts=2024-04-16T15:10:43.834752462Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*404e8595-2e9f-4fcf-9495-925f6d245e20/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"222222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org3\", plan=\"free\", pod=\"org3-grafana-7fd6786f4b-242cb\", pod_template_hash=\"7fd6786f4b\", resource_version=\"167282051\", slug=\"org3\", stackId=\"1111111\"}"
ts=2024-04-16T15:10:43.834460948Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*404e8595-2e9f-4fcf-9495-925f6d245e20/hg-plugins/*.log
ts=2024-04-16T15:10:43.833997181Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hg-plugins/0.log
ts=2024-04-16T15:10:43.833967761Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hg-plugins/0.log
ts=2024-04-16T15:10:43.83395078Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hg-plugins/0.log
ts=2024-04-16T15:10:43.83393401Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hg-plugins/0.log reason=null
ts=2024-04-16T15:10:43.8338997Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hg-plugins/0.log
ts=2024-04-16T15:10:43.8338763Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*404e8595-2e9f-4fcf-9495-925f6d245e20/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"222222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org3\", plan=\"free\", pod=\"org3-grafana-7fd6786f4b-242cb\", pod_template_hash=\"7fd6786f4b\", resource_version=\"167282051\", slug=\"org3\", stackId=\"1111111\"}"
ts=2024-04-16T15:10:43.833819339Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*404e8595-2e9f-4fcf-9495-925f6d245e20/hgrun/*.log
ts=2024-04-16T15:10:43.833794999Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hgrun/0.log
ts=2024-04-16T15:10:43.838457484Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.837161525Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.833775169Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hgrun/0.log
ts=2024-04-16T15:10:43.833756968Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hgrun/0.log
ts=2024-04-16T15:10:43.833697477Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hgrun/0.log reason=null
ts=2024-04-16T15:10:43.833662116Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hgrun/0.log
ts=2024-04-16T15:10:43.833629856Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*404e8595-2e9f-4fcf-9495-925f6d245e20/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"222222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org3\", plan=\"free\", pod=\"org3-grafana-7fd6786f4b-242cb\", pod_template_hash=\"7fd6786f4b\", resource_version=\"167282051\", slug=\"org3\", stackId=\"1111111\"}"
ts=2024-04-16T15:10:43.833457274Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*404e8595-2e9f-4fcf-9495-925f6d245e20/hosted-grafana-security/*.log
ts=2024-04-16T15:10:43.833433624Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.833400573Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.833382792Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.833364072Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hosted-grafana-security/0.log reason=null
ts=2024-04-16T15:10:43.833319402Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.833226031Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*404e8595-2e9f-4fcf-9495-925f6d245e20/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"222222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org3\", plan=\"free\", pod=\"org3-grafana-7fd6786f4b-242cb\", pod_template_hash=\"7fd6786f4b\", resource_version=\"167282051\", slug=\"org3\", stackId=\"1111111\"}"
ts=2024-04-16T15:10:43.820104119Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/hosted-grafana_.something-grafana-7c67b65cdd-mk82w_496ab238-1eed-4b76-8345-de4537cb7c0e/grafana/0.log"
ts=2024-04-16T15:10:43.819972587Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7c67b65cdd-mk82w_496ab238-1eed-4b76-8345-de4537cb7c0e/grafana/0.log ..."
ts=2024-04-16T15:10:43.825845873Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.82429901Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.812714489Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/grafana-ruler_grafana-ruler-7cb758bfd6-hcq2d_c3259533-8d14-4b65-a6d0-c3880de4feba/grafana-ruler/0.log.20240416-150642.tmp op=CREATE
ts=2024-04-16T15:10:43.811775736Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.789499114Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-d89d8b5fd-lqw92_e60bfc3b-9967-4a9d-8370-ace8d046123b/grafana/0.log to appear..."
ts=2024-04-16T15:10:43.789387503Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-d89d8b5fd-lqw92_e60bfc3b-9967-4a9d-8370-ace8d046123b/grafana/0.log ..."
ts=2024-04-16T15:10:43.763317681Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.73769616Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/tempo-prod_ingester-18_e3cf5e0a-b9ba-4be0-b3b0-6157f3cca0f7/ingester/0.log.20240416-150613.tmp op=CREATE
ts=2024-04-16T15:10:43.717654564Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod-012_query-frontend-7f5bf94766-r6nx7_cf30b042-cf38-4562-a431-26e7e37665c2/query-frontend/4.log.20240416-151033.tmp op=CREATE
ts=2024-04-16T15:10:43.715625202Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.710462108Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.706609368Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.699754134Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-10_ruler-c957f4f69-stmzv_cfa34492-c676-42ce-b71c-5793aa5d1488/ruler/0.log.20240416-151033.tmp op=CREATE
ts=2024-04-16T15:10:43.689150897Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.671351341Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.66739874Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/mimir-dedicated-48_ingester-zone-b-72_1f08c7df-e68c-4101-afba-3cb84acb34f0/ingester/0.log"
ts=2024-04-16T15:10:43.667327798Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/mimir-dedicated-48_ingester-zone-b-72_1f08c7df-e68c-4101-afba-3cb84acb34f0/ingester/0.log ..."
ts=2024-04-16T15:10:43.665742041Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/insight-logs_promtail-insight-logs-pxhrc_c86ff41c-f07b-4d03-aef7-05d9e243cef6/promtail/0.log"
ts=2024-04-16T15:10:43.665664607Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/insight-logs_promtail-insight-logs-pxhrc_c86ff41c-f07b-4d03-aef7-05d9e243cef6/promtail/0.log ..."
ts=2024-04-16T15:10:43.624354483Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-7c67b65cdd-mk82w_496ab238-1eed-4b76-8345-de4537cb7c0e/grafana/0.log op=CREATE
ts=2024-04-16T15:10:43.618647073Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-7c67b65cdd-mk82w_496ab238-1eed-4b76-8345-de4537cb7c0e/grafana/0.log.20240416-151043 op=CREATE
ts=2024-04-16T15:10:43.614330743Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-85ddb8cd7b-qdw4m_700c6bab-d080-42d2-b8bf-7d6b3180395a/grafana/0.log.20240416-150932.tmp op=CREATE
ts=2024-04-16T15:10:43.613342016Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-58687d789-49wfb_269bf9a8-7811-4259-b1fc-ff487ffc9a9a/grafana/0.log to appear..."
ts=2024-04-16T15:10:43.613114354Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-58687d789-49wfb_269bf9a8-7811-4259-b1fc-ff487ffc9a9a/grafana/0.log ..."
ts=2024-04-16T15:10:43.612632997Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-7c67b65cdd-mk82w_496ab238-1eed-4b76-8345-de4537cb7c0e/grafana/0.log.20240416-151011.gz op=CREATE
ts=2024-04-16T15:10:43.608713774Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-d95d68499-wsddd_b19cb3ac-7212-48e8-a74c-573d5732f2a9/grafana/0.log op=CREATE
ts=2024-04-16T15:10:43.607662962Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-d95d68499-wsddd_b19cb3ac-7212-48e8-a74c-573d5732f2a9/grafana/0.log.20240416-151043 op=CREATE
ts=2024-04-16T15:10:43.604544245Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-d95d68499-wsddd_b19cb3ac-7212-48e8-a74c-573d5732f2a9/grafana/0.log.20240416-151023.gz op=CREATE
ts=2024-04-16T15:10:43.593678478Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.59255473Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-66bd77fc5d-w6frp_e21695f7-52de-495e-8d78-741519657226/hgrun/0.log to appear..."
ts=2024-04-16T15:10:43.592407837Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-66bd77fc5d-w6frp_e21695f7-52de-495e-8d78-741519657226/hgrun/0.log ..."
ts=2024-04-16T15:10:43.587217309Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.5862021Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.585137415Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-58687d789-49wfb_269bf9a8-7811-4259-b1fc-ff487ffc9a9a/hosted-grafana-security/0.log to appear..."
ts=2024-04-16T15:10:43.584968803Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-58687d789-49wfb_269bf9a8-7811-4259-b1fc-ff487ffc9a9a/hosted-grafana-security/0.log ..."
ts=2024-04-16T15:10:43.585003039Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.580159324Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.579992546Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.569307522Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.56817841Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.56492517Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-gateway-78cb874c8f-2pmlc_26b57c76-eabc-4d29-9305-3f034c24e147/istio-proxy/0.log.20240416-151013.tmp op=CREATE
ts=2024-04-16T15:10:43.563399768Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-66bd77fc5d-w6frp_e21695f7-52de-495e-8d78-741519657226/hosted-grafana-security/0.log to appear..."
ts=2024-04-16T15:10:43.563325868Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-66bd77fc5d-w6frp_e21695f7-52de-495e-8d78-741519657226/hosted-grafana-security/0.log ..."
ts=2024-04-16T15:10:43.555169973Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.553816164Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/mimir-dedicated-48_ingester-zone-b-72_1f08c7df-e68c-4101-afba-3cb84acb34f0/ingester/0.log op=CREATE
ts=2024-04-16T15:10:43.553398207Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/mimir-dedicated-48_ingester-zone-b-72_1f08c7df-e68c-4101-afba-3cb84acb34f0/ingester/0.log.20240416-151043 op=CREATE
ts=2024-04-16T15:10:43.552323479Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/mimir-dedicated-48_ingester-zone-b-72_1f08c7df-e68c-4101-afba-3cb84acb34f0/ingester/0.log.20240416-150452.gz op=CREATE
ts=2024-04-16T15:10:43.55175804Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-6766cd4c64-v7ccz_c8ec4c12-78c6-4f20-b382-8ae7e9f0f4af/hgrun/0.log to appear..."
ts=2024-04-16T15:10:43.55169237Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-6766cd4c64-v7ccz_c8ec4c12-78c6-4f20-b382-8ae7e9f0f4af/hgrun/0.log ..."
ts=2024-04-16T15:10:43.551819984Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*35649bfd-52ff-4281-9294-5f65fd5a89fc/marketplaces-api/*.log
ts=2024-04-16T15:10:43.551782223Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/grafana-com_marketplaces-api-f67ff7567-gqrvb_35649bfd-52ff-4281-9294-5f65fd5a89fc/marketplaces-api/0.log
ts=2024-04-16T15:10:43.551748001Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/grafana-com_marketplaces-api-f67ff7567-gqrvb_35649bfd-52ff-4281-9294-5f65fd5a89fc/marketplaces-api/0.log
ts=2024-04-16T15:10:43.551719538Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/grafana-com_marketplaces-api-f67ff7567-gqrvb_35649bfd-52ff-4281-9294-5f65fd5a89fc/marketplaces-api/0.log
ts=2024-04-16T15:10:43.551673641Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/grafana-com_marketplaces-api-f67ff7567-gqrvb_35649bfd-52ff-4281-9294-5f65fd5a89fc/marketplaces-api/0.log reason=null
ts=2024-04-16T15:10:43.551621071Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/grafana-com_marketplaces-api-f67ff7567-gqrvb_35649bfd-52ff-4281-9294-5f65fd5a89fc/marketplaces-api/0.log
ts=2024-04-16T15:10:43.551543875Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*35649bfd-52ff-4281-9294-5f65fd5a89fc/marketplaces-api/*.log:{container=\"marketplaces-api\", job=\"grafana-com/marketplaces-api\", name=\"marketplaces-api\", namespace=\"grafana-com\", pod=\"marketplaces-api-f67ff7567-gqrvb\", pod_template_hash=\"f67ff7567\"}"
ts=2024-04-16T15:10:43.551598289Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.539801042Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-58687d789-49wfb_269bf9a8-7811-4259-b1fc-ff487ffc9a9a/hgrun/0.log to appear..."
ts=2024-04-16T15:10:43.53959404Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-58687d789-49wfb_269bf9a8-7811-4259-b1fc-ff487ffc9a9a/hgrun/0.log ..."
ts=2024-04-16T15:10:43.532863601Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-67d8c765c5-hsdpm_6cbab356-d034-4ff8-9879-077d7f363956/hosted-grafana-security/0.log to appear..."
ts=2024-04-16T15:10:43.53265924Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-67d8c765c5-hsdpm_6cbab356-d034-4ff8-9879-077d7f363956/hosted-grafana-security/0.log ..."
ts=2024-04-16T15:10:43.524399551Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/kube-system_calico-node-7zxvh_a1ad8906-41f3-46de-9fb7-64656624556b/calico-node/0.log to appear..."
ts=2024-04-16T15:10:43.524115935Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/kube-system_calico-node-7zxvh_a1ad8906-41f3-46de-9fb7-64656624556b/calico-node/0.log ..."
ts=2024-04-16T15:10:43.523998271Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/kube-system_calico-node-7zxvh_a1ad8906-41f3-46de-9fb7-64656624556b/install-cni/0.log to appear..."
ts=2024-04-16T15:10:43.523842724Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/kube-system_calico-node-7zxvh_a1ad8906-41f3-46de-9fb7-64656624556b/install-cni/0.log ..."
ts=2024-04-16T15:10:43.522251853Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-66bd77fc5d-w6frp_e21695f7-52de-495e-8d78-741519657226/grafana/0.log to appear..."
ts=2024-04-16T15:10:43.522135262Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-66bd77fc5d-w6frp_e21695f7-52de-495e-8d78-741519657226/grafana/0.log ..."
ts=2024-04-16T15:10:43.513279228Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/kube-system_calico-node-jwmqj_894554c7-64d9-4221-9032-1d2966ce987f/install-cni/0.log to appear..."
ts=2024-04-16T15:10:43.513186063Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/kube-system_calico-node-jwmqj_894554c7-64d9-4221-9032-1d2966ce987f/install-cni/0.log ..."
ts=2024-04-16T15:10:43.508349919Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*043372d7-9411-443f-ba7f-80988f77d8b3/hgrun/*.log
ts=2024-04-16T15:10:43.508329009Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hgrun/0.log
ts=2024-04-16T15:10:43.508309139Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hgrun/0.log
ts=2024-04-16T15:10:43.508287738Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hgrun/0.log
ts=2024-04-16T15:10:43.508267968Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hgrun/0.log reason=null
ts=2024-04-16T15:10:43.508237718Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hgrun/0.log
ts=2024-04-16T15:10:43.508173806Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*043372d7-9411-443f-ba7f-80988f77d8b3/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i3333\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org4\", plan=\"free\", pod=\"org4-grafana-b4f87fcc5-fflgn\", pod_template_hash=\"b4f87fcc5\", resource_version=\"167289888\", slug=\"org4\", stackId=\"333333\"}"
ts=2024-04-16T15:10:43.507960012Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.507831693Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*043372d7-9411-443f-ba7f-80988f77d8b3/grafana/*.log
ts=2024-04-16T15:10:43.507797611Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/grafana/0.log
ts=2024-04-16T15:10:43.507781221Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/grafana/0.log
ts=2024-04-16T15:10:43.507765151Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/grafana/0.log
ts=2024-04-16T15:10:43.507730981Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/grafana/0.log reason=null
ts=2024-04-16T15:10:43.5076889Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/grafana/0.log
ts=2024-04-16T15:10:43.50762108Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*043372d7-9411-443f-ba7f-80988f77d8b3/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i3333\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org4\", plan=\"free\", pod=\"org4-grafana-b4f87fcc5-fflgn\", pod_template_hash=\"b4f87fcc5\", resource_version=\"167289888\", slug=\"org4\", stackId=\"333333\"}"
ts=2024-04-16T15:10:43.507460368Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*043372d7-9411-443f-ba7f-80988f77d8b3/hosted-grafana-security/*.log
ts=2024-04-16T15:10:43.507397087Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.507344086Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.507308256Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.507283825Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hosted-grafana-security/0.log reason=null
ts=2024-04-16T15:10:43.507259015Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.507208275Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*043372d7-9411-443f-ba7f-80988f77d8b3/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i3333\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org4\", plan=\"free\", pod=\"org4-grafana-b4f87fcc5-fflgn\", pod_template_hash=\"b4f87fcc5\", resource_version=\"167289888\", slug=\"org4\", stackId=\"333333\"}"
ts=2024-04-16T15:10:43.507100714Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*043372d7-9411-443f-ba7f-80988f77d8b3/hg-plugins/*.log
ts=2024-04-16T15:10:43.507027892Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hg-plugins/0.log
ts=2024-04-16T15:10:43.506996362Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hg-plugins/0.log
ts=2024-04-16T15:10:43.506977991Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hg-plugins/0.log
ts=2024-04-16T15:10:43.506953691Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hg-plugins/0.log reason=null
ts=2024-04-16T15:10:43.506923011Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hg-plugins/0.log
ts=2024-04-16T15:10:43.50681944Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*043372d7-9411-443f-ba7f-80988f77d8b3/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i3333\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org4\", plan=\"free\", pod=\"org4-grafana-b4f87fcc5-fflgn\", pod_template_hash=\"b4f87fcc5\", resource_version=\"167289888\", slug=\"org4\", stackId=\"333333\"}"
ts=2024-04-16T15:10:43.503581356Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/insight-logs_promtail-insight-logs-pxhrc_c86ff41c-f07b-4d03-aef7-05d9e243cef6/promtail/0.log op=CREATE
ts=2024-04-16T15:10:43.50308684Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/insight-logs_promtail-insight-logs-pxhrc_c86ff41c-f07b-4d03-aef7-05d9e243cef6/promtail/0.log.20240416-151043 op=CREATE
ts=2024-04-16T15:10:43.499582153Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/insight-logs_promtail-insight-logs-pxhrc_c86ff41c-f07b-4d03-aef7-05d9e243cef6/promtail/0.log.20240416-145410.gz op=CREATE
ts=2024-04-16T15:10:43.491065781Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-5f848b765-hmj42_31c37ccc-044a-4bed-a85f-358870a3a02c/hg-plugins/0.log - &{Offset:0 Whence:0}"
ts=2024-04-16T15:10:43.491039691Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-5f848b765-hmj42_31c37ccc-044a-4bed-a85f-358870a3a02c/hg-plugins/0.log
ts=2024-04-16T15:10:43.490892989Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-5f848b765-hmj42_31c37ccc-044a-4bed-a85f-358870a3a02c/hg-plugins
ts=2024-04-16T15:10:43.485641743Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-5f848b765-hmj42_31c37ccc-044a-4bed-a85f-358870a3a02c/hgrun/0.log
ts=2024-04-16T15:10:43.485611252Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-5f848b765-hmj42_31c37ccc-044a-4bed-a85f-358870a3a02c/hgrun/0.log - &{Offset:0 Whence:0}"
ts=2024-04-16T15:10:43.485462621Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-5f848b765-hmj42_31c37ccc-044a-4bed-a85f-358870a3a02c/hgrun
ts=2024-04-16T15:10:43.485235288Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-5f848b765-hmj42_31c37ccc-044a-4bed-a85f-358870a3a02c/grafana/0.log
ts=2024-04-16T15:10:43.485241618Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-5f848b765-hmj42_31c37ccc-044a-4bed-a85f-358870a3a02c/grafana/0.log - &{Offset:0 Whence:0}"
ts=2024-04-16T15:10:43.484833293Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-5f848b765-hmj42_31c37ccc-044a-4bed-a85f-358870a3a02c/grafana
ts=2024-04-16T15:10:43.485316721Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/pyroscope-ebpf_profiler-jt8kd_f4785188-1d36-4950-a6f5-328218c73e59/profiler/0.log"
ts=2024-04-16T15:10:43.485223151Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/pyroscope-ebpf_profiler-jt8kd_f4785188-1d36-4950-a6f5-328218c73e59/profiler/0.log ..."
ts=2024-04-16T15:10:43.483728092Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.475061435Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.471990067Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.47184237Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.471539735Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.459291911Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.454815097Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.442284255Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-d95d68499-wsddd_b19cb3ac-7212-48e8-a74c-573d5732f2a9/grafana/0.log.20240416-151023.tmp op=CREATE
ts=2024-04-16T15:10:43.440096821Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.424305955Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.421737727Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-7d484c79c6-hzqs8_daf4dc8d-7371-48e5-8a18-cda6289c7f2a/grafana/0.log op=CREATE
ts=2024-04-16T15:10:43.420163091Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-7d484c79c6-hzqs8_daf4dc8d-7371-48e5-8a18-cda6289c7f2a/grafana/0.log.20240416-151043 op=CREATE
ts=2024-04-16T15:10:43.416217316Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/mimir-dedicated-48_ingester-zone-b-72_1f08c7df-e68c-4101-afba-3cb84acb34f0/ingester/0.log.20240416-150452.tmp op=CREATE
ts=2024-04-16T15:10:43.413967547Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-7d484c79c6-hzqs8_daf4dc8d-7371-48e5-8a18-cda6289c7f2a/grafana/0.log.20240416-151032.gz op=CREATE
ts=2024-04-16T15:10:43.409009601Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/cortex-prod-06_aggregator-40-7d876c497f-qlb2t_f99739bd-bd1f-4605-aaf0-61ef442f4610/aggregator/0.log"
ts=2024-04-16T15:10:43.408920067Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/cortex-prod-06_aggregator-40-7d876c497f-qlb2t_f99739bd-bd1f-4605-aaf0-61ef442f4610/aggregator/0.log ..."
ts=2024-04-16T15:10:43.401825217Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/insight-logs_promtail-insight-logs-pxhrc_c86ff41c-f07b-4d03-aef7-05d9e243cef6/promtail/0.log.20240416-145410.tmp op=CREATE
ts=2024-04-16T15:10:43.382088048Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.382307404Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.381990902Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-67d8c765c5-hsdpm_6cbab356-d034-4ff8-9879-077d7f363956/hg-plugins/0.log to appear..."
ts=2024-04-16T15:10:43.38181857Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-67d8c765c5-hsdpm_6cbab356-d034-4ff8-9879-077d7f363956/hg-plugins/0.log ..."
ts=2024-04-16T15:10:43.381333325Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.378767789Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-67d8c765c5-hsdpm_6cbab356-d034-4ff8-9879-077d7f363956/grafana/0.log to appear..."
ts=2024-04-16T15:10:43.378415347Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-67d8c765c5-hsdpm_6cbab356-d034-4ff8-9879-077d7f363956/grafana/0.log ..."
ts=2024-04-16T15:10:43.375853671Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-b9fc57d4d-sctgm_af150893-4992-4f5e-aa4d-5f178e76f33a/hg-plugins/0.log to appear..."
ts=2024-04-16T15:10:43.375689458Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-b9fc57d4d-sctgm_af150893-4992-4f5e-aa4d-5f178e76f33a/hg-plugins/0.log ..."
ts=2024-04-16T15:10:43.370564138Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/profiles-ops-001_fire-compactor-45_bd1d50cc-1edf-4070-b4e9-aca09955fbc2/compactor/0.log.20240416-150722.tmp op=CREATE
ts=2024-04-16T15:10:43.361010102Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/loki-prod3_distributor-76c559cb69-4v4k4_8a587464-bdea-4777-83a4-fbb81df911ec/distributor/0.log"
ts=2024-04-16T15:10:43.368284509Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/loki-prod3_distributor-76c559cb69-4v4k4_8a587464-bdea-4777-83a4-fbb81df911ec/distributor/0.log ..."
ts=2024-04-16T15:10:43.366399734Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-67d8c765c5-hsdpm_6cbab356-d034-4ff8-9879-077d7f363956/hgrun/0.log to appear..."
ts=2024-04-16T15:10:43.366192752Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-67d8c765c5-hsdpm_6cbab356-d034-4ff8-9879-077d7f363956/hgrun/0.log ..."
ts=2024-04-16T15:10:43.349652417Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hosted-grafana-security/0.log to appear..."
ts=2024-04-16T15:10:43.349452285Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hosted-grafana-security/0.log ..."
ts=2024-04-16T15:10:43.344818625Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-b9fc57d4d-sctgm_af150893-4992-4f5e-aa4d-5f178e76f33a/grafana/0.log to appear..."
ts=2024-04-16T15:10:43.344791104Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-b9fc57d4d-sctgm_af150893-4992-4f5e-aa4d-5f178e76f33a/hgrun/0.log to appear..."
ts=2024-04-16T15:10:43.344771214Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-b9fc57d4d-sctgm_af150893-4992-4f5e-aa4d-5f178e76f33a/grafana/0.log ..."
ts=2024-04-16T15:10:43.344715123Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-b9fc57d4d-sctgm_af150893-4992-4f5e-aa4d-5f178e76f33a/hgrun/0.log ..."
ts=2024-04-16T15:10:43.339861011Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/alertmanager_alertmanager-3_a068a2ab-d658-44d2-9b78-d9d7eb47e7cf/alertmanager/0.log op=CREATE
ts=2024-04-16T15:10:43.339463329Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/alertmanager_alertmanager-3_a068a2ab-d658-44d2-9b78-d9d7eb47e7cf/alertmanager/0.log.20240416-151043 op=CREATE
ts=2024-04-16T15:10:43.338895596Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.338376505Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/alertmanager_alertmanager-3_a068a2ab-d658-44d2-9b78-d9d7eb47e7cf/alertmanager/0.log.20240416-150933.gz op=CREATE
ts=2024-04-16T15:10:43.33378223Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-66bd77fc5d-w6frp_e21695f7-52de-495e-8d78-741519657226/hg-plugins/0.log to appear..."
ts=2024-04-16T15:10:43.333585597Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-66bd77fc5d-w6frp_e21695f7-52de-495e-8d78-741519657226/hg-plugins/0.log ..."
ts=2024-04-16T15:10:43.332851752Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/gr-prod-04_cortex-gw-6579fd68cf-wxfdz_47df6fbe-43af-4cf7-a77f-edc3de06197c/cortex-gw/0.log"
ts=2024-04-16T15:10:43.332775064Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/gr-prod-04_cortex-gw-6579fd68cf-wxfdz_47df6fbe-43af-4cf7-a77f-edc3de06197c/cortex-gw/0.log ..."
ts=2024-04-16T15:10:43.328013161Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.327593586Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.323077753Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/grafana/0.log to appear..."
ts=2024-04-16T15:10:43.322751558Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/grafana/0.log ..."
ts=2024-04-16T15:10:43.319886478Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.320037046Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-8c9f94cf-4npds_48df0ae1-00a4-4c5f-9d89-8c55b50c08f4/grafana/0.log to appear..."
ts=2024-04-16T15:10:43.320023386Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-8c9f94cf-4npds_48df0ae1-00a4-4c5f-9d89-8c55b50c08f4/hg-plugins/0.log to appear..."
ts=2024-04-16T15:10:43.320011954Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-8c9f94cf-4npds_48df0ae1-00a4-4c5f-9d89-8c55b50c08f4/hosted-grafana-security/0.log to appear..."
ts=2024-04-16T15:10:43.319989552Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-8c9f94cf-4npds_48df0ae1-00a4-4c5f-9d89-8c55b50c08f4/hg-plugins/0.log ..."
ts=2024-04-16T15:10:43.319945602Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-8c9f94cf-4npds_48df0ae1-00a4-4c5f-9d89-8c55b50c08f4/hosted-grafana-security/0.log ..."
ts=2024-04-16T15:10:43.319933664Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-8c9f94cf-4npds_48df0ae1-00a4-4c5f-9d89-8c55b50c08f4/grafana/0.log ..."
ts=2024-04-16T15:10:43.314580717Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-58687d789-49wfb_269bf9a8-7811-4259-b1fc-ff487ffc9a9a/hg-plugins/0.log to appear..."
ts=2024-04-16T15:10:43.314408635Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-58687d789-49wfb_269bf9a8-7811-4259-b1fc-ff487ffc9a9a/hg-plugins/0.log ..."
ts=2024-04-16T15:10:43.314451491Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-8c9f94cf-4npds_48df0ae1-00a4-4c5f-9d89-8c55b50c08f4/hgrun/0.log to appear..."
ts=2024-04-16T15:10:43.314350274Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-8c9f94cf-4npds_48df0ae1-00a4-4c5f-9d89-8c55b50c08f4/hgrun/0.log ..."
ts=2024-04-16T15:10:43.308822084Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.306240475Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-06_aggregator-40-7d876c497f-qlb2t_f99739bd-bd1f-4605-aaf0-61ef442f4610/aggregator/0.log op=CREATE
ts=2024-04-16T15:10:43.305755943Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-06_aggregator-40-7d876c497f-qlb2t_f99739bd-bd1f-4605-aaf0-61ef442f4610/aggregator/0.log.20240416-151043 op=CREATE
ts=2024-04-16T15:10:43.304574308Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.302851526Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-6766cd4c64-v7ccz_c8ec4c12-78c6-4f20-b382-8ae7e9f0f4af/hg-plugins/0.log to appear..."
ts=2024-04-16T15:10:43.302759616Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-6766cd4c64-v7ccz_c8ec4c12-78c6-4f20-b382-8ae7e9f0f4af/hg-plugins/0.log ..."
ts=2024-04-16T15:10:43.302148378Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-06_aggregator-40-7d876c497f-qlb2t_f99739bd-bd1f-4605-aaf0-61ef442f4610/aggregator/0.log.20240416-150742.gz op=CREATE
ts=2024-04-16T15:10:43.297925853Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*9afcdf84-163b-402e-bdd2-cfb711593385/hosted-grafana-security/*.log
ts=2024-04-16T15:10:43.297912585Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.297901068Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.297890333Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.297880399Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hosted-grafana-security/0.log reason=null
ts=2024-04-16T15:10:43.297866745Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.297841906Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*9afcdf84-163b-402e-bdd2-cfb711593385/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i4444\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org5\", plan=\"free\", pod=\"org5-grafana-ddbf649cc-zgtf6\", pod_template_hash=\"ddbf649cc\", resource_version=\"95783554\", slug=\"org5\", stackId=\"555555\"}"
ts=2024-04-16T15:10:43.297823221Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*9afcdf84-163b-402e-bdd2-cfb711593385/grafana/*.log
ts=2024-04-16T15:10:43.297808137Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/grafana/0.log
ts=2024-04-16T15:10:43.297793002Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/grafana/0.log
ts=2024-04-16T15:10:43.29777652Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/grafana/0.log
ts=2024-04-16T15:10:43.297763874Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/grafana/0.log reason=null
ts=2024-04-16T15:10:43.297751195Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/grafana/0.log
ts=2024-04-16T15:10:43.297733011Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*9afcdf84-163b-402e-bdd2-cfb711593385/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i4444\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org5\", plan=\"free\", pod=\"org5-grafana-ddbf649cc-zgtf6\", pod_template_hash=\"ddbf649cc\", resource_version=\"95783554\", slug=\"org5\", stackId=\"555555\"}"
ts=2024-04-16T15:10:43.297678961Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*9afcdf84-163b-402e-bdd2-cfb711593385/hg-plugins/*.log
ts=2024-04-16T15:10:43.29766299Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hg-plugins/0.log
ts=2024-04-16T15:10:43.297646746Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hg-plugins/0.log
ts=2024-04-16T15:10:43.297636834Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hg-plugins/0.log
ts=2024-04-16T15:10:43.297624194Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hg-plugins/0.log reason=null
ts=2024-04-16T15:10:43.297610295Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hg-plugins/0.log
ts=2024-04-16T15:10:43.297594201Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*9afcdf84-163b-402e-bdd2-cfb711593385/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i4444\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org5\", plan=\"free\", pod=\"org5-grafana-ddbf649cc-zgtf6\", pod_template_hash=\"ddbf649cc\", resource_version=\"95783554\", slug=\"org5\", stackId=\"555555\"}"
ts=2024-04-16T15:10:43.297572495Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*9afcdf84-163b-402e-bdd2-cfb711593385/hgrun/*.log
ts=2024-04-16T15:10:43.297555846Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hgrun/0.log
ts=2024-04-16T15:10:43.29753931Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hgrun/0.log
ts=2024-04-16T15:10:43.297520943Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hgrun/0.log
ts=2024-04-16T15:10:43.29750834Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hgrun/0.log reason=null
ts=2024-04-16T15:10:43.297478804Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hgrun/0.log
ts=2024-04-16T15:10:43.297420444Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*9afcdf84-163b-402e-bdd2-cfb711593385/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i4444\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org5\", plan=\"free\", pod=\"org5-grafana-ddbf649cc-zgtf6\", pod_template_hash=\"ddbf649cc\", resource_version=\"95783554\", slug=\"org5\", stackId=\"555555\"}"
ts=2024-04-16T15:10:43.296157354Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hosted-grafana-security/*.log
ts=2024-04-16T15:10:43.296139225Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i5555\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org6\", plan=\"free\", pod=\"org6-grafana-fdfdc64bb-srrx6\", pod_template_hash=\"fdfdc64bb\", resource_version=\"95745089\", slug=\"org6\", stackId=\"666666\"}"
ts=2024-04-16T15:10:43.29611646Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.296116023Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hgrun/*.log
ts=2024-04-16T15:10:43.296101716Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i5555\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org6\", plan=\"free\", pod=\"org6-grafana-fdfdc64bb-srrx6\", pod_template_hash=\"fdfdc64bb\", resource_version=\"95745089\", slug=\"org6\", stackId=\"666666\"}"
ts=2024-04-16T15:10:43.296098008Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.296083728Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.2960726Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hosted-grafana-security/0.log reason=null
ts=2024-04-16T15:10:43.296071337Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*0ecafb81-c168-4bc4-99e3-e8b2315a09b8/grafana/*.log
ts=2024-04-16T15:10:43.296059925Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*0ecafb81-c168-4bc4-99e3-e8b2315a09b8/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i5555\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org6\", plan=\"free\", pod=\"org6-grafana-fdfdc64bb-srrx6\", pod_template_hash=\"fdfdc64bb\", resource_version=\"95745089\", slug=\"org6\", stackId=\"666666\"}"
ts=2024-04-16T15:10:43.296036364Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.296037858Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hg-plugins/*.log
ts=2024-04-16T15:10:43.296011595Z caller=filetarget.go:326 level=info component=logs logs_config=default msg="removing directory from watcher" directory=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hosted-grafana-security
ts=2024-04-16T15:10:43.296005269Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i5555\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org6\", plan=\"free\", pod=\"org6-grafana-fdfdc64bb-srrx6\", pod_template_hash=\"fdfdc64bb\", resource_version=\"95745089\", slug=\"org6\", stackId=\"666666\"}"
ts=2024-04-16T15:10:43.295945337Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hg-plugins/0.log
ts=2024-04-16T15:10:43.295931888Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hg-plugins/0.log
ts=2024-04-16T15:10:43.295920924Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hg-plugins/0.log
ts=2024-04-16T15:10:43.295903261Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hg-plugins/0.log reason=null
ts=2024-04-16T15:10:43.295876551Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hg-plugins/0.log
ts=2024-04-16T15:10:43.295857948Z caller=filetarget.go:326 level=info component=logs logs_config=default msg="removing directory from watcher" directory=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hg-plugins
ts=2024-04-16T15:10:43.295670559Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-7c67b65cdd-mk82w_496ab238-1eed-4b76-8345-de4537cb7c0e/grafana/0.log.20240416-151011.tmp op=CREATE
ts=2024-04-16T15:10:43.295672083Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hgrun/0.log
ts=2024-04-16T15:10:43.295659368Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hgrun/0.log
ts=2024-04-16T15:10:43.295649791Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hgrun/0.log
ts=2024-04-16T15:10:43.295626379Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hgrun/0.log reason=null
ts=2024-04-16T15:10:43.295589808Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/pyroscope-ebpf_profiler-jt8kd_f4785188-1d36-4950-a6f5-328218c73e59/profiler/0.log op=CREATE
ts=2024-04-16T15:10:43.295587723Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hgrun/0.log
ts=2024-04-16T15:10:43.295545284Z caller=filetarget.go:326 level=info component=logs logs_config=default msg="removing directory from watcher" directory=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hgrun
ts=2024-04-16T15:10:43.295472158Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.295436643Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/grafana/0.log
ts=2024-04-16T15:10:43.295422636Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/grafana/0.log
ts=2024-04-16T15:10:43.295411202Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/grafana/0.log
ts=2024-04-16T15:10:43.295395484Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/grafana/0.log reason=null
ts=2024-04-16T15:10:43.295370159Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hg-plugins/0.log
ts=2024-04-16T15:10:43.295364051Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/grafana/0.log
ts=2024-04-16T15:10:43.295336559Z caller=filetarget.go:326 level=info component=logs logs_config=default msg="removing directory from watcher" directory=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/grafana
ts=2024-04-16T15:10:43.295169248Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/pyroscope-ebpf_profiler-jt8kd_f4785188-1d36-4950-a6f5-328218c73e59/profiler/0.log.20240416-151043 op=CREATE
ts=2024-04-16T15:10:43.294932482Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hgrun/0.log
ts=2024-04-16T15:10:43.294837816Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/grafana/0.log
ts=2024-04-16T15:10:43.294695226Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/loki-prod-026_query-frontend-7fc658ff96-xhbns_91d07b6a-76db-4267-a421-9616df7ab198/query-frontend/0.log"
ts=2024-04-16T15:10:43.294614823Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/loki-prod-026_query-frontend-7fc658ff96-xhbns_91d07b6a-76db-4267-a421-9616df7ab198/query-frontend/0.log ..."
ts=2024-04-16T15:10:43.291684718Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/pyroscope-ebpf_profiler-jt8kd_f4785188-1d36-4950-a6f5-328218c73e59/profiler/0.log.20240416-135703.gz op=CREATE
ts=2024-04-16T15:10:43.291663946Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/profiles-ops-001_fire-compactor-91_2be1ca60-4f1b-4f90-90bc-d235de54176b/compactor/0.log"
ts=2024-04-16T15:10:43.291549231Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/profiles-ops-001_fire-compactor-91_2be1ca60-4f1b-4f90-90bc-d235de54176b/compactor/0.log ..."
ts=2024-04-16T15:10:43.285237795Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/grafana-ruler_grafana-ruler-78bc75f5d6-p98cc_e082d5d3-bb47-435b-9c42-9304ea43d9a5/grafana-ruler/0.log"
ts=2024-04-16T15:10:43.28515592Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/grafana-ruler_grafana-ruler-78bc75f5d6-p98cc_e082d5d3-bb47-435b-9c42-9304ea43d9a5/grafana-ruler/0.log ..."
ts=2024-04-16T15:10:43.267196378Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-04_cortex-gw-6579fd68cf-wxfdz_47df6fbe-43af-4cf7-a77f-edc3de06197c/cortex-gw/0.log op=CREATE
ts=2024-04-16T15:10:43.266859931Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-04_cortex-gw-6579fd68cf-wxfdz_47df6fbe-43af-4cf7-a77f-edc3de06197c/cortex-gw/0.log.20240416-151043 op=CREATE
ts=2024-04-16T15:10:43.256583646Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*64872ae4-62eb-4757-b148-72bab4a9e880/hosted-grafana-security/*.log
ts=2024-04-16T15:10:43.256553405Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.256428125Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.256408755Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hosted-grafana-security/0.log
ts=2024-04-16T15:10:43.256365475Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hosted-grafana-security/0.log reason=null
ts=2024-04-16T15:10:43.256261263Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*64872ae4-62eb-4757-b148-72bab4a9e880/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i6666\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org7\", plan=\"free\", pod=\"org7-grafana-647dc5b44f-pmz8j\", pod_template_hash=\"647dc5b44f\", resource_version=\"167297262\", slug=\"org7\", stackId=\"777777\"}"
ts=2024-04-16T15:10:43.256183643Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*64872ae4-62eb-4757-b148-72bab4a9e880/hgrun/*.log
ts=2024-04-16T15:10:43.256166953Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hgrun/0.log
ts=2024-04-16T15:10:43.256150033Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hgrun/0.log
ts=2024-04-16T15:10:43.256132803Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hgrun/0.log
ts=2024-04-16T15:10:43.256115503Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hgrun/0.log reason=null
ts=2024-04-16T15:10:43.256072092Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*64872ae4-62eb-4757-b148-72bab4a9e880/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i6666\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org7\", plan=\"free\", pod=\"org7-grafana-647dc5b44f-pmz8j\", pod_template_hash=\"647dc5b44f\", resource_version=\"167297262\", slug=\"org7\", stackId=\"777777\"}"
ts=2024-04-16T15:10:43.256024602Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*64872ae4-62eb-4757-b148-72bab4a9e880/hg-plugins/*.log
ts=2024-04-16T15:10:43.256006782Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hg-plugins/0.log
ts=2024-04-16T15:10:43.255989251Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hg-plugins/0.log
ts=2024-04-16T15:10:43.255974991Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hg-plugins/0.log
ts=2024-04-16T15:10:43.255960211Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hg-plugins/0.log reason=null
ts=2024-04-16T15:10:43.255933401Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hg-plugins/0.log
ts=2024-04-16T15:10:43.255907671Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*64872ae4-62eb-4757-b148-72bab4a9e880/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i6666\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org7\", plan=\"free\", pod=\"org7-grafana-647dc5b44f-pmz8j\", pod_template_hash=\"647dc5b44f\", resource_version=\"167297262\", slug=\"org7\", stackId=\"777777\"}"
ts=2024-04-16T15:10:43.25582646Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*64872ae4-62eb-4757-b148-72bab4a9e880/grafana/*.log
ts=2024-04-16T15:10:43.25580358Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/grafana/0.log
ts=2024-04-16T15:10:43.25578448Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/grafana/0.log
ts=2024-04-16T15:10:43.25576612Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/grafana/0.log
ts=2024-04-16T15:10:43.25574394Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/grafana/0.log reason=null
ts=2024-04-16T15:10:43.255643719Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*64872ae4-62eb-4757-b148-72bab4a9e880/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i6666\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org7\", plan=\"free\", pod=\"org7-grafana-647dc5b44f-pmz8j\", pod_template_hash=\"647dc5b44f\", resource_version=\"167297262\", slug=\"org7\", stackId=\"777777\"}"
ts=2024-04-16T15:10:43.263042477Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-04_cortex-gw-6579fd68cf-wxfdz_47df6fbe-43af-4cf7-a77f-edc3de06197c/cortex-gw/0.log.20240416-150933.gz op=CREATE
ts=2024-04-16T15:10:43.259110088Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.257819383Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.244818982Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.239151767Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/loki-cd_loki-cd-loki-create-deployment-tuesday-prod-1713254400-wait-for-github-pr-862956382_20556f0a-8a98-4361-bb91-baa2ffb08c41/main/0.log - &{Offset:0 Whence:0}"
ts=2024-04-16T15:10:43.239080083Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/loki-cd_loki-cd-loki-create-deployment-tuesday-prod-1713254400-wait-for-github-pr-862956382_20556f0a-8a98-4361-bb91-baa2ffb08c41/main/0.log
ts=2024-04-16T15:10:43.238913661Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/loki-cd_loki-cd-loki-create-deployment-tuesday-prod-1713254400-wait-for-github-pr-862956382_20556f0a-8a98-4361-bb91-baa2ffb08c41/main
ts=2024-04-16T15:10:43.234330247Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-dd8bf9cd-s8k4m_02f6e4f5-aad2-445f-a04c-f666f0d54ee4/hgrun/0.log to appear..."
ts=2024-04-16T15:10:43.234253033Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-dd8bf9cd-s8k4m_02f6e4f5-aad2-445f-a04c-f666f0d54ee4/hgrun/0.log ..."
ts=2024-04-16T15:10:43.231425217Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-64dc5cbf86-nvgx5_b5380684-df51-45a4-bacb-ca0cfdf7a05f/hgrun/0.log to appear..."
ts=2024-04-16T15:10:43.231145214Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-64dc5cbf86-nvgx5_b5380684-df51-45a4-bacb-ca0cfdf7a05f/hgrun/0.log ..."
ts=2024-04-16T15:10:43.220000621Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_distributor-d9d7957b8-w7pnf_6cfc70ca-26c6-4421-9483-ffe123fe3a6b/distributor/0.log op=CREATE
ts=2024-04-16T15:10:43.21939049Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_distributor-d9d7957b8-w7pnf_6cfc70ca-26c6-4421-9483-ffe123fe3a6b/distributor/0.log.20240416-151043 op=CREATE
ts=2024-04-16T15:10:43.219314914Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.215848733Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 93 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.215641301Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_distributor-d9d7957b8-w7pnf_6cfc70ca-26c6-4421-9483-ffe123fe3a6b/distributor/0.log.20240416-150450.gz op=CREATE
ts=2024-04-16T15:10:43.213328712Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.212676387Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.212298621Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.209036752Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/alertmanager_alertmanager-3_a068a2ab-d658-44d2-9b78-d9d7eb47e7cf/alertmanager/0.log.20240416-150933.tmp op=CREATE
ts=2024-04-16T15:10:43.207714985Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/hosted-grafana_.something-grafana-d95d68499-sqqh8_9c491b2c-2dff-4a72-a3ef-c5719f5cf696/grafana/0.log"
ts=2024-04-16T15:10:43.207654185Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-d95d68499-sqqh8_9c491b2c-2dff-4a72-a3ef-c5719f5cf696/grafana/0.log ..."
ts=2024-04-16T15:10:43.207288069Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/loki-prod_distributor-59f8b8d6f-cf5s5_e3faafed-b644-460f-aa64-0c5195eb2fac/distributor/0.log"
ts=2024-04-16T15:10:43.207202372Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/loki-prod_distributor-59f8b8d6f-cf5s5_e3faafed-b644-460f-aa64-0c5195eb2fac/distributor/0.log ..."
ts=2024-04-16T15:10:43.206926893Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/tempo-prod_ingester-65_02bae376-3900-4aaf-bc1c-717824388403/ingester/0.log"
ts=2024-04-16T15:10:43.206845577Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/tempo-prod_ingester-65_02bae376-3900-4aaf-bc1c-717824388403/ingester/0.log ..."
ts=2024-04-16T15:10:43.205820894Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 93 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.205713872Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.202126267Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.196717804Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/k6-cloud_testcoordinator-job-2665838-9g8ds_19a1cce8-5f04-46e0-a124-292b0dd9b343/testcoordinator/0.log
ts=2024-04-16T15:10:43.196713431Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/k6-cloud_testcoordinator-job-2665838-9g8ds_19a1cce8-5f04-46e0-a124-292b0dd9b343/testcoordinator/0.log - &{Offset:0 Whence:0}"
ts=2024-04-16T15:10:43.19648143Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/k6-cloud_testcoordinator-job-2665838-9g8ds_19a1cce8-5f04-46e0-a124-292b0dd9b343/testcoordinator
ts=2024-04-16T15:10:43.196210056Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-06_aggregator-40-7d876c497f-qlb2t_f99739bd-bd1f-4605-aaf0-61ef442f4610/aggregator/0.log.20240416-150742.tmp op=CREATE
ts=2024-04-16T15:10:43.192290389Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*19a1cce8-5f04-46e0-a124-292b0dd9b343/testcoordinator/*.log:{batch_kubernetes_io_controller_uid=\"25ec5edf-f78e-468b-b6f3-3b9685f0cc8f\", batch_kubernetes_io_job_name=\"testcoordinator-job-2665838\", container=\"testcoordinator\", controller_uid=\"25ec5edf-f78e-468b-b6f3-3b9685f0cc8f\", job=\"k6-cloud/testcoordinator\", job_name=\"testcoordinator-job-2665838\", name=\"testcoordinator\", namespace=\"k6-cloud\", pod=\"testcoordinator-job-2665838-9g8ds\"}"
ts=2024-04-16T15:10:43.177328136Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-dd8bf9cd-s8k4m_02f6e4f5-aad2-445f-a04c-f666f0d54ee4/hg-plugins/0.log to appear..."
ts=2024-04-16T15:10:43.177232412Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-dd8bf9cd-s8k4m_02f6e4f5-aad2-445f-a04c-f666f0d54ee4/hg-plugins/0.log ..."
ts=2024-04-16T15:10:43.187183919Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/kube-system_calico-node-8dlq6_07f72b39-55dd-4e63-903d-0120cd9b3cda/install-cni/0.log to appear..."
ts=2024-04-16T15:10:43.187062724Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/kube-system_calico-node-8dlq6_07f72b39-55dd-4e63-903d-0120cd9b3cda/install-cni/0.log ..."
ts=2024-04-16T15:10:43.18059824Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.174053022Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.172105378Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.170592658Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/loki-prod_query-frontend-9b95769c4-4kkh4_0cc79d0a-6c37-414d-8a2a-0b91b0b3e6c2/query-frontend/4.log"
ts=2024-04-16T15:10:43.170496061Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/loki-prod_query-frontend-9b95769c4-4kkh4_0cc79d0a-6c37-414d-8a2a-0b91b0b3e6c2/query-frontend/4.log ..."
ts=2024-04-16T15:10:43.170424393Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod3_distributor-76c559cb69-4v4k4_8a587464-bdea-4777-83a4-fbb81df911ec/distributor/0.log op=CREATE
ts=2024-04-16T15:10:43.170071302Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod3_distributor-76c559cb69-4v4k4_8a587464-bdea-4777-83a4-fbb81df911ec/distributor/0.log.20240416-151043 op=CREATE
ts=2024-04-16T15:10:43.168507042Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/profiles-ops-001_fire-compactor-91_2be1ca60-4f1b-4f90-90bc-d235de54176b/compactor/0.log op=CREATE
ts=2024-04-16T15:10:43.167505496Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/profiles-ops-001_fire-compactor-91_2be1ca60-4f1b-4f90-90bc-d235de54176b/compactor/0.log.20240416-151043 op=CREATE
ts=2024-04-16T15:10:43.16743356Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod3_distributor-76c559cb69-4v4k4_8a587464-bdea-4777-83a4-fbb81df911ec/distributor/0.log.20240416-150602.gz op=CREATE
ts=2024-04-16T15:10:43.16346356Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/profiles-ops-001_fire-compactor-91_2be1ca60-4f1b-4f90-90bc-d235de54176b/compactor/0.log.20240416-150902.gz op=CREATE
ts=2024-04-16T15:10:43.162766766Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.162659451Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.154757178Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.154349425Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.144637923Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-64dc5cbf86-nvgx5_b5380684-df51-45a4-bacb-ca0cfdf7a05f/hg-plugins/0.log to appear..."
ts=2024-04-16T15:10:43.144559282Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-64dc5cbf86-nvgx5_b5380684-df51-45a4-bacb-ca0cfdf7a05f/hg-plugins/0.log ..."
ts=2024-04-16T15:10:43.136197313Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-7f47dc8c57-7xkbj_94f05c74-2cbd-48ba-9109-b1966920123b/hosted-grafana-security/0.log to appear..."
ts=2024-04-16T15:10:43.136128662Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7f47dc8c57-7xkbj_94f05c74-2cbd-48ba-9109-b1966920123b/hosted-grafana-security/0.log ..."
ts=2024-04-16T15:10:43.135428948Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:43.13241598Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-64dc5cbf86-nvgx5_b5380684-df51-45a4-bacb-ca0cfdf7a05f/grafana/0.log to appear..."
ts=2024-04-16T15:10:43.1323331Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-64dc5cbf86-nvgx5_b5380684-df51-45a4-bacb-ca0cfdf7a05f/grafana/0.log ..."
ts=2024-04-16T15:10:43.12931936Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-7d484c79c6-hzqs8_daf4dc8d-7371-48e5-8a18-cda6289c7f2a/grafana/0.log.20240416-151032.tmp op=CREATE
ts=2024-04-16T15:10:43.127063794Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-58bc4b54f9-v2hs8_64317ae0-8f2c-4861-8629-4363b46f550d/hgrun/0.log - &{Offset:0 Whence:0}"
ts=2024-04-16T15:10:43.127036834Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-58bc4b54f9-v2hs8_64317ae0-8f2c-4861-8629-4363b46f550d/hgrun/0.log
ts=2024-04-16T15:10:43.126942786Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-58bc4b54f9-v2hs8_64317ae0-8f2c-4861-8629-4363b46f550d/hg-plugins/0.log
ts=2024-04-16T15:10:43.126913616Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-58bc4b54f9-v2hs8_64317ae0-8f2c-4861-8629-4363b46f550d/hgrun
ts=2024-04-16T15:10:43.126921946Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-58bc4b54f9-v2hs8_64317ae0-8f2c-4861-8629-4363b46f550d/hg-plugins/0.log - &{Offset:0 Whence:0}"
ts=2024-04-16T15:10:43.126750144Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-58bc4b54f9-v2hs8_64317ae0-8f2c-4861-8629-4363b46f550d/hg-plugins
ts=2024-04-16T15:10:43.126157093Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.125022226Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/pyroscope-ebpf_profiler-jt8kd_f4785188-1d36-4950-a6f5-328218c73e59/profiler/0.log.20240416-135703.tmp op=CREATE
ts=2024-04-16T15:10:43.120379101Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-04_cortex-gw-6579fd68cf-wxfdz_47df6fbe-43af-4cf7-a77f-edc3de06197c/cortex-gw/0.log.20240416-150933.tmp op=CREATE
ts=2024-04-16T15:10:43.120021897Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/tempo-prod_ingester-65_02bae376-3900-4aaf-bc1c-717824388403/ingester/0.log op=CREATE
ts=2024-04-16T15:10:43.119451076Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/tempo-prod_ingester-65_02bae376-3900-4aaf-bc1c-717824388403/ingester/0.log.20240416-151043 op=CREATE
ts=2024-04-16T15:10:43.116318645Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/kube-system_calico-node-jwmqj_894554c7-64d9-4221-9032-1d2966ce987f/calico-node/0.log to appear..."
ts=2024-04-16T15:10:43.115865529Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/kube-system_calico-node-jwmqj_894554c7-64d9-4221-9032-1d2966ce987f/calico-node/0.log ..."
ts=2024-04-16T15:10:43.115024147Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/tempo-prod_ingester-65_02bae376-3900-4aaf-bc1c-717824388403/ingester/0.log.20240416-150952.gz op=CREATE
ts=2024-04-16T15:10:43.108835306Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hg-plugins/0.log to appear..."
ts=2024-04-16T15:10:43.108765136Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hg-plugins/0.log ..."
ts=2024-04-16T15:10:43.08743031Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_distributor-59f8b8d6f-cf5s5_e3faafed-b644-460f-aa64-0c5195eb2fac/distributor/0.log op=CREATE
ts=2024-04-16T15:10:43.087145122Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/gr-prod-04_graphite-write-proxy-5c4768b4d-694rv_5159bfec-768a-4597-b065-9f39abfbc85d/graphite-write-proxy/0.log"
ts=2024-04-16T15:10:43.086905187Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_distributor-59f8b8d6f-cf5s5_e3faafed-b644-460f-aa64-0c5195eb2fac/distributor/0.log.20240416-151043 op=CREATE
ts=2024-04-16T15:10:43.086995387Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/gr-prod-04_graphite-write-proxy-5c4768b4d-694rv_5159bfec-768a-4597-b065-9f39abfbc85d/graphite-write-proxy/0.log ..."
ts=2024-04-16T15:10:43.083530447Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_distributor-59f8b8d6f-cf5s5_e3faafed-b644-460f-aa64-0c5195eb2fac/distributor/0.log.20240416-150319.gz op=CREATE
ts=2024-04-16T15:10:43.082142726Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.081175944Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-7f47dc8c57-7xkbj_94f05c74-2cbd-48ba-9109-b1966920123b/hgrun/0.log to appear..."
ts=2024-04-16T15:10:43.081142393Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7f47dc8c57-7xkbj_94f05c74-2cbd-48ba-9109-b1966920123b/hgrun/0.log ..."
ts=2024-04-16T15:10:43.081118603Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-7f47dc8c57-7xkbj_94f05c74-2cbd-48ba-9109-b1966920123b/hg-plugins/0.log to appear..."
ts=2024-04-16T15:10:43.081069412Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7f47dc8c57-7xkbj_94f05c74-2cbd-48ba-9109-b1966920123b/hg-plugins/0.log ..."
ts=2024-04-16T15:10:43.078778917Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-7f47dc8c57-7xkbj_94f05c74-2cbd-48ba-9109-b1966920123b/grafana/0.log to appear..."
ts=2024-04-16T15:10:43.078661804Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7f47dc8c57-7xkbj_94f05c74-2cbd-48ba-9109-b1966920123b/grafana/0.log ..."
ts=2024-04-16T15:10:43.065663262Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_query-frontend-9b95769c4-4kkh4_0cc79d0a-6c37-414d-8a2a-0b91b0b3e6c2/query-frontend/4.log op=CREATE
ts=2024-04-16T15:10:43.065151425Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_query-frontend-9b95769c4-4kkh4_0cc79d0a-6c37-414d-8a2a-0b91b0b3e6c2/query-frontend/4.log.20240416-151043 op=CREATE
ts=2024-04-16T15:10:43.062372224Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_query-frontend-9b95769c4-4kkh4_0cc79d0a-6c37-414d-8a2a-0b91b0b3e6c2/query-frontend/4.log.20240416-150942.gz op=CREATE
ts=2024-04-16T15:10:43.057467305Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*6e95d7c0-d863-461c-a6d1-68653e438e3e/kube-proxy/*.log:{component=\"kube-proxy\", container=\"kube-proxy\", job=\"kube-system/gke-ops-us-east-0-main-n2s32-1-1dd39c-32ae1dde-hmhw\", namespace=\"kube-system\", pod=\"kube-proxy-gke-ops-us-east-0-main-n2s32-1-1dd39c-32ae1dde-hmhw\", tier=\"node\"}"
ts=2024-04-16T15:10:43.053333798Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/loki-prod_parallel-querier-54566cbb6c-d5tbl_b5e4705e-f54a-44c2-8e3b-9d6417be4208/parallel-querier/0.log"
ts=2024-04-16T15:10:43.053227989Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/loki-prod_parallel-querier-54566cbb6c-d5tbl_b5e4705e-f54a-44c2-8e3b-9d6417be4208/parallel-querier/0.log ..."
ts=2024-04-16T15:10:43.047219694Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/kube-system_kube-proxy-gke-ops-us-east-0-main-n2s32-1-1dd39c-32ae1dde-hmhw_1954fd4ff7221e619e2d202bfb2c4ab9/kube-proxy/0.log - &{Offset:0 Whence:0}"
ts=2024-04-16T15:10:43.047188126Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/kube-system_kube-proxy-gke-ops-us-east-0-main-n2s32-1-1dd39c-32ae1dde-hmhw_1954fd4ff7221e619e2d202bfb2c4ab9/kube-proxy/0.log
ts=2024-04-16T15:10:43.047043424Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/kube-system_kube-proxy-gke-ops-us-east-0-main-n2s32-1-1dd39c-32ae1dde-hmhw_1954fd4ff7221e619e2d202bfb2c4ab9/kube-proxy
ts=2024-04-16T15:10:43.044285087Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*1954fd4ff7221e619e2d202bfb2c4ab9/kube-proxy/*.log:{component=\"kube-proxy\", container=\"kube-proxy\", job=\"kube-system/kube-proxy\", namespace=\"kube-system\", pod=\"kube-proxy-gke-ops-us-east-0-main-n2s32-1-1dd39c-32ae1dde-hmhw\", tier=\"node\"}"
ts=2024-04-16T15:10:43.020577028Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hg-plugins/0.log to appear..."
ts=2024-04-16T15:10:43.020377695Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hg-plugins/0.log ..."
ts=2024-04-16T15:10:43.020680098Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/profiles-ops-001_fire-compactor-91_2be1ca60-4f1b-4f90-90bc-d235de54176b/compactor/0.log.20240416-150902.tmp op=CREATE
ts=2024-04-16T15:10:43.015377842Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hgrun/0.log to appear..."
ts=2024-04-16T15:10:43.015197479Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hgrun/0.log ..."
ts=2024-04-16T15:10:43.008239745Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-67d4884694-zttck_be16c84b-12ba-4590-8e1d-22bbf77c01a4/hosted-grafana-security/0.log to appear..."
ts=2024-04-16T15:10:43.008154235Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-67d4884694-zttck_be16c84b-12ba-4590-8e1d-22bbf77c01a4/hosted-grafana-security/0.log ..."
ts=2024-04-16T15:10:43.010216068Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:43.003786403Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-d95d68499-sqqh8_9c491b2c-2dff-4a72-a3ef-c5719f5cf696/grafana/0.log op=CREATE
ts=2024-04-16T15:10:43.002376901Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-d95d68499-sqqh8_9c491b2c-2dff-4a72-a3ef-c5719f5cf696/grafana/0.log.20240416-151043 op=CREATE
ts=2024-04-16T15:10:42.999925258Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-d95d68499-sqqh8_9c491b2c-2dff-4a72-a3ef-c5719f5cf696/grafana/0.log.20240416-151012.gz op=CREATE
ts=2024-04-16T15:10:42.99578715Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/loki-prod_query-frontend-9b95769c4-88ptz_c65aaf2a-b17d-43d7-ae47-592a90086d0e/query-frontend/2.log"
ts=2024-04-16T15:10:42.995688711Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/loki-prod_query-frontend-9b95769c4-88ptz_c65aaf2a-b17d-43d7-ae47-592a90086d0e/query-frontend/2.log ..."
ts=2024-04-16T15:10:42.992974648Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/tempo-prod_ingester-65_02bae376-3900-4aaf-bc1c-717824388403/ingester/0.log.20240416-150952.tmp op=CREATE
ts=2024-04-16T15:10:42.988556779Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/cortex-prod-10_ingester-zone-c-116_4655218c-833b-460a-879b-c5ccc57b70c4/ingester/0.log"
ts=2024-04-16T15:10:42.988475443Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/cortex-prod-10_ingester-zone-c-116_4655218c-833b-460a-879b-c5ccc57b70c4/ingester/0.log ..."
ts=2024-04-16T15:10:42.988115995Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_distributor-d9d7957b8-w7pnf_6cfc70ca-26c6-4421-9483-ffe123fe3a6b/distributor/0.log.20240416-150450.tmp op=CREATE
ts=2024-04-16T15:10:42.98719306Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod3_distributor-76c559cb69-4v4k4_8a587464-bdea-4777-83a4-fbb81df911ec/distributor/0.log.20240416-150602.tmp op=CREATE
ts=2024-04-16T15:10:42.981209224Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/gr-prod-03_graphite-querier-644cb88c5-k6mbl_bba9f1fd-60ff-432d-a788-19cd2e04af92/querier/0.log"
ts=2024-04-16T15:10:42.981121512Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/gr-prod-03_graphite-querier-644cb88c5-k6mbl_bba9f1fd-60ff-432d-a788-19cd2e04af92/querier/0.log ..."
ts=2024-04-16T15:10:42.972859196Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:42.970735558Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:42.968185291Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:42.968148784Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:42.961558674Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/hosted-grafana_.something-grafana-97c647648-whbkm_74aa1a7c-2b5b-4986-a95b-3cd30627d846/grafana/0.log"
ts=2024-04-16T15:10:42.961469548Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-97c647648-whbkm_74aa1a7c-2b5b-4986-a95b-3cd30627d846/grafana/0.log ..."
ts=2024-04-16T15:10:42.960769391Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:42.95311867Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-67d4884694-zttck_be16c84b-12ba-4590-8e1d-22bbf77c01a4/grafana/0.log to appear..."
ts=2024-04-16T15:10:42.95302964Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-67d4884694-zttck_be16c84b-12ba-4590-8e1d-22bbf77c01a4/grafana/0.log ..."
ts=2024-04-16T15:10:42.959146686Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-04_graphite-write-proxy-5c4768b4d-694rv_5159bfec-768a-4597-b065-9f39abfbc85d/graphite-write-proxy/0.log op=CREATE
ts=2024-04-16T15:10:42.95884468Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-04_graphite-write-proxy-5c4768b4d-694rv_5159bfec-768a-4597-b065-9f39abfbc85d/graphite-write-proxy/0.log.20240416-151042 op=CREATE
ts=2024-04-16T15:10:42.956246032Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*c3c249a2-c8ff-40f4-a66d-9d746b39110b/hg-plugins/*.log
ts=2024-04-16T15:10:42.95623547Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hg-plugins/0.log
ts=2024-04-16T15:10:42.956222919Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hg-plugins/0.log
ts=2024-04-16T15:10:42.956211852Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hg-plugins/0.log
ts=2024-04-16T15:10:42.956202082Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hg-plugins/0.log reason=null
ts=2024-04-16T15:10:42.956185492Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hg-plugins/0.log
ts=2024-04-16T15:10:42.956170942Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*c3c249a2-c8ff-40f4-a66d-9d746b39110b/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i7777\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org8\", plan=\"free\", pod=\"org8-grafana-6c66686686-mqtcr\", pod_template_hash=\"6c66686686\", resource_version=\"95723800\", slug=\"org8\", stackId=\"888888\"}"
ts=2024-04-16T15:10:42.956145757Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*c3c249a2-c8ff-40f4-a66d-9d746b39110b/grafana/*.log
ts=2024-04-16T15:10:42.956134619Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/grafana/0.log
ts=2024-04-16T15:10:42.956120932Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/grafana/0.log
ts=2024-04-16T15:10:42.956108765Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/grafana/0.log
ts=2024-04-16T15:10:42.956098278Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/grafana/0.log reason=null
ts=2024-04-16T15:10:42.956083968Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/grafana/0.log
ts=2024-04-16T15:10:42.956067383Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*c3c249a2-c8ff-40f4-a66d-9d746b39110b/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i7777\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org8\", plan=\"free\", pod=\"org8-grafana-6c66686686-mqtcr\", pod_template_hash=\"6c66686686\", resource_version=\"95723800\", slug=\"org8\", stackId=\"888888\"}"
ts=2024-04-16T15:10:42.95599928Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*c3c249a2-c8ff-40f4-a66d-9d746b39110b/hosted-grafana-security/*.log
ts=2024-04-16T15:10:42.95598394Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hosted-grafana-security/0.log
ts=2024-04-16T15:10:42.955972198Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hosted-grafana-security/0.log
ts=2024-04-16T15:10:42.955959785Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hosted-grafana-security/0.log
ts=2024-04-16T15:10:42.955948953Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hosted-grafana-security/0.log reason=null
ts=2024-04-16T15:10:42.955934083Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hosted-grafana-security/0.log
ts=2024-04-16T15:10:42.955913962Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*c3c249a2-c8ff-40f4-a66d-9d746b39110b/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i7777\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org8\", plan=\"free\", pod=\"org8-grafana-6c66686686-mqtcr\", pod_template_hash=\"6c66686686\", resource_version=\"95723800\", slug=\"org8\", stackId=\"888888\"}"
ts=2024-04-16T15:10:42.955890015Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*c3c249a2-c8ff-40f4-a66d-9d746b39110b/hgrun/*.log
ts=2024-04-16T15:10:42.955870893Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hgrun/0.log
ts=2024-04-16T15:10:42.95585467Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hgrun/0.log
ts=2024-04-16T15:10:42.955842788Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hgrun/0.log
ts=2024-04-16T15:10:42.955828203Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hgrun/0.log reason=null
ts=2024-04-16T15:10:42.95580573Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hgrun/0.log
ts=2024-04-16T15:10:42.95572753Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*c3c249a2-c8ff-40f4-a66d-9d746b39110b/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i7777\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org8\", plan=\"free\", pod=\"org8-grafana-6c66686686-mqtcr\", pod_template_hash=\"6c66686686\", resource_version=\"95723800\", slug=\"org8\", stackId=\"888888\"}"
ts=2024-04-16T15:10:42.95600412Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:42.9540311Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-04_graphite-write-proxy-5c4768b4d-694rv_5159bfec-768a-4597-b065-9f39abfbc85d/graphite-write-proxy/0.log.20240416-150912.gz op=CREATE
ts=2024-04-16T15:10:42.943541527Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_query-frontend-9b95769c4-4kkh4_0cc79d0a-6c37-414d-8a2a-0b91b0b3e6c2/query-frontend/4.log.20240416-150942.tmp op=CREATE
ts=2024-04-16T15:10:42.926967398Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:42.90779516Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/kube-system_calico-node-8dlq6_07f72b39-55dd-4e63-903d-0120cd9b3cda/calico-node/0.log to appear..."
ts=2024-04-16T15:10:42.90688742Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/kube-system_calico-node-8dlq6_07f72b39-55dd-4e63-903d-0120cd9b3cda/calico-node/0.log ..."
ts=2024-04-16T15:10:42.904408857Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-97c647648-whbkm_74aa1a7c-2b5b-4986-a95b-3cd30627d846/grafana/0.log op=CREATE
ts=2024-04-16T15:10:42.902260189Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-97c647648-whbkm_74aa1a7c-2b5b-4986-a95b-3cd30627d846/grafana/0.log.20240416-151042 op=CREATE
ts=2024-04-16T15:10:42.88299518Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:42.880612088Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-d95d68499-sqqh8_9c491b2c-2dff-4a72-a3ef-c5719f5cf696/grafana/0.log.20240416-151012.tmp op=CREATE
ts=2024-04-16T15:10:42.878870475Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-67d4884694-zttck_be16c84b-12ba-4590-8e1d-22bbf77c01a4/hgrun/0.log to appear..."
ts=2024-04-16T15:10:42.878760804Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-67d4884694-zttck_be16c84b-12ba-4590-8e1d-22bbf77c01a4/hgrun/0.log ..."
ts=2024-04-16T15:10:42.87611453Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/loki-prod-028_querier-5864bffd9f-pr8x4_f823f483-b830-4ebb-9bb3-fd6f35b49cfb/querier/1.log"
ts=2024-04-16T15:10:42.876032602Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/loki-prod-028_querier-5864bffd9f-pr8x4_f823f483-b830-4ebb-9bb3-fd6f35b49cfb/querier/1.log ..."
ts=2024-04-16T15:10:42.857009279Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:42.85475629Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*bd6a9960-6f03-46ee-b646-1456cff50d71/hosted-grafana-security/*.log
ts=2024-04-16T15:10:42.85473729Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hosted-grafana-security/0.log
ts=2024-04-16T15:10:42.85471274Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hosted-grafana-security/0.log
ts=2024-04-16T15:10:42.85469692Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hosted-grafana-security/0.log
ts=2024-04-16T15:10:42.854665989Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hosted-grafana-security/0.log reason=null
ts=2024-04-16T15:10:42.853457993Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*bd6a9960-6f03-46ee-b646-1456cff50d71/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i8888\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org9\", plan=\"free\", pod=\"org9-grafana-cccfb8fd6-cktl7\", pod_template_hash=\"cccfb8fd6\", resource_version=\"167320979\", slug=\"org9\", stackId=\"999999\"}"
ts=2024-04-16T15:10:42.853309562Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*bd6a9960-6f03-46ee-b646-1456cff50d71/hgrun/*.log
ts=2024-04-16T15:10:42.853292992Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hgrun/0.log
ts=2024-04-16T15:10:42.853277081Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hgrun/0.log
ts=2024-04-16T15:10:42.853262091Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hgrun/0.log
ts=2024-04-16T15:10:42.853248801Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hgrun/0.log reason=null
ts=2024-04-16T15:10:42.853229701Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hgrun/0.log
ts=2024-04-16T15:10:42.853210051Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*bd6a9960-6f03-46ee-b646-1456cff50d71/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i8888\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org9\", plan=\"free\", pod=\"org9-grafana-cccfb8fd6-cktl7\", pod_template_hash=\"cccfb8fd6\", resource_version=\"167320979\", slug=\"org9\", stackId=\"999999\"}"
ts=2024-04-16T15:10:42.85315126Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*bd6a9960-6f03-46ee-b646-1456cff50d71/grafana/*.log
ts=2024-04-16T15:10:42.853125369Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/grafana/0.log
ts=2024-04-16T15:10:42.853110049Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/grafana/0.log
ts=2024-04-16T15:10:42.853094689Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/grafana/0.log
ts=2024-04-16T15:10:42.853080408Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/grafana/0.log reason=null
ts=2024-04-16T15:10:42.853059658Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/grafana/0.log
ts=2024-04-16T15:10:42.853040568Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*bd6a9960-6f03-46ee-b646-1456cff50d71/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i8888\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org9\", plan=\"free\", pod=\"org9-grafana-cccfb8fd6-cktl7\", pod_template_hash=\"cccfb8fd6\", resource_version=\"167320979\", slug=\"org9\", stackId=\"999999\"}"
ts=2024-04-16T15:10:42.853013388Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*bd6a9960-6f03-46ee-b646-1456cff50d71/hg-plugins/*.log
ts=2024-04-16T15:10:42.852994747Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hg-plugins/0.log
ts=2024-04-16T15:10:42.852976767Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hg-plugins/0.log
ts=2024-04-16T15:10:42.852962817Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hg-plugins/0.log
ts=2024-04-16T15:10:42.852946987Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hg-plugins/0.log reason=null
ts=2024-04-16T15:10:42.852918397Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hg-plugins/0.log
ts=2024-04-16T15:10:42.852862236Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*bd6a9960-6f03-46ee-b646-1456cff50d71/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i8888\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org9\", plan=\"free\", pod=\"org9-grafana-cccfb8fd6-cktl7\", pod_template_hash=\"cccfb8fd6\", resource_version=\"167320979\", slug=\"org9\", stackId=\"999999\"}"
ts=2024-04-16T15:10:42.852799278Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*97381017-8002-430b-84fc-ed98c0becff1/hg-plugins/*.log
ts=2024-04-16T15:10:42.852780068Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hg-plugins/0.log
ts=2024-04-16T15:10:42.852742378Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hg-plugins/0.log
ts=2024-04-16T15:10:42.852725798Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hg-plugins/0.log
ts=2024-04-16T15:10:42.852704968Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hg-plugins/0.log reason=null
ts=2024-04-16T15:10:42.852619797Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*97381017-8002-430b-84fc-ed98c0becff1/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i9999\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org10\", plan=\"free\", pod=\"org10-grafana-67b899f4c-6jkpm\", pod_template_hash=\"67b899f4c\", resource_version=\"167379037\", slug=\"org10\", stackId=\"101010\"}"
ts=2024-04-16T15:10:42.852585297Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*97381017-8002-430b-84fc-ed98c0becff1/hgrun/*.log
ts=2024-04-16T15:10:42.852563836Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hgrun/0.log
ts=2024-04-16T15:10:42.852542956Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hgrun/0.log
ts=2024-04-16T15:10:42.852526406Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hgrun/0.log
ts=2024-04-16T15:10:42.852503976Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hgrun/0.log reason=null
ts=2024-04-16T15:10:42.852335605Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*97381017-8002-430b-84fc-ed98c0becff1/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i9999\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org10\", plan=\"free\", pod=\"org10-grafana-67b899f4c-6jkpm\", pod_template_hash=\"67b899f4c\", resource_version=\"167379037\", slug=\"org10\", stackId=\"101010\"}"
ts=2024-04-16T15:10:42.852191634Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*97381017-8002-430b-84fc-ed98c0becff1/grafana/*.log
ts=2024-04-16T15:10:42.852171694Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/grafana/0.log
ts=2024-04-16T15:10:42.852154564Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/grafana/0.log
ts=2024-04-16T15:10:42.852139114Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/grafana/0.log
ts=2024-04-16T15:10:42.852119214Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/grafana/0.log reason=null
ts=2024-04-16T15:10:42.852054393Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*97381017-8002-430b-84fc-ed98c0becff1/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i9999\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org10\", plan=\"free\", pod=\"org10-grafana-67b899f4c-6jkpm\", pod_template_hash=\"67b899f4c\", resource_version=\"167379037\", slug=\"org10\", stackId=\"101010\"}"
ts=2024-04-16T15:10:42.851876072Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*97381017-8002-430b-84fc-ed98c0becff1/hosted-grafana-security/*.log
ts=2024-04-16T15:10:42.851851762Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hosted-grafana-security/0.log
ts=2024-04-16T15:10:42.851827431Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hosted-grafana-security/0.log
ts=2024-04-16T15:10:42.851787641Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hosted-grafana-security/0.log
ts=2024-04-16T15:10:42.851700581Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hosted-grafana-security/0.log reason=null
ts=2024-04-16T15:10:42.85160466Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*97381017-8002-430b-84fc-ed98c0becff1/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i9999\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org10\", plan=\"free\", pod=\"org10-grafana-67b899f4c-6jkpm\", pod_template_hash=\"67b899f4c\", resource_version=\"167379037\", slug=\"org10\", stackId=\"101010\"}"
ts=2024-04-16T15:10:42.844520154Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hgrun/0.log to appear..."
ts=2024-04-16T15:10:42.84425242Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hgrun/0.log ..."
ts=2024-04-16T15:10:42.843430857Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_query-frontend-9b95769c4-88ptz_c65aaf2a-b17d-43d7-ae47-592a90086d0e/query-frontend/2.log op=CREATE
ts=2024-04-16T15:10:42.843006913Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_query-frontend-9b95769c4-88ptz_c65aaf2a-b17d-43d7-ae47-592a90086d0e/query-frontend/2.log.20240416-151042 op=CREATE
ts=2024-04-16T15:10:42.839982384Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_query-frontend-9b95769c4-88ptz_c65aaf2a-b17d-43d7-ae47-592a90086d0e/query-frontend/2.log.20240416-150912.gz op=CREATE
ts=2024-04-16T15:10:42.838989799Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:42.8353956Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:42.834326829Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_distributor-59f8b8d6f-cf5s5_e3faafed-b644-460f-aa64-0c5195eb2fac/distributor/0.log.20240416-150319.tmp op=CREATE
ts=2024-04-16T15:10:42.834328514Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:42.831554341Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:42.816059502Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/grafana/0.log to appear..."
ts=2024-04-16T15:10:42.805204253Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/grafana/0.log ..."
ts=2024-04-16T15:10:42.82002393Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:42.813772951Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/loki-prod3_parallel-querier-burst-68d978b6bd-tqtt7_53706fa4-7ae2-4000-9905-5c2289ffcb80/parallel-querier/0.log"
ts=2024-04-16T15:10:42.813717925Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/loki-prod3_parallel-querier-burst-68d978b6bd-tqtt7_53706fa4-7ae2-4000-9905-5c2289ffcb80/parallel-querier/0.log ..."
ts=2024-04-16T15:10:42.81324777Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:42.812324173Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-04_graphite-write-proxy-5c4768b4d-694rv_5159bfec-768a-4597-b065-9f39abfbc85d/graphite-write-proxy/0.log.20240416-150912.tmp op=CREATE
ts=2024-04-16T15:10:42.801709333Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hgrun/*.log
ts=2024-04-16T15:10:42.801692403Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hgrun/0.log
ts=2024-04-16T15:10:42.801677883Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hgrun/0.log
ts=2024-04-16T15:10:42.801666533Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hgrun/0.log
ts=2024-04-16T15:10:42.801649933Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hgrun/0.log reason=null
ts=2024-04-16T15:10:42.801570113Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i1010\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org11\", plan=\"free\", pod=\"org11-grafana-55fdb5b7bc-npz4b\", pod_template_hash=\"55fdb5b7bc\", resource_version=\"167301760\", slug=\"org11\", stackId=\"111111\"}"
ts=2024-04-16T15:10:42.801410661Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hg-plugins/*.log
ts=2024-04-16T15:10:42.801392301Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hg-plugins/0.log
ts=2024-04-16T15:10:42.80136025Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hg-plugins/0.log
ts=2024-04-16T15:10:42.80134193Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hg-plugins/0.log
ts=2024-04-16T15:10:42.80130589Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hg-plugins/0.log reason=null
ts=2024-04-16T15:10:42.80124087Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i1010\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org11\", plan=\"free\", pod=\"org11-grafana-55fdb5b7bc-npz4b\", pod_template_hash=\"55fdb5b7bc\", resource_version=\"167301760\", slug=\"org11\", stackId=\"111111\"}"
ts=2024-04-16T15:10:42.801144688Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hosted-grafana-security/*.log
ts=2024-04-16T15:10:42.801130058Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hosted-grafana-security/0.log
ts=2024-04-16T15:10:42.801111398Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hosted-grafana-security/0.log
ts=2024-04-16T15:10:42.801092908Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hosted-grafana-security/0.log
ts=2024-04-16T15:10:42.801059578Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hosted-grafana-security/0.log reason=null
ts=2024-04-16T15:10:42.800965347Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i1010\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org11\", plan=\"free\", pod=\"org11-grafana-55fdb5b7bc-npz4b\", pod_template_hash=\"55fdb5b7bc\", resource_version=\"167301760\", slug=\"org11\", stackId=\"111111\"}"
ts=2024-04-16T15:10:42.800900727Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/grafana/*.log
ts=2024-04-16T15:10:42.800843615Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/grafana/0.log
ts=2024-04-16T15:10:42.800805775Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/grafana/0.log
ts=2024-04-16T15:10:42.800785075Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/grafana/0.log
ts=2024-04-16T15:10:42.800761935Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/grafana/0.log reason=null
ts=2024-04-16T15:10:42.800648554Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i1010\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org11\", plan=\"free\", pod=\"org11-grafana-55fdb5b7bc-npz4b\", pod_template_hash=\"55fdb5b7bc\", resource_version=\"167301760\", slug=\"org11\", stackId=\"111111\"}"
ts=2024-04-16T15:10:42.794244605Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/grafana-ruler_grafana-ruler-78bc75f5d6-p98cc_e082d5d3-bb47-435b-9c42-9304ea43d9a5/grafana-ruler/0.log op=CREATE
ts=2024-04-16T15:10:42.793880616Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/grafana-ruler_grafana-ruler-78bc75f5d6-p98cc_e082d5d3-bb47-435b-9c42-9304ea43d9a5/grafana-ruler/0.log.20240416-151042 op=CREATE
ts=2024-04-16T15:10:42.800115014Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:42.786968877Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:42.780224258Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-76c955c777-glpnp_8e9456f5-b97c-4b8e-b602-0354fc36abc5/grafana/0.log to appear..."
ts=2024-04-16T15:10:42.780195748Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-76c955c777-glpnp_8e9456f5-b97c-4b8e-b602-0354fc36abc5/grafana/0.log ..."
ts=2024-04-16T15:10:42.780195348Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-76c955c777-glpnp_8e9456f5-b97c-4b8e-b602-0354fc36abc5/hg-plugins/0.log to appear..."
ts=2024-04-16T15:10:42.780186827Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-76c955c777-glpnp_8e9456f5-b97c-4b8e-b602-0354fc36abc5/hgrun/0.log to appear..."
ts=2024-04-16T15:10:42.780165857Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-76c955c777-glpnp_8e9456f5-b97c-4b8e-b602-0354fc36abc5/hg-plugins/0.log ..."
ts=2024-04-16T15:10:42.780151287Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-76c955c777-glpnp_8e9456f5-b97c-4b8e-b602-0354fc36abc5/hgrun/0.log ..."
ts=2024-04-16T15:10:42.780050945Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-76c955c777-glpnp_8e9456f5-b97c-4b8e-b602-0354fc36abc5/hosted-grafana-security/0.log to appear..."
ts=2024-04-16T15:10:42.779989823Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-76c955c777-glpnp_8e9456f5-b97c-4b8e-b602-0354fc36abc5/hosted-grafana-security/0.log ..."
ts=2024-04-16T15:10:42.779827758Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-10_ingester-zone-c-116_4655218c-833b-460a-879b-c5ccc57b70c4/ingester/0.log op=CREATE
ts=2024-04-16T15:10:42.778158862Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/grafana-ruler_grafana-ruler-78bc75f5d6-p98cc_e082d5d3-bb47-435b-9c42-9304ea43d9a5/grafana-ruler/0.log.20240416-151022.gz op=CREATE
ts=2024-04-16T15:10:42.777863082Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-10_ingester-zone-c-116_4655218c-833b-460a-879b-c5ccc57b70c4/ingester/0.log.20240416-151042 op=CREATE
ts=2024-04-16T15:10:42.777503455Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-10_ingester-zone-c-116_4655218c-833b-460a-879b-c5ccc57b70c4/ingester/0.log.20240416-144237.gz op=CREATE
ts=2024-04-16T15:10:42.768846823Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*045e328e-c799-47f0-80ed-193ee19a7618/hgrun/*.log
ts=2024-04-16T15:10:42.768825903Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hgrun/0.log
ts=2024-04-16T15:10:42.768805432Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hgrun/0.log
ts=2024-04-16T15:10:42.768782331Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hgrun/0.log
ts=2024-04-16T15:10:42.768724271Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hgrun/0.log reason=null
ts=2024-04-16T15:10:42.76861648Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*045e328e-c799-47f0-80ed-193ee19a7618/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i1212\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org12\", plan=\"free\", pod=\"org12-grafana-54444b9976-ttcfk\", pod_template_hash=\"54444b9976\", resource_version=\"167262602\", slug=\"org12\", stackId=\"121212\"}"
ts=2024-04-16T15:10:42.768356606Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*045e328e-c799-47f0-80ed-193ee19a7618/hosted-grafana-security/*.log
ts=2024-04-16T15:10:42.768267666Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hosted-grafana-security/0.log
ts=2024-04-16T15:10:42.768230715Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hosted-grafana-security/0.log
ts=2024-04-16T15:10:42.768212805Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hosted-grafana-security/0.log
ts=2024-04-16T15:10:42.768176105Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hosted-grafana-security/0.log reason=null
ts=2024-04-16T15:10:42.768095804Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*045e328e-c799-47f0-80ed-193ee19a7618/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i1212\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org12\", plan=\"free\", pod=\"org12-grafana-54444b9976-ttcfk\", pod_template_hash=\"54444b9976\", resource_version=\"167262602\", slug=\"org12\", stackId=\"121212\"}"
ts=2024-04-16T15:10:42.768020902Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*045e328e-c799-47f0-80ed-193ee19a7618/grafana/*.log
ts=2024-04-16T15:10:42.767966672Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/grafana/0.log
ts=2024-04-16T15:10:42.767931071Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/grafana/0.log
ts=2024-04-16T15:10:42.767895471Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/grafana/0.log
ts=2024-04-16T15:10:42.767873841Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/grafana/0.log reason=null
ts=2024-04-16T15:10:42.76773819Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*045e328e-c799-47f0-80ed-193ee19a7618/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i1212\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org12\", plan=\"free\", pod=\"org12-grafana-54444b9976-ttcfk\", pod_template_hash=\"54444b9976\", resource_version=\"167262602\", slug=\"org12\", stackId=\"121212\"}"
ts=2024-04-16T15:10:42.767444506Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*045e328e-c799-47f0-80ed-193ee19a7618/hg-plugins/*.log
ts=2024-04-16T15:10:42.767415776Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hg-plugins/0.log
ts=2024-04-16T15:10:42.767389305Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hg-plugins/0.log
ts=2024-04-16T15:10:42.767367185Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hg-plugins/0.log
ts=2024-04-16T15:10:42.767323154Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hg-plugins/0.log reason=null
ts=2024-04-16T15:10:42.768149996Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:42.767208786Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-67d4884694-zttck_be16c84b-12ba-4590-8e1d-22bbf77c01a4/hg-plugins/0.log to appear..."
ts=2024-04-16T15:10:42.766909734Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-67d4884694-zttck_be16c84b-12ba-4590-8e1d-22bbf77c01a4/hg-plugins/0.log ..."
ts=2024-04-16T15:10:42.766936439Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hg-plugins/0.log
ts=2024-04-16T15:10:42.766841898Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*045e328e-c799-47f0-80ed-193ee19a7618/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i1212\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org12\", plan=\"free\", pod=\"org12-grafana-54444b9976-ttcfk\", pod_template_hash=\"54444b9976\", resource_version=\"167262602\", slug=\"org12\", stackId=\"121212\"}"
ts=2024-04-16T15:10:42.749268494Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hg-plugins/0.log to appear..."
ts=2024-04-16T15:10:42.749193363Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hg-plugins/0.log ..."
ts=2024-04-16T15:10:42.749456785Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:42.728605579Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-65c44c9895-6zvtx_a12648d7-6689-4c79-9831-c366cce8d104/hosted-grafana-security/0.log to appear..."
ts=2024-04-16T15:10:42.728528002Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-65c44c9895-6zvtx_a12648d7-6689-4c79-9831-c366cce8d104/hosted-grafana-security/0.log ..."
ts=2024-04-16T15:10:42.73359549Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-03_graphite-querier-644cb88c5-k6mbl_bba9f1fd-60ff-432d-a788-19cd2e04af92/querier/0.log op=CREATE
ts=2024-04-16T15:10:42.733215247Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-03_graphite-querier-644cb88c5-k6mbl_bba9f1fd-60ff-432d-a788-19cd2e04af92/querier/0.log.20240416-151042 op=CREATE
ts=2024-04-16T15:10:42.728105968Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-03_graphite-querier-644cb88c5-k6mbl_bba9f1fd-60ff-432d-a788-19cd2e04af92/querier/0.log.20240416-151012.gz op=CREATE
ts=2024-04-16T15:10:42.729086881Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_query-frontend-9b95769c4-88ptz_c65aaf2a-b17d-43d7-ae47-592a90086d0e/query-frontend/2.log.20240416-150912.tmp op=CREATE
ts=2024-04-16T15:10:42.72344954Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-65c44c9895-6zvtx_a12648d7-6689-4c79-9831-c366cce8d104/grafana/0.log to appear..."
ts=2024-04-16T15:10:42.723192811Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-65c44c9895-6zvtx_a12648d7-6689-4c79-9831-c366cce8d104/grafana/0.log ..."
ts=2024-04-16T15:10:42.723512942Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod-011_querier-burst-67f76b84bf-jp5r2_0fb2bd0f-7fd3-42a1-a2bc-f4b1173dca16/querier/0.log op=CREATE
ts=2024-04-16T15:10:42.722517505Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod-011_querier-burst-67f76b84bf-jp5r2_0fb2bd0f-7fd3-42a1-a2bc-f4b1173dca16/querier/0.log.20240416-151042 op=CREATE
ts=2024-04-16T15:10:42.72053387Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod3_parallel-querier-burst-68d978b6bd-tqtt7_53706fa4-7ae2-4000-9905-5c2289ffcb80/parallel-querier/0.log op=CREATE
ts=2024-04-16T15:10:42.720198477Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod3_parallel-querier-burst-68d978b6bd-tqtt7_53706fa4-7ae2-4000-9905-5c2289ffcb80/parallel-querier/0.log.20240416-151042 op=CREATE
ts=2024-04-16T15:10:42.719530886Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod-011_querier-burst-67f76b84bf-jp5r2_0fb2bd0f-7fd3-42a1-a2bc-f4b1173dca16/querier/0.log.20240416-150752.gz op=CREATE
ts=2024-04-16T15:10:42.718029955Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod3_parallel-querier-burst-68d978b6bd-tqtt7_53706fa4-7ae2-4000-9905-5c2289ffcb80/parallel-querier/0.log.20240416-145001.gz op=CREATE
ts=2024-04-16T15:10:42.712336239Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-d4f89f5f4-wvb7x_2c16ac56-b4fb-42db-b091-5feaafc58aae/hosted-grafana-security/0.log to appear..."
ts=2024-04-16T15:10:42.712265048Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-d4f89f5f4-wvb7x_2c16ac56-b4fb-42db-b091-5feaafc58aae/hosted-grafana-security/0.log ..."
ts=2024-04-16T15:10:42.716568736Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:42.716271215Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:42.713422105Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-7b5df5b97b-chz7b_c41f2190-4e4f-4361-b8ff-56c9a1e1bcf9/grafana/0.log to appear..."
ts=2024-04-16T15:10:42.713328587Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-7b5df5b97b-chz7b_c41f2190-4e4f-4361-b8ff-56c9a1e1bcf9/hosted-grafana-security/0.log to appear..."
ts=2024-04-16T15:10:42.713234075Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7b5df5b97b-chz7b_c41f2190-4e4f-4361-b8ff-56c9a1e1bcf9/hosted-grafana-security/0.log ..."
ts=2024-04-16T15:10:42.713174528Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7b5df5b97b-chz7b_c41f2190-4e4f-4361-b8ff-56c9a1e1bcf9/grafana/0.log ..."
ts=2024-04-16T15:10:42.71291811Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:42.711689109Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:42.710581512Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:42.70678986Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:42.703742148Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:42.702399352Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hosted-grafana-security/0.log to appear..."
ts=2024-04-16T15:10:42.702151739Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hosted-grafana-security/0.log ..."
ts=2024-04-16T15:10:42.699127208Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/grafana/0.log to appear..."
ts=2024-04-16T15:10:42.699053206Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/grafana/0.log ..."
ts=2024-04-16T15:10:42.692820048Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*32bb6165-f27e-4160-8068-54f665f5b240/grafana/*.log
ts=2024-04-16T15:10:42.692785128Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*32bb6165-f27e-4160-8068-54f665f5b240/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i1313\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org13\", plan=\"free\", pod=\"org13-grafana-5c8b54fd6c-9lp4b\", pod_template_hash=\"5c8b54fd6c\", resource_version=\"143617487\", slug=\"org13\", stackId=\"131313\"}"
ts=2024-04-16T15:10:42.692719938Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*32bb6165-f27e-4160-8068-54f665f5b240/hosted-grafana-security/*.log
ts=2024-04-16T15:10:42.692680177Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*32bb6165-f27e-4160-8068-54f665f5b240/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i1313\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org13\", plan=\"free\", pod=\"org13-grafana-5c8b54fd6c-9lp4b\", pod_template_hash=\"5c8b54fd6c\", resource_version=\"143617487\", slug=\"org13\", stackId=\"131313\"}"
ts=2024-04-16T15:10:42.692596936Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*32bb6165-f27e-4160-8068-54f665f5b240/hgrun/*.log
ts=2024-04-16T15:10:42.692534796Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*32bb6165-f27e-4160-8068-54f665f5b240/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i1313\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org13\", plan=\"free\", pod=\"org13-grafana-5c8b54fd6c-9lp4b\", pod_template_hash=\"5c8b54fd6c\", resource_version=\"143617487\", slug=\"org13\", stackId=\"131313\"}"
ts=2024-04-16T15:10:42.692373454Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*32bb6165-f27e-4160-8068-54f665f5b240/hg-plugins/*.log
ts=2024-04-16T15:10:42.692271523Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*32bb6165-f27e-4160-8068-54f665f5b240/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i1313\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org13\", plan=\"free\", pod=\"org13-grafana-5c8b54fd6c-9lp4b\", pod_template_hash=\"5c8b54fd6c\", resource_version=\"143617487\", slug=\"org13\", stackId=\"131313\"}"
ts=2024-04-16T15:10:42.691652875Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-7b5df5b97b-chz7b_c41f2190-4e4f-4361-b8ff-56c9a1e1bcf9/hg-plugins/0.log to appear..."
ts=2024-04-16T15:10:42.691549847Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7b5df5b97b-chz7b_c41f2190-4e4f-4361-b8ff-56c9a1e1bcf9/hg-plugins/0.log ..."
ts=2024-04-16T15:10:42.689356411Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hgrun/0.log to appear..."
ts=2024-04-16T15:10:42.68929355Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hgrun/0.log ..."
ts=2024-04-16T15:10:42.681178579Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hg-plugins/0.log to appear..."
ts=2024-04-16T15:10:42.681073898Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hg-plugins/0.log ..."
ts=2024-04-16T15:10:42.684055816Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/loki-prod3_distributor-76c559cb69-9s6gv_e844bf19-fdc1-4b1a-8eaa-59338915688c/distributor/0.log"
ts=2024-04-16T15:10:42.683990983Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/loki-prod3_distributor-76c559cb69-9s6gv_e844bf19-fdc1-4b1a-8eaa-59338915688c/distributor/0.log ..."
ts=2024-04-16T15:10:42.68284694Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:42.680210253Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hgrun/0.log
ts=2024-04-16T15:10:42.680182593Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hgrun/0.log
ts=2024-04-16T15:10:42.680157683Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hgrun/0.log
ts=2024-04-16T15:10:42.680129951Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hgrun/0.log reason=null
ts=2024-04-16T15:10:42.680039191Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hgrun/0.log
ts=2024-04-16T15:10:42.679805009Z caller=filetarget.go:326 level=info component=logs logs_config=default msg="removing directory from watcher" directory=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hgrun
ts=2024-04-16T15:10:42.677990299Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hgrun/0.log
ts=2024-04-16T15:10:42.677695535Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_parallel-querier-54566cbb6c-d5tbl_b5e4705e-f54a-44c2-8e3b-9d6417be4208/parallel-querier/0.log op=CREATE
ts=2024-04-16T15:10:42.67757253Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/cortex-prod-06_ruler-586cbc8869-f8n6l_edecd605-8103-49b9-8c74-ea64d43fff32/ruler/0.log"
ts=2024-04-16T15:10:42.677525677Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/cortex-prod-06_ruler-586cbc8869-f8n6l_edecd605-8103-49b9-8c74-ea64d43fff32/ruler/0.log ..."
ts=2024-04-16T15:10:42.677253034Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_parallel-querier-54566cbb6c-d5tbl_b5e4705e-f54a-44c2-8e3b-9d6417be4208/parallel-querier/0.log.20240416-151042 op=CREATE
ts=2024-04-16T15:10:42.673854007Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_parallel-querier-54566cbb6c-d5tbl_b5e4705e-f54a-44c2-8e3b-9d6417be4208/parallel-querier/0.log.20240416-150950.gz op=CREATE
ts=2024-04-16T15:10:42.673361577Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=32.606839ms
ts=2024-04-16T15:10:42.673347703Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=12.306545ms
ts=2024-04-16T15:10:42.673335684Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.582085ms
ts=2024-04-16T15:10:42.673321039Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.570803ms
ts=2024-04-16T15:10:42.67330676Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.500669ms
ts=2024-04-16T15:10:42.673292934Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.557649ms
ts=2024-04-16T15:10:42.673273766Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.549069ms
ts=2024-04-16T15:10:42.673260645Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.416831ms
ts=2024-04-16T15:10:42.673245436Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=4.29634ms
ts=2024-04-16T15:10:42.673229231Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=18.413586ms
ts=2024-04-16T15:10:42.67319054Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=10.120498ms
ts=2024-04-16T15:10:42.67317492Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=8.245874ms
ts=2024-04-16T15:10:42.673158204Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=10.098523ms
ts=2024-04-16T15:10:42.673140834Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=10.06335ms
ts=2024-04-16T15:10:42.673108764Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=10.051817ms
ts=2024-04-16T15:10:42.673076835Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=19.877726ms
ts=2024-04-16T15:10:42.673058533Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=21.882173ms
ts=2024-04-16T15:10:42.67304026Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=32.263441ms
ts=2024-04-16T15:10:42.673013718Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=29.986533ms
ts=2024-04-16T15:10:42.672994786Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=32.232997ms
ts=2024-04-16T15:10:42.672984226Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=32.130626ms
ts=2024-04-16T15:10:42.672971725Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=32.171497ms
ts=2024-04-16T15:10:42.672959904Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=32.176985ms
ts=2024-04-16T15:10:42.672942987Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=32.084013ms
ts=2024-04-16T15:10:42.672926073Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=11.896444ms
ts=2024-04-16T15:10:42.672910999Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=3.998797ms
ts=2024-04-16T15:10:42.672894776Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=11.850326ms
ts=2024-04-16T15:10:42.672894249Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=14.171954ms
ts=2024-04-16T15:10:42.672866028Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=32.086994ms
ts=2024-04-16T15:10:42.67283067Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=36.096322ms
ts=2024-04-16T15:10:42.672814991Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=40.058304ms
ts=2024-04-16T15:10:42.670517273Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=36.032523ms
ts=2024-04-16T15:10:42.668933743Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=36.208371ms
ts=2024-04-16T15:10:42.66886626Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=36.05808ms
ts=2024-04-16T15:10:42.667280183Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hosted-grafana-security/0.log
ts=2024-04-16T15:10:42.667255583Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hosted-grafana-security/0.log
ts=2024-04-16T15:10:42.667241643Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hosted-grafana-security/0.log
ts=2024-04-16T15:10:42.667227123Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hosted-grafana-security/0.log reason=null
ts=2024-04-16T15:10:42.667185182Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hosted-grafana-security/0.log
ts=2024-04-16T15:10:42.667135012Z caller=filetarget.go:326 level=info component=logs logs_config=default msg="removing directory from watcher" directory=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hosted-grafana-security
ts=2024-04-16T15:10:42.665722097Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hosted-grafana-security/0.log
ts=2024-04-16T15:10:42.664904534Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=13.781937ms
ts=2024-04-16T15:10:42.663064079Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hg-plugins/0.log
ts=2024-04-16T15:10:42.663029038Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hg-plugins/0.log
ts=2024-04-16T15:10:42.662977628Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hg-plugins/0.log
ts=2024-04-16T15:10:42.662936316Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hg-plugins/0.log reason=null
ts=2024-04-16T15:10:42.662857596Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hg-plugins/0.log
ts=2024-04-16T15:10:42.662782495Z caller=filetarget.go:326 level=info component=logs logs_config=default msg="removing directory from watcher" directory=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hg-plugins
ts=2024-04-16T15:10:42.663057068Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.044531ms
ts=2024-04-16T15:10:42.663048272Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=32.234907ms
ts=2024-04-16T15:10:42.66304184Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=13.994973ms
ts=2024-04-16T15:10:42.663029813Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=50.032359ms
ts=2024-04-16T15:10:42.661585692Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/grafana/0.log
ts=2024-04-16T15:10:42.661559292Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/grafana/0.log
ts=2024-04-16T15:10:42.661516881Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/grafana/0.log
ts=2024-04-16T15:10:42.661496161Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/grafana/0.log reason=null
ts=2024-04-16T15:10:42.661448091Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/grafana/0.log
ts=2024-04-16T15:10:42.66137395Z caller=filetarget.go:326 level=info component=logs logs_config=default msg="removing directory from watcher" directory=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/grafana
ts=2024-04-16T15:10:42.661240579Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hg-plugins/0.log
ts=2024-04-16T15:10:42.661012758Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=40.387272ms
ts=2024-04-16T15:10:42.661004266Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=36.237323ms
ts=2024-04-16T15:10:42.660997233Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=32.218555ms
ts=2024-04-16T15:10:42.659763033Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/grafana/0.log
ts=2024-04-16T15:10:42.658807921Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=39.820002ms
ts=2024-04-16T15:10:42.658702944Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=39.742622ms
ts=2024-04-16T15:10:42.658726236Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=31.767787ms
ts=2024-04-16T15:10:42.658714221Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=35.841792ms
ts=2024-04-16T15:10:42.658714776Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=33.927979ms
ts=2024-04-16T15:10:42.658695601Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=39.81259ms
ts=2024-04-16T15:10:42.65866213Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=41.858297ms
ts=2024-04-16T15:10:42.654781676Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=41.792949ms
ts=2024-04-16T15:10:42.653169395Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=40.091185ms
ts=2024-04-16T15:10:42.651144688Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=41.288699ms
ts=2024-04-16T15:10:42.651057845Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=6.125234ms
ts=2024-04-16T15:10:42.649021734Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=41.269305ms
ts=2024-04-16T15:10:42.648955137Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=37.689603ms
ts=2024-04-16T15:10:42.644901985Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=37.188226ms
ts=2024-04-16T15:10:42.642995943Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=39.294935ms
ts=2024-04-16T15:10:42.640846468Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=37.01759ms
ts=2024-04-16T15:10:42.640837803Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=37.015617ms
ts=2024-04-16T15:10:42.640745996Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=37.018571ms
ts=2024-04-16T15:10:42.640760648Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=37.027827ms
ts=2024-04-16T15:10:42.640752425Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=40.427749ms
ts=2024-04-16T15:10:42.640753435Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=55.809823ms
ts=2024-04-16T15:10:42.640738991Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=42.238747ms
ts=2024-04-16T15:10:42.640721969Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=44.181487ms
ts=2024-04-16T15:10:42.640415221Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-65649b6fd6-4fxxr_02a8ddd6-b563-4890-a8d5-8b53ba00111b/hosted-grafana-security/0.log to appear..."
ts=2024-04-16T15:10:42.639945597Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-65649b6fd6-4fxxr_02a8ddd6-b563-4890-a8d5-8b53ba00111b/hosted-grafana-security/0.log ..."
ts=2024-04-16T15:10:42.636702915Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=42.405442ms
ts=2024-04-16T15:10:42.634457562Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=37.933939ms
ts=2024-04-16T15:10:42.632745858Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=41.592749ms
ts=2024-04-16T15:10:42.632704401Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=36.257142ms
ts=2024-04-16T15:10:42.632702581Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=41.462693ms
ts=2024-04-16T15:10:42.630777393Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=52.719537ms
ts=2024-04-16T15:10:42.630640151Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/otlp-gateway_otlp-gateway-7869b8fc46-ggpjr_d51b35fe-fa02-42cc-9587-ba9e6cf2a372/otlp-gateway/0.log"
ts=2024-04-16T15:10:42.630556426Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/otlp-gateway_otlp-gateway-7869b8fc46-ggpjr_d51b35fe-fa02-42cc-9587-ba9e6cf2a372/otlp-gateway/0.log ..."
ts=2024-04-16T15:10:42.629476701Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:42.628748536Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=41.898809ms
ts=2024-04-16T15:10:42.627663582Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod-011_querier-burst-67f76b84bf-jp5r2_0fb2bd0f-7fd3-42a1-a2bc-f4b1173dca16/querier/0.log.20240416-150752.tmp op=CREATE
ts=2024-04-16T15:10:42.613959129Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-10_ingester-zone-c-116_4655218c-833b-460a-879b-c5ccc57b70c4/ingester/0.log.20240416-144237.tmp op=CREATE
ts=2024-04-16T15:10:42.626926123Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=41.961295ms
ts=2024-04-16T15:10:42.624776836Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=39.828771ms
ts=2024-04-16T15:10:42.624736788Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=39.814769ms
ts=2024-04-16T15:10:42.622840931Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=37.933798ms
ts=2024-04-16T15:10:42.620593182Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=42.477123ms
ts=2024-04-16T15:10:42.620515824Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:42.618872355Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=40.789061ms
ts=2024-04-16T15:10:42.618873556Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=40.76289ms
ts=2024-04-16T15:10:42.618845099Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=35.695961ms
ts=2024-04-16T15:10:42.61677274Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=38.670924ms
ts=2024-04-16T15:10:42.613055386Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=34.959848ms
ts=2024-04-16T15:10:42.61296789Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=34.845608ms
ts=2024-04-16T15:10:42.612956175Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=34.849789ms
ts=2024-04-16T15:10:42.612326411Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-5796686bc-lwgkl_6f827ff8-26e9-43ee-b131-31037198f332/grafana/0.log to appear..."
ts=2024-04-16T15:10:42.611233748Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=33.140862ms
ts=2024-04-16T15:10:42.61085982Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-5796686bc-lwgkl_6f827ff8-26e9-43ee-b131-31037198f332/grafana/0.log ..."
ts=2024-04-16T15:10:42.609822629Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=31.748642ms
ts=2024-04-16T15:10:42.608846393Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/gr-prod-04_cortex-gw-6579fd68cf-6vkzk_69ffa46d-5698-4d3a-aa3e-a218bddab1e7/cortex-gw/0.log"
ts=2024-04-16T15:10:42.608739764Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/gr-prod-04_cortex-gw-6579fd68cf-6vkzk_69ffa46d-5698-4d3a-aa3e-a218bddab1e7/cortex-gw/0.log ..."
ts=2024-04-16T15:10:42.60772791Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=29.664548ms
ts=2024-04-16T15:10:42.607682174Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=29.589004ms
ts=2024-04-16T15:10:42.607073897Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/cortex-prod-06_ruler-586cbc8869-n7g68_7127c2b9-28a3-4111-b151-4480358a1638/ruler/0.log"
ts=2024-04-16T15:10:42.606988671Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/cortex-prod-06_ruler-586cbc8869-n7g68_7127c2b9-28a3-4111-b151-4480358a1638/ruler/0.log ..."
ts=2024-04-16T15:10:42.603802514Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=18.738931ms
ts=2024-04-16T15:10:42.603794604Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=25.717986ms
ts=2024-04-16T15:10:42.603705733Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=25.680015ms
ts=2024-04-16T15:10:42.603664662Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=18.695855ms
ts=2024-04-16T15:10:42.603661227Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=25.564163ms
ts=2024-04-16T15:10:42.601446876Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod3_parallel-querier-burst-68d978b6bd-tqtt7_53706fa4-7ae2-4000-9905-5c2289ffcb80/parallel-querier/0.log.20240416-145001.tmp op=CREATE
ts=2024-04-16T15:10:42.600671271Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:42.600294563Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=22.235405ms
ts=2024-04-16T15:10:42.598467331Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=21.178172ms
ts=2024-04-16T15:10:42.596495024Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=19.632352ms
ts=2024-04-16T15:10:42.596487701Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=19.632253ms
ts=2024-04-16T15:10:42.596411984Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=28.429057ms
ts=2024-04-16T15:10:42.59426318Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_92276_cloudwatch_tags duration=16.22324ms
ts=2024-04-16T15:10:42.591096003Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=2.040881ms
ts=2024-04-16T15:10:42.591104598Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=13.034134ms
ts=2024-04-16T15:10:42.589029314Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=19.10715ms
ts=2024-04-16T15:10:42.586876248Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:42.586817954Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=16.887584ms
ts=2024-04-16T15:10:42.586376081Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/loki-prod3_querier-688cf9ff66-ckh64_88063351-e149-431f-806d-52987df865ca/querier/0.log"
ts=2024-04-16T15:10:42.586278511Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/loki-prod3_querier-688cf9ff66-ckh64_88063351-e149-431f-806d-52987df865ca/querier/0.log ..."
ts=2024-04-16T15:10:42.58487305Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=16.89909ms
ts=2024-04-16T15:10:42.584928018Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.98178ms
ts=2024-04-16T15:10:42.584921953Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=21.178899ms
ts=2024-04-16T15:10:42.584925445Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=16.858537ms
ts=2024-04-16T15:10:42.584913112Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.920848ms
ts=2024-04-16T15:10:42.584894716Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=21.098321ms
ts=2024-04-16T15:10:42.584867384Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=16.892285ms
ts=2024-04-16T15:10:42.584760426Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/kube-system_calico-node-nbpjs_9c0b6492-afdc-4657-b99d-a7503d2fcb00/calico-node/0.log - &{Offset:0 Whence:0}"
ts=2024-04-16T15:10:42.584727732Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/kube-system_calico-node-nbpjs_9c0b6492-afdc-4657-b99d-a7503d2fcb00/calico-node/0.log
ts=2024-04-16T15:10:42.584643207Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/kube-system_calico-node-nbpjs_9c0b6492-afdc-4657-b99d-a7503d2fcb00/calico-node
ts=2024-04-16T15:10:42.583982669Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 93 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:42.583107672Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=21.504842ms
ts=2024-04-16T15:10:42.581700829Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-5796686bc-lwgkl_6f827ff8-26e9-43ee-b131-31037198f332/hosted-grafana-security/0.log to appear..."
ts=2024-04-16T15:10:42.581418535Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-5796686bc-lwgkl_6f827ff8-26e9-43ee-b131-31037198f332/hosted-grafana-security/0.log ..."
ts=2024-04-16T15:10:42.57663139Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:42.576603054Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=15.196731ms
ts=2024-04-16T15:10:42.576588908Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=13.068655ms
ts=2024-04-16T15:10:42.576571188Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=2.277991ms
ts=2024-04-16T15:10:42.576553041Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=15.109411ms
ts=2024-04-16T15:10:42.576537264Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=4.424369ms
ts=2024-04-16T15:10:42.576491264Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=8.515501ms
ts=2024-04-16T15:10:42.576522639Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=15.178204ms
ts=2024-04-16T15:10:42.576504949Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=6.470666ms
ts=2024-04-16T15:10:42.576458896Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=21.195099ms
ts=2024-04-16T15:10:42.576470384Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=6.413532ms
ts=2024-04-16T15:10:42.576461807Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=15.098093ms
ts=2024-04-16T15:10:42.576436074Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=21.129507ms
ts=2024-04-16T15:10:42.576432522Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=35.78072ms
ts=2024-04-16T15:10:42.576423296Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=19.320399ms
ts=2024-04-16T15:10:42.576433546Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=21.127352ms
ts=2024-04-16T15:10:42.57642251Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=17.134007ms
ts=2024-04-16T15:10:42.576403643Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=24.447434ms
ts=2024-04-16T15:10:42.576332073Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=26.680906ms
ts=2024-04-16T15:10:42.576321491Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.457388ms
ts=2024-04-16T15:10:42.576310653Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=17.008867ms
ts=2024-04-16T15:10:42.57627133Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=26.554435ms
ts=2024-04-16T15:10:42.576029821Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/grafana-ruler_grafana-ruler-78bc75f5d6-p98cc_e082d5d3-bb47-435b-9c42-9304ea43d9a5/grafana-ruler/0.log.20240416-151022.tmp op=CREATE
ts=2024-04-16T15:10:42.574265315Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=33.719897ms
ts=2024-04-16T15:10:42.573530517Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-06_ruler-586cbc8869-f8n6l_edecd605-8103-49b9-8c74-ea64d43fff32/ruler/0.log op=CREATE
ts=2024-04-16T15:10:42.573182782Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-06_ruler-586cbc8869-f8n6l_edecd605-8103-49b9-8c74-ea64d43fff32/ruler/0.log.20240416-151042 op=CREATE
ts=2024-04-16T15:10:42.572085566Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=22.417183ms
ts=2024-04-16T15:10:42.565828672Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/hg-plugins/0.log
ts=2024-04-16T15:10:42.565798661Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/hg-plugins/0.log - &{Offset:0 Whence:0}"
ts=2024-04-16T15:10:42.565656449Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/hg-plugins
ts=2024-04-16T15:10:42.564661119Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/grafana/0.log
ts=2024-04-16T15:10:42.564618808Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/grafana/0.log - &{Offset:0 Whence:0}"
ts=2024-04-16T15:10:42.564392566Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/hosted-grafana-security/0.log
ts=2024-04-16T15:10:42.564308205Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/hosted-grafana-security/0.log - &{Offset:0 Whence:0}"
ts=2024-04-16T15:10:42.564046973Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/hosted-grafana-security
ts=2024-04-16T15:10:42.571355512Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"
ts=2024-04-16T15:10:42.569999783Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=29.446773ms
ts=2024-04-16T15:10:42.569966615Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=22.473028ms
ts=2024-04-16T15:10:42.569895328Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.619424ms
ts=2024-04-16T15:10:42.569919882Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=29.276926ms
ts=2024-04-16T15:10:42.569907592Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=29.29022ms
ts=2024-04-16T15:10:42.569892451Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=29.350764ms
ts=2024-04-16T15:10:42.569343595Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-06_ruler-586cbc8869-f8n6l_edecd605-8103-49b9-8c74-ea64d43fff32/ruler/0.log.20240416-151002.gz op=CREATE
ts=2024-04-16T15:10:42.56856052Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:42.567938235Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=27.39652ms
ts=2024-04-16T15:10:42.568026014Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'"
ts=2024-04-16T15:10:42.567957297Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=27.431728ms
ts=2024-04-16T15:10:42.567953817Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=25.298132ms
ts=2024-04-16T15:10:42.567957895Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=27.448889ms
ts=2024-04-16T15:10:42.56793769Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=29.70719ms
ts=2024-04-16T15:10:42.563669017Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=51.443269ms
ts=2024-04-16T15:10:42.563592311Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=40.019732ms
ts=2024-04-16T15:10:42.563619118Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/hgrun/0.log
ts=2024-04-16T15:10:42.563576238Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/hgrun/0.log - &{Offset:0 Whence:0}"
ts=2024-04-16T15:10:42.563404315Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/grafana
ts=2024-04-16T15:10:42.56347333Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=35.794868ms
ts=2024-04-16T15:10:42.56199166Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/hgrun
ts=2024-04-16T15:10:42.561385592Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=31.672584ms
ts=2024-04-16T15:10:42.561572233Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=37.96656ms
ts=2024-04-16T15:10:42.561408375Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=44.74634ms
ts=2024-04-16T15:10:42.561376386Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=25.453761ms
ts=2024-04-16T15:10:42.561342238Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=29.564864ms
ts=2024-04-16T15:10:42.561317318Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=31.684575ms
ts=2024-04-16T15:10:42.557102408Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*b92ee988-5c26-4c64-bba3-ff6a01723759/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i1111\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"orgnamehere\", plan=\"free\", pod=\"orgnamehere-grafana-7c65678f86-9zhlb\", pod_template_hash=\"7c65678f86\", resource_version=\"143638246\", slug=\"orgnamehere\", stackId=\"866772\"}"
ts=2024-04-16T15:10:42.556930066Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*b92ee988-5c26-4c64-bba3-ff6a01723759/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i1111\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"orgnamehere\", plan=\"free\", pod=\"orgnamehere-grafana-7c65678f86-9zhlb\", pod_template_hash=\"7c65678f86\", resource_version=\"143638246\", slug=\"orgnamehere\", stackId=\"866772\"}"
ts=2024-04-16T15:10:42.556706613Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*b92ee988-5c26-4c64-bba3-ff6a01723759/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i1111\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"orgnamehere\", plan=\"free\", pod=\"orgnamehere-grafana-7c65678f86-9zhlb\", pod_template_hash=\"7c65678f86\", resource_version=\"143638246\", slug=\"orgnamehere\", stackId=\"866772\"}"
ts=2024-04-16T15:10:42.556278698Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*b92ee988-5c26-4c64-bba3-ff6a01723759/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i1111\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"orgnamehere\", plan=\"free\", pod=\"orgnamehere-grafana-7c65678f86-9zhlb\", pod_template_hash=\"7c65678f86\", resource_version=\"143638246\", slug=\"orgnamehere\", stackId=\"866772\"}"
ts=2024-04-16T15:10:42.559283249Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=33.523804ms
ts=2024-04-16T15:10:42.559263621Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=40.654994ms
ts=2024-04-16T15:10:42.557068004Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=40.414993ms
ts=2024-04-16T15:10:42.55528547Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.748545ms
ts=2024-04-16T15:10:42.555273483Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.735015ms
ts=2024-04-16T15:10:42.555218448Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=38.592855ms
ts=2024-04-16T15:10:42.555230437Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=38.545339ms