[exporter/awsemf] Log events in a single PutLogEvents request must be in chronological order. #37762
Closed as not planned
Description
Component(s)
exporter/awsemf
What happened?
Description
After upgrading from collector version 0.108.0 to 0.119.0, we have started seeing the following errors in our collector logs:
2025-02-06T23:12:47.012Z error [email protected]/cwlog_client.go:101 cwlog_client: Error occurs in PutLogEvents, will not retry the request {"kind": "exporter", "data_type": "metrics", "name": "awsemf", "error": "InvalidParameterException: Log events in a single PutLogEvents request must be in chronological order.", "LogGroupName": "log-group-name", "LogStreamName": "log-stream-name"}
github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs.(*Client).PutLogEvents
github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/[email protected]/cwlog_client.go:101
github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs.(*logPusher).pushEventBatch
github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/[email protected]/pusher.go:264
github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs.(*logPusher).AddLogEntry
github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/[email protected]/pusher.go:239
github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter.(*emfExporter).pushMetricsData
github.com/open-telemetry/opentelemetry-collector-contrib/exporter/[email protected]/emf_exporter.go:140
go.opentelemetry.io/collector/exporter/exporterhelper.(*metricsRequest).Export
go.opentelemetry.io/collector/[email protected]/exporterhelper/metrics.go:64
go.opentelemetry.io/collector/exporter/exporterhelper/internal.(*TimeoutSender).Send
go.opentelemetry.io/collector/[email protected]/exporterhelper/internal/timeout_sender.go:53
go.opentelemetry.io/collector/exporter/exporterhelper/internal.(*obsReportSender[...]).Send
go.opentelemetry.io/collector/[email protected]/exporterhelper/internal/obs_report_sender.go:28
go.opentelemetry.io/collector/exporter/exporterhelper/internal.(*BaseExporter).Send
go.opentelemetry.io/collector/[email protected]/exporterhelper/internal/base_exporter.go:129
go.opentelemetry.io/collector/exporter/exporterhelper.NewMetricsRequest.func1
go.opentelemetry.io/collector/[email protected]/exporterhelper/metrics.go:142
go.opentelemetry.io/collector/consumer.ConsumeMetricsFunc.ConsumeMetrics
go.opentelemetry.io/collector/[email protected]/metrics.go:26
go.opentelemetry.io/collector/consumer.ConsumeMetricsFunc.ConsumeMetrics
go.opentelemetry.io/collector/[email protected]/metrics.go:26
go.opentelemetry.io/collector/receiver/otlpreceiver/internal/metrics.(*Receiver).Export
go.opentelemetry.io/collector/receiver/[email protected]/internal/metrics/otlp.go:41
go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp.rawMetricsServer.Export
go.opentelemetry.io/collector/[email protected]/pmetric/pmetricotlp/grpc.go:88
go.opentelemetry.io/collector/pdata/internal/data/protogen/collector/metrics/v1._MetricsService_Export_Handler.func1
go.opentelemetry.io/collector/[email protected]/internal/data/protogen/collector/metrics/v1/metrics_service.pb.go:311
go.opentelemetry.io/collector/config/configgrpc.(*ServerConfig).getGrpcServerOptions.enhanceWithClientInformation.func9
go.opentelemetry.io/collector/config/[email protected]/configgrpc.go:514
go.opentelemetry.io/collector/pdata/internal/data/protogen/collector/metrics/v1._MetricsService_Export_Handler
go.opentelemetry.io/collector/[email protected]/internal/data/protogen/collector/metrics/v1/metrics_service.pb.go:313
google.golang.org/grpc.(*Server).processUnaryRPC
google.golang.org/[email protected]/server.go:1400
google.golang.org/grpc.(*Server).handleStream
google.golang.org/[email protected]/server.go:1810
google.golang.org/grpc.(*Server).serveStreams.func2.1
google.golang.org/[email protected]/server.go:1030
2025-02-06T23:12:47.012Z error internal/base_exporter.go:131 Exporting failed. Rejecting data. {"kind": "exporter", "data_type": "metrics", "name": "awsemf", "error": "Permanent error: InvalidParameterException: Log events in a single PutLogEvents request must be in chronological order.", "rejected_items": 5214}
go.opentelemetry.io/collector/exporter/exporterhelper/internal.(*BaseExporter).Send
go.opentelemetry.io/collector/[email protected]/exporterhelper/internal/base_exporter.go:131
go.opentelemetry.io/collector/exporter/exporterhelper.NewMetricsRequest.func1
go.opentelemetry.io/collector/[email protected]/exporterhelper/metrics.go:142
go.opentelemetry.io/collector/consumer.ConsumeMetricsFunc.ConsumeMetrics
go.opentelemetry.io/collector/[email protected]/metrics.go:26
go.opentelemetry.io/collector/consumer.ConsumeMetricsFunc.ConsumeMetrics
go.opentelemetry.io/collector/[email protected]/metrics.go:26
go.opentelemetry.io/collector/receiver/otlpreceiver/internal/metrics.(*Receiver).Export
go.opentelemetry.io/collector/receiver/[email protected]/internal/metrics/otlp.go:41
go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp.rawMetricsServer.Export
go.opentelemetry.io/collector/[email protected]/pmetric/pmetricotlp/grpc.go:88
go.opentelemetry.io/collector/pdata/internal/data/protogen/collector/metrics/v1._MetricsService_Export_Handler.func1
go.opentelemetry.io/collector/[email protected]/internal/data/protogen/collector/metrics/v1/metrics_service.pb.go:311
go.opentelemetry.io/collector/config/configgrpc.(*ServerConfig).getGrpcServerOptions.enhanceWithClientInformation.func9
go.opentelemetry.io/collector/config/[email protected]/configgrpc.go:514
go.opentelemetry.io/collector/pdata/internal/data/protogen/collector/metrics/v1._MetricsService_Export_Handler
go.opentelemetry.io/collector/[email protected]/internal/data/protogen/collector/metrics/v1/metrics_service.pb.go:313
google.golang.org/grpc.(*Server).processUnaryRPC
google.golang.org/[email protected]/server.go:1400
google.golang.org/grpc.(*Server).handleStream
google.golang.org/[email protected]/server.go:1810
google.golang.org/grpc.(*Server).serveStreams.func2.1
google.golang.org/[email protected]/server.go:1030
Our collector configuration:
extensions:
health_check:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
processors:
batch:
exporters:
awsemf:
log_group_name: "log-group-name"
log_stream_name: "log-stream-name"
region: "us-west-2"
endpoint: "logs.us-west-2.amazonaws.com"
log_retention: 7
loadbalancing:
[...]
service:
pipelines:
traces:
receivers: [otlp]
exporters: [loadbalancing]
metrics:
receivers: [otlp]
exporters: [awsemf]
Steps to Reproduce
Using docker image from ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib, upgrade from version 0.108.
Expected Result
No errors in collector logs
Actual Result
InvalidParameterException errors in collector logs
Collector version
0.119.0
Environment information
Environment
OS: Alpine
Compiler(if manually compiled): (e.g., "go 14.2")
Docker Image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.119.0
OpenTelemetry Collector configuration
extensions:
health_check:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
processors:
batch:
exporters:
awsemf:
log_group_name: "log-group-name"
log_stream_name: "log-stream-name"
region: "us-west-2"
endpoint: "logs.us-west-2.amazonaws.com"
log_retention: 7
loadbalancing:
[...]
service:
pipelines:
traces:
receivers: [otlp]
exporters: [loadbalancing]
metrics:
receivers: [otlp]
exporters: [awsemf]Log output
2025-02-06T23:12:47.012Z error [email protected]/cwlog_client.go:101 cwlog_client: Error occurs in PutLogEvents, will not retry the request {"kind": "exporter", "data_type": "metrics", "name": "awsemf", "error": "InvalidParameterException: Log events in a single PutLogEvents request must be in chronological order.", "LogGroupName": "log-group-name", "LogStreamName": "log-stream-name"}
github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs.(*Client).PutLogEvents
github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/[email protected]/cwlog_client.go:101
github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs.(*logPusher).pushEventBatch
github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/[email protected]/pusher.go:264
github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs.(*logPusher).AddLogEntry
github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/[email protected]/pusher.go:239
github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter.(*emfExporter).pushMetricsData
github.com/open-telemetry/opentelemetry-collector-contrib/exporter/[email protected]/emf_exporter.go:140
go.opentelemetry.io/collector/exporter/exporterhelper.(*metricsRequest).Export
go.opentelemetry.io/collector/[email protected]/exporterhelper/metrics.go:64
go.opentelemetry.io/collector/exporter/exporterhelper/internal.(*TimeoutSender).Send
go.opentelemetry.io/collector/[email protected]/exporterhelper/internal/timeout_sender.go:53
go.opentelemetry.io/collector/exporter/exporterhelper/internal.(*obsReportSender[...]).Send
go.opentelemetry.io/collector/[email protected]/exporterhelper/internal/obs_report_sender.go:28
go.opentelemetry.io/collector/exporter/exporterhelper/internal.(*BaseExporter).Send
go.opentelemetry.io/collector/[email protected]/exporterhelper/internal/base_exporter.go:129
go.opentelemetry.io/collector/exporter/exporterhelper.NewMetricsRequest.func1
go.opentelemetry.io/collector/[email protected]/exporterhelper/metrics.go:142
go.opentelemetry.io/collector/consumer.ConsumeMetricsFunc.ConsumeMetrics
go.opentelemetry.io/collector/[email protected]/metrics.go:26
go.opentelemetry.io/collector/consumer.ConsumeMetricsFunc.ConsumeMetrics
go.opentelemetry.io/collector/[email protected]/metrics.go:26
go.opentelemetry.io/collector/receiver/otlpreceiver/internal/metrics.(*Receiver).Export
go.opentelemetry.io/collector/receiver/[email protected]/internal/metrics/otlp.go:41
go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp.rawMetricsServer.Export
go.opentelemetry.io/collector/[email protected]/pmetric/pmetricotlp/grpc.go:88
go.opentelemetry.io/collector/pdata/internal/data/protogen/collector/metrics/v1._MetricsService_Export_Handler.func1
go.opentelemetry.io/collector/[email protected]/internal/data/protogen/collector/metrics/v1/metrics_service.pb.go:311
go.opentelemetry.io/collector/config/configgrpc.(*ServerConfig).getGrpcServerOptions.enhanceWithClientInformation.func9
go.opentelemetry.io/collector/config/[email protected]/configgrpc.go:514
go.opentelemetry.io/collector/pdata/internal/data/protogen/collector/metrics/v1._MetricsService_Export_Handler
go.opentelemetry.io/collector/[email protected]/internal/data/protogen/collector/metrics/v1/metrics_service.pb.go:313
google.golang.org/grpc.(*Server).processUnaryRPC
google.golang.org/[email protected]/server.go:1400
google.golang.org/grpc.(*Server).handleStream
google.golang.org/[email protected]/server.go:1810
google.golang.org/grpc.(*Server).serveStreams.func2.1
google.golang.org/[email protected]/server.go:1030
2025-02-06T23:12:47.012Z error internal/base_exporter.go:131 Exporting failed. Rejecting data. {"kind": "exporter", "data_type": "metrics", "name": "awsemf", "error": "Permanent error: InvalidParameterException: Log events in a single PutLogEvents request must be in chronological order.", "rejected_items": 5214}
go.opentelemetry.io/collector/exporter/exporterhelper/internal.(*BaseExporter).Send
go.opentelemetry.io/collector/[email protected]/exporterhelper/internal/base_exporter.go:131
go.opentelemetry.io/collector/exporter/exporterhelper.NewMetricsRequest.func1
go.opentelemetry.io/collector/[email protected]/exporterhelper/metrics.go:142
go.opentelemetry.io/collector/consumer.ConsumeMetricsFunc.ConsumeMetrics
go.opentelemetry.io/collector/[email protected]/metrics.go:26
go.opentelemetry.io/collector/consumer.ConsumeMetricsFunc.ConsumeMetrics
go.opentelemetry.io/collector/[email protected]/metrics.go:26
go.opentelemetry.io/collector/receiver/otlpreceiver/internal/metrics.(*Receiver).Export
go.opentelemetry.io/collector/receiver/[email protected]/internal/metrics/otlp.go:41
go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp.rawMetricsServer.Export
go.opentelemetry.io/collector/[email protected]/pmetric/pmetricotlp/grpc.go:88
go.opentelemetry.io/collector/pdata/internal/data/protogen/collector/metrics/v1._MetricsService_Export_Handler.func1
go.opentelemetry.io/collector/[email protected]/internal/data/protogen/collector/metrics/v1/metrics_service.pb.go:311
go.opentelemetry.io/collector/config/configgrpc.(*ServerConfig).getGrpcServerOptions.enhanceWithClientInformation.func9
go.opentelemetry.io/collector/config/[email protected]/configgrpc.go:514
go.opentelemetry.io/collector/pdata/internal/data/protogen/collector/metrics/v1._MetricsService_Export_Handler
go.opentelemetry.io/collector/[email protected]/internal/data/protogen/collector/metrics/v1/metrics_service.pb.go:313
google.golang.org/grpc.(*Server).processUnaryRPC
google.golang.org/[email protected]/server.go:1400
google.golang.org/grpc.(*Server).handleStream
google.golang.org/[email protected]/server.go:1810
google.golang.org/grpc.(*Server).serveStreams.func2.1
google.golang.org/[email protected]/server.go:1030Additional context
No response