Use Adaptive Metrics to reduce cardinality

Both Grafana Alloy and OpenTelemetry Collector recommended configurations for scaling include adding an extra label collector_id to exported metrics to differentiate series generated by different instances of Grafana Alloy of OpenTelemetry Collector.

Adaptive Metrics in Grafana Cloud could be used to solve increased cardinality issues.

We recommend using Adaptive Metrics HTTP API in order to update aggregation rules. To interact with API please follow steps first. Then follow the Upload new aggregation rules instruction modifying current rules with next rules:

{
  "rules": [
    {
      "metric": "traces_spanmetrics_latency_sum",
      "drop_labels": ["collector_id"],
      "aggregations": ["sum:counter"]
    },
    {
      "metric": "traces_spanmetrics_latency_count",
      "drop_labels": ["collector_id"],
      "aggregations": ["sum:counter"]
    },
    {
      "metric": "traces_spanmetrics_latency_bucket",
      "drop_labels": ["collector_id"],
      "aggregations": ["sum:counter"]
    },
    {
      "metric": "traces_spanmetrics_calls_total",
      "drop_labels": ["collector_id"],
      "aggregations": ["sum:counter"]
    },
    {
      "metric": "traces_spanmetrics_size_total",
      "drop_labels": ["collector_id"],
      "aggregations": ["sum:counter"]
    },
    {
      "metric": "traces_service_graph_request_total",
      "drop_labels": ["collector_id"],
      "aggregations": ["sum:counter"]
    },
    {
      "metric": "traces_service_graph_request_failed_total",
      "drop_labels": ["collector_id"],
      "aggregations": ["sum:counter"]
    },
    {
      "metric": "traces_service_graph_request_server_seconds_sum",
      "drop_labels": ["collector_id"],
      "aggregations": ["sum:counter"]
    },
    {
      "metric": "traces_service_graph_request_server_seconds_count",
      "drop_labels": ["collector_id"],
      "aggregations": ["sum:counter"]
    },
    {
      "metric": "traces_service_graph_request_server_seconds_bucket",
      "drop_labels": ["collector_id"],
      "aggregations": ["sum:counter"]
    },
    {
      "metric": "traces_service_graph_request_client_seconds_bucket",
      "drop_labels": ["collector_id", "namespace", "source"],
      "aggregations": ["sum:counter"]
    },
    {
      "metric": "traces_service_graph_request_client_seconds_sum",
      "drop_labels": ["collector_id", "namespace", "source"],
      "aggregations": ["sum:counter"]
    },
    {
      "metric": "traces_service_graph_request_client_seconds_count",
      "drop_labels": ["collector_id", "namespace", "source"],
      "aggregations": ["sum:counter"]
    },
    {
      "metric": "target_info",
      "drop_labels": ["collector_id"],
      "aggregations": ["sum:counter"]
    }
  ]
}